lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1212830 [2/2] - in /lucene/dev/branches/lucene3622: ./ lucene/ lucene/contrib/ lucene/src/java/org/apache/lucene/index/ lucene/src/test/org/apache/lucene/index/ solr/ solr/contrib/ solr/contrib/clustering/ solr/contrib/clustering/src/java/...
Date Sat, 10 Dec 2011 18:05:09 GMT
Modified: lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java
(original)
+++ lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java
Sat Dec 10 18:05:08 2011
@@ -15,6 +15,7 @@ package org.apache.solr.handler.clusteri
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.util.Collections;
 import java.util.List;
 
 import org.carrot2.core.Cluster;
@@ -48,6 +49,12 @@ public class EchoClusteringAlgorithm ext
   @Attribute(key = AttributeNames.CLUSTERS)
   private List<Cluster> clusters;
 
+  @Input
+  @Processing
+  @Attribute(key = "custom-fields")
+  private String customFields = "";
+
+  
   @Override
   public void process() throws ProcessingException {
     clusters = Lists.newArrayListWithCapacity(documents.size());
@@ -55,6 +62,15 @@ public class EchoClusteringAlgorithm ext
     for (Document document : documents) {
       final Cluster cluster = new Cluster();
       cluster.addPhrases(document.getTitle(), document.getSummary());
+      if (document.getLanguage() != null) {
+        cluster.addPhrases(document.getLanguage().name());
+      }
+      for (String field : customFields.split(",")) {
+        Object value = document.getField(field);
+        if (value != null) {
+          cluster.addPhrases(value.toString());
+        }
+      }
       cluster.addDocuments(document);
       clusters.add(cluster);
     }

Modified: lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java
(original)
+++ lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java
Sat Dec 10 18:05:08 2011
@@ -25,9 +25,7 @@ import org.carrot2.core.ProcessingCompon
 import org.carrot2.core.ProcessingException;
 import org.carrot2.core.attribute.AttributeNames;
 import org.carrot2.core.attribute.Processing;
-import org.carrot2.text.linguistic.DefaultLexicalDataFactory;
 import org.carrot2.text.linguistic.ILexicalData;
-import org.carrot2.text.linguistic.ILexicalDataFactory;
 import org.carrot2.text.preprocessing.pipeline.BasicPreprocessingPipeline;
 import org.carrot2.text.util.MutableCharArray;
 import org.carrot2.util.attribute.Attribute;
@@ -46,37 +44,37 @@ import com.google.common.collect.Lists;
  */
 @Bindable(prefix = "LexicalResourcesCheckClusteringAlgorithm")
 public class LexicalResourcesCheckClusteringAlgorithm extends
-		ProcessingComponentBase implements IClusteringAlgorithm {
+    ProcessingComponentBase implements IClusteringAlgorithm {
 
-	@Output
-	@Processing
-	@Attribute(key = AttributeNames.CLUSTERS)
-	private List<Cluster> clusters;
-
-	@Input
-	@Processing
-	@Attribute
-	private String wordsToCheck;
-
-	private BasicPreprocessingPipeline preprocessing = new BasicPreprocessingPipeline();
-
-	@Override
-	public void process() throws ProcessingException {
-		clusters = Lists.newArrayList();
-		if (wordsToCheck == null) {
-			return;
-		}
-
-		// Test with Maltese so that the English clustering performed in other tests
-		// is not affected by the test stopwords and stoplabels.
-		ILexicalData lexicalData = preprocessing.lexicalDataFactory
-				.getLexicalData(LanguageCode.MALTESE);
-
-		for (String word : wordsToCheck.split(",")) {
-			if (!lexicalData.isCommonWord(new MutableCharArray(word))
-					&& !lexicalData.isStopLabel(word)) {
-				clusters.add(new Cluster(word));
-			}
-		}
-	}
+  @Output
+  @Processing
+  @Attribute(key = AttributeNames.CLUSTERS)
+  private List<Cluster> clusters;
+
+  @Input
+  @Processing
+  @Attribute
+  private String wordsToCheck;
+
+  private BasicPreprocessingPipeline preprocessing = new BasicPreprocessingPipeline();
+
+  @Override
+  public void process() throws ProcessingException {
+    clusters = Lists.newArrayList();
+    if (wordsToCheck == null) {
+      return;
+    }
+
+    // Test with Maltese so that the English clustering performed in other tests
+    // is not affected by the test stopwords and stoplabels.
+    ILexicalData lexicalData = preprocessing.lexicalDataFactory
+        .getLexicalData(LanguageCode.MALTESE);
+
+    for (String word : wordsToCheck.split(",")) {
+      if (!lexicalData.isCommonWord(new MutableCharArray(word))
+          && !lexicalData.isStopLabel(word)) {
+        clusters.add(new Cluster(word));
+      }
+    }
+  }
 }

Modified: lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
(original)
+++ lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
Sat Dec 10 18:05:08 2011
@@ -17,6 +17,7 @@
 package org.apache.solr.handler.dataimport;
 
 import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.update.AddUpdateCommand;
 import org.apache.solr.update.CommitUpdateCommand;
@@ -27,8 +28,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
-import java.util.Map;
-import java.util.Set;
 
 /**
  * <p> Writes documents to SOLR. </p>
@@ -43,12 +42,14 @@ public class SolrWriter extends DIHWrite
   static final String LAST_INDEX_KEY = "last_index_time";
 
   private final UpdateRequestProcessor processor;
-
+  private final int commitWithin;
+  
   SolrQueryRequest req;
 
   public SolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
     this.processor = processor;
     this.req = req;
+    commitWithin = (req != null) ? req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1):
-1;
   }
   
   @Override
@@ -65,6 +66,7 @@ public class SolrWriter extends DIHWrite
     try {
       AddUpdateCommand command = new AddUpdateCommand(req);
       command.solrDoc = d;
+      command.commitWithin = commitWithin;
       processor.processAdd(command);
     } catch (Exception e) {
       log.warn("Error creating document : " + d, e);

Modified: lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
(original)
+++ lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
Sat Dec 10 18:05:08 2011
@@ -24,6 +24,7 @@ import org.apache.solr.client.solrj.resp
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.UpdateParams;
 
 import org.junit.After;
 import org.junit.Before;
@@ -80,6 +81,33 @@ public class TestContentStreamDataSource
     assertEquals("Hello C1", ((List)doc.getFieldValue("desc")).get(0));
   }
 
+  @Test
+  public void testCommitWithin() throws Exception {
+    DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
+    ModifiableSolrParams params = params("command", "full-import", 
+        "clean", "false", UpdateParams.COMMIT, "false", 
+        UpdateParams.COMMIT_WITHIN, "1000");
+    req.setParams(params);
+    String url = "http://localhost:" + jetty.getLocalPort() + "/solr";
+    CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(url);
+    solrServer.request(req);
+    Thread.sleep(100);
+    ModifiableSolrParams queryAll = params("q", "*");
+    QueryResponse qres = solrServer.query(queryAll);
+    SolrDocumentList results = qres.getResults();
+    assertEquals(0, results.getNumFound());
+    Thread.sleep(1000);
+    for (int i = 0; i < 10; i++) {
+      qres = solrServer.query(queryAll);
+      results = qres.getResults();
+      if (2 == results.getNumFound()) {
+        return;
+      }
+      Thread.sleep(500);
+    }
+    fail("Commit should have occured but it did not");
+  }
+  
   private class SolrInstance {
     String name;
     Integer port;

Modified: lucene/dev/branches/lucene3622/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
(original)
+++ lucene/dev/branches/lucene3622/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
Sat Dec 10 18:05:08 2011
@@ -162,13 +162,18 @@ public class QueryComponent extends Sear
 
     //TODO: move weighting of sort
     Sort groupSort = searcher.weightSort(cmd.getSort());
-    // groupSort defaults to sort
-    String groupSortStr = params.get(GroupParams.GROUP_SORT);
     if (groupSort == null) {
-      groupSort = new Sort();
+      groupSort = Sort.RELEVANCE;
     }
+
+    // groupSort defaults to sort
+    String groupSortStr = params.get(GroupParams.GROUP_SORT);
     //TODO: move weighting of sort
     Sort sortWithinGroup = groupSortStr == null ?  groupSort : searcher.weightSort(QueryParsing.parseSort(groupSortStr,
req));
+    if (sortWithinGroup == null) {
+      sortWithinGroup = Sort.RELEVANCE;
+    }
+
     groupingSpec.setSortWithinGroup(sortWithinGroup);
     groupingSpec.setGroupSort(groupSort);
 

Modified: lucene/dev/branches/lucene3622/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
(original)
+++ lucene/dev/branches/lucene3622/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
Sat Dec 10 18:05:08 2011
@@ -140,6 +140,7 @@ public class TestDistributedGrouping ext
     query("q", "*:*", "fq", s1 + ":a", "rows", 100, "fl", "id," + i1, "group", "true", "group.field",
i1, "group.limit", 10, "sort", i1 + " asc, id asc", "group.truncate", "true", "facet", "true",
"facet.field", t1);
 
     // We cannot validate distributed grouping with scoring as first sort. since there is
no global idf. We can check if no errors occur
+    simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field",
i1, "group.limit", 10, "sort", i1 + " desc", "group.sort", "score desc"); // SOLR-2955
     simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field",
i1, "group.limit", 10, "sort", "score desc, _docid_ asc, id asc");
     simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field",
i1, "group.limit", 10);
   }
@@ -149,6 +150,7 @@ public class TestDistributedGrouping ext
     for (int i = 0; i < queryParams.length; i += 2) {
       params.add(queryParams[i].toString(), queryParams[i + 1].toString());
     }
+    params.set("shards", shards);
     queryServer(params);
   }
 



Mime
View raw message