lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m..@apache.org
Subject svn commit: r1171968 [1/2] - in /lucene/dev/branches/branch_3x: lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/ lucene/src/java/org/apache/lucene/search/ solr/ solr/core/src/java/org/apache/solr/handler/component/ solr/core/src/java...
Date Sat, 17 Sep 2011 12:46:08 GMT
Author: mvg
Date: Sat Sep 17 12:46:06 2011
New Revision: 1171968

URL: http://svn.apache.org/viewvc?rev=1171968&view=rev
Log:
SOLR-2066: Added support for distributed grouping.

Added:
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/Command.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/GroupingSpecification.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/collector/
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardRequestFactory.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardResponseProcessor.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommandResult.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/SearchGroupsFieldCommand.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/TopGroupsFieldCommand.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/SearchGroupsRequestFactory.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/StoredFieldsShardRequestFactory.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/TopGroupsShardRequestFactory.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/StoredFieldsShardResponseProcessor.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/ShardResultTransformer.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/EndResultTransformer.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java
    lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
Removed:
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetHitCollector.java
Modified:
    lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java
    lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/TopDocs.java
    lucene/dev/branches/branch_3x/solr/CHANGES.txt
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/QueryUtils.java
    lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
    lucene/dev/branches/branch_3x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java

Modified: lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java Sat Sep 17 12:46:06 2011
@@ -17,13 +17,13 @@ package org.apache.lucene.search.groupin
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.*;
-
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 
+import java.io.IOException;
+import java.util.*;
+
 /**
  * Represents a group that is found during the first pass search.
  *
@@ -46,6 +46,29 @@ public class SearchGroup<GROUP_VALUE_TYP
     return("SearchGroup(groupValue=" + groupValue + " sortValues=" + Arrays.toString(sortValues) + ")");
   }
 
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    SearchGroup that = (SearchGroup) o;
+
+    if (groupValue == null) {
+      if (that.groupValue != null) {
+        return false;
+      }
+    } else if (!groupValue.equals(that.groupValue)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    return groupValue != null ? groupValue.hashCode() : 0;
+  }
+
   private static class ShardIter<T> {
     public final Iterator<SearchGroup<T>> iter;
     public final int shardIndex;

Modified: lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TopGroups.java Sat Sep 17 12:46:06 2011
@@ -17,13 +17,13 @@ package org.apache.lucene.search.groupin
  * limitations under the License.
  */
 
-import java.io.IOException;
-
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TopDocs;
 
+import java.io.IOException;
+
 /** Represents result returned by a grouping search.
  *
  * @lucene.experimental */
@@ -70,8 +70,13 @@ public class TopGroups<GROUP_VALUE_TYPE>
    *  same groupSort and docSort, and the top groups passed
    *  to all second-pass collectors must be the same.
    *
-   * <b>NOTE</b>: this cannot merge totalGroupCount; ie the
-   * returned TopGroups will have null totalGroupCount.
+   * <b>NOTE</b>: We can't always compute an exact totalGroupCount.
+   * Documents belonging to a group may occur on more than
+   * one shard and thus the merged totalGroupCount can be
+   * higher than the actual totalGroupCount. In this case the
+   * totalGroupCount represents a upper bound. If the documents
+   * of one group do only reside in one shard then the
+   * totalGroupCount is exact.
    *
    * <b>NOTE</b>: the topDocs in each GroupDocs is actually
    * an instance of TopDocsAndShards
@@ -87,6 +92,8 @@ public class TopGroups<GROUP_VALUE_TYPE>
 
     int totalHitCount = 0;
     int totalGroupedHitCount = 0;
+    // Optionally merge the totalGroupCount.
+    Integer totalGroupCount = null;
 
     final int numGroups = shardGroups[0].groups.length;
     for(TopGroups<T> shard : shardGroups) {
@@ -95,6 +102,13 @@ public class TopGroups<GROUP_VALUE_TYPE>
       }
       totalHitCount += shard.totalHitCount;
       totalGroupedHitCount += shard.totalGroupedHitCount;
+      if (shard.totalGroupCount != null) {
+        if (totalGroupCount == null) {
+          totalGroupCount = 0;
+        }
+
+        totalGroupCount += shard.totalGroupCount;
+      }
     }
 
     @SuppressWarnings("unchecked")
@@ -156,10 +170,19 @@ public class TopGroups<GROUP_VALUE_TYPE>
                                                    shardGroups[0].groups[groupIDX].groupSortValues);
     }
 
-    return new TopGroups<T>(groupSort.getSort(),
-                            docSort == null ? null : docSort.getSort(),
-                            totalHitCount,
-                            totalGroupedHitCount,
-                            mergedGroupDocs);
+    if (totalGroupCount != null) {
+      TopGroups<T> result = new TopGroups<T>(groupSort.getSort(),
+                              docSort == null ? null : docSort.getSort(),
+                              totalHitCount,
+                              totalGroupedHitCount,
+                              mergedGroupDocs);
+      return new TopGroups<T>(result, totalGroupCount);
+    } else {
+      return new TopGroups<T>(groupSort.getSort(),
+                              docSort == null ? null : docSort.getSort(),
+                              totalHitCount,
+                              totalGroupedHitCount,
+                              mergedGroupDocs);
+    }
   }
 }

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java Sat Sep 17 12:46:06 2011
@@ -178,7 +178,17 @@ public abstract class FieldComparator<T>
    *  if your values may sometimes be null */
   @SuppressWarnings("unchecked")
   public int compareValues(T first, T second) {
-    return ((Comparable<T>) first).compareTo(second);
+    if (first == null) {
+      if (second == null) {
+        return 0;
+      } else {
+        return -1;
+      }
+    } else if (second == null) {
+      return 1;
+    } else {
+      return ((Comparable<T>) first).compareTo(second);
+    }
   }
 
   /** Parses field's values as byte (using {@link

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/TopDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/TopDocs.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/TopDocs.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/TopDocs.java Sat Sep 17 12:46:06 2011
@@ -232,11 +232,8 @@ public class TopDocs implements java.io.
       assert queue.size() > 0;
       ShardRef ref = queue.pop();
       final ScoreDoc hit = shardHits[ref.shardIndex].scoreDocs[ref.hitIndex++];
-      if (sort == null) {
-        hits[hitUpto] = new ScoreDoc(hit.doc, hit.score, ref.shardIndex);
-      } else {
-        hits[hitUpto] = new FieldDoc(hit.doc, hit.score, ((FieldDoc) hit).fields, ref.shardIndex);
-      }
+      hit.shardIndex = ref.shardIndex;
+      hits[hitUpto] = hit;
 
       //System.out.println("  hitUpto=" + hitUpto);
       //System.out.println("    doc=" + hits[hitUpto].doc + " score=" + hits[hitUpto].score);

Modified: lucene/dev/branches/branch_3x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/CHANGES.txt?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/CHANGES.txt Sat Sep 17 12:46:06 2011
@@ -36,6 +36,9 @@ New Features
   can be specified with a name in solrconfig.xml, and use hl.boundaryScanner=name
   parameter to specify the named <boundaryScanner/>. (koji)
 
+* SOLR-2066: Added support for distributed grouping.
+  (Martijn van Groningen, Jasper van Veghel, Matt Beaumont)
+
 
 Bug Fixes
 ----------------------

Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Sat Sep 17 12:46:06 2011
@@ -20,10 +20,11 @@ package org.apache.solr.handler.componen
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.*;
+import org.apache.lucene.search.grouping.GroupDocs;
+import org.apache.lucene.search.grouping.SearchGroup;
+import org.apache.lucene.search.grouping.TopGroups;
+import org.apache.lucene.util.BytesRef;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
@@ -35,6 +36,25 @@ import org.apache.solr.response.SolrQuer
 import org.apache.solr.schema.FieldType;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.*;
+import org.apache.solr.search.grouping.CommandHandler;
+import org.apache.solr.search.grouping.GroupingSpecification;
+import org.apache.solr.search.grouping.distributed.ShardRequestFactory;
+import org.apache.solr.search.grouping.distributed.ShardResponseProcessor;
+import org.apache.solr.search.grouping.distributed.command.QueryCommand;
+import org.apache.solr.search.grouping.distributed.command.SearchGroupsFieldCommand;
+import org.apache.solr.search.grouping.distributed.command.TopGroupsFieldCommand;
+import org.apache.solr.search.grouping.distributed.requestfactory.SearchGroupsRequestFactory;
+import org.apache.solr.search.grouping.distributed.requestfactory.StoredFieldsShardRequestFactory;
+import org.apache.solr.search.grouping.distributed.requestfactory.TopGroupsShardRequestFactory;
+import org.apache.solr.search.grouping.distributed.responseprocessor.SearchGroupShardResponseProcessor;
+import org.apache.solr.search.grouping.distributed.responseprocessor.StoredFieldsShardResponseProcessor;
+import org.apache.solr.search.grouping.distributed.responseprocessor.TopGroupsShardResponseProcessor;
+import org.apache.solr.search.grouping.distributed.shardresultserializer.SearchGroupsResultTransformer;
+import org.apache.solr.search.grouping.distributed.shardresultserializer.TopGroupsResultTransformer;
+import org.apache.solr.search.grouping.endresulttransformer.EndResultTransformer;
+import org.apache.solr.search.grouping.endresulttransformer.GroupedEndResultTransformer;
+import org.apache.solr.search.grouping.endresulttransformer.MainEndResultTransformer;
+import org.apache.solr.search.grouping.endresulttransformer.SimpleEndResultTransformer;
 import org.apache.solr.util.SolrPluginUtils;
 
 import java.io.IOException;
@@ -119,6 +139,46 @@ public class QueryComponent extends Sear
     if(shards_start != null) {
       rb.shards_start = Integer.parseInt(shards_start);
     }
+
+    boolean grouping = params.getBool(GroupParams.GROUP, false);
+    if (!grouping) {
+      return;
+    }
+    SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
+    SolrIndexSearcher searcher = rb.req.getSearcher();
+    GroupingSpecification groupingSpec = new GroupingSpecification();
+    rb.setGroupingSpec(groupingSpec);
+
+    //TODO: move weighting of sort
+    Sort groupSort = searcher.weightSort(cmd.getSort());
+    // groupSort defaults to sort
+    String groupSortStr = params.get(GroupParams.GROUP_SORT);
+    if (groupSort == null) {
+      groupSort = new Sort();
+    }
+    //TODO: move weighting of sort
+    Sort sortWithinGroup = groupSortStr == null ?  groupSort : searcher.weightSort(QueryParsing.parseSort(groupSortStr, req));
+    groupingSpec.setSortWithinGroup(sortWithinGroup);
+    groupingSpec.setGroupSort(groupSort);
+
+    String formatStr = params.get(GroupParams.GROUP_FORMAT, Grouping.Format.grouped.name());
+    Grouping.Format responseFormat;
+    try {
+       responseFormat = Grouping.Format.valueOf(formatStr);
+    } catch (IllegalArgumentException e) {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.format("Illegal %s parameter", GroupParams.GROUP_FORMAT));
+    }
+    groupingSpec.setResponseFormat(responseFormat);
+
+    groupingSpec.setFields(params.getParams(GroupParams.GROUP_FIELD));
+    groupingSpec.setQueries(params.getParams(GroupParams.GROUP_QUERY));
+    groupingSpec.setGroupOffset(params.getInt(GroupParams.GROUP_OFFSET, 0));
+    groupingSpec.setGroupLimit(params.getInt(GroupParams.GROUP_LIMIT, 1));
+    groupingSpec.setOffset(rb.getSortSpec().getOffset());
+    groupingSpec.setLimit(rb.getSortSpec().getCount());
+    groupingSpec.setIncludeGroupCount(params.getBool(GroupParams.GROUP_TOTAL_COUNT, false));
+    groupingSpec.setMain(params.getBool(GroupParams.GROUP_MAIN, false));
+    groupingSpec.setNeedScore((cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0);
   }
 
   /**
@@ -181,75 +241,131 @@ public class QueryComponent extends Sear
     //
     // grouping / field collapsing
     //
-    boolean doGroup = params.getBool(GroupParams.GROUP, false);
-    if (doGroup) {
+    GroupingSpecification groupingSpec = rb.getGroupingSpec();
+    if (groupingSpec != null) {
       try {
-        int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
-        boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
-        String[] fields = params.getParams(GroupParams.GROUP_FIELD);
-        String[] queries = params.getParams(GroupParams.GROUP_QUERY);
-        String groupSortStr = params.get(GroupParams.GROUP_SORT);
-        boolean main = params.getBool(GroupParams.GROUP_MAIN, false);
-        boolean truncateGroups = params.getBool(GroupParams.GROUP_TRUNCATE, false);
+        boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
+        if (params.getBool("group.distibuted.first", false)) {
+          CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
+              .setQueryCommand(cmd)
+              .setNeedDocSet(false) // Order matters here
+              .setSearcher(searcher);
+
+          for (String field : groupingSpec.getFields()) {
+            topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
+                .setField(searcher.getSchema().getField(field))
+                .setGroupSort(groupingSpec.getGroupSort())
+                .setTopNGroups(cmd.getOffset() + cmd.getLen())
+                .build()
+            );
+          }
 
-        String formatStr = params.get(GroupParams.GROUP_FORMAT, Grouping.Format.grouped.name());
-        Grouping.Format defaultFormat;
-        try {
-          defaultFormat = Grouping.Format.valueOf(formatStr);
-        } catch (IllegalArgumentException e) {
-          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.format("Illegal %s parameter", GroupParams.GROUP_FORMAT));
-        }
+          CommandHandler commandHandler = topsGroupsActionBuilder.build();
+          commandHandler.execute();
+          SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
+          rsp.add("firstPhase", commandHandler.processResult(result, serializer));
+          rb.setResult(result);
+          return;
+        } else if (params.getBool("group.distibuted.second", false)) {
+          CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
+              .setQueryCommand(cmd)
+              .setSearcher(searcher);
+
+          for (String field : groupingSpec.getFields()) {
+            String[] topGroupsParam = params.getParams("group.topgroups." + field);
+            if (topGroupsParam == null) {
+              continue;
+            }
 
-        boolean includeTotalGroupCount = params.getBool(GroupParams.GROUP_TOTAL_COUNT, false);
-        Grouping.TotalCount defaultTotalCount = includeTotalGroupCount ? Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
-        Sort sort = searcher.weightSort(cmd.getSort());
-        // groupSort defaults to sort
-        Sort groupSort = groupSortStr == null ?  sort : searcher.weightSort(QueryParsing.parseSort(groupSortStr, req));
+            List<SearchGroup<String>> topGroups = new ArrayList<SearchGroup<String>>(topGroupsParam.length);
+            for (String topGroup : topGroupsParam) {
+              SearchGroup<String> searchGroup = new SearchGroup<String>();
+              if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
+                searchGroup.groupValue = searcher.getSchema().getField(field).getType().readableToIndexed(topGroup);
+              }
+              topGroups.add(searchGroup);
+            }
 
-        int limitDefault = cmd.getLen(); // this is normally from "rows"
-        int groupOffsetDefault = params.getInt(GroupParams.GROUP_OFFSET, 0);
-        int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);
+            secondPhaseBuilder.addCommandField(
+                new TopGroupsFieldCommand.Builder()
+                    .setField(searcher.getSchema().getField(field))
+                    .setGroupSort(groupingSpec.getGroupSort())
+                    .setSortWithinGroup(groupingSpec.getSortWithinGroup())
+                    .setFirstPhaseGroups(topGroups)
+                    .setMaxDocPerGroup(groupingSpec.getGroupOffset() + groupingSpec.getGroupLimit())
+                    .setNeedScores(needScores)
+                    .setNeedMaxScore(needScores)
+                    .setNeedGroupCount(groupingSpec.isIncludeGroupCount())
+                    .build()
+            );
+          }
+
+          for (String query : groupingSpec.getQueries()) {
+            secondPhaseBuilder.addCommandField(new QueryCommand.Builder()
+                .setDocsToCollect(groupingSpec.getOffset() + groupingSpec.getLimit())
+                .setSort(groupingSpec.getGroupSort())
+                .setQuery(query, rb.req)
+                .setDocSet(searcher)
+                .build()
+            );
+          }
+
+          CommandHandler commandHandler = secondPhaseBuilder.build();
+          commandHandler.execute();
+          TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
+          rsp.add("secondPhase", commandHandler.processResult(result, serializer));
+          rb.setResult(result);
+          return;
+        }
 
-        Grouping grouping = new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, main);
-        grouping.setSort(sort)
-            .setGroupSort(groupSort)
-            .setDefaultFormat(defaultFormat)
+        int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
+        boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
+        boolean truncateGroups = params.getBool(GroupParams.GROUP_TRUNCATE, false);
+        Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
+            Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
+        int limitDefault = cmd.getLen(); // this is normally from "rows"
+        Grouping grouping =
+            new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
+        grouping.setSort(groupingSpec.getGroupSort())
+            .setGroupSort(groupingSpec.getSortWithinGroup())
+            .setDefaultFormat(groupingSpec.getResponseFormat())
             .setLimitDefault(limitDefault)
             .setDefaultTotalCount(defaultTotalCount)
-            .setDocsPerGroupDefault(docsPerGroupDefault)
-            .setGroupOffsetDefault(groupOffsetDefault)
+            .setDocsPerGroupDefault(groupingSpec.getGroupLimit())
+            .setGroupOffsetDefault(groupingSpec.getGroupOffset())
             .setGetGroupedDocSet(truncateGroups);
 
-        if (fields != null) {
-          for (String field : fields) {
+        if (groupingSpec.getFields() != null) {
+          for (String field : groupingSpec.getFields()) {
             grouping.addFieldCommand(field, rb.req);
           }
         }
 
-        if (queries != null) {
-          for (String groupByStr : queries) {
+        if (groupingSpec.getQueries() != null) {
+          for (String groupByStr : groupingSpec.getQueries()) {
             grouping.addQueryCommand(groupByStr, rb.req);
           }
         }
 
-        if (rb.doHighlights || rb.isDebug()) {
+        if (rb.doHighlights || rb.isDebug() || params.getBool(MoreLikeThisParams.MLT, false)) {
           // we need a single list of the returned docs
           cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
         }
 
         grouping.execute();
-        rb.setResult( result );
         if (grouping.isSignalCacheWarning()) {
           rsp.add(
               "cacheWarning",
               String.format("Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
           );
         }
-        rsp.add("grouped", result.groupedResults);
+        rb.setResult(result);
+
         if (grouping.mainResult != null) {
           rsp.add("response", grouping.mainResult);
           rsp.getToLog().add("hits", grouping.mainResult.matches());
-        } else {
+        } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
+          rsp.add("grouped", result.groupedResults);
           rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
         }
         return;
@@ -280,7 +396,7 @@ public class QueryComponent extends Sear
     if(fsv){
       Sort sort = searcher.weightSort(rb.getSortSpec().getSort());
       SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
-      NamedList sortVals = new NamedList(); // order is important for the sort fields
+      NamedList<List> sortVals = new NamedList<List>(); // order is important for the sort fields
       Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field
 
       SolrIndexReader reader = searcher.getReader();
@@ -364,6 +480,48 @@ public class QueryComponent extends Sear
 
   @Override  
   public int distributedProcess(ResponseBuilder rb) throws IOException {
+    if (rb.grouping()) {
+      return groupedDistributedProcess(rb);
+    } else {
+      return regularDistributedProcess(rb);
+    }
+  }
+
+  private int groupedDistributedProcess(ResponseBuilder rb) {
+    int nextStage = ResponseBuilder.STAGE_DONE;
+    ShardRequestFactory shardRequestFactory = null;
+
+    if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY) {
+      nextStage = ResponseBuilder.STAGE_PARSE_QUERY;
+    } else if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) {
+      createDistributedIdf(rb);
+      nextStage = ResponseBuilder.STAGE_TOP_GROUPS;
+    } else if (rb.stage < ResponseBuilder.STAGE_TOP_GROUPS) {
+      nextStage = ResponseBuilder.STAGE_TOP_GROUPS;
+    } else if (rb.stage == ResponseBuilder.STAGE_TOP_GROUPS) {
+      shardRequestFactory = new SearchGroupsRequestFactory();
+      nextStage = ResponseBuilder.STAGE_EXECUTE_QUERY;
+    } else if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) {
+      nextStage = ResponseBuilder.STAGE_EXECUTE_QUERY;
+    } else if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
+      shardRequestFactory = new TopGroupsShardRequestFactory();
+      nextStage = ResponseBuilder.STAGE_GET_FIELDS;
+    } else if (rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+      nextStage = ResponseBuilder.STAGE_GET_FIELDS;
+    } else if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+      shardRequestFactory = new StoredFieldsShardRequestFactory();
+      nextStage = ResponseBuilder.STAGE_DONE;
+    }
+
+    if (shardRequestFactory != null) {
+      for (ShardRequest shardRequest : shardRequestFactory.constructRequest(rb)) {
+        rb.addRequest(this, shardRequest);
+      }
+    }
+    return nextStage;
+  }
+
+  private int regularDistributedProcess(ResponseBuilder rb) {
     if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY)
       return ResponseBuilder.STAGE_PARSE_QUERY;
     if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) {
@@ -383,33 +541,101 @@ public class QueryComponent extends Sear
     return ResponseBuilder.STAGE_DONE;
   }
 
-
   @Override
   public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
+    if (rb.grouping()) {
+      handleGroupedResponses(rb, sreq);
+    } else {
+      handleRegularResponses(rb, sreq);
+    }
+  }
+
+  private void handleGroupedResponses(ResponseBuilder rb, ShardRequest sreq) {
+    ShardResponseProcessor responseProcessor = null;
+    if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_GROUPS) != 0) {
+      responseProcessor = new SearchGroupShardResponseProcessor();
+    } else if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+      responseProcessor = new TopGroupsShardResponseProcessor();
+    } else if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
+      responseProcessor = new StoredFieldsShardResponseProcessor();
+    }
+
+    if (responseProcessor != null) {
+      responseProcessor.process(rb, sreq);
+    }
+  }
+
+  private void handleRegularResponses(ResponseBuilder rb, ShardRequest sreq) {
     if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
       mergeIds(rb, sreq);
     }
 
     if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
       returnFields(rb, sreq);
-      return;
     }
   }
 
   @Override
   public void finishStage(ResponseBuilder rb) {
-    if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
-      // We may not have been able to retrieve all the docs due to an
-      // index change.  Remove any null documents.
-      for (Iterator<SolrDocument> iter = rb._responseDocs.iterator(); iter.hasNext();) {
-        if (iter.next() == null) {
-          iter.remove();
-          rb._responseDocs.setNumFound(rb._responseDocs.getNumFound()-1);
-        }        
+    if (rb.stage != ResponseBuilder.STAGE_GET_FIELDS) {
+      return;
+    }
+    if (rb.grouping()) {
+      groupedFinishStage(rb);
+    } else {
+      regularFinishStage(rb);
+    }
+  }
+
+  private static final EndResultTransformer MAIN_END_RESULT_TRANSFORMER = new MainEndResultTransformer();
+  private static final EndResultTransformer SIMPLE_END_RESULT_TRANSFORMER = new SimpleEndResultTransformer();
+
+  @SuppressWarnings("unchecked")
+  private void groupedFinishStage(final ResponseBuilder rb) {
+    // To have same response as non-distributed request.
+    GroupingSpecification groupSpec = rb.getGroupingSpec();
+    if (rb.mergedTopGroups.isEmpty()) {
+      for (String field : groupSpec.getFields()) {
+        rb.mergedTopGroups.put(field, new TopGroups(null, null, 0, 0, new GroupDocs[]{}));
+      }
+      rb.resultIds = new HashMap<Object, ShardDoc>();
+    }
+
+    EndResultTransformer.SolrDocumentSource solrDocumentSource = new EndResultTransformer.SolrDocumentSource() {
+
+      public SolrDocument retrieve(ScoreDoc doc) {
+        ShardDoc solrDoc = (ShardDoc) doc;
+        return rb.retrievedDocuments.get(solrDoc.id);
       }
 
-      rb.rsp.add("response", rb._responseDocs);
+    };
+    EndResultTransformer endResultTransformer;
+    if (groupSpec.isMain()) {
+      endResultTransformer = MAIN_END_RESULT_TRANSFORMER;
+    } else if (Grouping.Format.grouped == groupSpec.getResponseFormat()) {
+      endResultTransformer = new GroupedEndResultTransformer(rb.req.getSearcher());
+    } else if (Grouping.Format.simple == groupSpec.getResponseFormat() && !groupSpec.isMain()) {
+      endResultTransformer = SIMPLE_END_RESULT_TRANSFORMER;
+    } else {
+      return;
+    }
+    Map<String, Object> combinedMap = new LinkedHashMap<String, Object>();
+    combinedMap.putAll(rb.mergedTopGroups);
+    combinedMap.putAll(rb.mergedQueryCommandResults);
+    endResultTransformer.transform(combinedMap, rb.rsp, rb.getGroupingSpec(), solrDocumentSource);
+  }
+
+  private void regularFinishStage(ResponseBuilder rb) {
+    // We may not have been able to retrieve all the docs due to an
+    // index change.  Remove any null documents.
+    for (Iterator<SolrDocument> iter = rb._responseDocs.iterator(); iter.hasNext();) {
+      if (iter.next() == null) {
+        iter.remove();
+        rb._responseDocs.setNumFound(rb._responseDocs.getNumFound()-1);
+      }
     }
+
+    rb.rsp.add("response", rb._responseDocs);
   }
 
 
@@ -547,7 +773,7 @@ public class QueryComponent extends Sear
 
       Map<Object,ShardDoc> resultIds = new HashMap<Object,ShardDoc>();
       for (int i=resultSize-1; i>=0; i--) {
-        ShardDoc shardDoc = (ShardDoc)queue.pop();
+        ShardDoc shardDoc = queue.pop();
         shardDoc.positionInResponse = i;
         // Need the toString() for correlation with other lists that must
         // be strings (like keys in highlighting, explain, etc)

Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java Sat Sep 17 12:46:06 2011
@@ -18,6 +18,10 @@
 package org.apache.solr.handler.component;
 
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.grouping.SearchGroup;
+import org.apache.lucene.search.grouping.TopGroups;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.RTimer;
@@ -28,7 +32,11 @@ import org.apache.solr.search.DocListAnd
 import org.apache.solr.search.QParser;
 import org.apache.solr.search.SortSpec;
 import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.grouping.GroupingSpecification;
+import org.apache.solr.search.grouping.distributed.command.QueryCommandResult;
 
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -57,6 +65,7 @@ public class ResponseBuilder
   private Query query = null;
   private List<Query> filters = null;
   private SortSpec sortSpec = null;
+  private GroupingSpecification groupingSpec;
 
   private DocListAndSet results = null;
   private NamedList<Object> debugInfo = null;
@@ -87,6 +96,7 @@ public class ResponseBuilder
 
   public static int STAGE_START           = 0;
   public static int STAGE_PARSE_QUERY     = 1000;
+  public static int STAGE_TOP_GROUPS = 1500;
   public static int STAGE_EXECUTE_QUERY   = 2000;
   public static int STAGE_GET_FIELDS      = 3000;
   public static int STAGE_DONE            = Integer.MAX_VALUE;
@@ -122,7 +132,7 @@ public class ResponseBuilder
 
   public GlobalCollectionStat globalCollectionStat;
 
-  Map<Object, ShardDoc> resultIds;
+  public Map<Object, ShardDoc> resultIds;
   // Maps uniqueKeyValue to ShardDoc, which may be used to
   // determine order of the doc or uniqueKey in the final
   // returned sequence.
@@ -135,6 +145,13 @@ public class ResponseBuilder
   StatsInfo _statsInfo;
   TermsComponent.TermsHelper _termsHelper;
 
+  // Context fields for grouping
+  public final Map<String, Collection<SearchGroup<String>>> mergedSearchGroups = new HashMap<String, Collection<SearchGroup<String>>>();
+  public final Map<String, Map<SearchGroup<String>, String>> searchGroupToShard = new HashMap<String, Map<SearchGroup<String>, String>>();
+  public final Map<String, TopGroups<String>> mergedTopGroups = new HashMap<String, TopGroups<String>>();
+  public final Map<String, QueryCommandResult> mergedQueryCommandResults = new HashMap<String, QueryCommandResult>();
+  public final Map<Object, SolrDocument> retrievedDocuments = new HashMap<Object, SolrDocument>();
+
   /**
    * Utility function to add debugging info.  This will make sure a valid
    * debugInfo exists before adding to it.
@@ -246,6 +263,18 @@ public class ResponseBuilder
     this.sortSpec = sort;
   }
 
+  public GroupingSpecification getGroupingSpec() {
+    return groupingSpec;
+  }
+
+  public void setGroupingSpec(GroupingSpecification groupingSpec) {
+    this.groupingSpec = groupingSpec;
+  }
+
+  public boolean grouping() {
+    return groupingSpec != null;
+  }
+
   public RTimer getTimer() {
     return timer;
   }

Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java Sat Sep 17 12:46:06 2011
@@ -16,19 +16,20 @@
  */
 package org.apache.solr.handler.component;
 
-import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.PriorityQueue;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.search.MissingStringLastComparatorSource;
 
 import java.text.Collator;
+import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.Locale;
 import java.util.List;
-import java.util.ArrayList;
+import java.util.Locale;
 
-public class ShardDoc {
+public class ShardDoc extends FieldDoc {
   public String shard;
   public String shardAddress;  // TODO
   
@@ -37,7 +38,7 @@ public class ShardDoc {
     // to short-circuit comparisons if the shard is equal, and can
     // also be used to break ties within the same shard.
 
-  Object id;
+  public Object id;
     // this is currently the uniqueKeyField but
     // may be replaced with internal docid in a future release.
 
@@ -53,9 +54,36 @@ public class ShardDoc {
   // but we shouldn't expose uniqueKey (have a map by it) until the stored-field
   // retrieval stage.
 
-  int positionInResponse;
+  public int positionInResponse;
   // the ordinal position in the merged response arraylist  
 
+  public ShardDoc(float score, Object[] fields, Object uniqueId, String shard) {
+      super(-1, score, fields);
+      this.id = uniqueId;
+      this.shard = shard;
+  }
+
+  public ShardDoc() {
+    super(-1, Float.NaN);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    ShardDoc shardDoc = (ShardDoc) o;
+
+    if (id != null ? !id.equals(shardDoc.id) : shardDoc.id != null) return false;
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    return id != null ? id.hashCode() : 0;
+  }
+
   @Override
   public String toString(){
     return "id="+id
@@ -70,7 +98,7 @@ public class ShardDoc {
 
 
 // used by distributed search to merge results.
-class ShardFieldSortedHitQueue extends PriorityQueue {
+class ShardFieldSortedHitQueue extends PriorityQueue<ShardDoc> {
 
   /** Stores a comparator corresponding to each field being sorted by */
   protected Comparator[] comparators;
@@ -112,10 +140,7 @@ class ShardFieldSortedHitQueue extends P
   }
 
   @Override
-  protected boolean lessThan(Object objA, Object objB) {
-    ShardDoc docA = (ShardDoc)objA;
-    ShardDoc docB = (ShardDoc)objB;
-
+  protected boolean lessThan(ShardDoc docA, ShardDoc docB) {
     // If these docs are from the same shard, then the relative order
     // is how they appeared in the response from that shard.    
     if (docA.shard == docB.shard) {
@@ -286,4 +311,4 @@ class ShardFieldSortedHitQueue extends P
     };
   }
 
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java Sat Sep 17 12:46:06 2011
@@ -16,11 +16,11 @@
  */
 package org.apache.solr.handler.component;
 
+import org.apache.solr.common.params.ModifiableSolrParams;
+
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.solr.common.params.ModifiableSolrParams;
-
 
 // todo... when finalized make accessors
 public class ShardRequest {
@@ -37,6 +37,7 @@ public class ShardRequest {
   public final static int PURPOSE_GET_DEBUG       =0x100;
   public final static int PURPOSE_GET_STATS       =0x200;
   public final static int PURPOSE_GET_TERMS       =0x400;
+  public final static int PURPOSE_GET_TOP_GROUPS  =0x800;
 
   public int purpose;  // the purpose of this request
 

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetCollector.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,95 @@
+package org.apache.solr.search;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.util.OpenBitSet;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+
+public class DocSetCollector extends Collector {
+  int pos=0;
+  OpenBitSet bits;
+  final int maxDoc;
+  final int smallSetSize;
+  int base;
+
+  // in case there aren't that many hits, we may not want a very sparse
+  // bit array.  Optimistically collect the first few docs in an array
+  // in case there are only a few.
+  final int[] scratch;
+
+  public DocSetCollector(int smallSetSize, int maxDoc) {
+    this.smallSetSize = smallSetSize;
+    this.maxDoc = maxDoc;
+    this.scratch = new int[smallSetSize];
+  }
+
+  @Override
+  public void collect(int doc) throws IOException {
+    doc += base;
+    // optimistically collect the first docs in an array
+    // in case the total number will be small enough to represent
+    // as a small set like SortedIntDocSet instead...
+    // Storing in this array will be quicker to convert
+    // than scanning through a potentially huge bit vector.
+    // FUTURE: when search methods all start returning docs in order, maybe
+    // we could have a ListDocSet() and use the collected array directly.
+    if (pos < scratch.length) {
+      scratch[pos]=doc;
+    } else {
+      // this conditional could be removed if BitSet was preallocated, but that
+      // would take up more memory, and add more GC time...
+      if (bits==null) bits = new OpenBitSet(maxDoc);
+      bits.fastSet(doc);
+    }
+
+    pos++;
+  }
+
+  public DocSet getDocSet() {
+    if (pos<=scratch.length) {
+      // assumes docs were collected in sorted order!
+      return new SortedIntDocSet(scratch, pos);
+    } else {
+      // set the bits for ids that were collected in the array
+      for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
+      return new BitDocSet(bits,pos);
+    }
+  }
+
+  @Override
+  public void setScorer(Scorer scorer) throws IOException {
+  }
+
+  @Override
+  public void setNextReader(IndexReader indexReader, int docBase) throws IOException {
+    this.base = docBase;
+  }
+
+  @Override
+  public boolean acceptsDocsOutOfOrder() {
+    return false;
+  }
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,84 @@
+package org.apache.solr.search;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.util.OpenBitSet;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class DocSetDelegateCollector extends DocSetCollector {
+  final Collector collector;
+
+  public DocSetDelegateCollector(int smallSetSize, int maxDoc, Collector collector) {
+    super(smallSetSize, maxDoc);
+    this.collector = collector;
+  }
+
+  @Override
+  public void collect(int doc) throws IOException {
+    collector.collect(doc);
+
+    doc += base;
+    // optimistically collect the first docs in an array
+    // in case the total number will be small enough to represent
+    // as a small set like SortedIntDocSet instead...
+    // Storing in this array will be quicker to convert
+    // than scanning through a potentially huge bit vector.
+    // FUTURE: when search methods all start returning docs in order, maybe
+    // we could have a ListDocSet() and use the collected array directly.
+    if (pos < scratch.length) {
+      scratch[pos]=doc;
+    } else {
+      // this conditional could be removed if BitSet was preallocated, but that
+      // would take up more memory, and add more GC time...
+      if (bits==null) bits = new OpenBitSet(maxDoc);
+      bits.fastSet(doc);
+    }
+
+    pos++;
+  }
+
+  @Override
+  public DocSet getDocSet() {
+    if (pos<=scratch.length) {
+      // assumes docs were collected in sorted order!
+      return new SortedIntDocSet(scratch, pos);
+    } else {
+      // set the bits for ids that were collected in the array
+      for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
+      return new BitDocSet(bits,pos);
+    }
+  }
+
+  @Override
+  public void setScorer(Scorer scorer) throws IOException {
+    collector.setScorer(scorer);
+  }
+
+  @Override
+  public void setNextReader(IndexReader indexReader, int docBase) throws IOException {
+    collector.setNextReader(indexReader, docBase);
+    this.base = docBase;
+  }
+}

Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java Sat Sep 17 12:46:06 2011
@@ -19,7 +19,6 @@ package org.apache.solr.search;
 
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.*;
 import org.apache.lucene.search.grouping.*;
@@ -35,6 +34,7 @@ import org.apache.solr.schema.StrFieldSo
 import org.apache.solr.search.function.OrdFieldSource;
 import org.apache.solr.search.function.ReverseOrdFieldSource;
 import org.apache.solr.search.function.ValueSource;
+import org.apache.solr.search.grouping.collector.FilterCollector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -774,7 +774,7 @@ public class Grouping {
      * {@inheritDoc}
      */
     protected void finish() throws IOException {
-      TopDocsCollector topDocsCollector = (TopDocsCollector) collector.collector;
+      TopDocsCollector topDocsCollector = (TopDocsCollector) collector.getDelegate();
       TopDocs topDocs = topDocsCollector.topDocs();
       GroupDocs<String> groupDocs = new GroupDocs<String>(topDocs.getMaxScore(), topDocs.totalHits, topDocs.scoreDocs, query.toString(), null);
       if (main) {
@@ -789,43 +789,7 @@ public class Grouping {
      * {@inheritDoc}
      */
     public int getMatches() {
-      return collector.matches;
-    }
-  }
-
-  /**
-   * A collector that filters incoming doc ids that are not in the filter
-   */
-  static class FilterCollector extends Collector {
-
-    final DocSet filter;
-    final Collector collector;
-    int docBase;
-    int matches;
-
-    public FilterCollector(DocSet filter, Collector collector) throws IOException {
-      this.filter = filter;
-      this.collector = collector;
-    }
-
-    public void setScorer(Scorer scorer) throws IOException {
-      collector.setScorer(scorer);
-    }
-
-    public void collect(int doc) throws IOException {
-      matches++;
-      if (filter.exists(doc + docBase)) {
-        collector.collect(doc);
-      }
-    }
-
-    public void setNextReader(IndexReader reader, int docBase) throws IOException {
-      this.docBase = docBase;
-      collector.setNextReader(reader, docBase);
-    }
-
-    public boolean acceptsDocsOutOfOrder() {
-      return collector.acceptsDocsOutOfOrder();
+      return collector.getMatches();
     }
   }
 

Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java Sat Sep 17 12:46:06 2011
@@ -104,6 +104,21 @@ public class MissingStringLastComparator
     }
 
     @Override
+    public int compareValues(String first, String second) {
+      if (first == null) {
+        if (second == null) {
+          return 0;
+        } else {
+          return 1;
+        }
+      } else if (second == null) {
+        return -1;
+      } else {
+        return first.compareTo(second);
+      }
+    }
+
+    @Override
     public int compareBottom(int doc) {
       assert bottomSlot != -1;
       int order = this.order[doc];

Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/QueryUtils.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/QueryUtils.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/QueryUtils.java Sat Sep 17 12:46:06 2011
@@ -17,10 +17,10 @@
 
 package org.apache.solr.search;
 
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
 
 import java.util.List;
 
@@ -30,7 +30,7 @@ import java.util.List;
 public class QueryUtils {
 
   /** return true if this query has no positive components */
-  static boolean isNegative(Query q) {
+  public static boolean isNegative(Query q) {
     if (!(q instanceof BooleanQuery)) return false;
     BooleanQuery bq = (BooleanQuery)q;
     List<BooleanClause> clauses = bq.clauses();
@@ -51,7 +51,7 @@ public class QueryUtils {
    * @param q
    * @return
    */
-  static Query getAbs(Query q) {
+  public static Query getAbs(Query q) {
     if (q instanceof WrappedQuery) {
       Query subQ = ((WrappedQuery)q).getWrappedQuery();
       Query absSubQ = getAbs(subQ);
@@ -95,7 +95,7 @@ public class QueryUtils {
   /** Makes negative queries suitable for querying by
    * lucene.
    */
-  static Query makeQueryable(Query q) {
+  public static Query makeQueryable(Query q) {
     if (q instanceof WrappedQuery) {
       return makeQueryable(((WrappedQuery)q).getWrappedQuery());
     }
@@ -105,7 +105,7 @@ public class QueryUtils {
   /** Fixes a negative query by adding a MatchAllDocs query clause.
    * The query passed in *must* be a negative query.
    */
-  static Query fixNegativeQuery(Query q) {
+  public static Query fixNegativeQuery(Query q) {
     BooleanQuery newBq = (BooleanQuery)q.clone();
     newBq.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
     return newBq;    

Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1171968&r1=1171967&r2=1171968&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Sat Sep 17 12:46:06 2011
@@ -640,10 +640,10 @@ public class SolrIndexSearcher extends I
   private static Query matchAllDocsQuery = new MatchAllDocsQuery();
 
 
-  static class ProcessedFilter {
-    DocSet answer;  // the answer, if non-null
-    Filter filter;
-    DelegatingCollector postFilter;
+  public static class ProcessedFilter {
+    public DocSet answer;  // the answer, if non-null
+    public Filter filter;
+    public DelegatingCollector postFilter;
   }
 
 

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/Command.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/Command.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/Command.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/Command.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,42 @@
+package org.apache.solr.search.grouping;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Sort;
+import org.apache.solr.schema.SchemaField;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ *
+ */
+public interface Command<T> {
+
+  List<Collector> create() throws IOException;
+
+  T result();
+
+  String getKey();
+
+  Sort getGroupSort();
+
+  Sort getSortWithinGroup();
+
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,142 @@
+package org.apache.solr.search.grouping;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.MultiCollector;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.search.*;
+import org.apache.solr.search.grouping.distributed.shardresultserializer.ShardResultTransformer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class CommandHandler {
+
+  public static class Builder {
+
+    private SolrIndexSearcher.QueryCommand queryCommand;
+    private List<Command> commands = new ArrayList<Command>();
+    private SolrIndexSearcher searcher;
+    private boolean needDocSet = false;
+
+    public Builder setQueryCommand(SolrIndexSearcher.QueryCommand queryCommand) {
+      this.queryCommand = queryCommand;
+      this.needDocSet = (queryCommand.getFlags() & SolrIndexSearcher.GET_DOCSET) != 0;
+      return this;
+    }
+
+    public Builder addCommandField(Command commandField) {
+      commands.add(commandField);
+      return this;
+    }
+
+    public Builder setSearcher(SolrIndexSearcher searcher) {
+      this.searcher = searcher;
+      return this;
+    }
+
+    /**
+     * Sets whether to compute a {@link DocSet}.
+     * May override the value set by {@link #setQueryCommand(org.apache.solr.search.SolrIndexSearcher.QueryCommand)}.
+     *
+     * @param needDocSet Whether to compute a {@link DocSet}
+     * @return this
+     */
+    public Builder setNeedDocSet(boolean needDocSet) {
+      this.needDocSet = needDocSet;
+      return this;
+    }
+
+    public CommandHandler build() {
+      if (queryCommand == null || searcher == null) {
+        throw new IllegalStateException("All fields must be set");
+      }
+
+      return new CommandHandler(queryCommand, commands, searcher, needDocSet);
+    }
+
+  }
+
+  private final SolrIndexSearcher.QueryCommand queryCommand;
+  private final List<Command> commands;
+  private final SolrIndexSearcher searcher;
+  private final boolean needDocset;
+
+  private DocSet docSet;
+
+  private CommandHandler(SolrIndexSearcher.QueryCommand queryCommand,
+                         List<Command> commands,
+                         SolrIndexSearcher searcher,
+                         boolean needDocset) {
+    this.queryCommand = queryCommand;
+    this.commands = commands;
+    this.searcher = searcher;
+    this.needDocset = needDocset;
+  }
+
+  @SuppressWarnings("unchecked")
+  public void execute() throws IOException {
+    final int nrOfCommands = commands.size();
+    List<Collector> collectors = new ArrayList<Collector>(nrOfCommands);
+    for (Command command : commands) {
+      collectors.addAll(command.create());
+    }
+
+    SolrIndexSearcher.ProcessedFilter pf = searcher.getProcessedFilter(
+        queryCommand.getFilter(), queryCommand.getFilterList()
+    );
+    Filter luceneFilter = pf.filter;
+    Query query = QueryUtils.makeQueryable(queryCommand.getQuery());
+    Collector wrappedCollectors;
+    if (collectors.isEmpty()) {
+      wrappedCollectors = null;
+    } else {
+      wrappedCollectors = MultiCollector.wrap(collectors.toArray(new Collector[nrOfCommands]));
+    }
+
+    if (wrappedCollectors == null && needDocset) {
+      int maxDoc = searcher.maxDoc();
+      DocSetCollector docSetCollector = new DocSetCollector(maxDoc >> 6, maxDoc);
+      searcher.search(query, luceneFilter, docSetCollector);
+      docSet = docSetCollector.getDocSet();
+    } else if (needDocset) {
+      int maxDoc = searcher.maxDoc();
+      DocSetCollector docSetCollector = new DocSetDelegateCollector(maxDoc >> 6, maxDoc, wrappedCollectors);
+      searcher.search(query, luceneFilter, docSetCollector);
+      docSet = docSetCollector.getDocSet();
+    } else {
+      searcher.search(query, luceneFilter, wrappedCollectors);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  public NamedList processResult(SolrIndexSearcher.QueryResult queryResult, ShardResultTransformer transformer) throws IOException {
+    if (needDocset) {
+      queryResult.setDocSet(docSet);
+    }
+    return transformer.transform(commands);
+  }
+
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/GroupingSpecification.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/GroupingSpecification.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/GroupingSpecification.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/GroupingSpecification.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,145 @@
+package org.apache.solr.search.grouping;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.search.Sort;
+import org.apache.solr.search.Grouping;
+
+/**
+ * Encapsulates the grouping options like fields group sort and more specified by clients.
+ */
+public class GroupingSpecification {
+
+  private String[] fields = new String[]{};
+  private String[] queries = new String[]{};
+  private int offset;
+  private int limit;
+  private int groupOffset;
+  private int groupLimit;
+  private Sort groupSort;
+  private Sort sortWithinGroup;
+  private boolean includeGroupCount;
+  private boolean main;
+  private Grouping.Format responseFormat;
+  private boolean needScore;
+
+  public String[] getFields() {
+    return fields;
+  }
+
+  public void setFields(String[] fields) {
+    if (fields == null) {
+      return;
+    }
+
+    this.fields = fields;
+  }
+
+  public String[] getQueries() {
+    return queries;
+  }
+
+  public void setQueries(String[] queries) {
+    if (queries == null) {
+      return;
+    }
+
+    this.queries = queries;
+  }
+
+  public int getGroupOffset() {
+    return groupOffset;
+  }
+
+  public void setGroupOffset(int groupOffset) {
+    this.groupOffset = groupOffset;
+  }
+
+  public int getGroupLimit() {
+    return groupLimit;
+  }
+
+  public void setGroupLimit(int groupLimit) {
+    this.groupLimit = groupLimit;
+  }
+
+  public int getOffset() {
+    return offset;
+  }
+
+  public void setOffset(int offset) {
+    this.offset = offset;
+  }
+
+  public int getLimit() {
+    return limit;
+  }
+
+  public void setLimit(int limit) {
+    this.limit = limit;
+  }
+
+  public Sort getGroupSort() {
+    return groupSort;
+  }
+
+  public void setGroupSort(Sort groupSort) {
+    this.groupSort = groupSort;
+  }
+
+  public Sort getSortWithinGroup() {
+    return sortWithinGroup;
+  }
+
+  public void setSortWithinGroup(Sort sortWithinGroup) {
+    this.sortWithinGroup = sortWithinGroup;
+  }
+
+  public boolean isIncludeGroupCount() {
+    return includeGroupCount;
+  }
+
+  public void setIncludeGroupCount(boolean includeGroupCount) {
+    this.includeGroupCount = includeGroupCount;
+  }
+
+  public boolean isMain() {
+    return main;
+  }
+
+  public void setMain(boolean main) {
+    this.main = main;
+  }
+
+  public Grouping.Format getResponseFormat() {
+    return responseFormat;
+  }
+
+  public void setResponseFormat(Grouping.Format responseFormat) {
+    this.responseFormat = responseFormat;
+  }
+
+  public boolean isNeedScore() {
+    return needScore;
+  }
+
+  public void setNeedScore(boolean needScore) {
+    this.needScore = needScore;
+  }
+
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/collector/FilterCollector.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,74 @@
+package org.apache.solr.search.grouping.collector;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Scorer;
+import org.apache.solr.search.DocSet;
+
+import java.io.IOException;
+
+/**
+ * A collector that filters incoming doc ids that are not in the filter
+ */
+public class FilterCollector extends Collector {
+
+  private final DocSet filter;
+  private final Collector delegate;
+  private int docBase;
+  private int matches;
+
+  public FilterCollector(DocSet filter, Collector delegate) throws IOException {
+    this.filter = filter;
+    this.delegate = delegate;
+  }
+
+  public void setScorer(Scorer scorer) throws IOException {
+    delegate.setScorer(scorer);
+  }
+
+  public void collect(int doc) throws IOException {
+    matches++;
+    if (filter.exists(doc + docBase)) {
+      delegate.collect(doc);
+    }
+  }
+
+  public void setNextReader(IndexReader indexReader, int docBase) throws IOException {
+    this.docBase = docBase;
+    delegate.setNextReader(indexReader, docBase);
+  }
+
+  public boolean acceptsDocsOutOfOrder() {
+    return delegate.acceptsDocsOutOfOrder();
+  }
+
+  public int getMatches() {
+    return matches;
+  }
+
+  /**
+   * Returns the delegate collector
+   *
+   * @return the delegate collector
+   */
+  public Collector getDelegate() {
+    return delegate;
+  }
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardRequestFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardRequestFactory.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardRequestFactory.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardRequestFactory.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,37 @@
+package org.apache.solr.search.grouping.distributed;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.ShardRequest;
+
+/**
+ * Responsible for creating shard requests to the shards in the cluster to perform distributed grouping.
+ */
+public interface ShardRequestFactory {
+
+  /**
+   * Returns {@link ShardRequest} instances.
+   * Never returns <code>null</code>. If no {@link ShardRequest} instances are constructed an empty array is returned.
+   *
+   * @param rb The response builder
+   * @return {@link ShardRequest} instances
+   */
+  ShardRequest[] constructRequest(ResponseBuilder rb);
+
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardResponseProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardResponseProcessor.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardResponseProcessor.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/ShardResponseProcessor.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,37 @@
+package org.apache.solr.search.grouping.distributed;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.ShardRequest;
+
+/**
+ * Responsible for processing shard responses.
+ */
+public interface ShardResponseProcessor {
+
+  /**
+   * Processes the responses from the specified shardRequest. The result is put into specific
+   * fields in the specified rb.
+   *
+   * @param rb The ResponseBuilder to put the merge result into
+   * @param shardRequest The shard request containing the responses from all shards.
+   */
+  void process(ResponseBuilder rb, ShardRequest shardRequest);
+
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,152 @@
+package org.apache.solr.search.grouping.distributed.command;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.*;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.grouping.Command;
+import org.apache.solr.search.grouping.collector.FilterCollector;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ *
+ */
+public class QueryCommand implements Command<QueryCommandResult> {
+
+  public static class Builder {
+
+    private Sort sort;
+    private String queryString;
+    private Query query;
+    private DocSet docSet;
+    private Integer docsToCollect;
+    private boolean needScores;
+
+    public Builder setSort(Sort sort) {
+      this.sort = sort;
+      return this;
+    }
+
+    public Builder setQuery(Query query) {
+      this.query = query;
+      return this;
+    }
+
+    /**
+     * Sets the group query from the specified groupQueryString.
+     * The groupQueryString is parsed into a query.
+     *
+     * @param groupQueryString The group query string to parse
+     * @param request The current request
+     * @return this
+     * @throws ParseException If parsing the groupQueryString failed
+     */
+    public Builder setQuery(String groupQueryString, SolrQueryRequest request) throws ParseException {
+      QParser parser = QParser.getParser(groupQueryString, null, request);
+      this.queryString = groupQueryString;
+      return setQuery(parser.getQuery());
+    }
+
+    public Builder setDocSet(DocSet docSet) {
+      this.docSet = docSet;
+      return this;
+    }
+
+    /**
+     * Sets the docSet based on the created {@link DocSet}
+     *
+     * @param searcher The searcher executing the
+     * @return
+     * @throws IOException
+     */
+    public Builder setDocSet(SolrIndexSearcher searcher) throws IOException {
+      return setDocSet(searcher.getDocSet(query));
+    }
+
+    public Builder setDocsToCollect(int docsToCollect) {
+      this.docsToCollect = docsToCollect;
+      return this;
+    }
+
+    public Builder setNeedScores(boolean needScores) {
+      this.needScores = needScores;
+      return this;
+    }
+
+    public QueryCommand build() {
+      if (sort == null || query == null || docSet == null || docsToCollect == null) {
+        throw new IllegalStateException("All fields must be set");
+      }
+
+      return new QueryCommand(sort, query, docsToCollect, needScores, docSet, queryString);
+    }
+
+  }
+
+  private final Sort sort;
+  private final Query query;
+  private final DocSet docSet;
+  private final int docsToCollect;
+  private final boolean needScores;
+  private final String queryString;
+
+  private TopDocsCollector collector;
+  private FilterCollector filterCollector;
+
+  private QueryCommand(Sort sort, Query query, int docsToCollect, boolean needScores, DocSet docSet, String queryString) {
+    this.sort = sort;
+    this.query = query;
+    this.docsToCollect = docsToCollect;
+    this.needScores = needScores;
+    this.docSet = docSet;
+    this.queryString = queryString;
+  }
+
+  public List<Collector> create() throws IOException {
+    if (sort == null || sort == Sort.RELEVANCE) {
+      collector = TopScoreDocCollector.create(docsToCollect, true);
+    } else {
+      collector = TopFieldCollector.create(sort, docsToCollect, true, needScores, needScores, true);
+    }
+    filterCollector = new FilterCollector(docSet, collector);
+    return Arrays.asList((Collector) filterCollector);
+  }
+
+  public QueryCommandResult result() {
+    return new QueryCommandResult(collector.topDocs(), filterCollector.getMatches());
+  }
+
+  public String getKey() {
+    return queryString != null ? queryString : query.toString();
+  }
+
+  public Sort getGroupSort() {
+    return sort;
+  }
+
+  public Sort getSortWithinGroup() {
+    return null;
+  }
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommandResult.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommandResult.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommandResult.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommandResult.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,42 @@
+package org.apache.solr.search.grouping.distributed.command;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.search.TopDocs;
+
+/**
+ * Encapsulates {@link TopDocs} and the number of matches.
+ */
+public class QueryCommandResult {
+
+  private final TopDocs topDocs;
+  private final int matches;
+
+  public QueryCommandResult(TopDocs topDocs, int matches) {
+    this.topDocs = topDocs;
+    this.matches = matches;
+  }
+
+  public TopDocs getTopDocs() {
+    return topDocs;
+  }
+
+  public int getMatches() {
+    return matches;
+  }
+}

Added: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/SearchGroupsFieldCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/SearchGroupsFieldCommand.java?rev=1171968&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/SearchGroupsFieldCommand.java (added)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/SearchGroupsFieldCommand.java Sat Sep 17 12:46:06 2011
@@ -0,0 +1,100 @@
+package org.apache.solr.search.grouping.distributed.command;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.grouping.SearchGroup;
+import org.apache.lucene.search.grouping.TermFirstPassGroupingCollector;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.grouping.Command;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ *
+ */
+public class SearchGroupsFieldCommand implements Command<Collection<SearchGroup<String>>> {
+
+  public static class Builder {
+
+    private SchemaField field;
+    private Sort groupSort;
+    private Integer topNGroups;
+
+    public Builder setField(SchemaField field) {
+      this.field = field;
+      return this;
+    }
+
+    public Builder setGroupSort(Sort groupSort) {
+      this.groupSort = groupSort;
+      return this;
+    }
+
+    public Builder setTopNGroups(int topNGroups) {
+      this.topNGroups = topNGroups;
+      return this;
+    }
+
+    public SearchGroupsFieldCommand build() {
+      if (field == null || groupSort == null || topNGroups == null) {
+        throw new IllegalStateException("All fields must be set");
+      }
+
+      return new SearchGroupsFieldCommand(field, groupSort, topNGroups);
+    }
+
+  }
+
+  private final SchemaField field;
+  private final Sort groupSort;
+  private final int topNGroups;
+
+  private TermFirstPassGroupingCollector collector;
+
+  private SearchGroupsFieldCommand(SchemaField field, Sort groupSort, int topNGroups) {
+    this.field = field;
+    this.groupSort = groupSort;
+    this.topNGroups = topNGroups;
+  }
+
+  public List<Collector> create() throws IOException {
+    collector = new TermFirstPassGroupingCollector(field.getName(), groupSort, topNGroups);
+    return Arrays.asList((Collector) collector);
+  }
+
+  public Collection<SearchGroup<String>> result() {
+    return collector.getTopGroups(0, true);
+  }
+
+  public Sort getSortWithinGroup() {
+    return null;
+  }
+
+  public Sort getGroupSort() {
+    return groupSort;
+  }
+
+  public String getKey() {
+    return field.getName();
+  }
+}



Mime
View raw message