lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [32/58] [abbrv] lucenenet git commit: WIP on Grouping
Date Thu, 10 Nov 2016 11:47:46 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/Function/FunctionAllGroupHeadsCollector.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/Function/FunctionAllGroupHeadsCollector.cs b/src/Lucene.Net.Grouping/Function/FunctionAllGroupHeadsCollector.cs
new file mode 100644
index 0000000..c50219d
--- /dev/null
+++ b/src/Lucene.Net.Grouping/Function/FunctionAllGroupHeadsCollector.cs
@@ -0,0 +1,155 @@
+using Lucene.Net.Index;
+using Lucene.Net.Queries.Function;
+using Lucene.Net.Search;
+using Lucene.Net.Util.Mutable;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.Search.Grouping.Function
+{
+    /// <summary>
+    /// An implementation of <see cref="AbstractAllGroupHeadsCollector"/> for retrieving the most relevant groups when grouping
+    /// by <see cref="ValueSource"/>.
+    /// 
+    /// @lucene.experimental
+    /// </summary>
+    public class FunctionAllGroupHeadsCollector : AbstractAllGroupHeadsCollector<FunctionAllGroupHeadsCollector.GroupHead>
+    {
+        private readonly ValueSource groupBy;
+        private readonly IDictionary /* Map<?, ?> */ vsContext;
+        private readonly IDictionary<MutableValue, GroupHead> groups;
+        private readonly Sort sortWithinGroup;
+
+        private FunctionValues.AbstractValueFiller filler;
+        private MutableValue mval;
+        private AtomicReaderContext readerContext;
+        private Scorer scorer;
+
+        /**
+         * Constructs a {@link FunctionAllGroupHeadsCollector} instance.
+         *
+         * @param groupBy The {@link ValueSource} to group by
+         * @param vsContext The ValueSource context
+         * @param sortWithinGroup The sort within a group
+         */
+        public FunctionAllGroupHeadsCollector(ValueSource groupBy, IDictionary /* Map<?, ?> */ vsContext, Sort sortWithinGroup)
+            : base(sortWithinGroup.GetSort().Length)
+        {
+            groups = new Dictionary<MutableValue, GroupHead>();
+            this.sortWithinGroup = sortWithinGroup;
+            this.groupBy = groupBy;
+            this.vsContext = vsContext;
+
+            SortField[] sortFields = sortWithinGroup.GetSort();
+            for (int i = 0; i < sortFields.Length; i++)
+            {
+                reversed[i] = sortFields[i].Reverse ? -1 : 1;
+            }
+        }
+
+        protected override void RetrieveGroupHeadAndAddIfNotExist(int doc)
+        {
+            filler.FillValue(doc);
+            GroupHead groupHead;
+            if (!groups.TryGetValue(mval, out groupHead))
+            {
+                MutableValue groupValue = mval.Duplicate();
+                groupHead = new GroupHead(this, groupValue, sortWithinGroup, doc);
+                groups[groupValue] = groupHead;
+                temporalResult.stop = true;
+            }
+            else
+            {
+                temporalResult.stop = false;
+            }
+            this.temporalResult.groupHead = groupHead;
+        }
+
+        protected override ICollection<GroupHead> GetCollectedGroupHeads()
+        {
+            return groups.Values;
+        }
+
+        public override Scorer Scorer
+        {
+            set
+            {
+                this.scorer = value;
+                foreach (GroupHead groupHead in groups.Values)
+                {
+                    foreach (FieldComparator comparator in groupHead.comparators)
+                    {
+                        comparator.Scorer = value;
+                    }
+                }
+            }
+        }
+
+        public override AtomicReaderContext NextReader
+        {
+            set
+            {
+                this.readerContext = value;
+                FunctionValues values = groupBy.GetValues(vsContext, value);
+                filler = values.ValueFiller;
+                mval = filler.Value;
+
+                foreach (GroupHead groupHead in groups.Values)
+                {
+                    for (int i = 0; i < groupHead.comparators.Length; i++)
+                    {
+                        groupHead.comparators[i] = groupHead.comparators[i].SetNextReader(value);
+                    }
+                }
+            }
+        }
+
+        /** Holds current head document for a single group.
+         *
+         * @lucene.experimental */
+        public class GroupHead : AbstractGroupHead /*AbstractAllGroupHeadsCollector.GroupHead<MutableValue>*/
+        {
+
+            private readonly FunctionAllGroupHeadsCollector outerInstance;
+            public readonly MutableValue groupValue;
+            internal readonly FieldComparator[] comparators;
+
+            internal GroupHead(FunctionAllGroupHeadsCollector outerInstance, MutableValue groupValue, Sort sort, int doc)
+                        /*: base(groupValue, doc + outerInstance.readerContext.DocBase)*/
+                        : base(doc + outerInstance.readerContext.DocBase)
+            {
+                this.outerInstance = outerInstance;
+                this.groupValue = groupValue;
+
+                SortField[] sortFields = sort.GetSort();
+                comparators = new FieldComparator[sortFields.Length];
+                for (int i = 0; i < sortFields.Length; i++)
+                {
+                    comparators[i] = sortFields[i].GetComparator(1, i).SetNextReader(outerInstance.readerContext);
+                    comparators[i].Scorer = outerInstance.scorer;
+                    comparators[i].Copy(0, doc);
+                    comparators[i].Bottom = 0;
+                }
+            }
+
+            public override int Compare(int compIDX, int doc)
+            {
+                return comparators[compIDX].CompareBottom(doc);
+            }
+
+            public override void UpdateDocHead(int doc)
+            {
+                foreach (FieldComparator comparator in comparators)
+                {
+                    comparator.Copy(0, doc);
+                    comparator.Bottom = 0;
+                }
+                this.Doc = doc + outerInstance.readerContext.DocBase;
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/Function/FunctionAllGroupsCollector.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/Function/FunctionAllGroupsCollector.cs b/src/Lucene.Net.Grouping/Function/FunctionAllGroupsCollector.cs
new file mode 100644
index 0000000..638f36b
--- /dev/null
+++ b/src/Lucene.Net.Grouping/Function/FunctionAllGroupsCollector.cs
@@ -0,0 +1,69 @@
+using Lucene.Net.Index;
+using Lucene.Net.Queries.Function;
+using Lucene.Net.Util.Mutable;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Search.Grouping.Function
+{
+    /// <summary>
+    /// A collector that collects all groups that match the
+    /// query. Only the group value is collected, and the order
+    /// is undefined.  This collector does not determine
+    /// the most relevant document of a group.
+    /// 
+    /// <para>
+    /// Implementation detail: Uses <see cref="ValueSource"/> and <see cref="FunctionValues"/> to retrieve the
+    /// field values to group by.
+    /// </para>
+    /// @lucene.experimental
+    /// </summary>
+    public class FunctionAllGroupsCollector : AbstractAllGroupsCollector<MutableValue>
+    {
+        private readonly IDictionary /* Map<?, ?> */ vsContext;
+        private readonly ValueSource groupBy;
+        private readonly SortedSet<MutableValue> groups = new SortedSet<MutableValue>();
+
+        private FunctionValues.AbstractValueFiller filler;
+        private MutableValue mval;
+
+        /**
+         * Constructs a {@link FunctionAllGroupsCollector} instance.
+         *
+         * @param groupBy The {@link ValueSource} to group by
+         * @param vsContext The ValueSource context
+         */
+        public FunctionAllGroupsCollector(ValueSource groupBy, IDictionary /* Map<?, ?> */ vsContext)
+        {
+            this.vsContext = vsContext;
+            this.groupBy = groupBy;
+        }
+
+        public override ICollection<MutableValue> Groups
+        {
+            get
+            {
+                return groups;
+            }
+        }
+
+        public override void Collect(int doc)
+        {
+            filler.FillValue(doc);
+            if (!groups.Contains(mval))
+            {
+                groups.Add(mval.Duplicate());
+            }
+        }
+
+        public override AtomicReaderContext NextReader
+        {
+            set
+            {
+                FunctionValues values = groupBy.GetValues(vsContext, value);
+                filler = values.ValueFiller;
+                mval = filler.Value;
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/Function/FunctionDistinctValuesCollector.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/Function/FunctionDistinctValuesCollector.cs b/src/Lucene.Net.Grouping/Function/FunctionDistinctValuesCollector.cs
new file mode 100644
index 0000000..2e84920
--- /dev/null
+++ b/src/Lucene.Net.Grouping/Function/FunctionDistinctValuesCollector.cs
@@ -0,0 +1,85 @@
+using Lucene.Net.Queries.Function;
+using Lucene.Net.Support;
+using Lucene.Net.Util.Mutable;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Lucene.Net.Index;
+
+namespace Lucene.Net.Search.Grouping.Function
+{
+    /// <summary>
+    /// Function based implementation of <see cref="AbstractDistinctValuesCollector"/>.
+    /// 
+    /// @lucene.experimental
+    /// </summary>
+    public class FunctionDistinctValuesCollector : AbstractDistinctValuesCollector<FunctionDistinctValuesCollector.GroupCount>
+    {
+        private readonly IDictionary /* Map<?, ?> */ vsContext;
+        private readonly ValueSource groupSource;
+        private readonly ValueSource countSource;
+        private readonly IDictionary<MutableValue, GroupCount> groupMap;
+
+        private FunctionValues.AbstractValueFiller groupFiller;
+        private FunctionValues.AbstractValueFiller countFiller;
+        private MutableValue groupMval;
+        private MutableValue countMval;
+
+        public FunctionDistinctValuesCollector(IDictionary /*Map<?, ?>*/ vsContext, ValueSource groupSource, ValueSource countSource, ICollection<SearchGroup<MutableValue>> groups)
+        {
+            this.vsContext = vsContext;
+            this.groupSource = groupSource;
+            this.countSource = countSource;
+            groupMap = new LurchTable<MutableValue, GroupCount>(1 << 4);
+            foreach (SearchGroup<MutableValue> group in groups)
+            {
+                groupMap[group.groupValue] = new GroupCount(group.groupValue);
+            }
+        }
+
+        public override List<GroupCount> GetGroups()
+        {
+            return new List<GroupCount>(groupMap.Values);
+        }
+
+        public override void Collect(int doc)
+        {
+            groupFiller.FillValue(doc);
+            GroupCount groupCount;
+            if (groupMap.TryGetValue(groupMval, out groupCount))
+            {
+                countFiller.FillValue(doc);
+                groupCount.uniqueValues.Add(countMval.Duplicate());
+            }
+        }
+
+        public override AtomicReaderContext NextReader
+        {
+            set
+            {
+                FunctionValues values = groupSource.GetValues(vsContext, value);
+                groupFiller = values.ValueFiller;
+                groupMval = groupFiller.Value;
+                values = countSource.GetValues(vsContext, value);
+                countFiller = values.ValueFiller;
+                countMval = countFiller.Value;
+            }
+        }
+
+
+
+        /** Holds distinct values for a single group.
+         *
+         * @lucene.experimental */
+        public class GroupCount : AbstractGroupCount<MutableValue> /*AbstractDistinctValuesCollector.GroupCount<MutableValue>*/
+        {
+            internal GroupCount(MutableValue groupValue)
+                : base(groupValue)
+            {
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/Function/FunctionFirstPassGroupingCollector.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/Function/FunctionFirstPassGroupingCollector.cs b/src/Lucene.Net.Grouping/Function/FunctionFirstPassGroupingCollector.cs
new file mode 100644
index 0000000..18f9fb6
--- /dev/null
+++ b/src/Lucene.Net.Grouping/Function/FunctionFirstPassGroupingCollector.cs
@@ -0,0 +1,71 @@
+using Lucene.Net.Index;
+using Lucene.Net.Queries.Function;
+using Lucene.Net.Util.Mutable;
+using System.Collections;
+
+namespace Lucene.Net.Search.Grouping.Function
+{
+    /// <summary>
+    /// Concrete implementation of <see cref="AbstractFirstPassGroupingCollector{TGroupValue}"/> that groups based on
+    /// <see cref="ValueSource"/> instances.
+    /// 
+    /// @lucene.experimental
+    /// </summary>
+    public class FunctionFirstPassGroupingCollector : AbstractFirstPassGroupingCollector<MutableValue>
+    {
+        private readonly ValueSource groupByVS;
+        private readonly IDictionary /* Map<?, ?> */ vsContext;
+
+        private FunctionValues.AbstractValueFiller filler;
+        private MutableValue mval;
+
+        /// <summary>
+        /// Creates a first pass collector.
+        /// </summary>
+        /// <param name="groupByVS">The <see cref="ValueSource"/> instance to group by</param>
+        /// <param name="vsContext">The <see cref="ValueSource"/> context</param>
+        /// <param name="groupSort">
+        /// The <see cref="Sort"/> used to sort the
+        /// groups.  The top sorted document within each group
+        /// according to groupSort, determines how that group
+        /// sorts against other groups.  This must be non-null,
+        /// ie, if you want to groupSort by relevance use
+        /// <see cref="Sort.RELEVANCE"/>.
+        /// </param>
+        /// <param name="topNGroups">How many top groups to keep.</param>
+        /// <exception cref="IOException">When I/O related errors occur</exception>
+        public FunctionFirstPassGroupingCollector(ValueSource groupByVS, IDictionary /* Map<?, ?> */ vsContext, Sort groupSort, int topNGroups)
+            : base(groupSort, topNGroups)
+        {
+            this.groupByVS = groupByVS;
+            this.vsContext = vsContext;
+        }
+
+        protected override MutableValue GetDocGroupValue(int doc)
+        {
+            filler.FillValue(doc);
+            return mval;
+        }
+
+        protected override MutableValue CopyDocGroupValue(MutableValue groupValue, MutableValue reuse)
+        {
+            if (reuse != null)
+            {
+                reuse.Copy(groupValue);
+                return reuse;
+            }
+            return groupValue.Duplicate();
+        }
+
+        public override AtomicReaderContext NextReader
+        {
+            set
+            {
+                base.NextReader = value;
+                FunctionValues values = groupByVS.GetValues(vsContext, value);
+                filler = values.ValueFiller;
+                mval = filler.Value;
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/Function/FunctionSecondPassGroupingCollector.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/Function/FunctionSecondPassGroupingCollector.cs b/src/Lucene.Net.Grouping/Function/FunctionSecondPassGroupingCollector.cs
new file mode 100644
index 0000000..23f04b0
--- /dev/null
+++ b/src/Lucene.Net.Grouping/Function/FunctionSecondPassGroupingCollector.cs
@@ -0,0 +1,64 @@
+using Lucene.Net.Index;
+using Lucene.Net.Queries.Function;
+using Lucene.Net.Util.Mutable;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Search.Grouping.Function
+{
+    /// <summary>
+    /// Concrete implementation of <see cref="AbstractSecondPassGroupingCollector{TGroupValue}"/> that groups based on
+    /// <see cref="ValueSource"/> instances.
+    /// 
+    /// @lucene.experimental
+    /// </summary>
+    public class FunctionSecondPassGroupingCollector : AbstractSecondPassGroupingCollector<MutableValue>
+    {
+        private readonly ValueSource groupByVS;
+        private readonly IDictionary /* Map<?, ?> */ vsContext;
+
+        private FunctionValues.AbstractValueFiller filler;
+        private MutableValue mval;
+
+        /// <summary>
+        /// Constructs a <see cref="FunctionSecondPassGroupingCollector"/> instance.
+        /// </summary>
+        /// <param name="searchGroups">The <see cref="SearchGroup{TGroupValue}"/> instances collected during the first phase.</param>
+        /// <param name="groupSort">The group sort</param>
+        /// <param name="withinGroupSort">The sort inside a group</param>
+        /// <param name="maxDocsPerGroup">The maximum number of documents to collect inside a group</param>
+        /// <param name="getScores">Whether to include the scores</param>
+        /// <param name="getMaxScores">Whether to include the maximum score</param>
+        /// <param name="fillSortFields">Whether to fill the sort values in <see cref="TopGroups{TGroupValueType}.WithinGroupSort"/></param>
+        /// <param name="groupByVS">The <see cref="ValueSource"/> to group by</param>
+        /// <param name="vsContext">The value source context</param>
+        /// <exception cref="IOException">When I/O related errors occur</exception>
+        public FunctionSecondPassGroupingCollector(ICollection<SearchGroup<MutableValue>> searchGroups, 
+            Sort groupSort, Sort withinGroupSort, int maxDocsPerGroup, bool getScores, bool getMaxScores, 
+            bool fillSortFields, ValueSource groupByVS, IDictionary /* Map<?, ?> */ vsContext)
+            : base(searchGroups, groupSort, withinGroupSort, maxDocsPerGroup, getScores, getMaxScores, fillSortFields)
+        {
+            this.groupByVS = groupByVS;
+            this.vsContext = vsContext;
+        }
+
+        protected override AbstractSecondPassGroupingCollector.SearchGroupDocs<MutableValue> RetrieveGroup(int doc)
+        {
+            filler.FillValue(doc);
+            AbstractSecondPassGroupingCollector.SearchGroupDocs<MutableValue> result;
+            groupMap.TryGetValue(mval, out result);
+            return result;
+        }
+
+        public override AtomicReaderContext NextReader
+        {
+            set
+            {
+                base.NextReader = value;
+                FunctionValues values = groupByVS.GetValues(vsContext, value);
+                filler = values.ValueFiller;
+                mval = filler.Value;
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/GroupDocs.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/GroupDocs.cs b/src/Lucene.Net.Grouping/GroupDocs.cs
index 00cdf83..29d227e 100644
--- a/src/Lucene.Net.Grouping/GroupDocs.cs
+++ b/src/Lucene.Net.Grouping/GroupDocs.cs
@@ -1,6 +1,6 @@
 using Lucene.Net.Search;
 
-namespace Lucene.Net.Grouping
+namespace Lucene.Net.Search.Grouping
 {
     /*
 	 * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -24,13 +24,13 @@ namespace Lucene.Net.Grouping
     /// 
     /// @lucene.experimental 
     /// </summary>
-    public class GroupDocs<TGroupValueType>
+    public class GroupDocs<TGroupValue>
     {
         /// <summary>
         /// The groupField value for all docs in this group; this
         /// may be null if hits did not have the groupField. 
         /// </summary>
-        public readonly TGroupValueType GroupValue;
+        public readonly TGroupValue GroupValue;
 
         /// <summary>
         /// Max score in this group
@@ -58,7 +58,7 @@ namespace Lucene.Net.Grouping
         /// </summary>
         public readonly object[] GroupSortValues;
 
-        public GroupDocs(float score, float maxScore, int totalHits, ScoreDoc[] scoreDocs, TGroupValueType groupValue, object[] groupSortValues)
+        public GroupDocs(float score, float maxScore, int totalHits, ScoreDoc[] scoreDocs, TGroupValue groupValue, object[] groupSortValues)
         {
             Score = score;
             MaxScore = maxScore;

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/GroupingSearch.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/GroupingSearch.cs b/src/Lucene.Net.Grouping/GroupingSearch.cs
new file mode 100644
index 0000000..28b5381
--- /dev/null
+++ b/src/Lucene.Net.Grouping/GroupingSearch.cs
@@ -0,0 +1,483 @@
+using Lucene.Net.Queries.Function;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Grouping;
+using Lucene.Net.Search.Grouping.Function;
+using Lucene.Net.Search.Grouping.Terms;
+using Lucene.Net.Util;
+using Lucene.Net.Util.Mutable;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.Search.Grouping
+{
+    /// <summary>
+    /// Convenience class to perform grouping in a non distributed environment.
+    /// @lucene.experimental
+    /// </summary>
+    public class GroupingSearch
+    {
+        private readonly string groupField;
+        private readonly ValueSource groupFunction;
+        private readonly IDictionary /* Map<?, ?> */ valueSourceContext;
+        private readonly Filter groupEndDocs;
+
+        private Sort groupSort = Sort.RELEVANCE;
+        private Sort sortWithinGroup;
+
+        private int groupDocsOffset;
+        private int groupDocsLimit = 1;
+        private bool fillSortFields;
+        private bool includeScores = true;
+        private bool includeMaxScore = true;
+
+        private double? maxCacheRAMMB;
+        private int? maxDocsToCache;
+        private bool cacheScores;
+        private bool allGroups;
+        private bool allGroupHeads;
+        private int initialSize = 128;
+
+        private ICollection /* Collection<?> */ matchingGroups;
+        private Bits matchingGroupHeads;
+
+        /**
+         * Constructs a <code>GroupingSearch</code> instance that groups documents by index terms using the {@link FieldCache}.
+         * The group field can only have one token per document. This means that the field must not be analysed.
+         *
+         * @param groupField The name of the field to group by.
+         */
+        public GroupingSearch(string groupField)
+            : this(groupField, null, null, null)
+        {
+        }
+
+        /**
+         * Constructs a <code>GroupingSearch</code> instance that groups documents by function using a {@link ValueSource}
+         * instance.
+         *
+         * @param groupFunction      The function to group by specified as {@link ValueSource}
+         * @param valueSourceContext The context of the specified groupFunction
+         */
+        public GroupingSearch(ValueSource groupFunction, IDictionary /* Map<?, ?> */ valueSourceContext)
+            : this(null, groupFunction, valueSourceContext, null)
+        {
+
+        }
+
+        /**
+         * Constructor for grouping documents by doc block.
+         * This constructor can only be used when documents belonging in a group are indexed in one block.
+         *
+         * @param groupEndDocs The filter that marks the last document in all doc blocks
+         */
+        public GroupingSearch(Filter groupEndDocs)
+            : this(null, null, null, groupEndDocs)
+        {
+        }
+
+        private GroupingSearch(string groupField, ValueSource groupFunction, IDictionary /* Map<?, ?> */ valueSourceContext, Filter groupEndDocs)
+        {
+            this.groupField = groupField;
+            this.groupFunction = groupFunction;
+            this.valueSourceContext = valueSourceContext;
+            this.groupEndDocs = groupEndDocs;
+        }
+
+        /**
+         * Executes a grouped search. Both the first pass and second pass are executed on the specified searcher.
+         *
+         * @param searcher    The {@link org.apache.lucene.search.IndexSearcher} instance to execute the grouped search on.
+         * @param query       The query to execute with the grouping
+         * @param groupOffset The group offset
+         * @param groupLimit  The number of groups to return from the specified group offset
+         * @return the grouped result as a {@link TopGroups} instance
+         * @throws IOException If any I/O related errors occur
+         */
+        public TopGroups<TGroupValue> Search<TGroupValue>(IndexSearcher searcher, Query query, int groupOffset, int groupLimit)
+        {
+            return Search<TGroupValue>(searcher, null, query, groupOffset, groupLimit);
+        }
+
+        /**
+         * Executes a grouped search. Both the first pass and second pass are executed on the specified searcher.
+         *
+         * @param searcher    The {@link org.apache.lucene.search.IndexSearcher} instance to execute the grouped search on.
+         * @param filter      The filter to execute with the grouping
+         * @param query       The query to execute with the grouping
+         * @param groupOffset The group offset
+         * @param groupLimit  The number of groups to return from the specified group offset
+         * @return the grouped result as a {@link TopGroups} instance
+         * @throws IOException If any I/O related errors occur
+         */
+        public TopGroups<TGroupValue> Search<TGroupValue>(IndexSearcher searcher, Filter filter, Query query, int groupOffset, int groupLimit)
+        {
+            if (groupField != null || groupFunction != null)
+            {
+                return GroupByFieldOrFunction<TGroupValue>(searcher, filter, query, groupOffset, groupLimit);
+            }
+            else if (groupEndDocs != null)
+            {
+                return (TopGroups<TGroupValue>)GroupByDocBlock<TGroupValue>(searcher, filter, query, groupOffset, groupLimit);
+            }
+            else
+            {
+                throw new InvalidOperationException("Either groupField, groupFunction or groupEndDocs must be set."); // This can't happen...
+            }
+        }
+
+        protected TopGroups<TGroupValue> GroupByFieldOrFunction<TGroupValue>(IndexSearcher searcher, Filter filter, Query query, int groupOffset, int groupLimit)
+        {
+            // LUCENENET TODO: Finish
+            return null;
+            //int topN = groupOffset + groupLimit;
+            //AbstractFirstPassGroupingCollector<TGroupValue> firstPassCollector;
+            //AbstractAllGroupsCollector<TGroupValue> allGroupsCollector;
+            //AbstractAllGroupHeadsCollector allGroupHeadsCollector;
+            //if (groupFunction != null)
+            //{
+            //    firstPassCollector = new FunctionFirstPassGroupingCollector(groupFunction, valueSourceContext, groupSort, topN);
+            //    if (allGroups)
+            //    {
+            //        allGroupsCollector = new FunctionAllGroupsCollector(groupFunction, valueSourceContext);
+            //    }
+            //    else
+            //    {
+            //        allGroupsCollector = null;
+            //    }
+            //    if (allGroupHeads)
+            //    {
+            //        allGroupHeadsCollector = new FunctionAllGroupHeadsCollector(groupFunction, valueSourceContext, sortWithinGroup);
+            //    }
+            //    else
+            //    {
+            //        allGroupHeadsCollector = null;
+            //    }
+            //}
+            //else
+            //{
+            //    firstPassCollector = new TermFirstPassGroupingCollector(groupField, groupSort, topN);
+            //    if (allGroups)
+            //    {
+            //        allGroupsCollector = new TermAllGroupsCollector(groupField, initialSize);
+            //    }
+            //    else
+            //    {
+            //        allGroupsCollector = null;
+            //    }
+            //    if (allGroupHeads)
+            //    {
+            //        allGroupHeadsCollector = TermAllGroupHeadsCollector.Create(groupField, sortWithinGroup, initialSize);
+            //    }
+            //    else
+            //    {
+            //        allGroupHeadsCollector = null;
+            //    }
+            //}
+
+            //Collector firstRound;
+            //if (allGroupHeads || allGroups)
+            //{
+            //    List<Collector> collectors = new List<Collector>();
+            //    collectors.Add(firstPassCollector);
+            //    if (allGroups)
+            //    {
+            //        collectors.Add(allGroupsCollector);
+            //    }
+            //    if (allGroupHeads)
+            //    {
+            //        collectors.Add(allGroupHeadsCollector);
+            //    }
+            //    firstRound = MultiCollector.Wrap(collectors.ToArray(/* new Collector[collectors.size()] */));
+            //}
+            //else
+            //{
+            //    firstRound = firstPassCollector;
+            //}
+
+            //CachingCollector cachedCollector = null;
+            //if (maxCacheRAMMB != null || maxDocsToCache != null)
+            //{
+            //    if (maxCacheRAMMB != null)
+            //    {
+            //        cachedCollector = CachingCollector.Create(firstRound, cacheScores, maxCacheRAMMB.Value);
+            //    }
+            //    else
+            //    {
+            //        cachedCollector = CachingCollector.Create(firstRound, cacheScores, maxDocsToCache.Value);
+            //    }
+            //    searcher.Search(query, filter, cachedCollector);
+            //}
+            //else
+            //{
+            //    searcher.Search(query, filter, firstRound);
+            //}
+
+            //if (allGroups)
+            //{
+            //    matchingGroups = (ICollection)allGroupsCollector.GetGroups();
+            //}
+            //else
+            //{
+            //    matchingGroups = (ICollection)Collections.EmptyList<TGroupValue>();
+            //}
+            //if (allGroupHeads)
+            //{
+            //    matchingGroupHeads = allGroupHeadsCollector.RetrieveGroupHeads(searcher.IndexReader.MaxDoc);
+            //}
+            //else
+            //{
+            //    matchingGroupHeads = new Bits_MatchNoBits(searcher.IndexReader.MaxDoc);
+            //}
+
+            //ICollection<SearchGroup<TGroupValue>> topSearchGroups = firstPassCollector.GetTopGroups(groupOffset, fillSortFields);
+            //if (topSearchGroups == null)
+            //{
+            //    return new TopGroups<TGroupValue>(new SortField[0], new SortField[0], 0, 0, new GroupDocs<TGroupValue>[0], float.NaN);
+            //}
+
+            //int topNInsideGroup = groupDocsOffset + groupDocsLimit;
+            //AbstractSecondPassGroupingCollector<TGroupValue> secondPassCollector;
+            //if (groupFunction != null)
+            //{
+            //    secondPassCollector = new FunctionSecondPassGroupingCollector(topSearchGroups as ICollection<SearchGroup<MutableValue>>, groupSort, sortWithinGroup, topNInsideGroup, includeScores, includeMaxScore, fillSortFields, groupFunction, valueSourceContext);
+            //}
+            //else
+            //{
+            //    secondPassCollector = new TermSecondPassGroupingCollector(groupField, topSearchGroups as ICollection<SearchGroup<BytesRef>>, groupSort, sortWithinGroup, topNInsideGroup, includeScores, includeMaxScore, fillSortFields);
+            //}
+
+            //if (cachedCollector != null && cachedCollector.Cached)
+            //{
+            //    cachedCollector.Replay(secondPassCollector);
+            //}
+            //else
+            //{
+            //    searcher.Search(query, filter, secondPassCollector);
+            //}
+
+            //if (allGroups)
+            //{
+            //    return new TopGroups<TGroupValue>(secondPassCollector.GetTopGroups(groupDocsOffset), matchingGroups.Count);
+            //}
+            //else
+            //{
+            //    return secondPassCollector.GetTopGroups(groupDocsOffset);
+            //}
+        }
+
+        protected TopGroups<T> GroupByDocBlock<T>(IndexSearcher searcher, Filter filter, Query query, int groupOffset, int groupLimit)
+        {
+            int topN = groupOffset + groupLimit;
+            BlockGroupingCollector c = new BlockGroupingCollector(groupSort, topN, includeScores, groupEndDocs);
+            searcher.Search(query, filter, c);
+            int topNInsideGroup = groupDocsOffset + groupDocsLimit;
+            return c.GetTopGroups<T>(sortWithinGroup, groupOffset, groupDocsOffset, topNInsideGroup, fillSortFields);
+        }
+
+        /**
+         * Enables caching for the second pass search. The cache will not grow over a specified limit in MB.
+         * The cache is filled during the first pass searched and then replayed during the second pass searched.
+         * If the cache grows beyond the specified limit, then the cache is purged and not used in the second pass search.
+         *
+         * @param maxCacheRAMMB The maximum amount in MB the cache is allowed to hold
+         * @param cacheScores   Whether to cache the scores
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetCachingInMB(double maxCacheRAMMB, bool cacheScores)
+        {
+            this.maxCacheRAMMB = maxCacheRAMMB;
+            this.maxDocsToCache = null;
+            this.cacheScores = cacheScores;
+            return this;
+        }
+
+        /**
+         * Enables caching for the second pass search. The cache will not contain more than the maximum specified documents.
+         * The cache is filled during the first pass searched and then replayed during the second pass searched.
+         * If the cache grows beyond the specified limit, then the cache is purged and not used in the second pass search.
+         *
+         * @param maxDocsToCache The maximum number of documents the cache is allowed to hold
+         * @param cacheScores    Whether to cache the scores
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetCaching(int maxDocsToCache, bool cacheScores)
+        {
+            this.maxDocsToCache = maxDocsToCache;
+            this.maxCacheRAMMB = null;
+            this.cacheScores = cacheScores;
+            return this;
+        }
+
+        /**
+         * Disables any enabled cache.
+         *
+         * @return <code>this</code>
+         */
+        public GroupingSearch DisableCaching()
+        {
+            this.maxCacheRAMMB = null;
+            this.maxDocsToCache = null;
+            return this;
+        }
+
+        /**
+         * Specifies how groups are sorted.
+         * Defaults to {@link Sort#RELEVANCE}.
+         *
+         * @param groupSort The sort for the groups.
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetGroupSort(Sort groupSort)
+        {
+            this.groupSort = groupSort;
+            return this;
+        }
+
+        /**
+         * Specified how documents inside a group are sorted.
+         * Defaults to {@link Sort#RELEVANCE}.
+         *
+         * @param sortWithinGroup The sort for documents inside a group
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetSortWithinGroup(Sort sortWithinGroup)
+        {
+            this.sortWithinGroup = sortWithinGroup;
+            return this;
+        }
+
+        /**
+         * Specifies the offset for documents inside a group.
+         *
+         * @param groupDocsOffset The offset for documents inside a
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetGroupDocsOffset(int groupDocsOffset)
+        {
+            this.groupDocsOffset = groupDocsOffset;
+            return this;
+        }
+
+        /**
+         * Specifies the number of documents to return inside a group from the specified groupDocsOffset.
+         *
+         * @param groupDocsLimit The number of documents to return inside a group
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetGroupDocsLimit(int groupDocsLimit)
+        {
+            this.groupDocsLimit = groupDocsLimit;
+            return this;
+        }
+
+        /**
+         * Whether to also fill the sort fields per returned group and groups docs.
+         *
+         * @param fillSortFields Whether to also fill the sort fields per returned group and groups docs
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetFillSortFields(bool fillSortFields)
+        {
+            this.fillSortFields = fillSortFields;
+            return this;
+        }
+
+        /**
+         * Whether to include the scores per doc inside a group.
+         *
+         * @param includeScores Whether to include the scores per doc inside a group
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetIncludeScores(bool includeScores)
+        {
+            this.includeScores = includeScores;
+            return this;
+        }
+
+        /**
+         * Whether to include the score of the most relevant document per group.
+         *
+         * @param includeMaxScore Whether to include the score of the most relevant document per group
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetIncludeMaxScore(bool includeMaxScore)
+        {
+            this.includeMaxScore = includeMaxScore;
+            return this;
+        }
+
+        /**
+         * Whether to also compute all groups matching the query.
+         * This can be used to determine the number of groups, which can be used for accurate pagination.
+         * <p/>
+         * When grouping by doc block the number of groups are automatically included in the {@link TopGroups} and this
+         * option doesn't have any influence.
+         *
+         * @param allGroups to also compute all groups matching the query
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetAllGroups(bool allGroups)
+        {
+            this.allGroups = allGroups;
+            return this;
+        }
+
+        /**
+         * If {@link #setAllGroups(boolean)} was set to <code>true</code> then all matching groups are returned, otherwise
+         * an empty collection is returned.
+         *
+         * @param <T> The group value type. This can be a {@link BytesRef} or a {@link MutableValue} instance. If grouping
+         *            by doc block this the group value is always <code>null</code>.
+         * @return all matching groups are returned, or an empty collection
+         */
+        public ICollection<T> GetAllMatchingGroups<T>()
+        {
+            return (ICollection<T>)matchingGroups;
+        }
+
+        /**
+         * Whether to compute all group heads (most relevant document per group) matching the query.
+         * <p/>
+         * This feature isn't enabled when grouping by doc block.
+         *
+         * @param allGroupHeads Whether to compute all group heads (most relevant document per group) matching the query
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetAllGroupHeads(bool allGroupHeads)
+        {
+            this.allGroupHeads = allGroupHeads;
+            return this;
+        }
+
+        /**
+         * Returns the matching group heads if {@link #setAllGroupHeads(boolean)} was set to true or an empty bit set.
+         *
+         * @return The matching group heads if {@link #setAllGroupHeads(boolean)} was set to true or an empty bit set
+         */
+        public Bits GetAllGroupHeads()
+        {
+            return matchingGroupHeads;
+        }
+
+        /**
+         * Sets the initial size of some internal used data structures.
+         * This prevents growing data structures many times. This can improve the performance of the grouping at the cost of
+         * more initial RAM.
+         * <p/>
+         * The {@link #setAllGroups} and {@link #setAllGroupHeads} features use this option.
+         * Defaults to 128.
+         *
+         * @param initialSize The initial size of some internal used data structures
+         * @return <code>this</code>
+         */
+        public GroupingSearch SetInitialSize(int initialSize)
+        {
+            this.initialSize = initialSize;
+            return this;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/Lucene.Net.Grouping.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/Lucene.Net.Grouping.csproj b/src/Lucene.Net.Grouping/Lucene.Net.Grouping.csproj
index 5d4fbe2..b10cd40 100644
--- a/src/Lucene.Net.Grouping/Lucene.Net.Grouping.csproj
+++ b/src/Lucene.Net.Grouping/Lucene.Net.Grouping.csproj
@@ -40,8 +40,29 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="AbstractAllGroupHeadsCollector.cs" />
+    <Compile Include="AbstractAllGroupsCollector.cs" />
+    <Compile Include="AbstractDistinctValuesCollector.cs" />
+    <Compile Include="AbstractFirstPassGroupingCollector.cs" />
+    <Compile Include="AbstractGroupFacetCollector.cs" />
+    <Compile Include="AbstractSecondPassGroupingCollector.cs" />
+    <Compile Include="BlockGroupingCollector.cs" />
+    <Compile Include="CollectedSearchGroup.cs" />
+    <Compile Include="Function\FunctionAllGroupHeadsCollector.cs" />
+    <Compile Include="Function\FunctionAllGroupsCollector.cs" />
+    <Compile Include="Function\FunctionDistinctValuesCollector.cs" />
+    <Compile Include="Function\FunctionFirstPassGroupingCollector.cs" />
+    <Compile Include="Function\FunctionSecondPassGroupingCollector.cs" />
     <Compile Include="GroupDocs.cs" />
+    <Compile Include="GroupingSearch.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="SearchGroup.cs" />
+    <Compile Include="Term\TermAllGroupHeadsCollector.cs" />
+    <Compile Include="Term\TermAllGroupsCollector.cs" />
+    <Compile Include="Term\TermDistinctValuesCollector.cs" />
+    <Compile Include="Term\TermFirstPassGroupingCollector.cs" />
+    <Compile Include="Term\TermGroupFacetCollector.cs" />
+    <Compile Include="Term\TermSecondPassGroupingCollector.cs" />
     <Compile Include="TopGroups.cs" />
   </ItemGroup>
   <ItemGroup>
@@ -49,7 +70,12 @@
       <Project>{5D4AD9BE-1FFB-41AB-9943-25737971BF57}</Project>
       <Name>Lucene.Net</Name>
     </ProjectReference>
+    <ProjectReference Include="..\Lucene.Net.Queries\Lucene.Net.Queries.csproj">
+      <Project>{69D7956C-C2CC-4708-B399-A188FEC384C4}</Project>
+      <Name>Lucene.Net.Queries</Name>
+    </ProjectReference>
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/9d72bcb3/src/Lucene.Net.Grouping/SearchGroup.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Grouping/SearchGroup.cs b/src/Lucene.Net.Grouping/SearchGroup.cs
new file mode 100644
index 0000000..6ab902f
--- /dev/null
+++ b/src/Lucene.Net.Grouping/SearchGroup.cs
@@ -0,0 +1,388 @@
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.Search.Grouping
+{
+    /// <summary>
+    /// Represents a group that is found during the first pass search.
+    /// @lucene.experimental
+    /// </summary>
+    /// <typeparam name="TGroupValue"></typeparam>
+    public class SearchGroup<TGroupValue>
+    {
+        /** The value that defines this group  */
+        public TGroupValue groupValue;
+
+        /** The sort values used during sorting. These are the
+         *  groupSort field values of the highest rank document
+         *  (by the groupSort) within the group.  Can be
+         * <code>null</code> if <code>fillFields=false</code> had
+         * been passed to {@link AbstractFirstPassGroupingCollector#getTopGroups} */
+        public object[] sortValues;
+
+        public override string ToString()
+        {
+            return ("SearchGroup(groupValue=" + groupValue + " sortValues=" + Arrays.ToString(sortValues) + ")");
+        }
+
+        public override bool Equals(object o)
+        {
+            if (this == o) return true;
+            if (o == null || GetType() != o.GetType()) return false;
+
+            SearchGroup<TGroupValue> that = (SearchGroup<TGroupValue>)o;
+
+            if (groupValue == null)
+            {
+                if (that.groupValue != null)
+                {
+                    return false;
+                }
+            }
+            else if (!groupValue.Equals(that.groupValue))
+            {
+                return false;
+            }
+
+            return true;
+        }
+
+        public override int GetHashCode()
+        {
+            return groupValue != null ? groupValue.GetHashCode() : 0;
+        }
+
+        private class ShardIter<T>
+        {
+            public readonly IEnumerator<SearchGroup<T>> iter;
+            public readonly int shardIndex;
+
+            public ShardIter(ICollection<SearchGroup<T>> shard, int shardIndex)
+            {
+                this.shardIndex = shardIndex;
+                iter = shard.GetEnumerator();
+                //Debug.Assert(iter.hasNext());
+            }
+
+            public SearchGroup<T> Next()
+            {
+                Debug.Assert(iter.MoveNext());
+                SearchGroup<T> group = iter.Current;
+                if (group.sortValues == null)
+                {
+                    throw new ArgumentException("group.sortValues is null; you must pass fillFields=true to the first pass collector");
+                }
+                return group;
+            }
+
+            public override string ToString()
+            {
+                return "ShardIter(shard=" + shardIndex + ")";
+            }
+        }
+
+        // Holds all shards currently on the same group
+        private class MergedGroup<T>
+        {
+
+            // groupValue may be null!
+            public readonly T groupValue;
+
+            public object[] topValues;
+            public readonly List<ShardIter<T>> shards = new List<ShardIter<T>>();
+            public int minShardIndex;
+            public bool processed;
+            public bool inQueue;
+
+            public MergedGroup(T groupValue)
+            {
+                this.groupValue = groupValue;
+            }
+
+            // Only for assert
+            private bool NeverEquals(object _other)
+            {
+                if (_other is MergedGroup<T>)
+                {
+                    MergedGroup<T> other = (MergedGroup<T>)_other;
+                    if (groupValue == null)
+                    {
+                        Debug.Assert(other.groupValue != null);
+                    }
+                    else
+                    {
+                        Debug.Assert(!groupValue.Equals(other.groupValue));
+                    }
+                }
+                return true;
+            }
+
+            public override bool Equals(object _other)
+            {
+                // We never have another MergedGroup instance with
+                // same groupValue
+                Debug.Assert(NeverEquals(_other));
+
+                if (_other is MergedGroup<T>)
+                {
+                    MergedGroup<T> other = (MergedGroup<T>)_other;
+                    if (groupValue == null)
+                    {
+                        return other == null;
+                    }
+                    else
+                    {
+                        return groupValue.Equals(other);
+                    }
+                }
+                else
+                {
+                    return false;
+                }
+            }
+
+            public override int GetHashCode()
+            {
+                if (groupValue == null)
+                {
+                    return 0;
+                }
+                else
+                {
+                    return groupValue.GetHashCode();
+                }
+            }
+        }
+
+        private class GroupComparator<T> : IComparer<MergedGroup<T>>
+        {
+
+            public readonly FieldComparator[] comparators;
+            public readonly int[] reversed;
+
+            public GroupComparator(Sort groupSort)
+            {
+                SortField[] sortFields = groupSort.GetSort();
+                comparators = new FieldComparator[sortFields.Length];
+                reversed = new int[sortFields.Length];
+                for (int compIDX = 0; compIDX < sortFields.Length; compIDX++)
+                {
+                    SortField sortField = sortFields[compIDX];
+                    comparators[compIDX] = sortField.GetComparator(1, compIDX);
+                    reversed[compIDX] = sortField.Reverse ? -1 : 1;
+                }
+            }
+
+            public virtual int Compare(MergedGroup<T> group, MergedGroup<T> other)
+            {
+                if (group == other)
+                {
+                    return 0;
+                }
+                //System.out.println("compare group=" + group + " other=" + other);
+                object[] groupValues = group.topValues;
+                object[] otherValues = other.topValues;
+                //System.out.println("  groupValues=" + groupValues + " otherValues=" + otherValues);
+                for (int compIDX = 0; compIDX < comparators.Length; compIDX++)
+                {
+                    int c = reversed[compIDX] * comparators[compIDX].CompareValues(groupValues[compIDX],
+                                                                                         otherValues[compIDX]);
+                    if (c != 0)
+                    {
+                        return c;
+                    }
+                }
+
+                // Tie break by min shard index:
+                Debug.Assert(group.minShardIndex != other.minShardIndex);
+                return group.minShardIndex - other.minShardIndex;
+            }
+        }
+
+        private class GroupMerger<T>
+        {
+
+            private readonly GroupComparator<T> groupComp;
+            private readonly ISet<MergedGroup<T>> queue;
+            private readonly IDictionary<T, MergedGroup<T>> groupsSeen;
+
+            public GroupMerger(Sort groupSort)
+            {
+                groupComp = new GroupComparator<T>(groupSort);
+                queue = new SortedSet<MergedGroup<T>>(groupComp);
+                groupsSeen = new Dictionary<T, MergedGroup<T>>();
+            }
+
+            private void UpdateNextGroup(int topN, ShardIter<T> shard)
+            {
+                while (shard.iter.MoveNext())
+                {
+                    SearchGroup<T> group = shard.Next();
+                    MergedGroup<T> mergedGroup = groupsSeen.ContainsKey(group.groupValue) ? groupsSeen[group.groupValue] : null;
+                    bool isNew = mergedGroup == null;
+                    //System.out.println("    next group=" + (group.groupValue == null ? "null" : ((BytesRef) group.groupValue).utf8ToString()) + " sort=" + Arrays.toString(group.sortValues));
+
+                    if (isNew)
+                    {
+                        // Start a new group:
+                        //System.out.println("      new");
+                        mergedGroup = new MergedGroup<T>(group.groupValue);
+                        mergedGroup.minShardIndex = shard.shardIndex;
+                        Debug.Assert(group.sortValues != null);
+                        mergedGroup.topValues = group.sortValues;
+                        groupsSeen[group.groupValue] = mergedGroup;
+                        mergedGroup.inQueue = true;
+                        queue.Add(mergedGroup);
+                    }
+                    else if (mergedGroup.processed)
+                    {
+                        // This shard produced a group that we already
+                        // processed; move on to next group...
+                        continue;
+                    }
+                    else
+                    {
+                        //System.out.println("      old");
+                        bool competes = false;
+                        for (int compIDX = 0; compIDX < groupComp.comparators.Length; compIDX++)
+                        {
+                            int cmp = groupComp.reversed[compIDX] * groupComp.comparators[compIDX].CompareValues(group.sortValues[compIDX],
+                                                                                                                       mergedGroup.topValues[compIDX]);
+                            if (cmp < 0)
+                            {
+                                // Definitely competes
+                                competes = true;
+                                break;
+                            }
+                            else if (cmp > 0)
+                            {
+                                // Definitely does not compete
+                                break;
+                            }
+                            else if (compIDX == groupComp.comparators.Length - 1)
+                            {
+                                if (shard.shardIndex < mergedGroup.minShardIndex)
+                                {
+                                    competes = true;
+                                }
+                            }
+                        }
+
+                        //System.out.println("      competes=" + competes);
+
+                        if (competes)
+                        {
+                            // Group's sort changed -- remove & re-insert
+                            if (mergedGroup.inQueue)
+                            {
+                                queue.Remove(mergedGroup);
+                            }
+                            mergedGroup.topValues = group.sortValues;
+                            mergedGroup.minShardIndex = shard.shardIndex;
+                            queue.Add(mergedGroup);
+                            mergedGroup.inQueue = true;
+                        }
+                    }
+
+                    mergedGroup.shards.Add(shard);
+                    break;
+                }
+
+                // Prune un-competitive groups:
+                while (queue.Count > topN)
+                {
+                    MergedGroup<T> group = queue.Last();
+                    queue.Remove(group);
+                    //System.out.println("PRUNE: " + group);
+                    group.inQueue = false;
+                }
+            }
+
+            public ICollection<SearchGroup<T>> Merge(IList<ICollection<SearchGroup<T>>> shards, int offset, int topN)
+            {
+
+                int maxQueueSize = offset + topN;
+
+                //System.out.println("merge");
+                // Init queue:
+                for (int shardIDX = 0; shardIDX < shards.Count; shardIDX++)
+                {
+                    ICollection<SearchGroup<T>> shard = shards[shardIDX];
+                    if (shard.Any())
+                    {
+                        //System.out.println("  insert shard=" + shardIDX);
+                        UpdateNextGroup(maxQueueSize, new ShardIter<T>(shard, shardIDX));
+                    }
+                }
+
+                // Pull merged topN groups:
+                List<SearchGroup<T>> newTopGroups = new List<SearchGroup<T>>();
+
+                int count = 0;
+
+                while (queue.Count != 0)
+                {
+                    MergedGroup<T> group = queue.First();
+                    queue.Remove(group);
+                    group.processed = true;
+                    //System.out.println("  pop: shards=" + group.shards + " group=" + (group.groupValue == null ? "null" : (((BytesRef) group.groupValue).utf8ToString())) + " sortValues=" + Arrays.toString(group.topValues));
+                    if (count++ >= offset)
+                    {
+                        SearchGroup<T> newGroup = new SearchGroup<T>();
+                        newGroup.groupValue = group.groupValue;
+                        newGroup.sortValues = group.topValues;
+                        newTopGroups.Add(newGroup);
+                        if (newTopGroups.Count == topN)
+                        {
+                            break;
+                        }
+                        //} else {
+                        // System.out.println("    skip < offset");
+                    }
+
+                    // Advance all iters in this group:
+                    foreach (ShardIter<T> shardIter in group.shards)
+                    {
+                        UpdateNextGroup(maxQueueSize, shardIter);
+                    }
+                }
+
+                if (newTopGroups.Count == 0)
+                {
+                    return null;
+                }
+                else
+                {
+                    return newTopGroups;
+                }
+            }
+        }
+
+        /** Merges multiple collections of top groups, for example
+         *  obtained from separate index shards.  The provided
+         *  groupSort must match how the groups were sorted, and
+         *  the provided SearchGroups must have been computed
+         *  with fillFields=true passed to {@link
+         *  AbstractFirstPassGroupingCollector#getTopGroups}.
+         *
+         * <p>NOTE: this returns null if the topGroups is empty.
+         */
+        public static ICollection<SearchGroup<T>> Merge<T>(IList<ICollection<SearchGroup<T>>> topGroups, int offset, int topN, Sort groupSort)
+        {
+            if (topGroups.Count == 0)
+            {
+                return null;
+            }
+            else
+            {
+                return new GroupMerger<T>(groupSort).Merge(topGroups, offset, topN);
+            }
+        }
+    }
+}


Mime
View raw message