lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From paulir...@apache.org
Subject [16/53] [abbrv] Finish Memory and VectorHighlighter
Date Thu, 07 Nov 2013 13:53:31 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/contrib/Memory/MemoryIndex.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Memory/MemoryIndex.cs b/src/contrib/Memory/MemoryIndex.cs
index 41c18fb..b383b08 100644
--- a/src/contrib/Memory/MemoryIndex.cs
+++ b/src/contrib/Memory/MemoryIndex.cs
@@ -29,6 +29,8 @@ using Lucene.Net.Analysis.Tokenattributes;
 using Lucene.Net.Documents;
 using Lucene.Net.Search;
 using Lucene.Net.Support;
+using Lucene.Net.Util;
+using Lucene.Net.Search.Similarities;
 
 namespace Lucene.Net.Index.Memory
 {
@@ -156,20 +158,27 @@ namespace Lucene.Net.Index.Memory
     public partial class MemoryIndex
     {
         /* info for each field: Map<String fieldName, Info field> */
-        private HashMap<String, Info> fields = new HashMap<String, Info>();
+        private readonly HashMap<String, Info> fields = new HashMap<String, Info>();
 
         /* fields sorted ascending by fieldName; lazily computed on demand */
-        [NonSerialized] private KeyValuePair<String, Info>[] sortedFields;
+        [NonSerialized]
+        private KeyValuePair<String, Info>[] sortedFields;
 
-        /* pos: positions[3*i], startOffset: positions[3*i +1], endOffset: positions[3*i +2] */
-        private int stride;
+        private readonly bool storeOffsets;
 
-        /* Could be made configurable; See {@link Document#setBoost(float)} */
-        private static float docBoost = 1.0f;
+        private const bool DEBUG = false;
 
-        private static long serialVersionUID = 2782195016849084649L;
+        private readonly ByteBlockPool byteBlockPool;
+        private readonly IntBlockPool intBlockPool;
+        //  private final IntBlockPool.SliceReader postingsReader;
+        private readonly IntBlockPool.SliceWriter postingsWriter;
 
-        private static bool DEBUG = false;
+        private HashMap<String, FieldInfo> fieldInfos = new HashMap<String, FieldInfo>();
+
+        private Counter bytesUsed;
+
+        // .NET: we're using the stuff in TermComparer.cs instead
+        //private static final Comparator<Object> termComparator ...
 
         /*
          * Constructs an empty instance.
@@ -191,9 +200,21 @@ namespace Lucene.Net.Index.Memory
          *            each token term in the text
          */
 
-        private MemoryIndex(bool storeOffsets)
+        public MemoryIndex(bool storeOffsets)
+            : this(storeOffsets, 0)
+        {
+        }
+
+        internal MemoryIndex(bool storeOffsets, long maxReusedBytes)
         {
-            this.stride = storeOffsets ? 3 : 1;
+            this.storeOffsets = storeOffsets;
+            this.bytesUsed = Counter.NewCounter();
+            int maxBufferedByteBlocks = (int)((maxReusedBytes / 2) / ByteBlockPool.BYTE_BLOCK_SIZE);
+            int maxBufferedIntBlocks = (int)((maxReusedBytes - (maxBufferedByteBlocks * ByteBlockPool.BYTE_BLOCK_SIZE)) / (IntBlockPool.INT_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT));
+            //assert (maxBufferedByteBlocks * ByteBlockPool.BYTE_BLOCK_SIZE) + (maxBufferedIntBlocks * IntBlockPool.INT_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT) <= maxReusedBytes;
+            byteBlockPool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, maxBufferedByteBlocks, bytesUsed));
+            intBlockPool = new IntBlockPool(new RecyclingIntBlockAllocator(IntBlockPool.INT_BLOCK_SIZE, maxBufferedIntBlocks, bytesUsed));
+            postingsWriter = new IntBlockPool.SliceWriter(intBlockPool);
         }
 
         /*
@@ -224,7 +245,7 @@ namespace Lucene.Net.Index.Memory
 
             TokenStream stream = analyzer.TokenStream(fieldName, new StringReader(text));
 
-            AddField(fieldName, stream);
+            AddField(fieldName, stream, 1.0f, analyzer.GetPositionIncrementGap(fieldName));
         }
 
         /*
@@ -261,6 +282,27 @@ namespace Lucene.Net.Index.Memory
             AddField(fieldName, stream, 1.0f);
         }
 
+        /**
+        * Iterates over the given token stream and adds the resulting terms to the index;
+        * Equivalent to adding a tokenized, indexed, termVectorStored, unstored,
+        * Lucene {@link org.apache.lucene.document.Field}.
+        * Finally closes the token stream. Note that untokenized keywords can be added with this method via 
+        * {@link #keywordTokenStream(Collection)}, the Lucene <code>KeywordTokenizer</code> or similar utilities.
+        * 
+        * @param fieldName
+        *            a name to be associated with the text
+        * @param stream
+        *            the token stream to retrieve tokens from.
+        * @param boost
+        *            the boost factor for hits for this field
+        *  
+        * @see org.apache.lucene.document.Field#setBoost(float)
+        */
+        public void AddField(String fieldName, TokenStream stream, float boost)
+        {
+            AddField(fieldName, stream, boost, 0);
+        }
+
         /*
          * Iterates over the given token stream and adds the resulting terms to the index;
          * Equivalent to adding a tokenized, indexed, termVectorStored, unstored,
@@ -276,7 +318,7 @@ namespace Lucene.Net.Index.Memory
          *            the boost factor for hits for this field
          * @see org.apache.lucene.document.Field#setBoost(float)
          */
-        public void AddField(String fieldName, TokenStream stream, float boost)
+        public void AddField(String fieldName, TokenStream stream, float boost, int positionIncrementGap)
         {
             try
             {
@@ -286,54 +328,80 @@ namespace Lucene.Net.Index.Memory
                     throw new ArgumentException("token stream must not be null");
                 if (boost <= 0.0f)
                     throw new ArgumentException("boost factor must be greater than 0.0");
-                if (fields[fieldName] != null)
-                    throw new ArgumentException("field must not be added more than once");
-
-                var terms = new HashMap<String, ArrayIntList>();
                 int numTokens = 0;
                 int numOverlapTokens = 0;
                 int pos = -1;
+                BytesRefHash terms;
+                SliceByteStartArray sliceArray;
+                Info info = null;
+                long sumTotalTermFreq = 0;
+                if ((info = fields[fieldName]) != null)
+                {
+                    numTokens = info.numTokens;
+                    numOverlapTokens = info.numOverlapTokens;
+                    pos = info.lastPosition + positionIncrementGap;
+                    terms = info.terms;
+                    boost *= info.boost;
+                    sliceArray = info.sliceArray;
+                    sumTotalTermFreq = info.sumTotalTermFreq;
+                }
+                else
+                {
+                    sliceArray = new SliceByteStartArray(BytesRefHash.DEFAULT_CAPACITY);
+                    terms = new BytesRefHash(byteBlockPool, BytesRefHash.DEFAULT_CAPACITY, sliceArray);
+                }
 
-                var termAtt = stream.AddAttribute<ITermAttribute>();
-                var posIncrAttribute = stream.AddAttribute<IPositionIncrementAttribute>();
-                var offsetAtt = stream.AddAttribute<IOffsetAttribute>();
-
+                if (!fieldInfos.ContainsKey(fieldName))
+                {
+                    fieldInfos[fieldName] =
+                        new FieldInfo(fieldName, true, fieldInfos.Count, false, false, false, this.storeOffsets ? FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, null, null);
+                }
+                ITermToBytesRefAttribute termAtt = stream.GetAttribute<ITermToBytesRefAttribute>();
+                IPositionIncrementAttribute posIncrAttribute = stream.AddAttribute<IPositionIncrementAttribute>();
+                IOffsetAttribute offsetAtt = stream.AddAttribute<IOffsetAttribute>();
+                BytesRef ref_renamed = termAtt.BytesRef;
                 stream.Reset();
+
                 while (stream.IncrementToken())
                 {
-                    String term = termAtt.Term;
-                    if (term.Length == 0) continue; // nothing to do
-                    //        if (DEBUG) System.Diagnostics.Debug.WriteLine("token='" + term + "'");
+                    termAtt.FillBytesRef();
+                    //if (DEBUG) System.err.println("token='" + term + "'");
                     numTokens++;
                     int posIncr = posIncrAttribute.PositionIncrement;
                     if (posIncr == 0)
                         numOverlapTokens++;
                     pos += posIncr;
-
-                    ArrayIntList positions = terms[term];
-                    if (positions == null)
+                    int ord = terms.Add(ref_renamed);
+                    if (ord < 0)
                     {
-                        // term not seen before
-                        positions = new ArrayIntList(stride);
-                        terms[term] = positions;
+                        ord = (-ord) - 1;
+                        postingsWriter.Reset(sliceArray.end[ord]);
                     }
-                    if (stride == 1)
+                    else
                     {
-                        positions.Add(pos);
+                        sliceArray.start[ord] = postingsWriter.StartNewSlice();
+                    }
+                    sliceArray.freq[ord]++;
+                    sumTotalTermFreq++;
+                    if (!storeOffsets)
+                    {
+                        postingsWriter.WriteInt(pos);
                     }
                     else
                     {
-                        positions.Add(pos, offsetAtt.StartOffset, offsetAtt.EndOffset);
+                        postingsWriter.WriteInt(pos);
+                        postingsWriter.WriteInt(offsetAtt.StartOffset);
+                        postingsWriter.WriteInt(offsetAtt.EndOffset);
                     }
+                    sliceArray.end[ord] = postingsWriter.CurrentOffset;
                 }
                 stream.End();
 
                 // ensure infos.numTokens > 0 invariant; needed for correct operation of terms()
                 if (numTokens > 0)
                 {
-                    boost = boost*docBoost; // see DocumentWriter.addDocument(...)
-                    fields[fieldName] = new Info(terms, numTokens, numOverlapTokens, boost);
-                    sortedFields = null; // invalidate sorted view, if any
+                    fields[fieldName] = new Info(terms, sliceArray, numTokens, numOverlapTokens, boost, pos, sumTotalTermFreq);
+                    sortedFields = null;    // invalidate sorted view, if any
                 }
             }
             catch (IOException e)
@@ -345,7 +413,7 @@ namespace Lucene.Net.Index.Memory
             {
                 try
                 {
-                    if (stream != null) stream.Close();
+                    if (stream != null) stream.Dispose();
                 }
                 catch (IOException e2)
                 {
@@ -386,7 +454,7 @@ namespace Lucene.Net.Index.Memory
             if (query == null)
                 throw new ArgumentException("query must not be null");
 
-            Searcher searcher = CreateSearcher();
+            IndexSearcher searcher = CreateSearcher();
             try
             {
                 float[] scores = new float[1]; // inits to 0.0f (no match)
@@ -425,44 +493,9 @@ namespace Lucene.Net.Index.Memory
          * 
          * @return the main memory consumption
          */
-        public int GetMemorySize()
+        public long GetMemorySize()
         {
-            // for example usage in a smart cache see nux.xom.pool.Pool    
-            int PTR = VM.PTR;
-            int INT = VM.INT;
-            int size = 0;
-            size += VM.SizeOfObject(2*PTR + INT); // memory index
-            if (sortedFields != null) size += VM.SizeOfObjectArray(sortedFields.Length);
-
-            size += VM.SizeOfHashMap(fields.Count);
-            foreach (var entry in fields)
-            {
-                // for each Field Info
-                Info info = entry.Value;
-                size += VM.SizeOfObject(2*INT + 3*PTR); // Info instance vars
-                if (info.SortedTerms != null) size += VM.SizeOfObjectArray(info.SortedTerms.Length);
-
-                int len = info.Terms.Count;
-                size += VM.SizeOfHashMap(len);
-
-                var iter2 = info.Terms.GetEnumerator();
-                while (--len >= 0)
-                {
-                    iter2.MoveNext();
-                    // for each term
-                    KeyValuePair<String, ArrayIntList> e = iter2.Current;
-                    size += VM.SizeOfObject(PTR + 3*INT); // assumes substring() memory overlay
-//        size += STR + 2 * ((String) e.getKey()).length();
-                    ArrayIntList positions = e.Value;
-                    size += VM.SizeOfArrayIntList(positions.Size());
-                }
-            }
-            return size;
-        }
-
-        private int NumPositions(ArrayIntList positions)
-        {
-            return positions.Size()/stride;
+            return RamUsageEstimator.SizeOf(this);
         }
 
         /* sorts into ascending order (on demand), reusing memory along the way */
@@ -495,9 +528,9 @@ namespace Lucene.Net.Index.Memory
         {
             StringBuilder result = new StringBuilder(256);
             SortFields();
-            int sumChars = 0;
             int sumPositions = 0;
             int sumTerms = 0;
+            BytesRef spare = new BytesRef();
 
             for (int i = 0; i < sortedFields.Length; i++)
             {
@@ -507,33 +540,54 @@ namespace Lucene.Net.Index.Memory
                 info.SortTerms();
                 result.Append(fieldName + ":\n");
 
-                int numChars = 0;
-                int numPos = 0;
-                for (int j = 0; j < info.SortedTerms.Length; j++)
+                SliceByteStartArray sliceArray = info.sliceArray;
+                int numPositions = 0;
+                IntBlockPool.SliceReader postingsReader = new IntBlockPool.SliceReader(intBlockPool);
+                for (int j = 0; j < info.terms.Size; j++)
                 {
-                    KeyValuePair<String, ArrayIntList> e = info.SortedTerms[j];
-                    String term = e.Key;
-                    ArrayIntList positions = e.Value;
-                    result.Append("\t'" + term + "':" + NumPositions(positions) + ":");
-                    result.Append(positions.ToString(stride)); // ignore offsets
+                    int ord = info.sortedTerms[j];
+                    info.terms.Get(ord, spare);
+                    int freq = sliceArray.freq[ord];
+                    result.Append("\t'" + spare + "':" + freq + ":");
+                    postingsReader.Reset(sliceArray.start[ord], sliceArray.end[ord]);
+                    result.Append(" [");
+                    int iters = storeOffsets ? 3 : 1;
+                    while (!postingsReader.EndOfSlice())
+                    {
+                        result.Append("(");
+
+                        for (int k = 0; k < iters; k++)
+                        {
+                            result.Append(postingsReader.ReadInt());
+                            if (k < iters - 1)
+                            {
+                                result.Append(", ");
+                            }
+                        }
+                        result.Append(")");
+                        if (!postingsReader.EndOfSlice())
+                        {
+                            result.Append(",");
+                        }
+
+                    }
+                    result.Append("]");
                     result.Append("\n");
-                    numPos += NumPositions(positions);
-                    numChars += term.Length;
+                    numPositions += freq;
                 }
 
-                result.Append("\tterms=" + info.SortedTerms.Length);
-                result.Append(", positions=" + numPos);
-                result.Append(", Kchars=" + (numChars/1000.0f));
+                result.Append("\tterms=" + info.sortedTerms.Length);
+                result.Append(", positions=" + numPositions);
+                result.Append(", memory=" + RamUsageEstimator.HumanReadableUnits(RamUsageEstimator.SizeOf(info)));
                 result.Append("\n");
-                sumPositions += numPos;
-                sumChars += numChars;
-                sumTerms += info.SortedTerms.Length;
+                sumPositions += numPositions;
+                sumTerms += info.terms.Size;
             }
 
             result.Append("\nfields=" + sortedFields.Length);
             result.Append(", terms=" + sumTerms);
             result.Append(", positions=" + sumPositions);
-            result.Append(", Kchars=" + (sumChars/1000.0f));
+            result.Append(", memory=" + RamUsageEstimator.HumanReadableUnits(GetMemorySize()));
             return result.ToString();
         }
 
@@ -549,91 +603,66 @@ namespace Lucene.Net.Index.Memory
         [Serializable]
         private sealed class Info
         {
-            public static readonly IComparer<KeyValuePair<string, Info>> InfoComparer = new TermComparer<Info>();
-            public static readonly IComparer<KeyValuePair<string, ArrayIntList>> ArrayIntListComparer = new TermComparer<ArrayIntList>(); 
-            /*
-             * Term strings and their positions for this field: Map <String
-             * termText, ArrayIntList positions>
-             */
-            private HashMap<String, ArrayIntList> terms;
+            /**
+            * Term strings and their positions for this field: Map String
+            * termText, ArrayIntList positions
+            */
+            internal readonly BytesRefHash terms;
+
+            internal readonly SliceByteStartArray sliceArray;
 
             /* Terms sorted ascending by term text; computed on demand */
-            [NonSerialized] private KeyValuePair<String, ArrayIntList>[] sortedTerms;
+            [NonSerialized]
+            internal int[] sortedTerms;
 
             /* Number of added tokens for this field */
-            private int numTokens;
+            internal readonly int numTokens;
 
             /* Number of overlapping tokens for this field */
-            private int numOverlapTokens;
+            internal readonly int numOverlapTokens;
 
             /* Boost factor for hits for this field */
-            private float boost;
+            internal readonly float boost;
 
-            /* Term for this field's fieldName, lazily computed on demand */
-            [NonSerialized] public Term template;
+            internal readonly long sumTotalTermFreq;
 
-            private static long serialVersionUID = 2882195016849084649L;
+            /** the last position encountered in this field for multi field support*/
+            internal int lastPosition;
 
-            public Info(HashMap<String, ArrayIntList> terms, int numTokens, int numOverlapTokens, float boost)
+            public Info(BytesRefHash terms, SliceByteStartArray sliceArray, int numTokens, int numOverlapTokens, float boost, int lastPosition, long sumTotalTermFreq)
             {
                 this.terms = terms;
+                this.sliceArray = sliceArray;
                 this.numTokens = numTokens;
-                this.NumOverlapTokens = numOverlapTokens;
+                this.numOverlapTokens = numOverlapTokens;
                 this.boost = boost;
+                this.sumTotalTermFreq = sumTotalTermFreq;
+                this.lastPosition = lastPosition;
             }
 
-            public HashMap<string, ArrayIntList> Terms
+            public long SumTotalTermFreq
             {
-                get { return terms; }
-            }
-
-            public int NumTokens
-            {
-                get { return numTokens; }
-            }
-
-            public int NumOverlapTokens
-            {
-                get { return numOverlapTokens; }
-                set { numOverlapTokens = value; }
-            }
-
-            public float Boost
-            {
-                get { return boost; }
-            }
-
-            public KeyValuePair<string, ArrayIntList>[] SortedTerms
-            {
-                get { return sortedTerms; }
+                get { return sumTotalTermFreq; }
             }
 
             /*
-         * Sorts hashed terms into ascending order, reusing memory along the
-         * way. Note that sorting is lazily delayed until required (often it's
-         * not required at all). If a sorted view is required then hashing +
-         * sort + binary search is still faster and smaller than TreeMap usage
-         * (which would be an alternative and somewhat more elegant approach,
-         * apart from more sophisticated Tries / prefix trees).
-         */
+            * Sorts hashed terms into ascending order, reusing memory along the
+            * way. Note that sorting is lazily delayed until required (often it's
+            * not required at all). If a sorted view is required then hashing +
+            * sort + binary search is still faster and smaller than TreeMap usage
+            * (which would be an alternative and somewhat more elegant approach,
+            * apart from more sophisticated Tries / prefix trees).
+            */
 
             public void SortTerms()
             {
-                if (SortedTerms == null) sortedTerms = Sort(Terms);
+                if (sortedTerms == null)
+                    sortedTerms = terms.Sort(BytesRef.UTF8SortedAsUnicodeComparer);
             }
 
-            /* note that the frequency can be calculated as numPosition(getPositions(x)) */
-
-            public ArrayIntList GetPositions(String term)
-            {
-                return Terms[term];
-            }
-
-            /* note that the frequency can be calculated as numPosition(getPositions(x)) */
-
-            public ArrayIntList GetPositions(int pos)
+            public float Boost
             {
-                return SortedTerms[pos].Value;
+                get { return boost; }
             }
         }
 
@@ -641,329 +670,196 @@ namespace Lucene.Net.Index.Memory
         ///////////////////////////////////////////////////////////////////////////////
         // Nested classes:
         ///////////////////////////////////////////////////////////////////////////////
+
         /*
-         * Efficient resizable auto-expanding list holding <c>int</c> elements;
-         * implemented with arrays.
+         * Search support for Lucene framework integration; implements all methods
+         * required by the Lucene IndexReader contracts.
          */
 
-        [Serializable]
-        private sealed class ArrayIntList
+        private sealed partial class MemoryIndexReader : AtomicReader
         {
+            private readonly MemoryIndex _index;
 
-            private int[] elements;
-            private int size = 0;
-
-            private static long serialVersionUID = 2282195016849084649L;
+            private IndexSearcher searcher; // needed to find searcher.getSimilarity() 
 
-            private ArrayIntList()
-                : this(10)
+            internal MemoryIndexReader(MemoryIndex index)
             {
-
+                _index = index;
             }
 
-            public ArrayIntList(int initialCapacity)
+            private Info GetInfo(String fieldName)
             {
-                elements = new int[initialCapacity];
+                return _index.fields[fieldName];
             }
 
-            public void Add(int elem)
+            private Info GetInfo(int pos)
             {
-                if (size == elements.Length) EnsureCapacity(size + 1);
-                elements[size++] = elem;
+                return _index.sortedFields[pos].Value;
             }
 
-            public void Add(int pos, int start, int end)
+            public override IBits LiveDocs
             {
-                if (size + 3 > elements.Length) EnsureCapacity(size + 3);
-                elements[size] = pos;
-                elements[size + 1] = start;
-                elements[size + 2] = end;
-                size += 3;
+                get { return null; }
             }
 
-            public int Get(int index)
+            public override FieldInfos FieldInfos
             {
-                if (index >= size) ThrowIndex(index);
-                return elements[index];
+                get { return new FieldInfos(_index.fieldInfos.Values.ToArray()); }
             }
 
-            public int Size()
+            public override NumericDocValues GetNumericDocValues(string field)
             {
-                return size;
+                return null;
             }
 
-            public int[] ToArray(int stride)
+            public override BinaryDocValues GetBinaryDocValues(string field)
             {
-                int[] arr = new int[Size()/stride];
-                if (stride == 1)
-                {
-                    Array.Copy(elements, 0, arr, 0, size);
-                }
-                else
-                {
-                    for (int i = 0, j = 0; j < size; i++, j += stride) arr[i] = elements[j];
-                }
-                return arr;
+                return null;
             }
 
-            private void EnsureCapacity(int minCapacity)
+            public override SortedDocValues GetSortedDocValues(string field)
             {
-                int newCapacity = Math.Max(minCapacity, (elements.Length*3)/2 + 1);
-                int[] newElements = new int[newCapacity];
-                Array.Copy(elements, 0, newElements, 0, size);
-                elements = newElements;
+                return null;
             }
 
-            private void ThrowIndex(int index)
+            public override SortedSetDocValues GetSortedSetDocValues(string field)
             {
-                throw new IndexOutOfRangeException("index: " + index
-                                                   + ", size: " + size);
+                return null;
             }
 
-            /* returns the first few positions (without offsets); debug only */
-
-            public string ToString(int stride)
+            private sealed class MemoryFields : Fields
             {
-                int s = Size()/stride;
-                int len = Math.Min(10, s); // avoid printing huge lists
-                StringBuilder buf = new StringBuilder(4*len);
-                buf.Append("[");
-                for (int i = 0; i < len; i++)
+                private readonly MemoryIndexReader parent;
+
+                public MemoryFields(MemoryIndexReader parent)
                 {
-                    buf.Append(Get(i*stride));
-                    if (i < len - 1) buf.Append(", ");
+                    this.parent = parent;
                 }
-                if (len != s) buf.Append(", ..."); // and some more...
-                buf.Append("]");
-                return buf.ToString();
-            }
-        }
 
+                public override IEnumerator<string> GetEnumerator()
+                {
+                    return parent._index.sortedFields.Select(i => i.Key).GetEnumerator();
+                }
 
-        ///////////////////////////////////////////////////////////////////////////////
-        // Nested classes:
-        ///////////////////////////////////////////////////////////////////////////////
-        private static readonly Term MATCH_ALL_TERM = new Term("");
+                public override Terms Terms(string field)
+                {
+                    int i = Array.BinarySearch(parent._index.sortedFields, field, new TermComparer<Info>());
+                    if (i < 0)
+                    {
+                        return null;
+                    }
+                    else
+                    {
+                        Info info = parent.GetInfo(i);
+                        info.SortTerms();
 
-        /*
-         * Search support for Lucene framework integration; implements all methods
-         * required by the Lucene IndexReader contracts.
-         */
+                        return new AnonymousTerms(this, info);
+                    }
+                }
 
-        private sealed partial class MemoryIndexReader : IndexReader
-        {
-            private readonly MemoryIndex _index;
+                private sealed class AnonymousTerms : Terms
+                {
+                    private readonly MemoryFields parent;
+                    private readonly Info info;
 
-            private Searcher searcher; // needed to find searcher.getSimilarity() 
+                    public AnonymousTerms(MemoryFields parent, Info info)
+                    {
+                        this.parent = parent;
+                        this.info = info;
+                    }
 
-            internal MemoryIndexReader(MemoryIndex index)
-            {
-                _index = index;
-            }
+                    public override TermsEnum Iterator(TermsEnum reuse)
+                    {
+                        return new MemoryTermsEnum(parent.parent._index, info);
+                    }
 
-            private Info GetInfo(String fieldName)
-            {
-                return _index.fields[fieldName];
-            }
+                    public override IComparer<BytesRef> Comparator
+                    {
+                        get { return BytesRef.UTF8SortedAsUnicodeComparer; }
+                    }
 
-            private Info GetInfo(int pos)
-            {
-                return _index.sortedFields[pos].Value;
-            }
+                    public override long Size
+                    {
+                        get { return info.terms.Size; }
+                    }
 
-            public override int DocFreq(Term term)
-            {
-                Info info = GetInfo(term.Field);
-                int freq = 0;
-                if (info != null) freq = info.GetPositions(term.Text) != null ? 1 : 0;
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.docFreq: " + term + ", freq:" + freq);
-                return freq;
-            }
+                    public override long SumTotalTermFreq
+                    {
+                        get { return info.SumTotalTermFreq; }
+                    }
 
-            public override TermEnum Terms()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.terms()");
-                return Terms(MATCH_ALL_TERM);
-            }
+                    public override long SumDocFreq
+                    {
+                        get { return info.terms.Size; }
+                    }
 
-            public override TermEnum Terms(Term term)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.terms: " + term);
+                    public override int DocCount
+                    {
+                        get { return info.terms.Size > 0 ? 1 : 0; }
+                    }
 
-                int i; // index into info.sortedTerms
-                int j; // index into sortedFields
+                    public override bool HasOffsets
+                    {
+                        get { return parent.parent._index.storeOffsets; }
+                    }
 
-                _index.SortFields();
-                if (_index.sortedFields.Length == 1 && _index.sortedFields[0].Key == term.Field)
-                {
-                    j = 0; // fast path
-                }
-                else
-                {
-                    j = Array.BinarySearch(_index.sortedFields, new KeyValuePair<string, Info>(term.Field, null), Info.InfoComparer);
-                }
+                    public override bool HasPositions
+                    {
+                        get { return true; }
+                    }
 
-                if (j < 0)
-                {
-                    // not found; choose successor
-                    j = -j - 1;
-                    i = 0;
-                    if (j < _index.sortedFields.Length) GetInfo(j).SortTerms();
-                }
-                else
-                {
-                    // found
-                    Info info = GetInfo(j);
-                    info.SortTerms();
-                    i = Array.BinarySearch(info.SortedTerms, new KeyValuePair<string, ArrayIntList>(term.Text, null), Info.ArrayIntListComparer);
-                    if (i < 0)
+                    public override bool HasPayloads
                     {
-                        // not found; choose successor
-                        i = -i - 1;
-                        if (i >= info.SortedTerms.Length)
-                        {
-                            // move to next successor
-                            j++;
-                            i = 0;
-                            if (j < _index.sortedFields.Length) GetInfo(j).SortTerms();
-                        }
+                        get { return false; }
                     }
                 }
-                int ix = i;
-                int jx = j;
-
-                return new MemoryTermEnum(_index, this, ix, jx);
-            }
 
-            public override TermPositions TermPositions()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.termPositions");
-
-                return new MemoryTermPositions(_index, this);
-            }
-
-
-            public override TermDocs TermDocs()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.termDocs");
-                return TermPositions();
-            }
-
-            public override ITermFreqVector[] GetTermFreqVectors(int docNumber)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVectors");
-                // This is okay, ToArray() is as optimized as writing it by hand
-                return _index.fields.Keys.Select(k => GetTermFreqVector(docNumber, k)).ToArray();
+                public override int Size
+                {
+                    get { return parent._index.sortedFields.Length; }
+                }
             }
 
-            public override void GetTermFreqVector(int docNumber, TermVectorMapper mapper)
+            public override Fields Fields
             {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVectors");
-
-                //      if (vectors.length == 0) return null;
-                foreach (String fieldName in _index.fields.Keys)
+                get
                 {
-                    GetTermFreqVector(docNumber, fieldName, mapper);
+                    _index.SortFields();
+                    return new MemoryFields(this);
                 }
             }
 
-            public override void GetTermFreqVector(int docNumber, String field, TermVectorMapper mapper)
+            public override Fields GetTermVectors(int docID)
             {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVector");
-                Info info = GetInfo(field);
-                if (info == null)
+                if (docID == 0)
                 {
-                    return;
+                    return Fields;
                 }
-                info.SortTerms();
-                mapper.SetExpectations(field, info.SortedTerms.Length, _index.stride != 1, true);
-                for (int i = info.SortedTerms.Length; --i >= 0;)
+                else
                 {
-
-                    ArrayIntList positions = info.SortedTerms[i].Value;
-                    int size = positions.Size();
-                    var offsets = new TermVectorOffsetInfo[size/_index.stride];
-
-                    for (int k = 0, j = 1; j < size; k++, j += _index.stride)
-                    {
-                        int start = positions.Get(j);
-                        int end = positions.Get(j + 1);
-                        offsets[k] = new TermVectorOffsetInfo(start, end);
-                    }
-                    mapper.Map(info.SortedTerms[i].Key, _index.NumPositions(info.SortedTerms[i].Value), offsets,
-                               (info.SortedTerms[i].Value).ToArray(_index.stride));
+                    return null;
                 }
             }
 
-            public override ITermFreqVector GetTermFreqVector(int docNumber, String fieldName)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVector");
-                Info info = GetInfo(fieldName);
-                if (info == null) return null; // TODO: or return empty vector impl???
-                info.SortTerms();
-
-                return new MemoryTermPositionVector(_index, info, fieldName);
-            }
-
             private Similarity GetSimilarity()
             {
                 if (searcher != null) return searcher.Similarity;
-                return Similarity.Default;
+                return IndexSearcher.DefaultSimilarity;
             }
 
-            internal void SetSearcher(Searcher searcher)
+            internal void SetSearcher(IndexSearcher searcher)
             {
                 this.searcher = searcher;
             }
 
-            /* performance hack: cache norms to avoid repeated expensive calculations */
-            private byte[] cachedNorms;
-            private String cachedFieldName;
-            private Similarity cachedSimilarity;
-
-            public override byte[] Norms(String fieldName)
+            public override int NumDocs
             {
-                byte[] norms = cachedNorms;
-                Similarity sim = GetSimilarity();
-                if (fieldName != cachedFieldName || sim != cachedSimilarity)
+                get
                 {
-                    // not cached?
-                    Info info = GetInfo(fieldName);
-                    int numTokens = info != null ? info.NumTokens : 0;
-                    int numOverlapTokens = info != null ? info.NumOverlapTokens : 0;
-                    float boost = info != null ? info.Boost : 1.0f;
-                    FieldInvertState invertState = new FieldInvertState(0, numTokens, numOverlapTokens, 0, boost);
-                    float n = sim.ComputeNorm(fieldName, invertState);
-                    byte norm = Similarity.EncodeNorm(n);
-                    norms = new byte[] {norm};
+                    if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.numDocs");
 
-                    // cache it for future reuse
-                    cachedNorms = norms;
-                    cachedFieldName = fieldName;
-                    cachedSimilarity = sim;
-                    if (DEBUG)
-                        System.Diagnostics.Debug.WriteLine("MemoryIndexReader.norms: " + fieldName + ":" + n + ":" +
-                                                           norm + ":" + numTokens);
+                    return 1;
                 }
-                return norms;
-            }
-
-            public override void Norms(String fieldName, byte[] bytes, int offset)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.norms*: " + fieldName);
-                byte[] norms = Norms(fieldName);
-                Buffer.BlockCopy(norms, 0, bytes, offset, norms.Length);
-            }
-
-            protected override void DoSetNorm(int doc, String fieldName, byte value)
-            {
-                throw new NotSupportedException();
-            }
-
-            public override int NumDocs()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.numDocs");
-                return _index.fields.Count > 0 ? 1 : 0;
             }
 
             public override int MaxDoc
@@ -975,164 +871,58 @@ namespace Lucene.Net.Index.Memory
                 }
             }
 
-            public override Document Document(int n)
+            public override void Document(int docID, StoredFieldVisitor visitor)
             {
                 if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.document");
-                return new Document(); // there are no stored fields
+                // no-op: there are no stored fields
             }
-
-            //When we convert to JDK 1.5 make this Set<String>
-            public override Document Document(int n, FieldSelector fieldSelector)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.document");
-                return new Document(); // there are no stored fields
-            }
-
-            public override bool IsDeleted(int n)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.isDeleted");
-                return false;
-            }
-
-            public override bool HasDeletions
-            {
-                get
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.hasDeletions");
-                    return false;
-                }
-            }
-
-            protected override void DoDelete(int docNum)
-            {
-                throw new NotSupportedException();
-            }
-
-            protected override void DoUndeleteAll()
-            {
-                throw new NotSupportedException();
-            }
-
-            protected override void DoCommit(IDictionary<String, String> commitUserData)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.doCommit");
-
-            }
-
+            
             protected override void DoClose()
             {
                 if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.doClose");
             }
 
-            // lucene >= 1.9 (remove this method for lucene-1.4.3)
-            public override ICollection<String> GetFieldNames(FieldOption fieldOption)
+            /* performance hack: cache norms to avoid repeated expensive calculations */
+            private NumericDocValues cachedNormValues;
+            private String cachedFieldName;
+            private Similarity cachedSimilarity;
+
+            public override NumericDocValues GetNormValues(string field)
             {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getFieldNamesOption");
-                if (fieldOption == FieldOption.UNINDEXED)
-                    return CollectionsHelper<string>.EmptyList();
-                if (fieldOption == FieldOption.INDEXED_NO_TERMVECTOR)
-                    return CollectionsHelper<string>.EmptyList();
-                if (fieldOption == FieldOption.TERMVECTOR_WITH_OFFSET && _index.stride == 1)
-                    return CollectionsHelper<string>.EmptyList();
-                if (fieldOption == FieldOption.TERMVECTOR_WITH_POSITION_OFFSET && _index.stride == 1)
-                    return CollectionsHelper<string>.EmptyList();
-
-                return _index.fields.Keys.AsReadOnly();
+                FieldInfo fieldInfo = _index.fieldInfos[field];
+                if (fieldInfo == null || fieldInfo.OmitsNorms)
+                    return null;
+                NumericDocValues norms = cachedNormValues;
+                Similarity sim = GetSimilarity();
+                if (!field.Equals(cachedFieldName) || sim != cachedSimilarity)
+                { // not cached?
+                    Info info = GetInfo(field);
+                    int numTokens = info != null ? info.numTokens : 0;
+                    int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
+                    float boost = info != null ? info.Boost : 1.0f;
+                    FieldInvertState invertState = new FieldInvertState(field, 0, numTokens, numOverlapTokens, 0, boost);
+                    long value = sim.ComputeNorm(invertState);
+                    norms = new MemoryIndexNormDocValues(value);
+                    // cache it for future reuse
+                    cachedNormValues = norms;
+                    cachedFieldName = field;
+                    cachedSimilarity = sim;
+                    if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.norms: " + field + ":" + value + ":" + numTokens);
+                }
+                return norms;
             }
         }
-
-
-        ///////////////////////////////////////////////////////////////////////////////
-        // Nested classes:
-        ///////////////////////////////////////////////////////////////////////////////
-        private static class VM
+        
+        /**
+        * Resets the {@link MemoryIndex} to its initial state and recycles all internal buffers.
+        */
+        public void Reset()
         {
-
-            public static readonly int PTR = Is64BitVM() ? 8 : 4;
-
-            // bytes occupied by primitive data types
-            public static readonly int BOOLEAN = 1;
-            public static readonly int BYTE = 1;
-            public static readonly int CHAR = 2;
-            public static readonly int SHORT = 2;
-            public static readonly int INT = 4;
-            public static readonly int LONG = 8;
-            public static readonly int FLOAT = 4;
-            public static readonly int DOUBLE = 8;
-
-            private static readonly int LOG_PTR = (int) Math.Round(Log2(PTR));
-
-            /*
-             * Object header of any heap allocated Java object. 
-             * ptr to class, info for monitor, gc, hash, etc.
-             */
-            private static readonly int OBJECT_HEADER = 2*PTR;
-
-            //  assumes n > 0
-            //  64 bit VM:
-            //    0     --> 0*PTR
-            //    1..8  --> 1*PTR
-            //    9..16 --> 2*PTR
-            private static int SizeOf(int n)
-            {
-                return (((n - 1) >> LOG_PTR) + 1) << LOG_PTR;
-            }
-
-            public static int SizeOfObject(int n)
-            {
-                return SizeOf(OBJECT_HEADER + n);
-            }
-
-            public static int SizeOfObjectArray(int len)
-            {
-                return SizeOfObject(INT + PTR*len);
-            }
-
-            public static int SizeOfCharArray(int len)
-            {
-                return SizeOfObject(INT + CHAR*len);
-            }
-
-            public static int SizeOfIntArray(int len)
-            {
-                return SizeOfObject(INT + INT*len);
-            }
-
-            public static int SizeOfString(int len)
-            {
-                return SizeOfObject(3*INT + PTR) + SizeOfCharArray(len);
-            }
-
-            public static int SizeOfHashMap(int len)
-            {
-                return SizeOfObject(4*PTR + 4*INT) + SizeOfObjectArray(len)
-                       + len*SizeOfObject(3*PTR + INT); // entries
-            }
-
-            // note: does not include referenced objects
-            public static int SizeOfArrayList(int len)
-            {
-                return SizeOfObject(PTR + 2*INT) + SizeOfObjectArray(len);
-            }
-
-            public static int SizeOfArrayIntList(int len)
-            {
-                return SizeOfObject(PTR + INT) + SizeOfIntArray(len);
-            }
-
-            private static bool Is64BitVM()
-            {
-                return IntPtr.Size == 8;
-            }
-
-            /* logarithm to the base 2. Example: log2(4) == 2, log2(8) == 3 */
-
-            private static double Log2(double value)
-            {
-                return Math.Log(value, 2);
-                //return Math.Log(value) / Math.Log(2);
-            }
+            this.fieldInfos.Clear();
+            this.fields.Clear();
+            this.sortedFields = null;
+            byteBlockPool.Reset(false, false); // no need to 0-fill the buffers
+            intBlockPool.Reset(true, false); // here must must 0-fill since we use slices
         }
-
     }
 }

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/contrib/Memory/MemoryIndexNormDocValues.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Memory/MemoryIndexNormDocValues.cs b/src/contrib/Memory/MemoryIndexNormDocValues.cs
new file mode 100644
index 0000000..3d55b3a
--- /dev/null
+++ b/src/contrib/Memory/MemoryIndexNormDocValues.cs
@@ -0,0 +1,45 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Lucene.Net.Index.Memory
+{
+    public class MemoryIndexNormDocValues : NumericDocValues
+    {
+        private readonly long value;
+
+        public MemoryIndexNormDocValues(long value)
+        {
+            this.value = value;
+        }
+
+        public override long Get(int docID)
+        {
+            if (docID != 0)
+                throw new IndexOutOfRangeException();
+            else
+                return value;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/contrib/Memory/MemoryTermEnum.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Memory/MemoryTermEnum.cs b/src/contrib/Memory/MemoryTermEnum.cs
deleted file mode 100644
index ba9436b..0000000
--- a/src/contrib/Memory/MemoryTermEnum.cs
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Lucene.Net.Index.Memory
-{
-    public partial class MemoryIndex
-    {
-        private sealed partial class MemoryIndexReader
-        {
-            private class MemoryTermEnum : TermEnum
-            {
-                private readonly MemoryIndex _index;
-                private readonly MemoryIndexReader _reader;
-                private int _i; // index into info.sortedTerms
-                private int _j; // index into sortedFields
-
-                public MemoryTermEnum(MemoryIndex index, MemoryIndexReader reader, int ix, int jx)
-                {
-                    _index = index;
-                    _reader = reader;
-                    _i = ix; // index into info.sortedTerms
-                    _j = jx; // index into sortedFields
-                }
-
-                public override bool Next()
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine("TermEnum.next");
-                    if (_j >= _index.sortedFields.Length) return false;
-                    Info info = _reader.GetInfo(_j);
-                    if (++_i < info.SortedTerms.Length) return true;
-
-                    // move to successor
-                    _j++;
-                    _i = 0;
-                    if (_j >= _index.sortedFields.Length) return false;
-                    _reader.GetInfo(_j).SortTerms();
-                    return true;
-                }
-
-                public override Term Term
-                {
-                    get
-                    {
-                        if (DEBUG) System.Diagnostics.Debug.WriteLine("TermEnum.term: " + _i);
-                        if (_j >= _index.sortedFields.Length) return null;
-                        Info info = _reader.GetInfo(_j);
-                        if (_i >= info.SortedTerms.Length) return null;
-                        //          if (DEBUG) System.Diagnostics.Debug.WriteLine("TermEnum.term: " + i + ", " + info.sortedTerms[i].getKey());
-                        return CreateTerm(info, _j, info.SortedTerms[_i].Key);
-                    }
-                }
-
-                public override int DocFreq()
-                {                
-                              if (DEBUG) System.Diagnostics.Debug.WriteLine("TermEnum.docFreq");
-                              if (_j >= _index.sortedFields.Length) return 0;
-                              Info info = _reader.GetInfo(_j);
-                              if (_i >= info.SortedTerms.Length) return 0;
-                              return _index.NumPositions(info.GetPositions(_i));
-                }
-
-                protected override void Dispose(bool disposing)
-                {
-                              if (DEBUG) System.Diagnostics.Debug.WriteLine("TermEnum.close");
-                }
-
-                private Term CreateTerm(Info info, int pos, string text)
-                {
-                    // Assertion: sortFields has already been called before
-                    Term template = info.template;
-                    if (template == null) { // not yet cached?
-                        String fieldName = _index.sortedFields[pos].Key;
-                    template = new Term(fieldName);
-                    info.template = template;
-                    }
-
-                    return template.CreateTerm(text);
-                }
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/contrib/Memory/MemoryTermPositionVector.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Memory/MemoryTermPositionVector.cs b/src/contrib/Memory/MemoryTermPositionVector.cs
deleted file mode 100644
index c4b2797..0000000
--- a/src/contrib/Memory/MemoryTermPositionVector.cs
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Lucene.Net.Index.Memory
-{
-    public partial class MemoryIndex
-    {
-        private sealed partial class MemoryIndexReader
-        {
-            private class MemoryTermPositionVector : TermPositionVector
-            {
-                private readonly MemoryIndex _index;
-                private readonly string _fieldName;
-                private readonly KeyValuePair<String,ArrayIntList>[] sortedTerms;
-
-                public MemoryTermPositionVector(MemoryIndex index, Info info, string fieldName)
-                {
-                    _index = index;
-                    _fieldName = fieldName;
-                    sortedTerms = info.SortedTerms;
-                }
-
-                public string Field
-                {
-                    get { return _fieldName; }
-                }
-
-                public int Size
-                {
-                    get { return sortedTerms.Length; }
-                }
-
-                public string[] GetTerms()
-                {
-                    var terms = new String[sortedTerms.Length];
-                    for (int i = sortedTerms.Length; --i >= 0; )
-                    {
-                        terms[i] = sortedTerms[i].Key;
-                    }
-                    return terms;
-                }
-
-                public int[] GetTermFrequencies()
-                {
-                    int[] freqs = new int[sortedTerms.Length];
-                    for (int i = sortedTerms.Length; --i >= 0; )
-                    {
-                        freqs[i] = _index.NumPositions(sortedTerms[i].Value);
-                    }
-                    return freqs;
-                }
-
-                public int IndexOf(string term)
-                {
-                    int i = Array.BinarySearch(sortedTerms, new KeyValuePair<string, ArrayIntList>(term, null), Info.ArrayIntListComparer);
-                    return i >= 0 ? i : -1;
-                }
-
-                public int[] IndexesOf(string[] terms, int start, int len)
-                {
-                    int[] indexes = new int[len];
-                    for (int i = 0; i < len; i++)
-                    {
-                        indexes[i] = IndexOf(terms[start++]);
-                    }
-                    return indexes;
-                }
-
-                public int[] GetTermPositions(int index)
-                {
-                    return sortedTerms[index].Value.ToArray(_index.stride);
-                }
-
-                public TermVectorOffsetInfo[] GetOffsets(int index)
-                {
-                    if (_index.stride == 1) return null; // no offsets stored
-
-                    ArrayIntList positions = sortedTerms[index].Value;
-                    int size = positions.Size();
-                    TermVectorOffsetInfo[] offsets = new TermVectorOffsetInfo[size / _index.stride];
-
-                    for (int i = 0, j = 1; j < size; i++, j += _index.stride)
-                    {
-                        int start = positions.Get(j);
-                        int end = positions.Get(j + 1);
-                        offsets[i] = new TermVectorOffsetInfo(start, end);
-                    }
-                    return offsets;
-                }
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/contrib/Memory/MemoryTermPositions.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Memory/MemoryTermPositions.cs b/src/contrib/Memory/MemoryTermPositions.cs
deleted file mode 100644
index 7096379..0000000
--- a/src/contrib/Memory/MemoryTermPositions.cs
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Lucene.Net.Index.Memory
-{
-    public partial class MemoryIndex
-    {
-        private sealed partial class MemoryIndexReader
-        {
-            private sealed class MemoryTermPositions : TermPositions
-            {
-                private readonly MemoryIndex _index;
-                private readonly MemoryIndexReader _reader;
-                private bool hasNext;
-                private int cursor = 0;
-                private ArrayIntList current;
-                private Term term;
-
-                public MemoryTermPositions(MemoryIndex index, MemoryIndexReader reader)
-                {
-                    _index = index;
-                    _reader = reader;
-                }
-
-                public void Seek(Term term)
-                {
-                    this.term = term;
-
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine(".seek: " + term);
-
-                    if (term == null)
-                    {
-                        hasNext = true; // term==null means match all docs
-                    }
-                    else
-                    {
-                        Info info = _reader.GetInfo(term.Field);
-                        current = info == null ? null : info.GetPositions(term.Text);
-                        hasNext = (current != null);
-                        cursor = 0;
-                    }
-                }
-
-                public void Seek(TermEnum termEnum)
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine(".seekEnum");
-                    Seek(termEnum.Term);
-                }
-
-                public int Doc
-                {
-                    get
-                    {
-                        if (DEBUG) System.Diagnostics.Debug.WriteLine(".doc");
-                        return 0;
-                    }
-                }
-
-                public int Freq
-                {
-                    get
-                    {
-                        int freq = current != null ? _index.NumPositions(current) : (term == null ? 1 : 0);
-                        if (DEBUG) System.Diagnostics.Debug.WriteLine(".freq: " + freq);
-                        return freq;
-                    }
-                }
-
-                public bool Next()
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine(".next: " + current + ", oldHasNext=" + hasNext);
-                    bool next = hasNext;
-                    hasNext = false;
-                    return next;
-                }
-
-                public int Read(int[] docs, int[] freqs)
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine(".read: " + docs.Length);
-                    if (!hasNext) return 0;
-                    hasNext = false;
-                    docs[0] = 0;
-                    freqs[0] = Freq;
-                    return 1;
-                }
-
-                public bool SkipTo(int target)
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine(".skipTo: " + target);
-                    return Next();
-                }
-
-                public void Close()
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine(".close");
-                }
-
-                public void Dispose()
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine(".close");
-                }
-
-                public int NextPosition()
-                {
-                    int pos = current.Get(cursor);
-                    cursor += _index.stride;
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine(".nextPosition: " + pos);
-                    return pos;
-                }
-
-                public int PayloadLength
-                {
-                    get { throw new NotSupportedException(); }
-                }
-
-                public byte[] GetPayload(byte[] data, int offset)
-                {
-                    throw new NotSupportedException();
-                }
-
-                public bool IsPayloadAvailable
-                {
-                    get { return false; }
-                }
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/contrib/Memory/MemoryTermsEnum.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Memory/MemoryTermsEnum.cs b/src/contrib/Memory/MemoryTermsEnum.cs
new file mode 100644
index 0000000..68f3c57
--- /dev/null
+++ b/src/contrib/Memory/MemoryTermsEnum.cs
@@ -0,0 +1,189 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Lucene.Net.Index.Memory
+{
+    public partial class MemoryIndex
+    {
+        private sealed partial class MemoryIndexReader
+        {
+            private class MemoryTermsEnum : TermsEnum
+            {
+                private readonly Info info;
+                private readonly BytesRef br = new BytesRef();
+                int termUpto = -1;
+
+                private readonly MemoryIndex index;
+
+                public MemoryTermsEnum(MemoryIndex index, Info info)
+                {
+                    this.index = index;
+                    this.info = info;
+                    info.SortTerms();
+                }
+
+                private int BinarySearch(BytesRef b, BytesRef bytesRef, int low,
+                    int high, BytesRefHash hash, int[] ords, IComparer<BytesRef> comparator)
+                {
+                    int mid = 0;
+                    while (low <= high)
+                    {
+                        mid = Number.URShift((low + high), 1);
+                        hash.Get(ords[mid], bytesRef);
+                        int cmp = comparator.Compare(bytesRef, b);
+                        if (cmp < 0)
+                        {
+                            low = mid + 1;
+                        }
+                        else if (cmp > 0)
+                        {
+                            high = mid - 1;
+                        }
+                        else
+                        {
+                            return mid;
+                        }
+                    }
+                    //assert comparator.compare(bytesRef, b) != 0;
+                    return -(low + 1);
+                }
+
+                public override bool SeekExact(BytesRef text, bool useCache)
+                {
+                    termUpto = BinarySearch(text, br, 0, info.terms.Size - 1, info.terms, info.sortedTerms, BytesRef.UTF8SortedAsUnicodeComparer);
+                    return termUpto >= 0;
+                }
+
+                public override SeekStatus SeekCeil(BytesRef text, bool useCache)
+                {
+                    termUpto = BinarySearch(text, br, 0, info.terms.Size - 1, info.terms, info.sortedTerms, BytesRef.UTF8SortedAsUnicodeComparer);
+                    if (termUpto < 0)
+                    { // not found; choose successor
+                        termUpto = -termUpto - 1;
+                        if (termUpto >= info.terms.Size)
+                        {
+                            return SeekStatus.END;
+                        }
+                        else
+                        {
+                            info.terms.Get(info.sortedTerms[termUpto], br);
+                            return SeekStatus.NOT_FOUND;
+                        }
+                    }
+                    else
+                    {
+                        return SeekStatus.FOUND;
+                    }
+                }
+
+                public override void SeekExact(long ord)
+                {
+                    //assert ord < info.terms.size();
+                    termUpto = (int)ord;
+                }
+
+                public override BytesRef Next()
+                {
+                    termUpto++;
+                    if (termUpto >= info.terms.Size)
+                    {
+                        return null;
+                    }
+                    else
+                    {
+                        info.terms.Get(info.sortedTerms[termUpto], br);
+                        return br;
+                    }
+                }
+
+                public override BytesRef Term
+                {
+                    get
+                    {
+                        return br;
+                    }
+                }
+
+                public override long Ord
+                {
+                    get { return termUpto; }
+                }
+
+                public override int DocFreq
+                {
+                    get { return 1; }
+                }
+
+                public override long TotalTermFreq
+                {
+                    get { return info.sliceArray.freq[info.sortedTerms[termUpto]]; }
+                }
+
+                public override DocsEnum Docs(IBits liveDocs, DocsEnum reuse, int flags)
+                {
+                    if (reuse == null || !(reuse is MemoryDocsEnum))
+                    {
+                        reuse = new MemoryDocsEnum();
+                    }
+                    return ((MemoryDocsEnum)reuse).Reset(liveDocs, info.sliceArray.freq[info.sortedTerms[termUpto]]);
+                }
+
+                public override DocsAndPositionsEnum DocsAndPositions(IBits liveDocs, DocsAndPositionsEnum reuse, int flags)
+                {
+                    if (reuse == null || !(reuse is MemoryDocsAndPositionsEnum))
+                    {
+                        reuse = new MemoryDocsAndPositionsEnum(index);
+                    }
+                    int ord = info.sortedTerms[termUpto];
+                    return ((MemoryDocsAndPositionsEnum)reuse).Reset(liveDocs, info.sliceArray.start[ord], info.sliceArray.end[ord], info.sliceArray.freq[ord]);
+                }
+
+                public override IComparer<BytesRef> Comparator
+                {
+                    get { return BytesRef.UTF8SortedAsUnicodeComparer; }
+                }
+
+                public override void SeekExact(BytesRef term, TermState state)
+                {
+                    //assert state != null;
+                    this.SeekExact(((OrdTermState)state).ord);
+                }
+
+                public override TermState TermState
+                {
+                    get
+                    {
+                        OrdTermState ts = new OrdTermState();
+                        ts.ord = termUpto;
+                        return ts;
+                    }
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/contrib/Memory/SliceByteStartArray.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Memory/SliceByteStartArray.cs b/src/contrib/Memory/SliceByteStartArray.cs
new file mode 100644
index 0000000..cd4c648
--- /dev/null
+++ b/src/contrib/Memory/SliceByteStartArray.cs
@@ -0,0 +1,77 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Lucene.Net.Index.Memory
+{
+    public partial class MemoryIndex
+    {
+        internal class SliceByteStartArray : BytesRefHash.DirectBytesStartArray
+        {
+            internal int[] start; // the start offset in the IntBlockPool per term
+            internal int[] end; // the end pointer in the IntBlockPool for the postings slice per term
+            internal int[] freq; // the term frequency
+
+            public SliceByteStartArray(int initSize)
+                : base(initSize)
+            {
+            }
+
+            public override int[] Init()
+            {
+                int[] ord = base.Init();
+                start = new int[ArrayUtil.Oversize(ord.Length, RamUsageEstimator.NUM_BYTES_INT)];
+                end = new int[ArrayUtil.Oversize(ord.Length, RamUsageEstimator.NUM_BYTES_INT)];
+                freq = new int[ArrayUtil.Oversize(ord.Length, RamUsageEstimator.NUM_BYTES_INT)];
+                //assert start.length >= ord.length;
+                //assert end.length >= ord.length;
+                //assert freq.length >= ord.length;
+                return ord;
+            }
+
+            public override int[] Grow()
+            {
+                int[] ord = base.Grow();
+                if (start.Length < ord.Length)
+                {
+                    start = ArrayUtil.Grow(start, ord.Length);
+                    end = ArrayUtil.Grow(end, ord.Length);
+                    freq = ArrayUtil.Grow(freq, ord.Length);
+                }
+                //assert start.length >= ord.length;
+                //assert end.length >= ord.length;
+                //assert freq.length >= ord.length;
+                return ord;
+            }
+
+            public override int[] Clear()
+            {
+                start = end = null;
+                return base.Clear();
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/contrib/Memory/TermComparer.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Memory/TermComparer.cs b/src/contrib/Memory/TermComparer.cs
index 0f3ce55..7576caf 100644
--- a/src/contrib/Memory/TermComparer.cs
+++ b/src/contrib/Memory/TermComparer.cs
@@ -20,13 +20,14 @@
 */
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
 namespace Lucene.Net.Index.Memory
 {
-    class TermComparer
+    internal class TermComparer
     {
         /// <summary>
         /// Sorts term entries into ascending order; also works for
@@ -42,11 +43,16 @@ namespace Lucene.Net.Index.Memory
         }
     }
 
-    sealed class TermComparer<T> : TermComparer, IComparer<KeyValuePair<string, T>>
+    sealed class TermComparer<T> : TermComparer, IComparer<KeyValuePair<string, T>>, IComparer
     {
         public int Compare(KeyValuePair<string, T> x, KeyValuePair<string, T> y)
         {
             return KeyComparer(x, y);
         }
+
+        int IComparer.Compare(object x, object y)
+        {
+            return KeyComparer((KeyValuePair<string, T>)x, (KeyValuePair<string, T>)y);
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/86d417ed/src/core/Support/Collections.cs
----------------------------------------------------------------------
diff --git a/src/core/Support/Collections.cs b/src/core/Support/Collections.cs
index 3e2ad2c..0256b73 100644
--- a/src/core/Support/Collections.cs
+++ b/src/core/Support/Collections.cs
@@ -275,7 +275,7 @@ namespace Lucene.Net.Support
         {
             list.Sort((a, b) => a.CompareTo(b));
         }
-
+        
         /// <summary>
         /// Fills the array with an specific value from an specific index to an specific index.
         /// </summary>


Mime
View raw message