lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject svn commit: r1344562 [2/4] - in /incubator/lucene.net/trunk: src/contrib/Analyzers/De/ src/contrib/Analyzers/Filters/ src/contrib/Core/Analysis/Ext/ src/contrib/Memory/ src/contrib/Memory/Properties/ src/contrib/Spatial/ src/contrib/Spatial/Prefix/ src...
Date Thu, 31 May 2012 04:33:43 GMT
Modified: incubator/lucene.net/trunk/src/contrib/Memory/MemoryIndex.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Memory/MemoryIndex.cs?rev=1344562&r1=1344561&r2=1344562&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Memory/MemoryIndex.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Memory/MemoryIndex.cs Thu May 31 04:33:41 2012
@@ -1,1123 +1,1123 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text;
-using Lucene.Net.Analysis;
-using Lucene.Net.Analysis.Tokenattributes;
-using Lucene.Net.Documents;
-using Lucene.Net.Search;
-using Lucene.Net.Support;
-
-namespace Lucene.Net.Index.Memory
-{
-    /// <summary>
-    /// High-performance single-document main memory Apache Lucene fulltext search index. 
-    /// 
-    /// <h4>Overview</h4>
-    /// 
-    /// This class is a replacement/substitute for a large subset of
-    /// {@link RAMDirectory} functionality. It is designed to
-    /// enable maximum efficiency for on-the-fly matchmaking combining structured and 
-    /// fuzzy fulltext search in realtime streaming applications such as Nux XQuery based XML 
-    /// message queues, publish-subscribe systems for Blogs/newsfeeds, text chat, data acquisition and 
-    /// distribution systems, application level routers, firewalls, classifiers, etc. 
-    /// Rather than targeting fulltext search of infrequent queries over huge persistent 
-    /// data archives (historic search), this class targets fulltext search of huge 
-    /// numbers of queries over comparatively small transient realtime data (prospective 
-    /// search). 
-    /// For example as in 
-    /// <pre>
-    /// float score = search(String text, Query query)
-    /// </pre>
-    /// <p/>
-    /// Each instance can hold at most one Lucene "document", with a document containing
-    /// zero or more "fields", each field having a name and a fulltext value. The
-    /// fulltext value is tokenized (split and transformed) into zero or more index terms 
-    /// (aka words) on <code>addField()</code>, according to the policy implemented by an
-    /// Analyzer. For example, Lucene analyzers can split on whitespace, normalize to lower case
-    /// for case insensitivity, ignore common terms with little discriminatory value such as "he", "in", "and" (stop
-    /// words), reduce the terms to their natural linguistic root form such as "fishing"
-    /// being reduced to "fish" (stemming), resolve synonyms/inflexions/thesauri 
-    /// (upon indexing and/or querying), etc. For details, see
-    /// <a target="_blank" href="http://today.java.net/pub/a/today/2003/07/30/LuceneIntro.html">Lucene Analyzer Intro</a>.
-    /// <p>
-    /// Arbitrary Lucene queries can be run against this class - see <a target="_blank" 
-    /// href="../../../../../../../queryparsersyntax.html">Lucene Query Syntax</a>
-    /// as well as <a target="_blank" 
-    /// href="http://today.java.net/pub/a/today/2003/11/07/QueryParserRules.html">Query Parser Rules</a>.
-    /// Note that a Lucene query selects on the field names and associated (indexed) 
-    /// tokenized terms, not on the original fulltext(s) - the latter are not stored 
-    /// but rather thrown away immediately after tokenization.
-    /// <p>
-    /// For some interesting background information on search technology, see Bob Wyman's
-    /// <a target="_blank" 
-    /// href="http://bobwyman.pubsub.com/main/2005/05/mary_hodder_poi.html">Prospective Search</a>, 
-    /// Jim Gray's
-    /// <a target="_blank" href="http://www.acmqueue.org/modules.php?name=Content&pa=showpage&pid=293&page=4">
-    /// A Call to Arms - Custom subscriptions</a>, and Tim Bray's
-    /// <a target="_blank" 
-    /// href="http://www.tbray.org/ongoing/When/200x/2003/07/30/OnSearchTOC">On Search, the Series</a>.
-    /// 
-    /// 
-    /// <h4>Example Usage</h4> 
-    /// 
-    /// <pre>
-    /// Analyzer analyzer = PatternAnalyzer.DEFAULT_ANALYZER;
-    /// //Analyzer analyzer = new SimpleAnalyzer();
-    /// MemoryIndex index = new MemoryIndex();
-    /// index.addField("content", "Readings about Salmons and other select Alaska fishing Manuals", analyzer);
-    /// index.addField("author", "Tales of James", analyzer);
-    /// QueryParser parser = new QueryParser("content", analyzer);
-    /// float score = index.search(parser.parse("+author:james +salmon~ +fish/// manual~"));
-    /// if (score &gt; 0.0f) {
-    ///     System.out.println("it's a match");
-    /// } else {
-    ///     System.out.println("no match found");
-    /// }
-    /// System.out.println("indexData=" + index.toString());
-    /// </pre>
-    /// 
-    /// 
-    /// <h4>Example XQuery Usage</h4> 
-    /// 
-    /// <pre>
-    /// (: An XQuery that finds all books authored by James that have something to do with "salmon fishing manuals", sorted by relevance :)
-    /// declare namespace lucene = "java:nux.xom.pool.FullTextUtil";
-    /// declare variable $query := "+salmon~ +fish/// manual~"; (: any arbitrary Lucene query can go here :)
-    /// 
-    /// for $book in /books/book[author="James" and lucene:match(abstract, $query) > 0.0]
-    /// let $score := lucene:match($book/abstract, $query)
-    /// order by $score descending
-    /// return $book
-    /// </pre>
-    /// 
-    /// 
-    /// <h4>No thread safety guarantees</h4>
-    /// 
-    /// An instance can be queried multiple times with the same or different queries,
-    /// but an instance is not thread-safe. If desired use idioms such as:
-    /// <pre>
-    /// MemoryIndex index = ...
-    /// synchronized (index) {
-    ///    // read and/or write index (i.e. add fields and/or query)
-    /// } 
-    /// </pre>
-    /// 
-    /// 
-    /// <h4>Performance Notes</h4>
-    /// 
-    /// Internally there's a new data structure geared towards efficient indexing 
-    /// and searching, plus the necessary support code to seamlessly plug into the Lucene 
-    /// framework.
-    /// <p/>
-    /// This class performs very well for very small texts (e.g. 10 chars) 
-    /// as well as for large texts (e.g. 10 MB) and everything in between. 
-    /// Typically, it is about 10-100 times faster than <code>RAMDirectory</code>.
-    /// Note that <code>RAMDirectory</code> has particularly 
-    /// large efficiency overheads for small to medium sized texts, both in time and space.
-    /// Indexing a field with N tokens takes O(N) in the best case, and O(N logN) in the worst 
-    /// case. Memory consumption is probably larger than for <code>RAMDirectory</code>.
-    /// <p/>
-    /// Example throughput of many simple term queries over a single MemoryIndex: 
-    /// ~500000 queries/sec on a MacBook Pro, jdk 1.5.0_06, server VM. 
-    /// As always, your mileage may vary.
-    /// <p/>
-    /// If you're curious about
-    /// the whereabouts of bottlenecks, run java 1.5 with the non-perturbing '-server
-    /// -agentlib:hprof=cpu=samples,depth=10' flags, then study the trace log and
-    /// correlate its hotspot trailer with its call stack headers (see <a
-    /// target="_blank" href="http://java.sun.com/developer/technicalArticles/Programming/HPROF.html">
-    /// hprof tracing </a>).
-    ///
-    ///</summary>
-    [Serializable]
-    public partial class MemoryIndex
-    {
-        /** info for each field: Map<String fieldName, Info field> */
-        private HashMap<String, Info> fields = new HashMap<String, Info>();
-
-        /** fields sorted ascending by fieldName; lazily computed on demand */
-        [NonSerialized] private KeyValuePair<String, Info>[] sortedFields;
-
-        /** pos: positions[3*i], startOffset: positions[3*i +1], endOffset: positions[3*i +2] */
-        private int stride;
-
-        /** Could be made configurable; See {@link Document#setBoost(float)} */
-        private static float docBoost = 1.0f;
-
-        private static long serialVersionUID = 2782195016849084649L;
-
-        private static bool DEBUG = false;
-
-        /**
-         * Constructs an empty instance.
-         */
-        public MemoryIndex()
-            : this(false)
-        {
-        }
-
-        /**
-         * Constructs an empty instance that can optionally store the start and end
-         * character offset of each token term in the text. This can be useful for
-         * highlighting of hit locations with the Lucene highlighter package.
-         * Private until the highlighter package matures, so that this can actually
-         * be meaningfully integrated.
-         * 
-         * @param storeOffsets
-         *            whether or not to store the start and end character offset of
-         *            each token term in the text
-         */
-
-        private MemoryIndex(bool storeOffsets)
-        {
-            this.stride = storeOffsets ? 3 : 1;
-        }
-
-        /**
-         * Convenience method; Tokenizes the given field text and adds the resulting
-         * terms to the index; Equivalent to adding an indexed non-keyword Lucene
-         * {@link org.apache.lucene.document.Field} that is
-         * {@link org.apache.lucene.document.Field.Index#ANALYZED tokenized},
-         * {@link org.apache.lucene.document.Field.Store#NO not stored},
-         * {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS termVectorStored with positions} (or
-         * {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS termVectorStored with positions and offsets}),
-         * 
-         * @param fieldName
-         *            a name to be associated with the text
-         * @param text
-         *            the text to tokenize and index.
-         * @param analyzer
-         *            the analyzer to use for tokenization
-         */
-
-        public void AddField(String fieldName, String text, Analyzer analyzer)
-        {
-            if (fieldName == null)
-                throw new ArgumentException("fieldName must not be null");
-            if (text == null)
-                throw new ArgumentException("text must not be null");
-            if (analyzer == null)
-                throw new ArgumentException("analyzer must not be null");
-
-            TokenStream stream = analyzer.TokenStream(fieldName, new StringReader(text));
-
-            AddField(fieldName, stream);
-        }
-
-        /**
-         * Convenience method; Creates and returns a token stream that generates a
-         * token for each keyword in the given collection, "as is", without any
-         * transforming text analysis. The resulting token stream can be fed into
-         * {@link #addField(String, TokenStream)}, perhaps wrapped into another
-         * {@link org.apache.lucene.analysis.TokenFilter}, as desired.
-         * 
-         * @param keywords
-         *            the keywords to generate tokens for
-         * @return the corresponding token stream
-         */
-
-        public TokenStream CreateKeywordTokenStream<T>(ICollection<T> keywords)
-        {
-            // TODO: deprecate & move this method into AnalyzerUtil?
-            if (keywords == null)
-                throw new ArgumentException("keywords must not be null");
-
-            return new KeywordTokenStream<T>(keywords);
-        }
-
-        /**
-         * Equivalent to <code>addField(fieldName, stream, 1.0f)</code>.
-         * 
-         * @param fieldName
-         *            a name to be associated with the text
-         * @param stream
-         *            the token stream to retrieve tokens from
-         */
-        public void AddField(String fieldName, TokenStream stream)
-        {
-            AddField(fieldName, stream, 1.0f);
-        }
-
-        /**
-         * Iterates over the given token stream and adds the resulting terms to the index;
-         * Equivalent to adding a tokenized, indexed, termVectorStored, unstored,
-         * Lucene {@link org.apache.lucene.document.Field}.
-         * Finally closes the token stream. Note that untokenized keywords can be added with this method via 
-         * {@link #CreateKeywordTokenStream(Collection)}, the Lucene contrib <code>KeywordTokenizer</code> or similar utilities.
-         * 
-         * @param fieldName
-         *            a name to be associated with the text
-         * @param stream
-         *            the token stream to retrieve tokens from.
-         * @param boost
-         *            the boost factor for hits for this field
-         * @see org.apache.lucene.document.Field#setBoost(float)
-         */
-        public void AddField(String fieldName, TokenStream stream, float boost)
-        {
-            try
-            {
-                if (fieldName == null)
-                    throw new ArgumentException("fieldName must not be null");
-                if (stream == null)
-                    throw new ArgumentException("token stream must not be null");
-                if (boost <= 0.0f)
-                    throw new ArgumentException("boost factor must be greater than 0.0");
-                if (fields[fieldName] != null)
-                    throw new ArgumentException("field must not be added more than once");
-
-                var terms = new HashMap<String, ArrayIntList>();
-                int numTokens = 0;
-                int numOverlapTokens = 0;
-                int pos = -1;
-
-                var termAtt = stream.AddAttribute<ITermAttribute>();
-                var posIncrAttribute = stream.AddAttribute<IPositionIncrementAttribute>();
-                var offsetAtt = stream.AddAttribute<IOffsetAttribute>();
-
-                stream.Reset();
-                while (stream.IncrementToken())
-                {
-                    String term = termAtt.Term();
-                    if (term.Length == 0) continue; // nothing to do
-                    //        if (DEBUG) System.Diagnostics.Debug.WriteLine("token='" + term + "'");
-                    numTokens++;
-                    int posIncr = posIncrAttribute.PositionIncrement;
-                    if (posIncr == 0)
-                        numOverlapTokens++;
-                    pos += posIncr;
-
-                    ArrayIntList positions = terms[term];
-                    if (positions == null)
-                    {
-                        // term not seen before
-                        positions = new ArrayIntList(stride);
-                        terms[term] = positions;
-                    }
-                    if (stride == 1)
-                    {
-                        positions.Add(pos);
-                    }
-                    else
-                    {
-                        positions.Add(pos, offsetAtt.StartOffset, offsetAtt.EndOffset);
-                    }
-                }
-                stream.End();
-
-                // ensure infos.numTokens > 0 invariant; needed for correct operation of terms()
-                if (numTokens > 0)
-                {
-                    boost = boost*docBoost; // see DocumentWriter.addDocument(...)
-                    fields[fieldName] = new Info(terms, numTokens, numOverlapTokens, boost);
-                    sortedFields = null; // invalidate sorted view, if any
-                }
-            }
-            catch (IOException e)
-            {
-                // can never happen
-                throw new SystemException(string.Empty, e);
-            }
-            finally
-            {
-                try
-                {
-                    if (stream != null) stream.Close();
-                }
-                catch (IOException e2)
-                {
-                    throw new SystemException(string.Empty, e2);
-                }
-            }
-        }
-
-        /**
-         * Creates and returns a searcher that can be used to execute arbitrary
-         * Lucene queries and to collect the resulting query results as hits.
-         * 
-         * @return a searcher
-         */
-
-        public IndexSearcher CreateSearcher()
-        {
-            MemoryIndexReader reader = new MemoryIndexReader(this);
-            IndexSearcher searcher = new IndexSearcher(reader); // ensures no auto-close !!
-            reader.SetSearcher(searcher); // to later get hold of searcher.getSimilarity()
-            return searcher;
-        }
-
-        /**
-         * Convenience method that efficiently returns the relevance score by
-         * matching this index against the given Lucene query expression.
-         * 
-         * @param query
-         *            an arbitrary Lucene query to run against this index
-         * @return the relevance score of the matchmaking; A number in the range
-         *         [0.0 .. 1.0], with 0.0 indicating no match. The higher the number
-         *         the better the match.
-         *
-         */
-
-        public float Search(Query query)
-        {
-            if (query == null)
-                throw new ArgumentException("query must not be null");
-
-            Searcher searcher = CreateSearcher();
-            try
-            {
-                float[] scores = new float[1]; // inits to 0.0f (no match)
-                searcher.Search(query, new FillingCollector(scores));
-                float score = scores[0];
-                return score;
-            }
-            catch (IOException e)
-            {
-                // can never happen (RAMDirectory)
-                throw new SystemException(string.Empty, e);
-            }
-            finally
-            {
-                // searcher.close();
-                /*
-                 * Note that it is harmless and important for good performance to
-                 * NOT close the index reader!!! This avoids all sorts of
-                 * unnecessary baggage and locking in the Lucene IndexReader
-                 * superclass, all of which is completely unnecessary for this main
-                 * memory index data structure without thread-safety claims.
-                 * 
-                 * Wishing IndexReader would be an interface...
-                 * 
-                 * Actually with the new tight createSearcher() API auto-closing is now
-                 * made impossible, hence searcher.close() would be harmless and also 
-                 * would not degrade performance...
-                 */
-            }
-        }
-
-        /**
-         * Returns a reasonable approximation of the main memory [bytes] consumed by
-         * this instance. Useful for smart memory sensititive caches/pools. Assumes
-         * fieldNames are interned, whereas tokenized terms are memory-overlaid.
-         * 
-         * @return the main memory consumption
-         */
-        public int GetMemorySize()
-        {
-            // for example usage in a smart cache see nux.xom.pool.Pool    
-            int PTR = VM.PTR;
-            int INT = VM.INT;
-            int size = 0;
-            size += VM.SizeOfObject(2*PTR + INT); // memory index
-            if (sortedFields != null) size += VM.SizeOfObjectArray(sortedFields.Length);
-
-            size += VM.SizeOfHashMap(fields.Count);
-            foreach (var entry in fields)
-            {
-                // for each Field Info
-                Info info = entry.Value;
-                size += VM.SizeOfObject(2*INT + 3*PTR); // Info instance vars
-                if (info.SortedTerms != null) size += VM.SizeOfObjectArray(info.SortedTerms.Length);
-
-                int len = info.Terms.Count;
-                size += VM.SizeOfHashMap(len);
-
-                var iter2 = info.Terms.GetEnumerator();
-                while (--len >= 0)
-                {
-                    iter2.MoveNext();
-                    // for each term
-                    KeyValuePair<String, ArrayIntList> e = iter2.Current;
-                    size += VM.SizeOfObject(PTR + 3*INT); // assumes substring() memory overlay
-//        size += STR + 2 * ((String) e.getKey()).length();
-                    ArrayIntList positions = e.Value;
-                    size += VM.SizeOfArrayIntList(positions.Size());
-                }
-            }
-            return size;
-        }
-
-        private int NumPositions(ArrayIntList positions)
-        {
-            return positions.Size()/stride;
-        }
-
-        /** sorts into ascending order (on demand), reusing memory along the way */
-
-        private void SortFields()
-        {
-            if (sortedFields == null) sortedFields = Sort(fields);
-        }
-
-        /** returns a view of the given map's entries, sorted ascending by key */
-
-        private static KeyValuePair<TKey, TValue>[] Sort<TKey, TValue>(HashMap<TKey, TValue> map)
-            where TKey : class, IComparable<TKey>
-        {
-            int size = map.Count;
-
-            var entries = map.ToArray();
-
-            if (size > 1) Array.Sort(entries, TermComparer.KeyComparer);
-            return entries;
-        }
-
-        /**
-         * Returns a String representation of the index data for debugging purposes.
-         * 
-         * @return the string representation
-         */
-
-        public override String ToString()
-        {
-            StringBuilder result = new StringBuilder(256);
-            SortFields();
-            int sumChars = 0;
-            int sumPositions = 0;
-            int sumTerms = 0;
-
-            for (int i = 0; i < sortedFields.Length; i++)
-            {
-                KeyValuePair<String, Info> entry = sortedFields[i];
-                String fieldName = entry.Key;
-                Info info = entry.Value;
-                info.SortTerms();
-                result.Append(fieldName + ":\n");
-
-                int numChars = 0;
-                int numPos = 0;
-                for (int j = 0; j < info.SortedTerms.Length; j++)
-                {
-                    KeyValuePair<String, ArrayIntList> e = info.SortedTerms[j];
-                    String term = e.Key;
-                    ArrayIntList positions = e.Value;
-                    result.Append("\t'" + term + "':" + NumPositions(positions) + ":");
-                    result.Append(positions.ToString(stride)); // ignore offsets
-                    result.Append("\n");
-                    numPos += NumPositions(positions);
-                    numChars += term.Length;
-                }
-
-                result.Append("\tterms=" + info.SortedTerms.Length);
-                result.Append(", positions=" + numPos);
-                result.Append(", Kchars=" + (numChars/1000.0f));
-                result.Append("\n");
-                sumPositions += numPos;
-                sumChars += numChars;
-                sumTerms += info.SortedTerms.Length;
-            }
-
-            result.Append("\nfields=" + sortedFields.Length);
-            result.Append(", terms=" + sumTerms);
-            result.Append(", positions=" + sumPositions);
-            result.Append(", Kchars=" + (sumChars/1000.0f));
-            return result.ToString();
-        }
-
-
-        ///////////////////////////////////////////////////////////////////////////////
-        // Nested classes:
-        ///////////////////////////////////////////////////////////////////////////////
-        /**
-         * Index data structure for a field; Contains the tokenized term texts and
-         * their positions.
-         */
-
-        [Serializable]
-        private sealed class Info
-        {
-            public static readonly IComparer<KeyValuePair<string, Info>> InfoComparer = new TermComparer<Info>();
-            public static readonly IComparer<KeyValuePair<string, ArrayIntList>> ArrayIntListComparer = new TermComparer<ArrayIntList>(); 
-            /**
-             * Term strings and their positions for this field: Map <String
-             * termText, ArrayIntList positions>
-             */
-            private HashMap<String, ArrayIntList> terms;
-
-            /** Terms sorted ascending by term text; computed on demand */
-            [NonSerialized] private KeyValuePair<String, ArrayIntList>[] sortedTerms;
-
-            /** Number of added tokens for this field */
-            private int numTokens;
-
-            /** Number of overlapping tokens for this field */
-            private int numOverlapTokens;
-
-            /** Boost factor for hits for this field */
-            private float boost;
-
-            /** Term for this field's fieldName, lazily computed on demand */
-            [NonSerialized] public Term template;
-
-            private static long serialVersionUID = 2882195016849084649L;
-
-            public Info(HashMap<String, ArrayIntList> terms, int numTokens, int numOverlapTokens, float boost)
-            {
-                this.terms = terms;
-                this.numTokens = numTokens;
-                this.NumOverlapTokens = numOverlapTokens;
-                this.boost = boost;
-            }
-
-            public HashMap<string, ArrayIntList> Terms
-            {
-                get { return terms; }
-            }
-
-            public int NumTokens
-            {
-                get { return numTokens; }
-            }
-
-            public int NumOverlapTokens
-            {
-                get { return numOverlapTokens; }
-                set { numOverlapTokens = value; }
-            }
-
-            public float Boost
-            {
-                get { return boost; }
-            }
-
-            public KeyValuePair<string, ArrayIntList>[] SortedTerms
-            {
-                get { return sortedTerms; }
-            }
-
-            /**
-         * Sorts hashed terms into ascending order, reusing memory along the
-         * way. Note that sorting is lazily delayed until required (often it's
-         * not required at all). If a sorted view is required then hashing +
-         * sort + binary search is still faster and smaller than TreeMap usage
-         * (which would be an alternative and somewhat more elegant approach,
-         * apart from more sophisticated Tries / prefix trees).
-         */
-
-            public void SortTerms()
-            {
-                if (SortedTerms == null) sortedTerms = Sort(Terms);
-            }
-
-            /** note that the frequency can be calculated as numPosition(getPositions(x)) */
-
-            public ArrayIntList GetPositions(String term)
-            {
-                return Terms[term];
-            }
-
-            /** note that the frequency can be calculated as numPosition(getPositions(x)) */
-
-            public ArrayIntList GetPositions(int pos)
-            {
-                return SortedTerms[pos].Value;
-            }
-
-            public float GetBoost()
-            {
-                return Boost;
-            }
-
-        }
-
-
-        ///////////////////////////////////////////////////////////////////////////////
-        // Nested classes:
-        ///////////////////////////////////////////////////////////////////////////////
-        /**
-         * Efficient resizable auto-expanding list holding <code>int</code> elements;
-         * implemented with arrays.
-         */
-
-        [Serializable]
-        private sealed class ArrayIntList
-        {
-
-            private int[] elements;
-            private int size = 0;
-
-            private static long serialVersionUID = 2282195016849084649L;
-
-            private ArrayIntList()
-                : this(10)
-            {
-
-            }
-
-            public ArrayIntList(int initialCapacity)
-            {
-                elements = new int[initialCapacity];
-            }
-
-            public void Add(int elem)
-            {
-                if (size == elements.Length) EnsureCapacity(size + 1);
-                elements[size++] = elem;
-            }
-
-            public void Add(int pos, int start, int end)
-            {
-                if (size + 3 > elements.Length) EnsureCapacity(size + 3);
-                elements[size] = pos;
-                elements[size + 1] = start;
-                elements[size + 2] = end;
-                size += 3;
-            }
-
-            public int Get(int index)
-            {
-                if (index >= size) ThrowIndex(index);
-                return elements[index];
-            }
-
-            public int Size()
-            {
-                return size;
-            }
-
-            public int[] ToArray(int stride)
-            {
-                int[] arr = new int[Size()/stride];
-                if (stride == 1)
-                {
-                    Array.Copy(elements, 0, arr, 0, size);
-                }
-                else
-                {
-                    for (int i = 0, j = 0; j < size; i++, j += stride) arr[i] = elements[j];
-                }
-                return arr;
-            }
-
-            private void EnsureCapacity(int minCapacity)
-            {
-                int newCapacity = Math.Max(minCapacity, (elements.Length*3)/2 + 1);
-                int[] newElements = new int[newCapacity];
-                Array.Copy(elements, 0, newElements, 0, size);
-                elements = newElements;
-            }
-
-            private void ThrowIndex(int index)
-            {
-                throw new IndexOutOfRangeException("index: " + index
-                                                   + ", size: " + size);
-            }
-
-            /** returns the first few positions (without offsets); debug only */
-
-            public string ToString(int stride)
-            {
-                int s = Size()/stride;
-                int len = Math.Min(10, s); // avoid printing huge lists
-                StringBuilder buf = new StringBuilder(4*len);
-                buf.Append("[");
-                for (int i = 0; i < len; i++)
-                {
-                    buf.Append(Get(i*stride));
-                    if (i < len - 1) buf.Append(", ");
-                }
-                if (len != s) buf.Append(", ..."); // and some more...
-                buf.Append("]");
-                return buf.ToString();
-            }
-        }
-
-
-        ///////////////////////////////////////////////////////////////////////////////
-        // Nested classes:
-        ///////////////////////////////////////////////////////////////////////////////
-        private static readonly Term MATCH_ALL_TERM = new Term("");
-
-        /**
-         * Search support for Lucene framework integration; implements all methods
-         * required by the Lucene IndexReader contracts.
-         */
-
-        private sealed partial class MemoryIndexReader : IndexReader
-        {
-            private readonly MemoryIndex _index;
-
-            private Searcher searcher; // needed to find searcher.getSimilarity() 
-
-            internal MemoryIndexReader(MemoryIndex index)
-            {
-                _index = index;
-            }
-
-            private Info GetInfo(String fieldName)
-            {
-                return _index.fields[fieldName];
-            }
-
-            private Info GetInfo(int pos)
-            {
-                return _index.sortedFields[pos].Value;
-            }
-
-            public override int DocFreq(Term term)
-            {
-                Info info = GetInfo(term.Field);
-                int freq = 0;
-                if (info != null) freq = info.GetPositions(term.Text) != null ? 1 : 0;
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.docFreq: " + term + ", freq:" + freq);
-                return freq;
-            }
-
-            public override TermEnum Terms()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.terms()");
-                return Terms(MATCH_ALL_TERM);
-            }
-
-            public override TermEnum Terms(Term term)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.terms: " + term);
-
-                int i; // index into info.sortedTerms
-                int j; // index into sortedFields
-
-                _index.SortFields();
-                if (_index.sortedFields.Length == 1 && _index.sortedFields[0].Key == term.Field)
-                {
-                    j = 0; // fast path
-                }
-                else
-                {
-                    j = Array.BinarySearch(_index.sortedFields, new KeyValuePair<string, Info>(term.Field, null), Info.InfoComparer);
-                }
-
-                if (j < 0)
-                {
-                    // not found; choose successor
-                    j = -j - 1;
-                    i = 0;
-                    if (j < _index.sortedFields.Length) GetInfo(j).SortTerms();
-                }
-                else
-                {
-                    // found
-                    Info info = GetInfo(j);
-                    info.SortTerms();
-                    i = Array.BinarySearch(info.SortedTerms, new KeyValuePair<string, ArrayIntList>(term.Text, null), Info.ArrayIntListComparer);
-                    if (i < 0)
-                    {
-                        // not found; choose successor
-                        i = -i - 1;
-                        if (i >= info.SortedTerms.Length)
-                        {
-                            // move to next successor
-                            j++;
-                            i = 0;
-                            if (j < _index.sortedFields.Length) GetInfo(j).SortTerms();
-                        }
-                    }
-                }
-                int ix = i;
-                int jx = j;
-
-                return new MemoryTermEnum(_index, this, ix, jx);
-            }
-
-            public override TermPositions TermPositions()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.termPositions");
-
-                return new MemoryTermPositions(_index, this);
-            }
-
-
-            public override TermDocs TermDocs()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.termDocs");
-                return TermPositions();
-            }
-
-            public override ITermFreqVector[] GetTermFreqVectors(int docNumber)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVectors");
-                // This is okay, ToArray() is as optimized as writing it by hand
-                return _index.fields.Keys.Select(k => GetTermFreqVector(docNumber, k)).ToArray();
-            }
-
-            public override void GetTermFreqVector(int docNumber, TermVectorMapper mapper)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVectors");
-
-                //      if (vectors.length == 0) return null;
-                foreach (String fieldName in _index.fields.Keys)
-                {
-                    GetTermFreqVector(docNumber, fieldName, mapper);
-                }
-            }
-
-            public override void GetTermFreqVector(int docNumber, String field, TermVectorMapper mapper)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVector");
-                Info info = GetInfo(field);
-                if (info == null)
-                {
-                    return;
-                }
-                info.SortTerms();
-                mapper.SetExpectations(field, info.SortedTerms.Length, _index.stride != 1, true);
-                for (int i = info.SortedTerms.Length; --i >= 0;)
-                {
-
-                    ArrayIntList positions = info.SortedTerms[i].Value;
-                    int size = positions.Size();
-                    var offsets = new TermVectorOffsetInfo[size/_index.stride];
-
-                    for (int k = 0, j = 1; j < size; k++, j += _index.stride)
-                    {
-                        int start = positions.Get(j);
-                        int end = positions.Get(j + 1);
-                        offsets[k] = new TermVectorOffsetInfo(start, end);
-                    }
-                    mapper.Map(info.SortedTerms[i].Key, _index.NumPositions(info.SortedTerms[i].Value), offsets,
-                               (info.SortedTerms[i].Value).ToArray(_index.stride));
-                }
-            }
-
-            public override ITermFreqVector GetTermFreqVector(int docNumber, String fieldName)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVector");
-                Info info = GetInfo(fieldName);
-                if (info == null) return null; // TODO: or return empty vector impl???
-                info.SortTerms();
-
-                return new MemoryTermPositionVector(_index, info, fieldName);
-            }
-
-            private Similarity GetSimilarity()
-            {
-                if (searcher != null) return searcher.Similarity;
-                return Similarity.Default;
-            }
-
-            internal void SetSearcher(Searcher searcher)
-            {
-                this.searcher = searcher;
-            }
-
-            /** performance hack: cache norms to avoid repeated expensive calculations */
-            private byte[] cachedNorms;
-            private String cachedFieldName;
-            private Similarity cachedSimilarity;
-
-            public override byte[] Norms(String fieldName)
-            {
-                byte[] norms = cachedNorms;
-                Similarity sim = GetSimilarity();
-                if (fieldName != cachedFieldName || sim != cachedSimilarity)
-                {
-                    // not cached?
-                    Info info = GetInfo(fieldName);
-                    int numTokens = info != null ? info.NumTokens : 0;
-                    int numOverlapTokens = info != null ? info.NumOverlapTokens : 0;
-                    float boost = info != null ? info.GetBoost() : 1.0f;
-                    FieldInvertState invertState = new FieldInvertState(0, numTokens, numOverlapTokens, 0, boost);
-                    float n = sim.ComputeNorm(fieldName, invertState);
-                    byte norm = Similarity.EncodeNorm(n);
-                    norms = new byte[] {norm};
-
-                    // cache it for future reuse
-                    cachedNorms = norms;
-                    cachedFieldName = fieldName;
-                    cachedSimilarity = sim;
-                    if (DEBUG)
-                        System.Diagnostics.Debug.WriteLine("MemoryIndexReader.norms: " + fieldName + ":" + n + ":" +
-                                                           norm + ":" + numTokens);
-                }
-                return norms;
-            }
-
-            public override void Norms(String fieldName, byte[] bytes, int offset)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.norms*: " + fieldName);
-                byte[] norms = Norms(fieldName);
-                Buffer.BlockCopy(norms, 0, bytes, offset, norms.Length);
-            }
-
-            protected override void DoSetNorm(int doc, String fieldName, byte value)
-            {
-                throw new NotSupportedException();
-            }
-
-            public override int NumDocs()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.numDocs");
-                return _index.fields.Count > 0 ? 1 : 0;
-            }
-
-            public override int MaxDoc
-            {
-                get
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.maxDoc");
-                    return 1;
-                }
-            }
-
-            public override Document Document(int n)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.document");
-                return new Document(); // there are no stored fields
-            }
-
-            //When we convert to JDK 1.5 make this Set<String>
-            public override Document Document(int n, FieldSelector fieldSelector)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.document");
-                return new Document(); // there are no stored fields
-            }
-
-            public override bool IsDeleted(int n)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.isDeleted");
-                return false;
-            }
-
-            public override bool HasDeletions
-            {
-                get
-                {
-                    if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.hasDeletions");
-                    return false;
-                }
-            }
-
-            protected override void DoDelete(int docNum)
-            {
-                throw new NotSupportedException();
-            }
-
-            protected override void DoUndeleteAll()
-            {
-                throw new NotSupportedException();
-            }
-
-            protected override void DoCommit(IDictionary<String, String> commitUserData)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.doCommit");
-
-            }
-
-            protected override void DoClose()
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.doClose");
-            }
-
-            // lucene >= 1.9 (remove this method for lucene-1.4.3)
-            public override ICollection<String> GetFieldNames(FieldOption fieldOption)
-            {
-                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getFieldNamesOption");
-                if (fieldOption == FieldOption.UNINDEXED)
-                    return CollectionsHelper<string>.EmptyList();
-                if (fieldOption == FieldOption.INDEXED_NO_TERMVECTOR)
-                    return CollectionsHelper<string>.EmptyList();
-                if (fieldOption == FieldOption.TERMVECTOR_WITH_OFFSET && _index.stride == 1)
-                    return CollectionsHelper<string>.EmptyList();
-                if (fieldOption == FieldOption.TERMVECTOR_WITH_POSITION_OFFSET && _index.stride == 1)
-                    return CollectionsHelper<string>.EmptyList();
-
-                return _index.fields.Keys.AsReadOnly();
-            }
-        }
-
-
-        ///////////////////////////////////////////////////////////////////////////////
-        // Nested classes:
-        ///////////////////////////////////////////////////////////////////////////////
-        private static class VM
-        {
-
-            public static readonly int PTR = Is64BitVM() ? 8 : 4;
-
-            // bytes occupied by primitive data types
-            public static readonly int BOOLEAN = 1;
-            public static readonly int BYTE = 1;
-            public static readonly int CHAR = 2;
-            public static readonly int SHORT = 2;
-            public static readonly int INT = 4;
-            public static readonly int LONG = 8;
-            public static readonly int FLOAT = 4;
-            public static readonly int DOUBLE = 8;
-
-            private static readonly int LOG_PTR = (int) Math.Round(Log2(PTR));
-
-            /**
-             * Object header of any heap allocated Java object. 
-             * ptr to class, info for monitor, gc, hash, etc.
-             */
-            private static readonly int OBJECT_HEADER = 2*PTR;
-
-            //  assumes n > 0
-            //  64 bit VM:
-            //    0     --> 0*PTR
-            //    1..8  --> 1*PTR
-            //    9..16 --> 2*PTR
-            private static int SizeOf(int n)
-            {
-                return (((n - 1) >> LOG_PTR) + 1) << LOG_PTR;
-            }
-
-            public static int SizeOfObject(int n)
-            {
-                return SizeOf(OBJECT_HEADER + n);
-            }
-
-            public static int SizeOfObjectArray(int len)
-            {
-                return SizeOfObject(INT + PTR*len);
-            }
-
-            public static int SizeOfCharArray(int len)
-            {
-                return SizeOfObject(INT + CHAR*len);
-            }
-
-            public static int SizeOfIntArray(int len)
-            {
-                return SizeOfObject(INT + INT*len);
-            }
-
-            public static int SizeOfString(int len)
-            {
-                return SizeOfObject(3*INT + PTR) + SizeOfCharArray(len);
-            }
-
-            public static int SizeOfHashMap(int len)
-            {
-                return SizeOfObject(4*PTR + 4*INT) + SizeOfObjectArray(len)
-                       + len*SizeOfObject(3*PTR + INT); // entries
-            }
-
-            // note: does not include referenced objects
-            public static int SizeOfArrayList(int len)
-            {
-                return SizeOfObject(PTR + 2*INT) + SizeOfObjectArray(len);
-            }
-
-            public static int SizeOfArrayIntList(int len)
-            {
-                return SizeOfObject(PTR + INT) + SizeOfIntArray(len);
-            }
-
-            private static bool Is64BitVM()
-            {
-                return IntPtr.Size == 8;
-            }
-
-            /** logarithm to the base 2. Example: log2(4) == 2, log2(8) == 3 */
-
-            private static double Log2(double value)
-            {
-                return Math.Log(value, 2);
-                //return Math.Log(value) / Math.Log(2);
-            }
-        }
-
-    }
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Documents;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+
+namespace Lucene.Net.Index.Memory
+{
+    /// <summary>
+    /// High-performance single-document main memory Apache Lucene fulltext search index. 
+    /// 
+    /// <h4>Overview</h4>
+    /// 
+    /// This class is a replacement/substitute for a large subset of
+    /// {@link RAMDirectory} functionality. It is designed to
+    /// enable maximum efficiency for on-the-fly matchmaking combining structured and 
+    /// fuzzy fulltext search in realtime streaming applications such as Nux XQuery based XML 
+    /// message queues, publish-subscribe systems for Blogs/newsfeeds, text chat, data acquisition and 
+    /// distribution systems, application level routers, firewalls, classifiers, etc. 
+    /// Rather than targeting fulltext search of infrequent queries over huge persistent 
+    /// data archives (historic search), this class targets fulltext search of huge 
+    /// numbers of queries over comparatively small transient realtime data (prospective 
+    /// search). 
+    /// For example as in 
+    /// <pre>
+    /// float score = search(String text, Query query)
+    /// </pre>
+    /// <p/>
+    /// Each instance can hold at most one Lucene "document", with a document containing
+    /// zero or more "fields", each field having a name and a fulltext value. The
+    /// fulltext value is tokenized (split and transformed) into zero or more index terms 
+    /// (aka words) on <code>addField()</code>, according to the policy implemented by an
+    /// Analyzer. For example, Lucene analyzers can split on whitespace, normalize to lower case
+    /// for case insensitivity, ignore common terms with little discriminatory value such as "he", "in", "and" (stop
+    /// words), reduce the terms to their natural linguistic root form such as "fishing"
+    /// being reduced to "fish" (stemming), resolve synonyms/inflexions/thesauri 
+    /// (upon indexing and/or querying), etc. For details, see
+    /// <a target="_blank" href="http://today.java.net/pub/a/today/2003/07/30/LuceneIntro.html">Lucene Analyzer Intro</a>.
+    /// <p>
+    /// Arbitrary Lucene queries can be run against this class - see <a target="_blank" 
+    /// href="../../../../../../../queryparsersyntax.html">Lucene Query Syntax</a>
+    /// as well as <a target="_blank" 
+    /// href="http://today.java.net/pub/a/today/2003/11/07/QueryParserRules.html">Query Parser Rules</a>.
+    /// Note that a Lucene query selects on the field names and associated (indexed) 
+    /// tokenized terms, not on the original fulltext(s) - the latter are not stored 
+    /// but rather thrown away immediately after tokenization.
+    /// <p>
+    /// For some interesting background information on search technology, see Bob Wyman's
+    /// <a target="_blank" 
+    /// href="http://bobwyman.pubsub.com/main/2005/05/mary_hodder_poi.html">Prospective Search</a>, 
+    /// Jim Gray's
+    /// <a target="_blank" href="http://www.acmqueue.org/modules.php?name=Content&pa=showpage&pid=293&page=4">
+    /// A Call to Arms - Custom subscriptions</a>, and Tim Bray's
+    /// <a target="_blank" 
+    /// href="http://www.tbray.org/ongoing/When/200x/2003/07/30/OnSearchTOC">On Search, the Series</a>.
+    /// 
+    /// 
+    /// <h4>Example Usage</h4> 
+    /// 
+    /// <pre>
+    /// Analyzer analyzer = PatternAnalyzer.DEFAULT_ANALYZER;
+    /// //Analyzer analyzer = new SimpleAnalyzer();
+    /// MemoryIndex index = new MemoryIndex();
+    /// index.addField("content", "Readings about Salmons and other select Alaska fishing Manuals", analyzer);
+    /// index.addField("author", "Tales of James", analyzer);
+    /// QueryParser parser = new QueryParser("content", analyzer);
+    /// float score = index.search(parser.parse("+author:james +salmon~ +fish/// manual~"));
+    /// if (score &gt; 0.0f) {
+    ///     System.out.println("it's a match");
+    /// } else {
+    ///     System.out.println("no match found");
+    /// }
+    /// System.out.println("indexData=" + index.toString());
+    /// </pre>
+    /// 
+    /// 
+    /// <h4>Example XQuery Usage</h4> 
+    /// 
+    /// <pre>
+    /// (: An XQuery that finds all books authored by James that have something to do with "salmon fishing manuals", sorted by relevance :)
+    /// declare namespace lucene = "java:nux.xom.pool.FullTextUtil";
+    /// declare variable $query := "+salmon~ +fish/// manual~"; (: any arbitrary Lucene query can go here :)
+    /// 
+    /// for $book in /books/book[author="James" and lucene:match(abstract, $query) > 0.0]
+    /// let $score := lucene:match($book/abstract, $query)
+    /// order by $score descending
+    /// return $book
+    /// </pre>
+    /// 
+    /// 
+    /// <h4>No thread safety guarantees</h4>
+    /// 
+    /// An instance can be queried multiple times with the same or different queries,
+    /// but an instance is not thread-safe. If desired use idioms such as:
+    /// <pre>
+    /// MemoryIndex index = ...
+    /// synchronized (index) {
+    ///    // read and/or write index (i.e. add fields and/or query)
+    /// } 
+    /// </pre>
+    /// 
+    /// 
+    /// <h4>Performance Notes</h4>
+    /// 
+    /// Internally there's a new data structure geared towards efficient indexing 
+    /// and searching, plus the necessary support code to seamlessly plug into the Lucene 
+    /// framework.
+    /// <p/>
+    /// This class performs very well for very small texts (e.g. 10 chars) 
+    /// as well as for large texts (e.g. 10 MB) and everything in between. 
+    /// Typically, it is about 10-100 times faster than <code>RAMDirectory</code>.
+    /// Note that <code>RAMDirectory</code> has particularly 
+    /// large efficiency overheads for small to medium sized texts, both in time and space.
+    /// Indexing a field with N tokens takes O(N) in the best case, and O(N logN) in the worst 
+    /// case. Memory consumption is probably larger than for <code>RAMDirectory</code>.
+    /// <p/>
+    /// Example throughput of many simple term queries over a single MemoryIndex: 
+    /// ~500000 queries/sec on a MacBook Pro, jdk 1.5.0_06, server VM. 
+    /// As always, your mileage may vary.
+    /// <p/>
+    /// If you're curious about
+    /// the whereabouts of bottlenecks, run java 1.5 with the non-perturbing '-server
+    /// -agentlib:hprof=cpu=samples,depth=10' flags, then study the trace log and
+    /// correlate its hotspot trailer with its call stack headers (see <a
+    /// target="_blank" href="http://java.sun.com/developer/technicalArticles/Programming/HPROF.html">
+    /// hprof tracing </a>).
+    ///
+    ///</summary>
+    [Serializable]
+    public partial class MemoryIndex
+    {
+        /** info for each field: Map<String fieldName, Info field> */
+        private HashMap<String, Info> fields = new HashMap<String, Info>();
+
+        /** fields sorted ascending by fieldName; lazily computed on demand */
+        [NonSerialized] private KeyValuePair<String, Info>[] sortedFields;
+
+        /** pos: positions[3*i], startOffset: positions[3*i +1], endOffset: positions[3*i +2] */
+        private int stride;
+
+        /** Could be made configurable; See {@link Document#setBoost(float)} */
+        private static float docBoost = 1.0f;
+
+        private static long serialVersionUID = 2782195016849084649L;
+
+        private static bool DEBUG = false;
+
+        /**
+         * Constructs an empty instance.
+         */
+        public MemoryIndex()
+            : this(false)
+        {
+        }
+
+        /**
+         * Constructs an empty instance that can optionally store the start and end
+         * character offset of each token term in the text. This can be useful for
+         * highlighting of hit locations with the Lucene highlighter package.
+         * Private until the highlighter package matures, so that this can actually
+         * be meaningfully integrated.
+         * 
+         * @param storeOffsets
+         *            whether or not to store the start and end character offset of
+         *            each token term in the text
+         */
+
+        private MemoryIndex(bool storeOffsets)
+        {
+            this.stride = storeOffsets ? 3 : 1;
+        }
+
+        /**
+         * Convenience method; Tokenizes the given field text and adds the resulting
+         * terms to the index; Equivalent to adding an indexed non-keyword Lucene
+         * {@link org.apache.lucene.document.Field} that is
+         * {@link org.apache.lucene.document.Field.Index#ANALYZED tokenized},
+         * {@link org.apache.lucene.document.Field.Store#NO not stored},
+         * {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS termVectorStored with positions} (or
+         * {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS termVectorStored with positions and offsets}),
+         * 
+         * @param fieldName
+         *            a name to be associated with the text
+         * @param text
+         *            the text to tokenize and index.
+         * @param analyzer
+         *            the analyzer to use for tokenization
+         */
+
+        public void AddField(String fieldName, String text, Analyzer analyzer)
+        {
+            if (fieldName == null)
+                throw new ArgumentException("fieldName must not be null");
+            if (text == null)
+                throw new ArgumentException("text must not be null");
+            if (analyzer == null)
+                throw new ArgumentException("analyzer must not be null");
+
+            TokenStream stream = analyzer.TokenStream(fieldName, new StringReader(text));
+
+            AddField(fieldName, stream);
+        }
+
+        /**
+         * Convenience method; Creates and returns a token stream that generates a
+         * token for each keyword in the given collection, "as is", without any
+         * transforming text analysis. The resulting token stream can be fed into
+         * {@link #addField(String, TokenStream)}, perhaps wrapped into another
+         * {@link org.apache.lucene.analysis.TokenFilter}, as desired.
+         * 
+         * @param keywords
+         *            the keywords to generate tokens for
+         * @return the corresponding token stream
+         */
+
+        public TokenStream CreateKeywordTokenStream<T>(ICollection<T> keywords)
+        {
+            // TODO: deprecate & move this method into AnalyzerUtil?
+            if (keywords == null)
+                throw new ArgumentException("keywords must not be null");
+
+            return new KeywordTokenStream<T>(keywords);
+        }
+
+        /**
+         * Equivalent to <code>addField(fieldName, stream, 1.0f)</code>.
+         * 
+         * @param fieldName
+         *            a name to be associated with the text
+         * @param stream
+         *            the token stream to retrieve tokens from
+         */
+        public void AddField(String fieldName, TokenStream stream)
+        {
+            AddField(fieldName, stream, 1.0f);
+        }
+
+        /**
+         * Iterates over the given token stream and adds the resulting terms to the index;
+         * Equivalent to adding a tokenized, indexed, termVectorStored, unstored,
+         * Lucene {@link org.apache.lucene.document.Field}.
+         * Finally closes the token stream. Note that untokenized keywords can be added with this method via 
+         * {@link #CreateKeywordTokenStream(Collection)}, the Lucene contrib <code>KeywordTokenizer</code> or similar utilities.
+         * 
+         * @param fieldName
+         *            a name to be associated with the text
+         * @param stream
+         *            the token stream to retrieve tokens from.
+         * @param boost
+         *            the boost factor for hits for this field
+         * @see org.apache.lucene.document.Field#setBoost(float)
+         */
+        public void AddField(String fieldName, TokenStream stream, float boost)
+        {
+            try
+            {
+                if (fieldName == null)
+                    throw new ArgumentException("fieldName must not be null");
+                if (stream == null)
+                    throw new ArgumentException("token stream must not be null");
+                if (boost <= 0.0f)
+                    throw new ArgumentException("boost factor must be greater than 0.0");
+                if (fields[fieldName] != null)
+                    throw new ArgumentException("field must not be added more than once");
+
+                var terms = new HashMap<String, ArrayIntList>();
+                int numTokens = 0;
+                int numOverlapTokens = 0;
+                int pos = -1;
+
+                var termAtt = stream.AddAttribute<ITermAttribute>();
+                var posIncrAttribute = stream.AddAttribute<IPositionIncrementAttribute>();
+                var offsetAtt = stream.AddAttribute<IOffsetAttribute>();
+
+                stream.Reset();
+                while (stream.IncrementToken())
+                {
+                    String term = termAtt.Term();
+                    if (term.Length == 0) continue; // nothing to do
+                    //        if (DEBUG) System.Diagnostics.Debug.WriteLine("token='" + term + "'");
+                    numTokens++;
+                    int posIncr = posIncrAttribute.PositionIncrement;
+                    if (posIncr == 0)
+                        numOverlapTokens++;
+                    pos += posIncr;
+
+                    ArrayIntList positions = terms[term];
+                    if (positions == null)
+                    {
+                        // term not seen before
+                        positions = new ArrayIntList(stride);
+                        terms[term] = positions;
+                    }
+                    if (stride == 1)
+                    {
+                        positions.Add(pos);
+                    }
+                    else
+                    {
+                        positions.Add(pos, offsetAtt.StartOffset, offsetAtt.EndOffset);
+                    }
+                }
+                stream.End();
+
+                // ensure infos.numTokens > 0 invariant; needed for correct operation of terms()
+                if (numTokens > 0)
+                {
+                    boost = boost*docBoost; // see DocumentWriter.addDocument(...)
+                    fields[fieldName] = new Info(terms, numTokens, numOverlapTokens, boost);
+                    sortedFields = null; // invalidate sorted view, if any
+                }
+            }
+            catch (IOException e)
+            {
+                // can never happen
+                throw new SystemException(string.Empty, e);
+            }
+            finally
+            {
+                try
+                {
+                    if (stream != null) stream.Close();
+                }
+                catch (IOException e2)
+                {
+                    throw new SystemException(string.Empty, e2);
+                }
+            }
+        }
+
+        /**
+         * Creates and returns a searcher that can be used to execute arbitrary
+         * Lucene queries and to collect the resulting query results as hits.
+         * 
+         * @return a searcher
+         */
+
+        public IndexSearcher CreateSearcher()
+        {
+            MemoryIndexReader reader = new MemoryIndexReader(this);
+            IndexSearcher searcher = new IndexSearcher(reader); // ensures no auto-close !!
+            reader.SetSearcher(searcher); // to later get hold of searcher.getSimilarity()
+            return searcher;
+        }
+
+        /**
+         * Convenience method that efficiently returns the relevance score by
+         * matching this index against the given Lucene query expression.
+         * 
+         * @param query
+         *            an arbitrary Lucene query to run against this index
+         * @return the relevance score of the matchmaking; A number in the range
+         *         [0.0 .. 1.0], with 0.0 indicating no match. The higher the number
+         *         the better the match.
+         *
+         */
+
+        public float Search(Query query)
+        {
+            if (query == null)
+                throw new ArgumentException("query must not be null");
+
+            Searcher searcher = CreateSearcher();
+            try
+            {
+                float[] scores = new float[1]; // inits to 0.0f (no match)
+                searcher.Search(query, new FillingCollector(scores));
+                float score = scores[0];
+                return score;
+            }
+            catch (IOException e)
+            {
+                // can never happen (RAMDirectory)
+                throw new SystemException(string.Empty, e);
+            }
+            finally
+            {
+                // searcher.close();
+                /*
+                 * Note that it is harmless and important for good performance to
+                 * NOT close the index reader!!! This avoids all sorts of
+                 * unnecessary baggage and locking in the Lucene IndexReader
+                 * superclass, all of which is completely unnecessary for this main
+                 * memory index data structure without thread-safety claims.
+                 * 
+                 * Wishing IndexReader would be an interface...
+                 * 
+                 * Actually with the new tight createSearcher() API auto-closing is now
+                 * made impossible, hence searcher.close() would be harmless and also 
+                 * would not degrade performance...
+                 */
+            }
+        }
+
+        /**
+         * Returns a reasonable approximation of the main memory [bytes] consumed by
+         * this instance. Useful for smart memory sensititive caches/pools. Assumes
+         * fieldNames are interned, whereas tokenized terms are memory-overlaid.
+         * 
+         * @return the main memory consumption
+         */
+        public int GetMemorySize()
+        {
+            // for example usage in a smart cache see nux.xom.pool.Pool    
+            int PTR = VM.PTR;
+            int INT = VM.INT;
+            int size = 0;
+            size += VM.SizeOfObject(2*PTR + INT); // memory index
+            if (sortedFields != null) size += VM.SizeOfObjectArray(sortedFields.Length);
+
+            size += VM.SizeOfHashMap(fields.Count);
+            foreach (var entry in fields)
+            {
+                // for each Field Info
+                Info info = entry.Value;
+                size += VM.SizeOfObject(2*INT + 3*PTR); // Info instance vars
+                if (info.SortedTerms != null) size += VM.SizeOfObjectArray(info.SortedTerms.Length);
+
+                int len = info.Terms.Count;
+                size += VM.SizeOfHashMap(len);
+
+                var iter2 = info.Terms.GetEnumerator();
+                while (--len >= 0)
+                {
+                    iter2.MoveNext();
+                    // for each term
+                    KeyValuePair<String, ArrayIntList> e = iter2.Current;
+                    size += VM.SizeOfObject(PTR + 3*INT); // assumes substring() memory overlay
+//        size += STR + 2 * ((String) e.getKey()).length();
+                    ArrayIntList positions = e.Value;
+                    size += VM.SizeOfArrayIntList(positions.Size());
+                }
+            }
+            return size;
+        }
+
+        private int NumPositions(ArrayIntList positions)
+        {
+            return positions.Size()/stride;
+        }
+
+        /** sorts into ascending order (on demand), reusing memory along the way */
+
+        private void SortFields()
+        {
+            if (sortedFields == null) sortedFields = Sort(fields);
+        }
+
+        /** returns a view of the given map's entries, sorted ascending by key */
+
+        private static KeyValuePair<TKey, TValue>[] Sort<TKey, TValue>(HashMap<TKey, TValue> map)
+            where TKey : class, IComparable<TKey>
+        {
+            int size = map.Count;
+
+            var entries = map.ToArray();
+
+            if (size > 1) Array.Sort(entries, TermComparer.KeyComparer);
+            return entries;
+        }
+
+        /**
+         * Returns a String representation of the index data for debugging purposes.
+         * 
+         * @return the string representation
+         */
+
+        public override String ToString()
+        {
+            StringBuilder result = new StringBuilder(256);
+            SortFields();
+            int sumChars = 0;
+            int sumPositions = 0;
+            int sumTerms = 0;
+
+            for (int i = 0; i < sortedFields.Length; i++)
+            {
+                KeyValuePair<String, Info> entry = sortedFields[i];
+                String fieldName = entry.Key;
+                Info info = entry.Value;
+                info.SortTerms();
+                result.Append(fieldName + ":\n");
+
+                int numChars = 0;
+                int numPos = 0;
+                for (int j = 0; j < info.SortedTerms.Length; j++)
+                {
+                    KeyValuePair<String, ArrayIntList> e = info.SortedTerms[j];
+                    String term = e.Key;
+                    ArrayIntList positions = e.Value;
+                    result.Append("\t'" + term + "':" + NumPositions(positions) + ":");
+                    result.Append(positions.ToString(stride)); // ignore offsets
+                    result.Append("\n");
+                    numPos += NumPositions(positions);
+                    numChars += term.Length;
+                }
+
+                result.Append("\tterms=" + info.SortedTerms.Length);
+                result.Append(", positions=" + numPos);
+                result.Append(", Kchars=" + (numChars/1000.0f));
+                result.Append("\n");
+                sumPositions += numPos;
+                sumChars += numChars;
+                sumTerms += info.SortedTerms.Length;
+            }
+
+            result.Append("\nfields=" + sortedFields.Length);
+            result.Append(", terms=" + sumTerms);
+            result.Append(", positions=" + sumPositions);
+            result.Append(", Kchars=" + (sumChars/1000.0f));
+            return result.ToString();
+        }
+
+
+        ///////////////////////////////////////////////////////////////////////////////
+        // Nested classes:
+        ///////////////////////////////////////////////////////////////////////////////
+        /**
+         * Index data structure for a field; Contains the tokenized term texts and
+         * their positions.
+         */
+
+        [Serializable]
+        private sealed class Info
+        {
+            public static readonly IComparer<KeyValuePair<string, Info>> InfoComparer = new TermComparer<Info>();
+            public static readonly IComparer<KeyValuePair<string, ArrayIntList>> ArrayIntListComparer = new TermComparer<ArrayIntList>(); 
+            /**
+             * Term strings and their positions for this field: Map <String
+             * termText, ArrayIntList positions>
+             */
+            private HashMap<String, ArrayIntList> terms;
+
+            /** Terms sorted ascending by term text; computed on demand */
+            [NonSerialized] private KeyValuePair<String, ArrayIntList>[] sortedTerms;
+
+            /** Number of added tokens for this field */
+            private int numTokens;
+
+            /** Number of overlapping tokens for this field */
+            private int numOverlapTokens;
+
+            /** Boost factor for hits for this field */
+            private float boost;
+
+            /** Term for this field's fieldName, lazily computed on demand */
+            [NonSerialized] public Term template;
+
+            private static long serialVersionUID = 2882195016849084649L;
+
+            public Info(HashMap<String, ArrayIntList> terms, int numTokens, int numOverlapTokens, float boost)
+            {
+                this.terms = terms;
+                this.numTokens = numTokens;
+                this.NumOverlapTokens = numOverlapTokens;
+                this.boost = boost;
+            }
+
+            public HashMap<string, ArrayIntList> Terms
+            {
+                get { return terms; }
+            }
+
+            public int NumTokens
+            {
+                get { return numTokens; }
+            }
+
+            public int NumOverlapTokens
+            {
+                get { return numOverlapTokens; }
+                set { numOverlapTokens = value; }
+            }
+
+            public float Boost
+            {
+                get { return boost; }
+            }
+
+            public KeyValuePair<string, ArrayIntList>[] SortedTerms
+            {
+                get { return sortedTerms; }
+            }
+
+            /**
+         * Sorts hashed terms into ascending order, reusing memory along the
+         * way. Note that sorting is lazily delayed until required (often it's
+         * not required at all). If a sorted view is required then hashing +
+         * sort + binary search is still faster and smaller than TreeMap usage
+         * (which would be an alternative and somewhat more elegant approach,
+         * apart from more sophisticated Tries / prefix trees).
+         */
+
+            public void SortTerms()
+            {
+                if (SortedTerms == null) sortedTerms = Sort(Terms);
+            }
+
+            /** note that the frequency can be calculated as numPosition(getPositions(x)) */
+
+            public ArrayIntList GetPositions(String term)
+            {
+                return Terms[term];
+            }
+
+            /** note that the frequency can be calculated as numPosition(getPositions(x)) */
+
+            public ArrayIntList GetPositions(int pos)
+            {
+                return SortedTerms[pos].Value;
+            }
+
+            public float GetBoost()
+            {
+                return Boost;
+            }
+
+        }
+
+
+        ///////////////////////////////////////////////////////////////////////////////
+        // Nested classes:
+        ///////////////////////////////////////////////////////////////////////////////
+        /**
+         * Efficient resizable auto-expanding list holding <code>int</code> elements;
+         * implemented with arrays.
+         */
+
+        [Serializable]
+        private sealed class ArrayIntList
+        {
+
+            private int[] elements;
+            private int size = 0;
+
+            private static long serialVersionUID = 2282195016849084649L;
+
+            private ArrayIntList()
+                : this(10)
+            {
+
+            }
+
+            public ArrayIntList(int initialCapacity)
+            {
+                elements = new int[initialCapacity];
+            }
+
+            public void Add(int elem)
+            {
+                if (size == elements.Length) EnsureCapacity(size + 1);
+                elements[size++] = elem;
+            }
+
+            public void Add(int pos, int start, int end)
+            {
+                if (size + 3 > elements.Length) EnsureCapacity(size + 3);
+                elements[size] = pos;
+                elements[size + 1] = start;
+                elements[size + 2] = end;
+                size += 3;
+            }
+
+            public int Get(int index)
+            {
+                if (index >= size) ThrowIndex(index);
+                return elements[index];
+            }
+
+            public int Size()
+            {
+                return size;
+            }
+
+            public int[] ToArray(int stride)
+            {
+                int[] arr = new int[Size()/stride];
+                if (stride == 1)
+                {
+                    Array.Copy(elements, 0, arr, 0, size);
+                }
+                else
+                {
+                    for (int i = 0, j = 0; j < size; i++, j += stride) arr[i] = elements[j];
+                }
+                return arr;
+            }
+
+            private void EnsureCapacity(int minCapacity)
+            {
+                int newCapacity = Math.Max(minCapacity, (elements.Length*3)/2 + 1);
+                int[] newElements = new int[newCapacity];
+                Array.Copy(elements, 0, newElements, 0, size);
+                elements = newElements;
+            }
+
+            private void ThrowIndex(int index)
+            {
+                throw new IndexOutOfRangeException("index: " + index
+                                                   + ", size: " + size);
+            }
+
+            /** returns the first few positions (without offsets); debug only */
+
+            public string ToString(int stride)
+            {
+                int s = Size()/stride;
+                int len = Math.Min(10, s); // avoid printing huge lists
+                StringBuilder buf = new StringBuilder(4*len);
+                buf.Append("[");
+                for (int i = 0; i < len; i++)
+                {
+                    buf.Append(Get(i*stride));
+                    if (i < len - 1) buf.Append(", ");
+                }
+                if (len != s) buf.Append(", ..."); // and some more...
+                buf.Append("]");
+                return buf.ToString();
+            }
+        }
+
+
+        ///////////////////////////////////////////////////////////////////////////////
+        // Nested classes:
+        ///////////////////////////////////////////////////////////////////////////////
+        private static readonly Term MATCH_ALL_TERM = new Term("");
+
+        /**
+         * Search support for Lucene framework integration; implements all methods
+         * required by the Lucene IndexReader contracts.
+         */
+
+        private sealed partial class MemoryIndexReader : IndexReader
+        {
+            private readonly MemoryIndex _index;
+
+            private Searcher searcher; // needed to find searcher.getSimilarity() 
+
+            internal MemoryIndexReader(MemoryIndex index)
+            {
+                _index = index;
+            }
+
+            private Info GetInfo(String fieldName)
+            {
+                return _index.fields[fieldName];
+            }
+
+            private Info GetInfo(int pos)
+            {
+                return _index.sortedFields[pos].Value;
+            }
+
+            public override int DocFreq(Term term)
+            {
+                Info info = GetInfo(term.Field);
+                int freq = 0;
+                if (info != null) freq = info.GetPositions(term.Text) != null ? 1 : 0;
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.docFreq: " + term + ", freq:" + freq);
+                return freq;
+            }
+
+            public override TermEnum Terms()
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.terms()");
+                return Terms(MATCH_ALL_TERM);
+            }
+
+            public override TermEnum Terms(Term term)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.terms: " + term);
+
+                int i; // index into info.sortedTerms
+                int j; // index into sortedFields
+
+                _index.SortFields();
+                if (_index.sortedFields.Length == 1 && _index.sortedFields[0].Key == term.Field)
+                {
+                    j = 0; // fast path
+                }
+                else
+                {
+                    j = Array.BinarySearch(_index.sortedFields, new KeyValuePair<string, Info>(term.Field, null), Info.InfoComparer);
+                }
+
+                if (j < 0)
+                {
+                    // not found; choose successor
+                    j = -j - 1;
+                    i = 0;
+                    if (j < _index.sortedFields.Length) GetInfo(j).SortTerms();
+                }
+                else
+                {
+                    // found
+                    Info info = GetInfo(j);
+                    info.SortTerms();
+                    i = Array.BinarySearch(info.SortedTerms, new KeyValuePair<string, ArrayIntList>(term.Text, null), Info.ArrayIntListComparer);
+                    if (i < 0)
+                    {
+                        // not found; choose successor
+                        i = -i - 1;
+                        if (i >= info.SortedTerms.Length)
+                        {
+                            // move to next successor
+                            j++;
+                            i = 0;
+                            if (j < _index.sortedFields.Length) GetInfo(j).SortTerms();
+                        }
+                    }
+                }
+                int ix = i;
+                int jx = j;
+
+                return new MemoryTermEnum(_index, this, ix, jx);
+            }
+
+            public override TermPositions TermPositions()
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.termPositions");
+
+                return new MemoryTermPositions(_index, this);
+            }
+
+
+            public override TermDocs TermDocs()
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.termDocs");
+                return TermPositions();
+            }
+
+            public override ITermFreqVector[] GetTermFreqVectors(int docNumber)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVectors");
+                // This is okay, ToArray() is as optimized as writing it by hand
+                return _index.fields.Keys.Select(k => GetTermFreqVector(docNumber, k)).ToArray();
+            }
+
+            public override void GetTermFreqVector(int docNumber, TermVectorMapper mapper)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVectors");
+
+                //      if (vectors.length == 0) return null;
+                foreach (String fieldName in _index.fields.Keys)
+                {
+                    GetTermFreqVector(docNumber, fieldName, mapper);
+                }
+            }
+
+            public override void GetTermFreqVector(int docNumber, String field, TermVectorMapper mapper)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVector");
+                Info info = GetInfo(field);
+                if (info == null)
+                {
+                    return;
+                }
+                info.SortTerms();
+                mapper.SetExpectations(field, info.SortedTerms.Length, _index.stride != 1, true);
+                for (int i = info.SortedTerms.Length; --i >= 0;)
+                {
+
+                    ArrayIntList positions = info.SortedTerms[i].Value;
+                    int size = positions.Size();
+                    var offsets = new TermVectorOffsetInfo[size/_index.stride];
+
+                    for (int k = 0, j = 1; j < size; k++, j += _index.stride)
+                    {
+                        int start = positions.Get(j);
+                        int end = positions.Get(j + 1);
+                        offsets[k] = new TermVectorOffsetInfo(start, end);
+                    }
+                    mapper.Map(info.SortedTerms[i].Key, _index.NumPositions(info.SortedTerms[i].Value), offsets,
+                               (info.SortedTerms[i].Value).ToArray(_index.stride));
+                }
+            }
+
+            public override ITermFreqVector GetTermFreqVector(int docNumber, String fieldName)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getTermFreqVector");
+                Info info = GetInfo(fieldName);
+                if (info == null) return null; // TODO: or return empty vector impl???
+                info.SortTerms();
+
+                return new MemoryTermPositionVector(_index, info, fieldName);
+            }
+
+            private Similarity GetSimilarity()
+            {
+                if (searcher != null) return searcher.Similarity;
+                return Similarity.Default;
+            }
+
+            internal void SetSearcher(Searcher searcher)
+            {
+                this.searcher = searcher;
+            }
+
+            /** performance hack: cache norms to avoid repeated expensive calculations */
+            private byte[] cachedNorms;
+            private String cachedFieldName;
+            private Similarity cachedSimilarity;
+
+            public override byte[] Norms(String fieldName)
+            {
+                byte[] norms = cachedNorms;
+                Similarity sim = GetSimilarity();
+                if (fieldName != cachedFieldName || sim != cachedSimilarity)
+                {
+                    // not cached?
+                    Info info = GetInfo(fieldName);
+                    int numTokens = info != null ? info.NumTokens : 0;
+                    int numOverlapTokens = info != null ? info.NumOverlapTokens : 0;
+                    float boost = info != null ? info.GetBoost() : 1.0f;
+                    FieldInvertState invertState = new FieldInvertState(0, numTokens, numOverlapTokens, 0, boost);
+                    float n = sim.ComputeNorm(fieldName, invertState);
+                    byte norm = Similarity.EncodeNorm(n);
+                    norms = new byte[] {norm};
+
+                    // cache it for future reuse
+                    cachedNorms = norms;
+                    cachedFieldName = fieldName;
+                    cachedSimilarity = sim;
+                    if (DEBUG)
+                        System.Diagnostics.Debug.WriteLine("MemoryIndexReader.norms: " + fieldName + ":" + n + ":" +
+                                                           norm + ":" + numTokens);
+                }
+                return norms;
+            }
+
+            public override void Norms(String fieldName, byte[] bytes, int offset)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.norms*: " + fieldName);
+                byte[] norms = Norms(fieldName);
+                Buffer.BlockCopy(norms, 0, bytes, offset, norms.Length);
+            }
+
+            protected override void DoSetNorm(int doc, String fieldName, byte value)
+            {
+                throw new NotSupportedException();
+            }
+
+            public override int NumDocs()
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.numDocs");
+                return _index.fields.Count > 0 ? 1 : 0;
+            }
+
+            public override int MaxDoc
+            {
+                get
+                {
+                    if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.maxDoc");
+                    return 1;
+                }
+            }
+
+            public override Document Document(int n)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.document");
+                return new Document(); // there are no stored fields
+            }
+
+            //When we convert to JDK 1.5 make this Set<String>
+            public override Document Document(int n, FieldSelector fieldSelector)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.document");
+                return new Document(); // there are no stored fields
+            }
+
+            public override bool IsDeleted(int n)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.isDeleted");
+                return false;
+            }
+
+            public override bool HasDeletions
+            {
+                get
+                {
+                    if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.hasDeletions");
+                    return false;
+                }
+            }
+
+            protected override void DoDelete(int docNum)
+            {
+                throw new NotSupportedException();
+            }
+
+            protected override void DoUndeleteAll()
+            {
+                throw new NotSupportedException();
+            }
+
+            protected override void DoCommit(IDictionary<String, String> commitUserData)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.doCommit");
+
+            }
+
+            protected override void DoClose()
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.doClose");
+            }
+
+            // lucene >= 1.9 (remove this method for lucene-1.4.3)
+            public override ICollection<String> GetFieldNames(FieldOption fieldOption)
+            {
+                if (DEBUG) System.Diagnostics.Debug.WriteLine("MemoryIndexReader.getFieldNamesOption");
+                if (fieldOption == FieldOption.UNINDEXED)
+                    return CollectionsHelper<string>.EmptyList();
+                if (fieldOption == FieldOption.INDEXED_NO_TERMVECTOR)
+                    return CollectionsHelper<string>.EmptyList();
+                if (fieldOption == FieldOption.TERMVECTOR_WITH_OFFSET && _index.stride == 1)
+                    return CollectionsHelper<string>.EmptyList();
+                if (fieldOption == FieldOption.TERMVECTOR_WITH_POSITION_OFFSET && _index.stride == 1)
+                    return CollectionsHelper<string>.EmptyList();
+
+                return _index.fields.Keys.AsReadOnly();
+            }
+        }
+
+
+        ///////////////////////////////////////////////////////////////////////////////
+        // Nested classes:
+        ///////////////////////////////////////////////////////////////////////////////
+        private static class VM
+        {
+
+            public static readonly int PTR = Is64BitVM() ? 8 : 4;
+
+            // bytes occupied by primitive data types
+            public static readonly int BOOLEAN = 1;
+            public static readonly int BYTE = 1;
+            public static readonly int CHAR = 2;
+            public static readonly int SHORT = 2;
+            public static readonly int INT = 4;
+            public static readonly int LONG = 8;
+            public static readonly int FLOAT = 4;
+            public static readonly int DOUBLE = 8;
+
+            private static readonly int LOG_PTR = (int) Math.Round(Log2(PTR));
+
+            /**
+             * Object header of any heap allocated Java object. 
+             * ptr to class, info for monitor, gc, hash, etc.
+             */

[... 72 lines stripped ...]


Mime
View raw message