lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Erik Hatcher <e...@ehatchersolutions.com>
Subject Re: svn commit: r208688 [1/2] - in /lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory: MemoryIndex.java PatternAnalyzer.java
Date Fri, 01 Jul 2005 02:19:42 GMT
Let's please keep formatting changes and functional changes separate  
in commits so that it's straightforward to read the differences.

     Erik

On Jun 30, 2005, at 5:40 PM, mharwood@apache.org wrote:

> Author: mharwood
> Date: Thu Jun 30 14:40:05 2005
> New Revision: 208688
>
> URL: http://svn.apache.org/viewcvs?rev=208688&view=rev
> Log:
> Updated version of MemoryIndex - reliant on new Term.createTerm()  
> method in Trunk
>
> Modified:
>     lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/ 
> index/memory/MemoryIndex.java
>     lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/ 
> index/memory/PatternAnalyzer.java
>
> Modified: lucene/java/trunk/contrib/memory/src/java/org/apache/ 
> lucene/index/memory/MemoryIndex.java
> URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/memory/ 
> src/java/org/apache/lucene/index/memory/MemoryIndex.java? 
> rev=208688&r1=208687&r2=208688&view=diff
> ====================================================================== 
> ========
> --- lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/ 
> index/memory/MemoryIndex.java (original)
> +++ lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/ 
> index/memory/MemoryIndex.java Thu Jun 30 14:40:05 2005
> @@ -1,1038 +1,1073 @@
> -package org.apache.lucene.index.memory;
> -
> -/**
> - * Copyright 2005 The Apache Software Foundation
> - *
> - * Licensed under the Apache License, Version 2.0 (the "License");
> - * you may not use this file except in compliance with the License.
> - * You may obtain a copy of the License at
> - *
> - *     http://www.apache.org/licenses/LICENSE-2.0
> - *
> - * Unless required by applicable law or agreed to in writing,  
> software
> - * distributed under the License is distributed on an "AS IS" BASIS,
> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  
> implied.
> - * See the License for the specific language governing permissions  
> and
> - * limitations under the License.
> - */
> -
> -import java.io.IOException;
> -import java.io.Serializable;
> -import java.io.StringReader;
> -import java.util.Arrays;
> -import java.util.Collection;
> -import java.util.Collections;
> -import java.util.Comparator;
> -import java.util.HashMap;
> -import java.util.Iterator;
> -import java.util.Map;
> -
> -import org.apache.lucene.analysis.Analyzer;
> -import org.apache.lucene.analysis.Token;
> -import org.apache.lucene.analysis.TokenStream;
> -import org.apache.lucene.document.Document;
> -import org.apache.lucene.index.IndexReader;
> -import org.apache.lucene.index.Term;
> -import org.apache.lucene.index.TermDocs;
> -import org.apache.lucene.index.TermEnum;
> -import org.apache.lucene.index.TermFreqVector;
> -import org.apache.lucene.index.TermPositionVector;
> -import org.apache.lucene.index.TermPositions;
> -import org.apache.lucene.search.HitCollector;
> -import org.apache.lucene.search.IndexSearcher;
> -import org.apache.lucene.search.Query;
> -import org.apache.lucene.search.Searcher;
> -import org.apache.lucene.search.Similarity;
> -
> -/**
> - * High-performance single-document main memory Apache Lucene  
> fulltext search index.
> - *
> - * <h4>Overview</h4>
> - *
> - * This class is a replacement/substitute for a large subset of
> - * {@link org.apache.lucene.store.RAMDirectory} functionality. It  
> is designed to
> - * enable maximum efficiency for on-the-fly matchmaking combining  
> structured and
> - * fuzzy full-text search in streaming applications such as Nux  
> XQuery based XML
> - * message queues, publish-subscribe systems for newsfeeds, data  
> acquisition and
> - * distribution systems, application level routers, firewalls,  
> classifiers, etc.
> - * For example as in <code>float score = query(String text, Query  
> query)</code>.
> - * <p>
> - * Each instance can hold at most one Lucene "document", with a  
> document containing
> - * zero or more "fields", each field having a name and a free text  
> value. The
> - * free text value is tokenized (split and transformed) into zero  
> or more index terms
> - * (aka words) on <code>addField()</code>, according to the policy  
> implemented by an
> - * Analyzer. For example, Lucene analyzers can split on  
> whitespace, normalize to lower case
> - * for case insensitivity, ignore common terms with little  
> discriminatory value such as "he", "in", "and" (stop
> - * words), reduce the terms to their natural linguistic root form  
> such as "fishing"
> - * being reduced to "fish" (stemming), resolve synonyms/inflexions/ 
> thesauri
> - * (upon indexing and/or querying), etc. For details, see
> - * <a target="_blank" href="http://today.java.net/pub/a/today/ 
> 2003/07/30/LuceneIntro.html">Lucene Analyzer Intro</a>.
> - * <p>
> - * Arbitrary Lucene queries can be run against this class - see <a  
> target="_blank"
> - * href="http://lucene.apache.org/java/docs/ 
> queryparsersyntax.html">Lucene Query Syntax</a>
> - * as well as <a target="_blank"
> - * href="http://today.java.net/pub/a/today/2003/11/07/ 
> QueryParserRules.html">Query Parser Rules</a>.
> - * Note that a Lucene query selects on the field names and  
> associated (indexed)
> - * tokenized terms, not on the original free text(s) - the latter  
> are not stored
> - * but rather thrown away immediately after tokenization.
> - * <p>
> - * For some interesting background information on search  
> technology, see
> - * <a target="_blank"
> - * href="http://www.tbray.org/ongoing/When/200x/2003/07/30/ 
> OnSearchTOC">On Search, the Series</a>.
> - *
> - *
> - * <h4>Example Usage</h4>
> - *
> - * <pre>
> - * Analyzer analyzer = new PatternAnalyzer.DEFAULT_ANALYZER;
> - * //Analyzer analyzer = new SimpleAnalyzer();
> - * MemoryIndex index = new MemoryIndex();
> - * index.addField("content", "James is in the woods", analyzer);
> - * index.addField("title", "Tales of James", analyzer);
> - * float score = index.query(QueryParser.parse("woods AND  
> title:tales", "content", analyzer));
> - * if (score &gt; 0.0f) {
> - *     System.out.println("it's a match");
> - * } else {
> - *     System.out.println("no match found");
> - * }
> - * score = index.query(QueryParser.parse("wood* AND  
> title:tale~0.2", "content", analyzer));
> - * System.out.println("score=" + score);
> - * System.out.println("indexData=" + index.toString());
> - * </pre>
> - *
> - *
> - * <h4>Example XQuery Usage</h4>
> - *
> - * <pre>
> - * (: An XQuery that finds all books authored by James that have  
> something to do with "fish", sorted by relevance :)
> - * declare namespace lucene = "java:nux.xom.pool.FullTextUtil";
> - * declare variable $query := "fish~"; (: any arbitrary Lucene  
> query can go here :)
> - *
> - * for $book in /books/book[author="James" and lucene:match(string 
> (./abstract), $query) > 0.0]
> - * let $score := lucene:match(string($book/abstract), $query)
> - * order by $score descending
> - * return (&lt;score>{$score}</score>, $book)
> - * </pre>
> - *
> - *
> - * <h4>No thread safety guarantees</h4>
> - *
> - * An instance can be queried multiple times with the same or  
> different queries,
> - * but an instance is not thread-safe. If desired use idioms such as:
> - * <pre>
> - * MemoryIndex index = ...
> - * synchronized (index) {
> - *    // read and/or write index (i.e. add fields and/or query)
> - * }
> - * </pre>
> - *
> - *
> - * <h4>Performance Notes</h4>
> - *
> - * Internally there's a new data structure geared towards  
> efficient indexing
> - * and searching, plus the necessary support code to seamlessly  
> plug into the Lucene
> - * framework.
> - * <p>
> - * This class performs very well for very small texts (e.g. 10 chars)
> - * as well as for large texts (e.g. 10 MB) and everything in between.
> - * Typically, it is about 10-100 times faster than  
> <code>RAMDirectory</code>.
> - * Note that <code>RAMDirectory</code> has particularly
> - * large efficiency overheads for small to medium sized texts,  
> both in time and space.
> - * Indexing a field with N tokens takes O(N) in the best case, and  
> O(N logN) in the worst
> - * case. Memory consumption is probably larger than for  
> <code>RAMDirectory</code>.
> - * <p>
> - * If you're curious about
> - * the whereabouts of bottlenecks, run java 1.5 with the non- 
> perturbing '-server
> - * -agentlib:hprof=cpu=samples,depth=10' flags, then study the  
> trace log and
> - * correlate its hotspot trailer with its call stack headers (see <a
> - * target="_blank"
> - * href="http://java.sun.com/developer/technicalArticles/ 
> Programming/HPROF.html">
> - * hprof tracing </a>).
> - *
> - * @author whoschek.AT.lbl.DOT.gov
> - */
> -public class MemoryIndex {
> -
> -    /** info for each field: Map<String fieldName, Info field> */
> -    private final HashMap fields = new HashMap();
> -
> -    /** fields sorted ascending by fieldName; lazily computed on  
> demand */
> -    private transient Map.Entry[] sortedFields;
> -
> -    /** pos: positions[3*i], startOffset: positions[3*(i+1)],  
> endOffset: positions[3*(i+2)] */
> -    private final int stride;
> -
> -    private static final long serialVersionUID =  
> 2782195016849084649L;
> -
> -    private static final boolean DEBUG = false;
> -
> -    /**
> -     * Sorts term entries into ascending order; also works for
> -     * Arrays.binarySearch() and Arrays.sort()
> -     */
> -    private static final Comparator termComparator = new Comparator 
> () {
> -        public int compare(Object o1, Object o2) {
> -            if (o1 instanceof Map.Entry) {
> -                o1 = ((Map.Entry) o1).getKey();
> -            }
> -            if (o2 instanceof Map.Entry) {
> -                o2 = ((Map.Entry) o2).getKey();
> -            }
> -            return ((String) o1).compareTo((String) o2);
> -        }
> -    };
> -
> -    /**
> -     * Constructs an instance.
> -     */
> -    public MemoryIndex() {
> -        this(false);
> -    }
> -
> -    /**
> -     * Constructs an instance that can optionally store the start  
> and end
> -     * character offset of each token term in the text. This can  
> be useful for
> -     * highlighting of hit locations with the Lucene highlighter  
> package.
> -     * Private until the highlighter package matures, so that this  
> can actually
> -     * be meaningfully integrated.
> -     *
> -     * @param storeOffsets
> -     *            whether or not to store the start and end  
> character offset of
> -     *            each token term in the text
> -     */
> -    private MemoryIndex(boolean storeOffsets) {
> -        this.stride = storeOffsets ? 3 : 1;
> -    }
> -
> -    /**
> -     * Convenience method; Tokenizes the given field text and adds  
> the resulting
> -     * terms to the index; Equivalent to adding a tokenized, indexed,
> -     * termVectorStored, unstored, non-keyword Lucene
> -     * {@link org.apache.lucene.document.Field}.
> -     *
> -     * @param fieldName
> -     *            a name to be associated with the text
> -     * @param text
> -     *            the text to tokenize and index.
> -     * @param analyzer
> -     *            the analyzer to use for tokenization
> -     */
> -    public void addField(String fieldName, String text, Analyzer  
> analyzer) {
> -        if (fieldName == null)
> -            throw new IllegalArgumentException("fieldName must not  
> be null");
> -        if (text == null)
> -            throw new IllegalArgumentException("text must not be  
> null");
> -        if (analyzer == null)
> -            throw new IllegalArgumentException("analyzer must not  
> be null");
> -
> -        TokenStream stream;
> -        if (analyzer instanceof PatternAnalyzer) {
> -            stream = ((PatternAnalyzer) analyzer).tokenStream 
> (fieldName, text);
> -        } else {
> -            stream = analyzer.tokenStream(fieldName, new  
> StringReader(text));
> -        }
> -        addField(fieldName, stream);
> -    }
> -
> -    /**
> -     * Iterates over the given token stream and adds the resulting  
> terms to the index;
> -     * Equivalent to adding a tokenized, indexed,  
> termVectorStored, unstored,
> -     * Lucene {@link org.apache.lucene.document.Field}.
> -     * Finally closes the token stream. Note that untokenized  
> keywords can be added with this method via
> -     * the Lucene contrib <code>KeywordTokenizer</code> or similar  
> utilities.
> -     *
> -     * @param fieldName
> -     *            a name to be associated with the text
> -     * @param stream
> -     *            the token stream to retrieve tokens from.
> -     */
> -    public void addField(String fieldName, TokenStream stream) {
> -        /*
> -         * Note that this method signature avoids having a user  
> call new
> -         * o.a.l.d.Field(...) which would be much too expensive  
> due to the
> -         * String.intern() usage of that class.
> -         *
> -         * More often than not, String.intern() leads to serious  
> performance
> -         * degradations rather than improvements! If you're  
> curious why, check
> -         * out the JDK's native code, see how it oscillates  
> multiple times back
> -         * and forth between Java code and native code on each  
> intern() call,
> -         * only to end up using a plain vanilla java.util.HashMap  
> on the Java
> -         * heap for it's interned strings! String.equals() has a  
> small cost
> -         * compared to String.intern(), trust me. Application  
> level interning
> -         * (e.g. a HashMap per Directory/Index) typically leads to  
> better
> -         * solutions than frequent hidden low-level calls to  
> String.intern().
> -         *
> -         * Perhaps with some luck, Lucene's Field.java (and  
> Term.java) and
> -         * cousins could be fixed to not use String.intern().  
> Sigh :-(
> -         */
> -        try {
> -            if (fieldName == null)
> -                throw new IllegalArgumentException("fieldName must  
> not be null");
> -            if (stream == null)
> -                throw new IllegalArgumentException("token stream  
> must not be null");
> -            if (fields.get(fieldName) != null)
> -                throw new IllegalArgumentException("field must not  
> be added more than once");
> -
> -            HashMap terms = new HashMap();
> -            int numTokens = 0;
> -            int pos = -1;
> -            Token token;
> -
> -            while ((token = stream.next()) != null) {
> -                numTokens++;
> -                pos += token.getPositionIncrement();
> -
> -                String term = token.termText();
> -                if (DEBUG) System.err.println("token='" + term +  
> "'");
> -                ArrayIntList positions = (ArrayIntList) terms.get 
> (term);
> -                if (positions == null) { // term not seen before
> -                    positions = new ArrayIntList(stride);
> -                    terms.put(term, positions);
> -                }
> -                if (stride == 1)
> -                    positions.add(pos);
> -                else
> -                    positions.add(pos, token.startOffset(),  
> token.endOffset());
> -            }
> -
> -            // ensure infos.numTokens > 0 invariant; needed for  
> correct operation of terms()
> -            if (numTokens > 0) {
> -                fields.put(fieldName, new Info(terms, numTokens));
> -                sortedFields = null; // invalidate sorted view, if  
> any
> -            }
> -        } catch (IOException e) { // can never happen
> -            throw new RuntimeException(e);
> -        } finally {
> -            try {
> -                if (stream != null) stream.close();
> -            } catch (IOException e2) {
> -                throw new RuntimeException(e2);
> -            }
> -        }
> -    }
> -
> -    /**
> -     * Creates and returns a searcher that can be used to execute  
> arbitrary
> -     * Lucene queries and to collect the resulting query results  
> as hits.
> -     */
> -    public IndexSearcher createSearcher() {
> -        MemoryIndexReader reader = new MemoryIndexReader();
> -        IndexSearcher searcher = new IndexSearcher(reader); //  
> ensures no auto-close !!
> -        reader.setSearcher(searcher); // to later get hold of  
> searcher.getSimilarity()
> -        return searcher;
> -    }
> -
> -    /**
> -     * Convenience method that efficiently returns the relevance  
> score by
> -     * matching this index against the given Lucene query expression.
> -     *
> -     * @param query
> -     *            an arbitrary Lucene query to run against this index
> -     * @return the relevance score of the matchmaking; A number in  
> the range
> -     *         [0.0 .. 1.0], with 0.0 indicating no match. The  
> higher the number
> -     *         the better the match.
> -     * @see org.apache.lucene.queryParser.QueryParser#parse 
> (String, String,
> -     *      Analyzer)
> -     */
> -    public float search(Query query) {
> -        if (query == null)
> -            throw new IllegalArgumentException("query must not be  
> null");
> -
> -        if (fields.size() == 0) return 0.0f; // nothing to do
> -        Searcher searcher = createSearcher();
> -        try {
> -            final float[] scores = new float[1]; // inits to 0.0f  
> (no match)
> -            searcher.search(query, new HitCollector() {
> -                public void collect(int doc, float score) {
> -                    scores[0] = score;
> -                }
> -            });
> -            float score = scores[0];
> -            return score;
> -        } catch (IOException e) { // can never happen (RAMDirectory)
> -            throw new RuntimeException(e);
> -        } finally {
> -            // searcher.close();
> -            /*
> -             * Note that it is harmless and important for good  
> performance to
> -             * NOT close the index reader!!! This avoids all sorts of
> -             * unnecessary baggage and locking in the Lucene  
> IndexReader
> -             * superclass, all of which is completely unnecessary  
> for this main
> -             * memory index data structure without thread-safety  
> claims.
> -             *
> -             * Wishing IndexReader would be an interface...
> -             *
> -             * Actually with the new tight createSearcher() API  
> auto-closing is now
> -             * made impossible, hence searcher.close() would be  
> harmless...
> -             */
> -        }
> -    }
> -
> -    /**
> -     * Returns a reasonable approximation of the main memory  
> [bytes] consumed by
> -     * this instance. Useful for smart memory sensititve caches/ 
> pools. Assumes
> -     * fieldNames are interned, whereas tokenized terms are memory- 
> overlaid. For
> -     * simplicity, assumes no VM word boundary alignment of  
> instance vars.
> -     */
> -    public int getMemorySize() {
> -        // for example usage in a smart cache see nux.xom.pool.Pool
> -        int HEADER = 12; // object header of any java object
> -        int PTR = 4; // pointer on 32 bit VMs
> -        int ARR = HEADER + 4;
> -        int STR = HEADER + 3*4 + PTR + ARR; // string
> -        int INTARRLIST = HEADER + 4 + PTR + ARR;
> -        int HASHMAP = HEADER + 4*PTR + 4*4 + ARR;
> -
> -        int size = 0;
> -        size += HEADER + 3*PTR; // memory index
> -        if (sortedFields != null) size += ARR + PTR *  
> sortedFields.length;
> -
> -        size += HASHMAP + fields.size() * (PTR + HEADER + 3*PTR +  
> 4); // Map.entries
> -        Iterator iter = fields.entrySet().iterator();
> -        while (iter.hasNext()) { // for each Field Info
> -            Map.Entry entry = (Map.Entry) iter.next();
> -            Info info = (Info) entry.getValue();
> -            size += HEADER + 4 + PTR + PTR; // Info instance vars
> -            if (info.sortedTerms != null) size += ARR + PTR *  
> info.sortedTerms.length;
> -
> -            int len = info.terms.size();
> -            size += HASHMAP + len * (PTR + HEADER + 3*PTR + 4); //  
> Map.entries
> -            Iterator iter2 = info.terms.entrySet().iterator();
> -            while (--len >= 0) { // for each term
> -                Map.Entry e = (Map.Entry) iter2.next();
> -                size += STR - ARR; // assumes substring() memory  
> overlay
> -//                size += STR + 2 * ((String) e.getKey()).length();
> -                ArrayIntList positions = (ArrayIntList) e.getValue();
> -                size += INTARRLIST + 4*positions.size();
> -            }
> -        }
> -        return size;
> -    }
> -
> -    private int numPositions(ArrayIntList positions) {
> -        return positions.size() / stride;
> -    }
> -
> -    /** sorts into ascending order (on demand), reusing memory  
> along the way */
> -    private void sortFields() {
> -        if (sortedFields == null) sortedFields = sort(fields);
> -    }
> -
> -    /** returns a view of the given map's entries, sorted  
> ascending by key */
> -    private static Map.Entry[] sort(HashMap map) {
> -        int size = map.size();
> -        Map.Entry[] entries = new Map.Entry[size];
> -
> -        Iterator iter = map.entrySet().iterator();
> -        for (int i=0; i < size; i++) {
> -            entries[i] = (Map.Entry) iter.next();
> -        }
> -
> -        if (size > 1) Arrays.sort(entries, termComparator);
> -        return entries;
> -    }
> -
> -  /**
> -  * Convenience method; Creates and returns a token stream that  
> generates a
> -  * token for each keyword in the given collection, "as is",  
> without any
> -  * transforming text analysis. The resulting token stream can be  
> fed into
> -  * {@link #addField(String, TokenStream)}, perhaps wrapped into  
> another
> -  * {@link org.apache.lucene.analysis.TokenFilter}, as desired.
> -  *
> -  * @param keywords
> -  *            the keywords to generate tokens for
> -  * @return the corresponding token stream
> -  */
> -  public TokenStream keywordTokenStream(final Collection keywords) {
> -    if (keywords == null)
> -      throw new IllegalArgumentException("keywords must not be  
> null");
> -
> -    return new TokenStream() {
> -      Iterator iter = keywords.iterator();
> -      int pos = 0;
> -      int start = 0;
> -      public Token next() {
> -        if (!iter.hasNext()) return null;
> -
> -        Object obj = iter.next();
> -        if (obj == null)
> -          throw new IllegalArgumentException("keyword must not be  
> null");
> -
> -        String term = obj.toString();
> -        Token token = new Token(term, start, start + term.length());
> -        start += term.length() + 1; // separate words by 1 (blank)  
> character
> -        pos++;
> -        return token;
> -      }
> -    };
> -  }
> -
> -    /** Returns a String representation of the index data for  
> debugging purposes. */
> -    public String toString() {
> -        StringBuffer result = new StringBuffer(256);
> -        sortFields();
> -        int sumChars = 0;
> -        int sumPositions = 0;
> -        int sumTerms = 0;
> -
> -        for (int i=0; i < sortedFields.length; i++) {
> -            Map.Entry entry = sortedFields[i];
> -            String fieldName = (String) entry.getKey();
> -            Info info = (Info) entry.getValue();
> -            info.sortTerms();
> -            result.append(fieldName + ":\n");
> -
> -            int numChars = 0;
> -            int numPositions = 0;
> -            for (int j=0; j < info.sortedTerms.length; j++) {
> -                Map.Entry e = info.sortedTerms[j];
> -                String term = (String) e.getKey();
> -                ArrayIntList positions = (ArrayIntList) e.getValue();
> -                result.append("\t'" + term + "':" + numPositions 
> (positions) + ":");
> -                result.append(positions.toString(stride)); //  
> ignore offsets
> -                result.append("\n");
> -                numPositions += numPositions(positions);
> -                numChars += term.length();
> -            }
> -
> -            result.append("\tterms=" + info.sortedTerms.length);
> -            result.append(", positions=" + numPositions);
> -            result.append(", Kchars=" + (numChars/1000.0f));
> -            result.append("\n");
> -            sumPositions += numPositions;
> -            sumChars += numChars;
> -            sumTerms += info.sortedTerms.length;
> -        }
> -
> -        result.append("\nfields=" + sortedFields.length);
> -        result.append(", terms=" + sumTerms);
> -        result.append(", positions=" + sumPositions);
> -        result.append(", Kchars=" + (sumChars/1000.0f));
> -        return result.toString();
> -    }
> -
> -
> -    ///////////////////////////////////////////////////////////////// 
> //////////////
> -    // Nested classes:
> -    ///////////////////////////////////////////////////////////////// 
> //////////////
> -    /**
> -     * Index data structure for a field; Contains the tokenized  
> term texts and
> -     * their positions.
> -     */
> -    private static final class Info implements Serializable {
> -
> -        /**
> -         * Term strings and their positions for this field: Map  
> <String
> -         * termText, ArrayIntList positions>
> -         */
> -        private final HashMap terms;
> -
> -        /** Terms sorted ascending by term text; computed on  
> demand */
> -        private transient Map.Entry[] sortedTerms;
> -
> -        /** Number of added tokens for this field */
> -        private final int numTokens;
> -
> -        private static final long serialVersionUID =  
> 2882195016849084649L;
> -
> -        public Info(HashMap terms, int numTokens) {
> -            this.terms = terms;
> -            this.numTokens = numTokens;
> -        }
> -
> -        /**
> -         * Sorts hashed terms into ascending order, reusing memory  
> along the
> -         * way. Note that sorting is lazily delayed until required  
> (often it's
> -         * not required at all). If a sorted view is required then  
> hashing +
> -         * sort + binary search is still faster and smaller than  
> TreeMap usage
> -         * (which would be an alternative and somewhat more  
> elegant approach,
> -         * apart from more sophisticated Tries / prefix trees).
> -         */
> -        public void sortTerms() {
> -            if (sortedTerms == null) sortedTerms = sort(terms);
> -        }
> -
> -        /** note that the frequency can be calculated as  
> numPosition(getPositions(x)) */
> -        public ArrayIntList getPositions(String term) {
> -            return (ArrayIntList) terms.get(term);
> -        }
> -
> -        /** note that the frequency can be calculated as  
> numPosition(getPositions(x)) */
> -        public ArrayIntList getPositions(int pos) {
> -            return (ArrayIntList) sortedTerms[pos].getValue();
> -        }
> -
> -    }
> -
> -
> -    ///////////////////////////////////////////////////////////////// 
> //////////////
> -    // Nested classes:
> -    ///////////////////////////////////////////////////////////////// 
> //////////////
> -    /**
> -     * Efficient resizable auto-expanding list holding <code>int</ 
> code> elements;
> -     * implemented with arrays.
> -     */
> -    private static final class ArrayIntList implements Serializable {
> -
> -        private int[] elements;
> -        private int size = 0;
> -
> -        private static final long serialVersionUID =  
> 2282195016849084649L;
> -
> -        public ArrayIntList() {
> -            this(10);
> -        }
> -
> -        public ArrayIntList(int initialCapacity) {
> -            elements = new int[initialCapacity];
> -        }
> -
> -        public void add(int elem) {
> -            if (size == elements.length) ensureCapacity(size + 1);
> -            elements[size++] = elem;
> -        }
> -
> -        public void add(int pos, int start, int end) {
> -            if (size + 3 > elements.length) ensureCapacity(size + 3);
> -            elements[size] = pos;
> -            elements[size+1] = start;
> -            elements[size+2] = end;
> -            size += 3;
> -        }
> -
> -        public int get(int index) {
> -            if (index >= size) throwIndex(index);
> -            return elements[index];
> -        }
> -
> -        public int size() {
> -            return size;
> -        }
> -
> -        public int[] toArray(int stride) {
> -            int[] arr = new int[size() / stride];
> -            if (stride == 1)
> -                System.arraycopy(elements, 0, arr, 0, size); //  
> fast path
> -            else
> -                for (int i=0, j=0; j < size; i++, j += stride) arr 
> [i] = elements[j];
> -            return arr;
> -        }
> -
> -        private void ensureCapacity(int minCapacity) {
> -            int newCapacity = Math.max(minCapacity,  
> (elements.length * 3) / 2 + 1);
> -            int[] newElements = new int[newCapacity];
> -            System.arraycopy(elements, 0, newElements, 0, size);
> -            elements = newElements;
> -        }
> -
> -        private void throwIndex(int index) {
> -            throw new IndexOutOfBoundsException("index: " + index
> -                        + ", size: " + size);
> -        }
> -
> -        /** returns the first few positions (without offsets);  
> debug only */
> -        public String toString(int stride) {
> -            int s = size() / stride;
> -            int len = Math.min(10, s); // avoid printing huge lists
> -            StringBuffer buf = new StringBuffer(4*len);
> -            buf.append("[");
> -            for (int i = 0; i < len; i++) {
> -                buf.append(get(i*stride));
> -                if (i < len-1) buf.append(", ");
> -            }
> -            if (len != s) buf.append(", ..."); // and some more...
> -            buf.append("]");
> -            return buf.toString();
> -        }
> -    }
> -
> -
> -    ///////////////////////////////////////////////////////////////// 
> //////////////
> -    // Nested classes:
> -    ///////////////////////////////////////////////////////////////// 
> //////////////
> -    private static final Term MATCH_ALL_TERM = new Term("", "");
> -
> -    /**
> -     * Search support for Lucene framework integration; implements  
> all methods
> -     * required by the Lucene IndexReader contracts.
> -     */
> -    private final class MemoryIndexReader extends IndexReader {
> -
> -        private Searcher searcher; // needed to find  
> searcher.getSimilarity()
> -
> -        private MemoryIndexReader() {
> -            super(null); // avoid as much superclass baggage as  
> possible
> -        }
> -
> -        // lucene >= 1.9 or lucene-1.4.3 with patch removing  
> "final" in superclass
> -        protected void finalize() {}
> -
> -        private Info getInfo(String fieldName) {
> -            return (Info) fields.get(fieldName);
> -        }
> -
> -        private Info getInfo(int pos) {
> -            return (Info) sortedFields[pos].getValue();
> -        }
> -
> -        public int docFreq(Term term) {
> -            Info info = getInfo(term.field());
> -            int freq = 0;
> -            if (info != null) freq = info.getPositions(term.text 
> ()) != null ? 1 : 0;
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.docFreq: " + term + ", freq:" + freq);
> -            return freq;
> -        }
> -
> -        public TermEnum terms() {
> -            if (DEBUG) System.err.println("MemoryIndexReader.terms 
> ()");
> -            return terms(MATCH_ALL_TERM);
> -        }
> -
> -        public TermEnum terms(Term term) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.terms: " + term);
> -
> -            int i; // index into info.sortedTerms
> -            int j; // index into sortedFields
> -
> -            sortFields();
> -            j = Arrays.binarySearch(sortedFields, term.field(),  
> termComparator);
> -            if (j < 0) { // not found; choose successor
> -                j = -j -1;
> -                i = 0;
> -                if (j < sortedFields.length) getInfo(j).sortTerms();
> -            }
> -            else { // found
> -                Info info = getInfo(j);
> -                info.sortTerms();
> -                i = Arrays.binarySearch(info.sortedTerms, term.text 
> (), termComparator);
> -                if (i < 0) { // not found; choose successor
> -                    i = -i -1;
> -                    if (i >= info.sortedTerms.length) { // move to  
> next successor
> -                        j++;
> -                        i = 0;
> -                        if (j < sortedFields.length) getInfo 
> (j).sortTerms();
> -                    }
> -                }
> -            }
> -            final int ix = i;
> -            final int jx = j;
> -
> -            return new TermEnum() {
> -
> -                private int i = ix; // index into info.sortedTerms
> -                private int j = jx; // index into sortedFields
> -
> -                public boolean next() {
> -                    if (DEBUG) System.err.println("TermEnum.next");
> -                    if (j >= sortedFields.length) return false;
> -                    Info info = getInfo(j);
> -                    if (++i < info.sortedTerms.length) return true;
> -
> -                    // move to successor
> -                    j++;
> -                    i = 0;
> -                    if (j >= sortedFields.length) return false;
> -                    info.sortTerms();
> -                    return true;
> -                }
> -
> -                public Term term() {
> -                    if (DEBUG) System.err.println("TermEnum.term:  
> " + i);
> -                    if (j >= sortedFields.length) return null;
> -                    Info info = getInfo(j);
> -                    if (i >= info.sortedTerms.length) return null;
> -                    String fieldName = (String) sortedFields 
> [j].getKey();
> -                    return new Term(fieldName, (String)  
> info.sortedTerms[i].getKey());
> -                }
> -
> -                public int docFreq() {
> -                    if (DEBUG) System.err.println 
> ("TermEnum.docFreq");
> -                    if (j >= sortedFields.length) return 0;
> -                    Info info = getInfo(j);
> -                    if (i >= info.sortedTerms.length) return 0;
> -                    return numPositions(info.getPositions(i));
> -                }
> -
> -                public void close() {
> -                    if (DEBUG) System.err.println("TermEnum.close");
> -                }
> -            };
> -        }
> -
> -        public TermPositions termPositions() {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.termPositions");
> -
> -            return new TermPositions() {
> -
> -                private boolean hasNext;
> -                private int cursor = 0;
> -                private ArrayIntList current;
> -
> -                public void seek(Term term) {
> -                    if (DEBUG) System.err.println(".seek: " + term);
> -                    Info info = getInfo(term.field());
> -                    current = info == null ? null :  
> info.getPositions(term.text());
> -                    hasNext = (current != null);
> -                    cursor = 0;
> -                }
> -
> -                public void seek(TermEnum termEnum) {
> -                    seek(termEnum.term());
> -                }
> -
> -                public int doc() {
> -                    if (DEBUG) System.err.println(".doc");
> -                    return 0;
> -                }
> -
> -                public int freq() {
> -                    int freq = current != null ? numPositions 
> (current) : 0;
> -                    if (DEBUG) System.err.println(".freq: " + freq);
> -                    return freq;
> -                }
> -
> -                public boolean next() {
> -                    if (DEBUG) System.err.println(".next: " +  
> current + ", oldHasNext=" + hasNext);
> -                    boolean next = hasNext;
> -                    hasNext = false;
> -                    return next;
> -                }
> -
> -                public int read(int[] docs, int[] freqs) {
> -                    if (DEBUG) System.err.println(".read: " +  
> docs.length);
> -                    if (!hasNext) return 0;
> -                    hasNext = false;
> -                    docs[0] = 0;
> -                    freqs[0] = freq();
> -                    return 1;
> -                }
> -
> -                public boolean skipTo(int target) {
> -                    if (DEBUG) System.err.println(".skipTo: " +  
> target);
> -                    return next();
> -                }
> -
> -                public void close() {
> -                    if (DEBUG) System.err.println(".close");
> -                }
> -
> -                public int nextPosition() { // implements  
> TermPositions
> -                    int pos = current.get(cursor);
> -                    cursor += stride;
> -                    if (DEBUG) System.err.println(".nextPosition:  
> " + pos);
> -                    return pos;
> -                }
> -            };
> -        }
> -
> -        public TermDocs termDocs() {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.termDocs");
> -            return termPositions();
> -        }
> -
> -        public TermFreqVector[] getTermFreqVectors(int docNumber) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getTermFreqVectors");
> -            TermFreqVector[] vectors = new TermFreqVector 
> [fields.size()];
> -            Iterator iter = fields.keySet().iterator();
> -            for (int i=0; i < vectors.length; i++) {
> -                String fieldName = (String) iter.next();
> -                vectors[i] = getTermFreqVector(docNumber, fieldName);
> -            }
> -            return vectors;
> -        }
> -
> -        public TermFreqVector getTermFreqVector(int docNumber,  
> final String fieldName) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getTermFreqVector");
> -            final Info info = getInfo(fieldName);
> -            if (info == null) return null; // TODO: or return  
> empty vector impl???
> -            info.sortTerms();
> -
> -            return new TermPositionVector() {
> -
> -                final Map.Entry[] sortedTerms = info.sortedTerms;
> -
> -                public String getField() {
> -                    return fieldName;
> -                }
> -
> -                public int size() {
> -                    return sortedTerms.length;
> -                }
> -
> -                public String[] getTerms() {
> -                    String[] terms = new String[sortedTerms.length];
> -                    for (int i=0; i < sortedTerms.length; i++) {
> -                        terms[i] = (String) sortedTerms[i].getKey();
> -                    }
> -                    return terms;
> -                }
> -
> -                public int[] getTermFrequencies() {
> -                    int[] freqs = new int[sortedTerms.length];
> -                    for (int i=0; i < sortedTerms.length; i++) {
> -                        freqs[i] = numPositions((ArrayIntList)  
> sortedTerms[i].getValue());
> -                    }
> -                    return freqs;
> -                }
> -
> -                public int indexOf(String term) {
> -                    int i = Arrays.binarySearch(sortedTerms, term,  
> termComparator);
> -                    return i >= 0 ? i : -1;
> -                }
> -
> -                public int[] indexesOf(String[] terms, int start,  
> int len) {
> -                    int[] indexes = new int[len];
> -                    for (int i=0; i < len; i++) {
> -                        indexes[i] = indexOf(terms[start++]);
> -                    }
> -                    return indexes;
> -                }
> -
> -                // lucene >= 1.4.3
> -                public int[] getTermPositions(int index) {
> -                    return ((ArrayIntList) sortedTerms 
> [index].getValue()).toArray(stride);
> -                }
> -
> -                // lucene >= 1.9 (remove this method for  
> lucene-1.4.3)
> -                public org.apache.lucene.index.TermVectorOffsetInfo 
> [] getOffsets(int index) {
> -                    if (stride == 1) return null; // no offsets  
> stored
> -
> -                    ArrayIntList positions = (ArrayIntList)  
> sortedTerms[index].getValue();
> -                    int size = positions.size();
> -                    org.apache.lucene.index.TermVectorOffsetInfo[]  
> offsets =
> -                        new  
> org.apache.lucene.index.TermVectorOffsetInfo[size / stride];
> -
> -                    for (int i=0, j=1; j < size; i++, j += stride) {
> -                        int start = positions.get(j);
> -                        int end = positions.get(j+1);
> -                        offsets[i] = new  
> org.apache.lucene.index.TermVectorOffsetInfo(start, end);
> -                    }
> -                    return offsets;
> -                }
> -
> -            };
> -        }
> -
> -        private Similarity getSimilarity() {
> -            return searcher.getSimilarity();
> -        }
> -
> -        private void setSearcher(Searcher searcher) {
> -            this.searcher = searcher;
> -        }
> -
> -                /** performance hack: cache norms to avoid  
> repeated expensive calculations */
> -                private byte[] cachedNorms;
> -                private String cachedFieldName;
> -                private Similarity cachedSimilarity;
> -
> -                public byte[] norms(String fieldName) {
> -                  byte[] norms = cachedNorms;
> -                  Similarity sim = getSimilarity();
> -                  if (fieldName != cachedFieldName || sim !=  
> cachedSimilarity) { // not cached?
> -                    Info info = getInfo(fieldName);
> -                    int numTokens = info != null ?  
> info.numTokens : 0;
> -                    float n = sim.lengthNorm(fieldName, numTokens);
> -                    byte norm = Similarity.encodeNorm(n);
> -                    norms = new byte[] {norm};
> -
> -                    cachedNorms = norms;
> -                    cachedFieldName = fieldName;
> -                    cachedSimilarity = sim;
> -                    if (DEBUG) System.err.println 
> ("MemoryIndexReader.norms: " + fieldName + ":" + n + ":" + norm +  
> ":" + numTokens);
> -                  }
> -                  return norms;
> -                }
> -
> -        public void norms(String fieldName, byte[] bytes, int  
> offset) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.norms: " + fieldName + "*");
> -            byte[] norms = norms(fieldName);
> -            System.arraycopy(norms, 0, bytes, offset, norms.length);
> -        }
> -
> -        protected void doSetNorm(int doc, String fieldName, byte  
> value) {
> -            throw new UnsupportedOperationException();
> -        }
> -
> -        public int numDocs() {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.numDocs");
> -            return fields.size() > 0 ? 1 : 0;
> -        }
> -
> -        public int maxDoc() {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.maxDoc");
> -            return 1;
> -        }
> -
> -        public Document document(int n) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.document");
> -            return new Document(); // there are no stored fields
> -        }
> -
> -        public boolean isDeleted(int n) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.isDeleted");
> -            return false;
> -        }
> -
> -        public boolean hasDeletions() {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.hasDeletions");
> -            return false;
> -        }
> -
> -        protected void doDelete(int docNum) {
> -            throw new UnsupportedOperationException();
> -        }
> -
> -        protected void doUndeleteAll() {
> -            throw new UnsupportedOperationException();
> -        }
> -
> -        protected void doCommit() {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.doCommit");
> -        }
> -
> -        protected void doClose() {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.doClose");
> -        }
> -
> -        // lucene <= 1.4.3
> -        public Collection getFieldNames() {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getFieldNames");
> -            return getFieldNames(true);
> -        }
> -
> -        // lucene <= 1.4.3
> -        public Collection getFieldNames(boolean indexed) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getFieldNames " + indexed);
> -            return indexed ? Collections.unmodifiableSet 
> (fields.keySet()) : Collections.EMPTY_SET;
> -        }
> -
> -        // lucene <= 1.4.3
> -        public Collection getIndexedFieldNames(boolean  
> storedTermVector) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getIndexedFieldNames " + storedTermVector);
> -            return getFieldNames(storedTermVector);
> -        }
> -
> -        // lucene >= 1.9 (deprecated) (remove this method for  
> lucene-1.4.3)
> -        public Collection getIndexedFieldNames 
> (org.apache.lucene.document.Field.TermVector tvSpec) {
> -            throw new UnsupportedOperationException(
> -                "Deprecated; replaced by getFieldNames 
> (IndexReader.FieldOption)");
> -        }
> -
> -        // lucene >= 1.9 (remove this method for lucene-1.4.3)
> -        public Collection getFieldNames(FieldOption fieldOption) {
> -            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getFieldNamesOption");
> -            if (fieldOption == FieldOption.UNINDEXED)
> -                return Collections.EMPTY_SET;
> -            if (fieldOption == FieldOption.INDEXED_NO_TERMVECTOR)
> -                return Collections.EMPTY_SET;
> -            if (fieldOption == FieldOption.TERMVECTOR_WITH_OFFSET  
> && stride == 1)
> -                return Collections.EMPTY_SET;
> -            if (fieldOption ==  
> FieldOption.TERMVECTOR_WITH_POSITION_OFFSET && stride == 1)
> -                return Collections.EMPTY_SET;
> -
> -            return Collections.unmodifiableSet(fields.keySet());
> -        }
> -    }
> -
> -}
> +package org.apache.lucene.index.memory;
> +
> +/**
> + * Copyright 2005 The Apache Software Foundation
> + *
> + * Licensed under the Apache License, Version 2.0 (the "License");
> + * you may not use this file except in compliance with the License.
> + * You may obtain a copy of the License at
> + *
> + *     http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,  
> software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  
> implied.
> + * See the License for the specific language governing permissions  
> and
> + * limitations under the License.
> + */
> +
> +import java.io.IOException;
> +import java.io.Serializable;
> +import java.util.Arrays;
> +import java.util.Collection;
> +import java.util.Collections;
> +import java.util.Comparator;
> +import java.util.HashMap;
> +import java.util.Iterator;
> +import java.util.Map;
> +
> +import org.apache.lucene.analysis.Analyzer;
> +import org.apache.lucene.analysis.Token;
> +import org.apache.lucene.analysis.TokenStream;
> +import org.apache.lucene.document.Document;
> +import org.apache.lucene.index.IndexReader;
> +import org.apache.lucene.index.Term;
> +import org.apache.lucene.index.TermDocs;
> +import org.apache.lucene.index.TermEnum;
> +import org.apache.lucene.index.TermFreqVector;
> +import org.apache.lucene.index.TermPositionVector;
> +import org.apache.lucene.index.TermPositions;
> +import org.apache.lucene.search.HitCollector;
> +import org.apache.lucene.search.IndexSearcher;
> +import org.apache.lucene.search.Query;
> +import org.apache.lucene.search.Searcher;
> +import org.apache.lucene.search.Similarity;
> +
> +/**
> + * High-performance single-document main memory Apache Lucene  
> fulltext search index.
> + *
> + * <h4>Overview</h4>
> + *
> + * This class is a replacement/substitute for a large subset of
> + * {@link org.apache.lucene.store.RAMDirectory} functionality. It  
> is designed to
> + * enable maximum efficiency for on-the-fly matchmaking combining  
> structured and
> + * fuzzy fulltext search in realtime streaming applications such  
> as Nux XQuery based XML
> + * message queues, publish-subscribe systems for Blogs/newsfeeds,  
> text chat, data acquisition and
> + * distribution systems, application level routers, firewalls,  
> classifiers, etc.
> + * Rather than targetting fulltext search of infrequent queries  
> over huge persistent
> + * data archives (historic search), this class targets fulltext  
> search of huge
> + * numbers of queries over comparatively small transient realtime  
> data (prospective
> + * search).
> + * For example as in <code>float score = search(String text, Query  
> query)</code>.
> + * <p>
> + * Each instance can hold at most one Lucene "document", with a  
> document containing
> + * zero or more "fields", each field having a name and a fulltext  
> value. The
> + * fulltext value is tokenized (split and transformed) into zero  
> or more index terms
> + * (aka words) on <code>addField()</code>, according to the policy  
> implemented by an
> + * Analyzer. For example, Lucene analyzers can split on  
> whitespace, normalize to lower case
> + * for case insensitivity, ignore common terms with little  
> discriminatory value such as "he", "in", "and" (stop
> + * words), reduce the terms to their natural linguistic root form  
> such as "fishing"
> + * being reduced to "fish" (stemming), resolve synonyms/inflexions/ 
> thesauri
> + * (upon indexing and/or querying), etc. For details, see
> + * <a target="_blank" href="http://today.java.net/pub/a/today/ 
> 2003/07/30/LuceneIntro.html">Lucene Analyzer Intro</a>.
> + * <p>
> + * Arbitrary Lucene queries can be run against this class - see <a  
> target="_blank"
> + * href="http://lucene.apache.org/java/docs/ 
> queryparsersyntax.html">Lucene Query Syntax</a>
> + * as well as <a target="_blank"
> + * href="http://today.java.net/pub/a/today/2003/11/07/ 
> QueryParserRules.html">Query Parser Rules</a>.
> + * Note that a Lucene query selects on the field names and  
> associated (indexed)
> + * tokenized terms, not on the original fulltext(s) - the latter  
> are not stored
> + * but rather thrown away immediately after tokenization.
> + * <p>
> + * For some interesting background information on search  
> technology, see Bob Wyman's
> + * <a target="_blank"
> + * href="http://bobwyman.pubsub.com/main/2005/05/ 
> mary_hodder_poi.html">Prospective Search</a>,
> + * Jim Gray's
> + * <a target="_blank" href="http://www.acmqueue.org/modules.php? 
> name=Content&pa=showpage&pid=293&page=4">
> + * A Call to Arms - Custom subscriptions</a>, and Tim Bray's
> + * <a target="_blank"
> + * href="http://www.tbray.org/ongoing/When/200x/2003/07/30/ 
> OnSearchTOC">On Search, the Series</a>.
> + *
> + *
> + * <h4>Example Usage</h4>
> + *
> + * <pre>
> + * Analyzer analyzer = PatternAnalyzer.DEFAULT_ANALYZER;
> + * //Analyzer analyzer = new SimpleAnalyzer();
> + * MemoryIndex index = new MemoryIndex();
> + * index.addField("content", "Readings about Salmons and other  
> select Alaska fishing Manuals", analyzer);
> + * index.addField("author", "Tales of James", analyzer);
> + * float score = index.search(QueryParser.parse("+author:james  
> +salmon~ +fish* manual~", "content", analyzer));
> + * if (score &gt; 0.0f) {
> + *     System.out.println("it's a match");
> + * } else {
> + *     System.out.println("no match found");
> + * }
> + * System.out.println("indexData=" + index.toString());
> + * </pre>
> + *
> + *
> + * <h4>Example XQuery Usage</h4>
> + *
> + * <pre>
> + * (: An XQuery that finds all books authored by James that have  
> something to do with "salmon fishing manuals", sorted by relevance :)
> + * declare namespace lucene = "java:nux.xom.pool.FullTextUtil";
> + * declare variable $query := "+salmon~ +fish* manual~"; (: any  
> arbitrary Lucene query can go here :)
> + *
> + * for $book in /books/book[author="James" and lucene:match 
> (abstract, $query) > 0.0]
> + * let $score := lucene:match($book/abstract, $query)
> + * order by $score descending
> + * return $book
> + * </pre>
> + *
> + *
> + * <h4>No thread safety guarantees</h4>
> + *
> + * An instance can be queried multiple times with the same or  
> different queries,
> + * but an instance is not thread-safe. If desired use idioms such as:
> + * <pre>
> + * MemoryIndex index = ...
> + * synchronized (index) {
> + *    // read and/or write index (i.e. add fields and/or query)
> + * }
> + * </pre>
> + *
> + *
> + * <h4>Performance Notes</h4>
> + *
> + * Internally there's a new data structure geared towards  
> efficient indexing
> + * and searching, plus the necessary support code to seamlessly  
> plug into the Lucene
> + * framework.
> + * <p>
> + * This class performs very well for very small texts (e.g. 10 chars)
> + * as well as for large texts (e.g. 10 MB) and everything in between.
> + * Typically, it is about 10-100 times faster than  
> <code>RAMDirectory</code>.
> + * Note that <code>RAMDirectory</code> has particularly
> + * large efficiency overheads for small to medium sized texts,  
> both in time and space.
> + * Indexing a field with N tokens takes O(N) in the best case, and  
> O(N logN) in the worst
> + * case. Memory consumption is probably larger than for  
> <code>RAMDirectory</code>.
> + * <p>
> + * If you're curious about
> + * the whereabouts of bottlenecks, run java 1.5 with the non- 
> perturbing '-server
> + * -agentlib:hprof=cpu=samples,depth=10' flags, then study the  
> trace log and
> + * correlate its hotspot trailer with its call stack headers (see <a
> + * target="_blank"
> + * href="http://java.sun.com/developer/technicalArticles/ 
> Programming/HPROF.html">
> + * hprof tracing </a>).
> + *
> + * @author whoschek.AT.lbl.DOT.gov
> + */
> +public class MemoryIndex {
> +
> +    /** info for each field: Map<String fieldName, Info field> */
> +    private final HashMap fields = new HashMap();
> +
> +    /** fields sorted ascending by fieldName; lazily computed on  
> demand */
> +    private transient Map.Entry[] sortedFields;
> +
> +    /** template terms sorted ascending by fieldName; lazily  
> computed on demand */
> +    private transient Term[] sortedTemplates;
> +
> +    /** pos: positions[3*i], startOffset: positions[3*i +1],  
> endOffset: positions[3*i +2] */
> +    private final int stride;
> +
> +    private static final long serialVersionUID =  
> 2782195016849084649L;
> +
> +    private static final boolean DEBUG = false;
> +
> +    /**
> +     * Sorts term entries into ascending order; also works for
> +     * Arrays.binarySearch() and Arrays.sort()
> +     */
> +    private static final Comparator termComparator = new Comparator 
> () {
> +        public int compare(Object o1, Object o2) {
> +            if (o1 instanceof Map.Entry) o1 = ((Map.Entry)  
> o1).getKey();
> +            if (o2 instanceof Map.Entry) o2 = ((Map.Entry)  
> o2).getKey();
> +            if (o1 == o2) return 0;
> +            return ((String) o1).compareTo((String) o2);
> +        }
> +    };
> +
> +    /**
> +     * Constructs an empty instance.
> +     */
> +    public MemoryIndex() {
> +        this(false);
> +    }
> +
> +    /**
> +     * Constructs an empty instance that can optionally store the  
> start and end
> +     * character offset of each token term in the text. This can  
> be useful for
> +     * highlighting of hit locations with the Lucene highlighter  
> package.
> +     * Private until the highlighter package matures, so that this  
> can actually
> +     * be meaningfully integrated.
> +     *
> +     * @param storeOffsets
> +     *            whether or not to store the start and end  
> character offset of
> +     *            each token term in the text
> +     */
> +    private MemoryIndex(boolean storeOffsets) {
> +        this.stride = storeOffsets ? 3 : 1;
> +    }
> +
> +    /**
> +     * Convenience method; Tokenizes the given field text and adds  
> the resulting
> +     * terms to the index; Equivalent to adding a tokenized, indexed,
> +     * termVectorStored, unstored, non-keyword Lucene
> +     * {@link org.apache.lucene.document.Field}.
> +     *
> +     * @param fieldName
> +     *            a name to be associated with the text
> +     * @param text
> +     *            the text to tokenize and index.
> +     * @param analyzer
> +     *            the analyzer to use for tokenization
> +     */
> +    public void addField(String fieldName, String text, Analyzer  
> analyzer) {
> +        if (fieldName == null)
> +            throw new IllegalArgumentException("fieldName must not  
> be null");
> +        if (text == null)
> +            throw new IllegalArgumentException("text must not be  
> null");
> +        if (analyzer == null)
> +            throw new IllegalArgumentException("analyzer must not  
> be null");
> +
> +        TokenStream stream;
> +        if (analyzer instanceof PatternAnalyzer) {
> +            stream = ((PatternAnalyzer) analyzer).tokenStream 
> (fieldName, text);
> +        } else {
> +            stream = analyzer.tokenStream(fieldName,
> +                    new PatternAnalyzer.FastStringReader(text));
> +        }
> +        addField(fieldName, stream);
> +    }
> +
> +    /**
> +     * Convenience method; Creates and returns a token stream that  
> generates a
> +     * token for each keyword in the given collection, "as is",  
> without any
> +     * transforming text analysis. The resulting token stream can  
> be fed into
> +     * {@link #addField(String, TokenStream)}, perhaps wrapped  
> into another
> +     * {@link org.apache.lucene.analysis.TokenFilter}, as desired.
> +     *
> +     * @param keywords
> +     *            the keywords to generate tokens for
> +     * @return the corresponding token stream
> +     */
> +    public TokenStream keywordTokenStream(final Collection  
> keywords) {
> +        if (keywords == null)
> +            throw new IllegalArgumentException("keywords must not  
> be null");
> +
> +        return new TokenStream() {
> +            private Iterator iter = keywords.iterator();
> +            private int start = 0;
> +            public Token next() {
> +                if (!iter.hasNext()) return null;
> +
> +                Object obj = iter.next();
> +                if (obj == null)
> +                    throw new IllegalArgumentException("keyword  
> must not be null");
> +
> +                String term = obj.toString();
> +                Token token = new Token(term, start, start +  
> term.length());
> +                start += term.length() + 1; // separate words by 1  
> (blank) character
> +                return token;
> +            }
> +        };
> +    }
> +
> +    /**
> +     * Iterates over the given token stream and adds the resulting  
> terms to the index;
> +     * Equivalent to adding a tokenized, indexed,  
> termVectorStored, unstored,
> +     * Lucene {@link org.apache.lucene.document.Field}.
> +     * Finally closes the token stream. Note that untokenized  
> keywords can be added with this method via
> +     * {@link #keywordTokenStream(Collection)}, the Lucene contrib  
> <code>KeywordTokenizer</code> or similar utilities.
> +     *
> +     * @param fieldName
> +     *            a name to be associated with the text
> +     * @param stream
> +     *            the token stream to retrieve tokens from.
> +     */
> +    public void addField(String fieldName, TokenStream stream) {
> +        /*
> +         * Note that this method signature avoids having a user  
> call new
> +         * o.a.l.d.Field(...) which would be much too expensive  
> due to the
> +         * String.intern() usage of that class.
> +         *
> +         * More often than not, String.intern() leads to serious  
> performance
> +         * degradations rather than improvements! If you're  
> curious why, check
> +         * out the JDK's native code, see how it oscillates  
> multiple times back
> +         * and forth between Java code and native code on each  
> intern() call,
> +         * only to end up using a plain vanilla java.util.HashMap  
> on the Java
> +         * heap for it's interned strings! String.equals() has a  
> small cost
> +         * compared to String.intern(), trust me. Application  
> level interning
> +         * (e.g. a HashMap per Directory/Index) typically leads to  
> better
> +         * solutions than frequent hidden low-level calls to  
> String.intern().
> +         *
> +         * Perhaps with some luck, Lucene's Field.java (and  
> Term.java) and
> +         * cousins could be fixed to not use String.intern().  
> Sigh :-(
> +         */
> +        try {
> +            if (fieldName == null)
> +                throw new IllegalArgumentException("fieldName must  
> not be null");
> +            if (stream == null)
> +                throw new IllegalArgumentException("token stream  
> must not be null");
> +            if (fields.get(fieldName) != null)
> +                throw new IllegalArgumentException("field must not  
> be added more than once");
> +
> +            HashMap terms = new HashMap();
> +            int numTokens = 0;
> +            int pos = -1;
> +            Token token;
> +
> +            while ((token = stream.next()) != null) {
> +                numTokens++;
> +                pos += token.getPositionIncrement();
> +
> +                String term = token.termText();
> +//                if (DEBUG) System.err.println("token='" + term +  
> "'");
> +                ArrayIntList positions = (ArrayIntList) terms.get 
> (term);
> +                if (positions == null) { // term not seen before
> +                    positions = new ArrayIntList(stride);
> +                    terms.put(term, positions);
> +                }
> +                if (stride == 1)
> +                    positions.add(pos);
> +                else
> +                    positions.add(pos, token.startOffset(),  
> token.endOffset());
> +            }
> +
> +            // ensure infos.numTokens > 0 invariant; needed for  
> correct operation of terms()
> +            if (numTokens > 0) {
> +                fields.put(fieldName, new Info(terms, numTokens));
> +                sortedFields = null;    // invalidate sorted view,  
> if any
> +                sortedTemplates = null; // invalidate sorted view,  
> if any
> +            }
> +        } catch (IOException e) { // can never happen
> +            throw new RuntimeException(e);
> +        } finally {
> +            try {
> +                if (stream != null) stream.close();
> +            } catch (IOException e2) {
> +                throw new RuntimeException(e2);
> +            }
> +        }
> +    }
> +
> +
> +    public IndexReader createReader(){
> +        return new MemoryIndexReader();
> +    }
> +
> +    /**
> +     * Creates and returns a searcher that can be used to execute  
> arbitrary
> +     * Lucene queries and to collect the resulting query results  
> as hits.
> +     */
> +    public IndexSearcher createSearcher() {
> +        MemoryIndexReader reader = new MemoryIndexReader();
> +        IndexSearcher searcher = new IndexSearcher(reader); //  
> ensures no auto-close !!
> +        reader.setSearcher(searcher); // to later get hold of  
> searcher.getSimilarity()
> +        return searcher;
> +    }
> +
> +    /**
> +     * Convenience method that efficiently returns the relevance  
> score by
> +     * matching this index against the given Lucene query expression.
> +     *
> +     * @param query
> +     *            an arbitrary Lucene query to run against this index
> +     * @return the relevance score of the matchmaking; A number in  
> the range
> +     *         [0.0 .. 1.0], with 0.0 indicating no match. The  
> higher the number
> +     *         the better the match.
> +     * @see org.apache.lucene.queryParser.QueryParser#parse 
> (String, String,
> +     *      Analyzer)
> +     */
> +    public float search(Query query) {
> +        if (query == null)
> +            throw new IllegalArgumentException("query must not be  
> null");
> +
> +        Searcher searcher = createSearcher();
> +        try {
> +            final float[] scores = new float[1]; // inits to 0.0f  
> (no match)
> +            searcher.search(query, new HitCollector() {
> +                public void collect(int doc, float score) {
> +                    scores[0] = score;
> +                }
> +            });
> +            float score = scores[0];
> +            return score;
> +        } catch (IOException e) { // can never happen (RAMDirectory)
> +            throw new RuntimeException(e);
> +        } finally {
> +            // searcher.close();
> +            /*
> +             * Note that it is harmless and important for good  
> performance to
> +             * NOT close the index reader!!! This avoids all sorts of
> +             * unnecessary baggage and locking in the Lucene  
> IndexReader
> +             * superclass, all of which is completely unnecessary  
> for this main
> +             * memory index data structure without thread-safety  
> claims.
> +             *
> +             * Wishing IndexReader would be an interface...
> +             *
> +             * Actually with the new tight createSearcher() API  
> auto-closing is now
> +             * made impossible, hence searcher.close() would be  
> harmless...
> +             */
> +        }
> +    }
> +
> +    /**
> +     * Returns a reasonable approximation of the main memory  
> [bytes] consumed by
> +     * this instance. Useful for smart memory sensititve caches/ 
> pools. Assumes
> +     * fieldNames are interned, whereas tokenized terms are memory- 
> overlaid. For
> +     * simplicity, assumes no VM word boundary alignment of  
> instance vars.
> +     */
> +    public int getMemorySize() {
> +        // for example usage in a smart cache see nux.xom.pool.Pool
> +        int HEADER = 12; // object header of any java object
> +        int PTR = 4; // pointer on 32 bit VMs
> +        int ARR = HEADER + 4;
> +        int STR = HEADER + 3*4 + PTR + ARR; // string
> +        int INTARRLIST = HEADER + 4 + PTR + ARR;
> +        int HASHMAP = HEADER + 4*PTR + 4*4 + ARR;
> +
> +        int size = 0;
> +        size += HEADER + 3*PTR; // memory index
> +        if (sortedFields != null) size += ARR + PTR *  
> sortedFields.length;
> +        if (sortedTemplates != null) size += ARR + PTR *  
> sortedTemplates.length;
> +
> +        size += HASHMAP + fields.size() * (PTR + HEADER + 3*PTR +  
> 4); // Map.entries
> +        Iterator iter = fields.entrySet().iterator();
> +        while (iter.hasNext()) { // for each Field Info
> +            Map.Entry entry = (Map.Entry) iter.next();
> +            Info info = (Info) entry.getValue();
> +            size += HEADER + 4 + PTR + PTR; // Info instance vars
> +            if (info.sortedTerms != null) size += ARR + PTR *  
> info.sortedTerms.length;
> +
> +            int len = info.terms.size();
> +            size += HASHMAP + len * (PTR + HEADER + 3*PTR + 4); //  
> Map.entries
> +            Iterator iter2 = info.terms.entrySet().iterator();
> +            while (--len >= 0) { // for each term
> +                Map.Entry e = (Map.Entry) iter2.next();
> +                size += STR - ARR; // assumes substring() memory  
> overlay
> +//                size += STR + 2 * ((String) e.getKey()).length();
> +                ArrayIntList positions = (ArrayIntList) e.getValue();
> +                size += INTARRLIST + 4*positions.size();
> +            }
> +        }
> +        return size;
> +    }
> +
> +    private int numPositions(ArrayIntList positions) {
> +        return positions.size() / stride;
> +    }
> +
> +    /** sorts into ascending order (on demand), reusing memory  
> along the way */
> +    private void sortFields() {
> +        if (sortedFields == null) sortedFields = sort(fields);
> +    }
> +
> +    /** returns a view of the given map's entries, sorted  
> ascending by key */
> +    private static Map.Entry[] sort(HashMap map) {
> +        int size = map.size();
> +        Map.Entry[] entries = new Map.Entry[size];
> +
> +        Iterator iter = map.entrySet().iterator();
> +        for (int i=0; i < size; i++) {
> +            entries[i] = (Map.Entry) iter.next();
> +        }
> +
> +        if (size > 1) Arrays.sort(entries, termComparator);
> +        return entries;
> +    }
> +
> +    /** Returns a new Term object, minimizing String.intern()  
> overheads. */
> +    private Term createTerm(int pos, String text) {
> +        // used by MemoryIndexReader.terms().term()
> +        // Assertion: sortFields has already been called before
> +        Term[] templates = sortedTemplates;
> +        if (templates == null) { // not yet initialized?
> +            templates = new Term[sortedFields.length];
> +            sortedTemplates = templates;
> +        }
> +        if (templates[pos] == null) { // not yet cached?
> +            String fieldName = (String) sortedFields[pos].getKey();
> +            templates[pos] = new Term(fieldName, "");
> +        }
> +
> +        return templates[pos].createTerm(text);
> +    }
> +
> +    /** Returns a String representation of the index data for  
> debugging purposes. */
> +    public String toString() {
> +        StringBuffer result = new StringBuffer(256);
> +        sortFields();
> +        int sumChars = 0;
> +        int sumPositions = 0;
> +        int sumTerms = 0;
> +
> +        for (int i=0; i < sortedFields.length; i++) {
> +            Map.Entry entry = sortedFields[i];
> +            String fieldName = (String) entry.getKey();
> +            Info info = (Info) entry.getValue();
> +            info.sortTerms();
> +            result.append(fieldName + ":\n");
> +
> +            int numChars = 0;
> +            int numPositions = 0;
> +            for (int j=0; j < info.sortedTerms.length; j++) {
> +                Map.Entry e = info.sortedTerms[j];
> +                String term = (String) e.getKey();
> +                ArrayIntList positions = (ArrayIntList) e.getValue();
> +                result.append("\t'" + term + "':" + numPositions 
> (positions) + ":");
> +                result.append(positions.toString(stride)); //  
> ignore offsets
> +                result.append("\n");
> +                numPositions += numPositions(positions);
> +                numChars += term.length();
> +            }
> +
> +            result.append("\tterms=" + info.sortedTerms.length);
> +            result.append(", positions=" + numPositions);
> +            result.append(", Kchars=" + (numChars/1000.0f));
> +            result.append("\n");
> +            sumPositions += numPositions;
> +            sumChars += numChars;
> +            sumTerms += info.sortedTerms.length;
> +        }
> +
> +        result.append("\nfields=" + sortedFields.length);
> +        result.append(", terms=" + sumTerms);
> +        result.append(", positions=" + sumPositions);
> +        result.append(", Kchars=" + (sumChars/1000.0f));
> +        return result.toString();
> +    }
> +
> +
> +    ///////////////////////////////////////////////////////////////// 
> //////////////
> +    // Nested classes:
> +    ///////////////////////////////////////////////////////////////// 
> //////////////
> +    /**
> +     * Index data structure for a field; Contains the tokenized  
> term texts and
> +     * their positions.
> +     */
> +    private static final class Info implements Serializable {
> +
> +        /**
> +         * Term strings and their positions for this field: Map  
> <String
> +         * termText, ArrayIntList positions>
> +         */
> +        private final HashMap terms;
> +
> +        /** Terms sorted ascending by term text; computed on  
> demand */
> +        private transient Map.Entry[] sortedTerms;
> +
> +        /** Number of added tokens for this field */
> +        private final int numTokens;
> +
> +        private static final long serialVersionUID =  
> 2882195016849084649L;
> +
> +        public Info(HashMap terms, int numTokens) {
> +            this.terms = terms;
> +            this.numTokens = numTokens;
> +        }
> +
> +        /**
> +         * Sorts hashed terms into ascending order, reusing memory  
> along the
> +         * way. Note that sorting is lazily delayed until required  
> (often it's
> +         * not required at all). If a sorted view is required then  
> hashing +
> +         * sort + binary search is still faster and smaller than  
> TreeMap usage
> +         * (which would be an alternative and somewhat more  
> elegant approach,
> +         * apart from more sophisticated Tries / prefix trees).
> +         */
> +        public void sortTerms() {
> +            if (sortedTerms == null) sortedTerms = sort(terms);
> +        }
> +
> +        /** note that the frequency can be calculated as  
> numPosition(getPositions(x)) */
> +        public ArrayIntList getPositions(String term) {
> +            return (ArrayIntList) terms.get(term);
> +        }
> +
> +        /** note that the frequency can be calculated as  
> numPosition(getPositions(x)) */
> +        public ArrayIntList getPositions(int pos) {
> +            return (ArrayIntList) sortedTerms[pos].getValue();
> +        }
> +
> +    }
> +
> +
> +    ///////////////////////////////////////////////////////////////// 
> //////////////
> +    // Nested classes:
> +    ///////////////////////////////////////////////////////////////// 
> //////////////
> +    /**
> +     * Efficient resizable auto-expanding list holding <code>int</ 
> code> elements;
> +     * implemented with arrays.
> +     */
> +    private static final class ArrayIntList implements Serializable {
> +
> +        private int[] elements;
> +        private int size = 0;
> +
> +        private static final long serialVersionUID =  
> 2282195016849084649L;
> +
> +        public ArrayIntList() {
> +            this(10);
> +        }
> +
> +        public ArrayIntList(int initialCapacity) {
> +            elements = new int[initialCapacity];
> +        }
> +
> +        public void add(int elem) {
> +            if (size == elements.length) ensureCapacity(size + 1);
> +            elements[size++] = elem;
> +        }
> +
> +        public void add(int pos, int start, int end) {
> +            if (size + 3 > elements.length) ensureCapacity(size + 3);
> +            elements[size] = pos;
> +            elements[size+1] = start;
> +            elements[size+2] = end;
> +            size += 3;
> +        }
> +
> +        public int get(int index) {
> +            if (index >= size) throwIndex(index);
> +            return elements[index];
> +        }
> +
> +        public int size() {
> +            return size;
> +        }
> +
> +        public int[] toArray(int stride) {
> +            int[] arr = new int[size() / stride];
> +            if (stride == 1)
> +                System.arraycopy(elements, 0, arr, 0, size); //  
> fast path
> +            else
> +                for (int i=0, j=0; j < size; i++, j += stride) arr 
> [i] = elements[j];
> +            return arr;
> +        }
> +
> +        private void ensureCapacity(int minCapacity) {
> +            int newCapacity = Math.max(minCapacity,  
> (elements.length * 3) / 2 + 1);
> +            int[] newElements = new int[newCapacity];
> +            System.arraycopy(elements, 0, newElements, 0, size);
> +            elements = newElements;
> +        }
> +
> +        private void throwIndex(int index) {
> +            throw new IndexOutOfBoundsException("index: " + index
> +                        + ", size: " + size);
> +        }
> +
> +        /** returns the first few positions (without offsets);  
> debug only */
> +        public String toString(int stride) {
> +            int s = size() / stride;
> +            int len = Math.min(10, s); // avoid printing huge lists
> +            StringBuffer buf = new StringBuffer(4*len);
> +            buf.append("[");
> +            for (int i = 0; i < len; i++) {
> +                buf.append(get(i*stride));
> +                if (i < len-1) buf.append(", ");
> +            }
> +            if (len != s) buf.append(", ..."); // and some more...
> +            buf.append("]");
> +            return buf.toString();
> +        }
> +    }
> +
> +
> +    ///////////////////////////////////////////////////////////////// 
> //////////////
> +    // Nested classes:
> +    ///////////////////////////////////////////////////////////////// 
> //////////////
> +    private static final Term MATCH_ALL_TERM = new Term("", "");
> +
> +    /**
> +     * Search support for Lucene framework integration; implements  
> all methods
> +     * required by the Lucene IndexReader contracts.
> +     */
> +    private final class MemoryIndexReader extends IndexReader {
> +
> +        private Searcher searcher; // needed to find  
> searcher.getSimilarity()
> +
> +        private MemoryIndexReader() {
> +            super(null); // avoid as much superclass baggage as  
> possible
> +        }
> +
> +        // lucene >= 1.9 or lucene-1.4.3 with patch removing  
> "final" in superclass
> +        protected void finalize() {}
> +
> +        private Info getInfo(String fieldName) {
> +            return (Info) fields.get(fieldName);
> +        }
> +
> +        private Info getInfo(int pos) {
> +            return (Info) sortedFields[pos].getValue();
> +        }
> +
> +        public int docFreq(Term term) {
> +            Info info = getInfo(term.field());
> +            int freq = 0;
> +            if (info != null) freq = info.getPositions(term.text 
> ()) != null ? 1 : 0;
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.docFreq: " + term + ", freq:" + freq);
> +            return freq;
> +        }
> +
> +        public TermEnum terms() {
> +            if (DEBUG) System.err.println("MemoryIndexReader.terms 
> ()");
> +            return terms(MATCH_ALL_TERM);
> +        }
> +
> +        public TermEnum terms(Term term) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.terms: " + term);
> +
> +            int i; // index into info.sortedTerms
> +            int j; // index into sortedFields
> +
> +            sortFields();
> +            if (sortedFields.length == 1 && sortedFields[0].getKey 
> () == term.field()) {
> +                j = 0; // fast path
> +            } else {
> +                j = Arrays.binarySearch(sortedFields, term.field 
> (), termComparator);
> +            }
> +
> +            if (j < 0) { // not found; choose successor
> +                j = -j -1;
> +                i = 0;
> +                if (j < sortedFields.length) getInfo(j).sortTerms();
> +            }
> +            else { // found
> +                Info info = getInfo(j);
> +                info.sortTerms();
> +                i = Arrays.binarySearch(info.sortedTerms, term.text 
> (), termComparator);
> +                if (i < 0) { // not found; choose successor
> +                    i = -i -1;
> +                    if (i >= info.sortedTerms.length) { // move to  
> next successor
> +                        j++;
> +                        i = 0;
> +                        if (j < sortedFields.length) getInfo 
> (j).sortTerms();
> +                    }
> +                }
> +            }
> +            final int ix = i;
> +            final int jx = j;
> +
> +            return new TermEnum() {
> +
> +                private int i = ix; // index into info.sortedTerms
> +                private int j = jx; // index into sortedFields
> +
> +                public boolean next() {
> +                    if (DEBUG) System.err.println("TermEnum.next");
> +                    if (j >= sortedFields.length) return false;
> +                    Info info = getInfo(j);
> +                    if (++i < info.sortedTerms.length) return true;
> +
> +                    // move to successor
> +                    j++;
> +                    i = 0;
> +                    if (j >= sortedFields.length) return false;
> +                    getInfo(j).sortTerms();
> +                    return true;
> +                }
> +
> +                public Term term() {
> +                    if (DEBUG) System.err.println("TermEnum.term:  
> " + i);
> +                    if (j >= sortedFields.length) return null;
> +                    Info info = getInfo(j);
> +                    if (i >= info.sortedTerms.length) return null;
> +//                    if (DEBUG) System.err.println 
> ("TermEnum.term: " + i + ", " + info.sortedTerms[i].getKey());
> +                    return createTerm(j, (String) info.sortedTerms 
> [i].getKey());
> +                }
> +
> +                public int docFreq() {
> +                    if (DEBUG) System.err.println 
> ("TermEnum.docFreq");
> +                    if (j >= sortedFields.length) return 0;
> +                    Info info = getInfo(j);
> +                    if (i >= info.sortedTerms.length) return 0;
> +                    return numPositions(info.getPositions(i));
> +                }
> +
> +                public void close() {
> +                    if (DEBUG) System.err.println("TermEnum.close");
> +                }
> +            };
> +        }
> +
> +        public TermPositions termPositions() {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.termPositions");
> +
> +            return new TermPositions() {
> +
> +                private boolean hasNext;
> +                private int cursor = 0;
> +                private ArrayIntList current;
> +
> +                public void seek(Term term) {
> +                    if (DEBUG) System.err.println(".seek: " + term);
> +                    Info info = getInfo(term.field());
> +                    current = info == null ? null :  
> info.getPositions(term.text());
> +                    hasNext = (current != null);
> +                    cursor = 0;
> +                }
> +
> +                public void seek(TermEnum termEnum) {
> +                    if (DEBUG) System.err.println(".seekEnum");
> +                    seek(termEnum.term());
> +                }
> +
> +                public int doc() {
> +                    if (DEBUG) System.err.println(".doc");
> +                    return 0;
> +                }
> +
> +                public int freq() {
> +                    int freq = current != null ? numPositions 
> (current) : 0;
> +                    if (DEBUG) System.err.println(".freq: " + freq);
> +                    return freq;
> +                }
> +
> +                public boolean next() {
> +                    if (DEBUG) System.err.println(".next: " +  
> current + ", oldHasNext=" + hasNext);
> +                    boolean next = hasNext;
> +                    hasNext = false;
> +                    return next;
> +                }
> +
> +                public int read(int[] docs, int[] freqs) {
> +                    if (DEBUG) System.err.println(".read: " +  
> docs.length);
> +                    if (!hasNext) return 0;
> +                    hasNext = false;
> +                    docs[0] = 0;
> +                    freqs[0] = freq();
> +                    return 1;
> +                }
> +
> +                public boolean skipTo(int target) {
> +                    if (DEBUG) System.err.println(".skipTo: " +  
> target);
> +                    return next();
> +                }
> +
> +                public void close() {
> +                    if (DEBUG) System.err.println(".close");
> +                }
> +
> +                public int nextPosition() { // implements  
> TermPositions
> +                    int pos = current.get(cursor);
> +                    cursor += stride;
> +                    if (DEBUG) System.err.println(".nextPosition:  
> " + pos);
> +                    return pos;
> +                }
> +            };
> +        }
> +
> +        public TermDocs termDocs() {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.termDocs");
> +            return termPositions();
> +        }
> +
> +        public TermFreqVector[] getTermFreqVectors(int docNumber) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getTermFreqVectors");
> +            TermFreqVector[] vectors = new TermFreqVector 
> [fields.size()];
> +            Iterator iter = fields.keySet().iterator();
> +            for (int i=0; i < vectors.length; i++) {
> +                String fieldName = (String) iter.next();
> +                vectors[i] = getTermFreqVector(docNumber, fieldName);
> +            }
> +            return vectors;
> +        }
> +
> +        public TermFreqVector getTermFreqVector(int docNumber,  
> final String fieldName) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getTermFreqVector");
> +            final Info info = getInfo(fieldName);
> +            if (info == null) return null; // TODO: or return  
> empty vector impl???
> +            info.sortTerms();
> +
> +            return new TermPositionVector() {
> +
> +                private final Map.Entry[] sortedTerms =  
> info.sortedTerms;
> +
> +                public String getField() {
> +                    return fieldName;
> +                }
> +
> +                public int size() {
> +                    return sortedTerms.length;
> +                }
> +
> +                public String[] getTerms() {
> +                    String[] terms = new String[sortedTerms.length];
> +                    for (int i=sortedTerms.length; --i >= 0; ) {
> +                        terms[i] = (String) sortedTerms[i].getKey();
> +                    }
> +                    return terms;
> +                }
> +
> +                public int[] getTermFrequencies() {
> +                    int[] freqs = new int[sortedTerms.length];
> +                    for (int i=sortedTerms.length; --i >= 0; ) {
> +                        freqs[i] = numPositions((ArrayIntList)  
> sortedTerms[i].getValue());
> +                    }
> +                    return freqs;
> +                }
> +
> +                public int indexOf(String term) {
> +                    int i = Arrays.binarySearch(sortedTerms, term,  
> termComparator);
> +                    return i >= 0 ? i : -1;
> +                }
> +
> +                public int[] indexesOf(String[] terms, int start,  
> int len) {
> +                    int[] indexes = new int[len];
> +                    for (int i=0; i < len; i++) {
> +                        indexes[i] = indexOf(terms[start++]);
> +                    }
> +                    return indexes;
> +                }
> +
> +                // lucene >= 1.4.3
> +                public int[] getTermPositions(int index) {
> +                    return ((ArrayIntList) sortedTerms 
> [index].getValue()).toArray(stride);
> +                }
> +
> +                // lucene >= 1.9 (remove this method for  
> lucene-1.4.3)
> +                public org.apache.lucene.index.TermVectorOffsetInfo 
> [] getOffsets(int index) {
> +                    if (stride == 1) return null; // no offsets  
> stored
> +
> +                    ArrayIntList positions = (ArrayIntList)  
> sortedTerms[index].getValue();
> +                    int size = positions.size();
> +                    org.apache.lucene.index.TermVectorOffsetInfo[]  
> offsets =
> +                        new  
> org.apache.lucene.index.TermVectorOffsetInfo[size / stride];
> +
> +                    for (int i=0, j=1; j < size; i++, j += stride) {
> +                        int start = positions.get(j);
> +                        int end = positions.get(j+1);
> +                        offsets[i] = new  
> org.apache.lucene.index.TermVectorOffsetInfo(start, end);
> +                    }
> +                    return offsets;
> +                }
> +
> +            };
> +        }
> +
> +        private Similarity getSimilarity() {
> +            return searcher.getSimilarity();
> +        }
> +
> +        private void setSearcher(Searcher searcher) {
> +            this.searcher = searcher;
> +        }
> +
> +        /** performance hack: cache norms to avoid repeated  
> expensive calculations */
> +        private byte[] cachedNorms;
> +        private String cachedFieldName;
> +        private Similarity cachedSimilarity;
> +
> +        public byte[] norms(String fieldName) {
> +            byte[] norms = cachedNorms;
> +            Similarity sim = getSimilarity();
> +            if (fieldName != cachedFieldName || sim !=  
> cachedSimilarity) { // not cached?
> +                Info info = getInfo(fieldName);
> +                int numTokens = info != null ? info.numTokens : 0;
> +                float n = sim.lengthNorm(fieldName, numTokens);
> +                byte norm = Similarity.encodeNorm(n);
> +                norms = new byte[] {norm};
> +
> +                cachedNorms = norms;
> +                cachedFieldName = fieldName;
> +                cachedSimilarity = sim;
> +                if (DEBUG) System.err.println 
> ("MemoryIndexReader.norms: " + fieldName + ":" + n + ":" + norm +  
> ":" + numTokens);
> +            }
> +            return norms;
> +        }
> +
> +        public void norms(String fieldName, byte[] bytes, int  
> offset) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.norms*: " + fieldName);
> +            byte[] norms = norms(fieldName);
> +            System.arraycopy(norms, 0, bytes, offset, norms.length);
> +        }
> +
> +        protected void doSetNorm(int doc, String fieldName, byte  
> value) {
> +            throw new UnsupportedOperationException();
> +        }
> +
> +        public int numDocs() {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.numDocs");
> +            return fields.size() > 0 ? 1 : 0;
> +        }
> +
> +        public int maxDoc() {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.maxDoc");
> +            return 1;
> +        }
> +
> +        public Document document(int n) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.document");
> +            return new Document(); // there are no stored fields
> +        }
> +
> +        public boolean isDeleted(int n) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.isDeleted");
> +            return false;
> +        }
> +
> +        public boolean hasDeletions() {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.hasDeletions");
> +            return false;
> +        }
> +
> +        protected void doDelete(int docNum) {
> +            throw new UnsupportedOperationException();
> +        }
> +
> +        protected void doUndeleteAll() {
> +            throw new UnsupportedOperationException();
> +        }
> +
> +        protected void doCommit() {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.doCommit");
> +        }
> +
> +        protected void doClose() {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.doClose");
> +        }
> +
> +        // lucene <= 1.4.3
> +        public Collection getFieldNames() {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getFieldNames");
> +            return getFieldNames(true);
> +        }
> +
> +        // lucene <= 1.4.3
> +        public Collection getFieldNames(boolean indexed) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getFieldNames " + indexed);
> +            return indexed ? Collections.unmodifiableSet 
> (fields.keySet()) : Collections.EMPTY_SET;
> +        }
> +
> +        // lucene <= 1.4.3
> +        public Collection getIndexedFieldNames(boolean  
> storedTermVector) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getIndexedFieldNames " + storedTermVector);
> +            return getFieldNames(storedTermVector);
> +        }
> +
> +        // lucene >= 1.9 (deprecated) (remove this method for  
> lucene-1.4.3)
> +        public Collection getIndexedFieldNames 
> (org.apache.lucene.document.Field.TermVector tvSpec) {
> +            throw new UnsupportedOperationException(
> +                "Deprecated; replaced by getFieldNames 
> (IndexReader.FieldOption)");
> +        }
> +
> +        // lucene >= 1.9 (remove this method for lucene-1.4.3)
> +        public Collection getFieldNames(FieldOption fieldOption) {
> +            if (DEBUG) System.err.println 
> ("MemoryIndexReader.getFieldNamesOption");
> +            if (fieldOption == FieldOption.UNINDEXED)
> +                return Collections.EMPTY_SET;
> +            if (fieldOption == FieldOption.INDEXED_NO_TERMVECTOR)
> +                return Collections.EMPTY_SET;
> +            if (fieldOption == FieldOption.TERMVECTOR_WITH_OFFSET  
> && stride == 1)
> +                return Collections.EMPTY_SET;
> +            if (fieldOption ==  
> FieldOption.TERMVECTOR_WITH_POSITION_OFFSET && stride == 1)
> +                return Collections.EMPTY_SET;
> +
> +            return Collections.unmodifiableSet(fields.keySet());
> +        }
> +    }
> +
> +}
> +
>
>


---------------------------------------------------------------------
To unsubscribe, e-mail: java-dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-dev-help@lucene.apache.org


Mime
View raw message