lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From busc...@apache.org
Subject svn commit: r533549 - in /lucene/java/trunk: ./ src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/document/ src/java/org/apache/lucene/index/ src/test/org/apache/lucene/analysis/ src/test/org/apache/lucene/index/
Date Sun, 29 Apr 2007 19:26:12 GMT
Author: buschmi
Date: Sun Apr 29 12:26:11 2007
New Revision: 533549

URL: http://svn.apache.org/viewvc?view=rev&rev=533549
Log:
LUCENE-580: 
- Added the public method reset() to TokenStream.
- Added a new constructor to Field that takes a TokenStream as argument, useful for pre-analyzed
fields.

Added:
    lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java
    lucene/java/trunk/src/java/org/apache/lucene/document/Field.java
    lucene/java/trunk/src/java/org/apache/lucene/document/Fieldable.java
    lucene/java/trunk/src/java/org/apache/lucene/index/DocumentWriter.java
    lucene/java/trunk/src/java/org/apache/lucene/index/FieldsReader.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?view=diff&rev=533549&r1=533548&r2=533549
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Sun Apr 29 12:26:11 2007
@@ -47,7 +47,16 @@
     (Chris Hostetter, Otis Gospodnetic)
 
  8. LUCENE-869: Changed FSIndexInput and FSIndexOutput to inner classes of FSDirectory
-    to enable extensibility of these classes.
+    to enable extensibility of these classes. (Michael Busch)
+
+ 9. LUCENE-580: Added the public method reset() to TokenStream. This method does
+    nothing by default, but may be overwritten by subclasses to support consuming
+    the TokenStream more than once. (Michael Busch)
+
+10. LUCENE-580: Added a new constructor to Field that takes a TokenStream as
+    argument, available as tokenStreamValue(). This is useful to avoid the need of 
+    "dummy analyzers" for pre-analyzed fields. (Karl Wettin, Michael Busch)
+
 
 Bug fixes
 

Added: lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java?view=auto&rev=533549
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java (added)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java Sun Apr
29 12:26:11 2007
@@ -0,0 +1,68 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * This class can be used if the Tokens of a TokenStream
+ * are intended to be consumed more than once. It caches
+ * all Tokens locally in a List.
+ * 
+ * CachingTokenFilter implements the optional method
+ * {@link TokenStream#reset()}, which repositions the
+ * stream to the first Token. 
+ *
+ */
+public class CachingTokenFilter extends TokenFilter {
+  private List cache;
+  private int index;
+  
+  public CachingTokenFilter(TokenStream input) {
+    super(input);
+  }
+  
+  public Token next() throws IOException {
+    if (cache == null) {
+      // fill cache lazily
+      cache = new LinkedList();
+      fillCache();
+    }
+    
+    if (index == cache.size()) {
+      // the cache is exhausted, return null
+      return null;
+    }
+    
+    return (Token) cache.get(index++);
+  }
+  
+  public void reset() throws IOException {
+    index = 0;
+  }
+  
+  private void fillCache() throws IOException {
+    Token token;
+    while ( (token = input.next()) != null) {
+      cache.add(token);
+    }
+  }
+
+}

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java?view=diff&rev=533549&r1=533548&r2=533549
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java Sun Apr 29 12:26:11
2007
@@ -35,6 +35,15 @@
   /** Returns the next token in the stream, or null at EOS. */
   public abstract Token next() throws IOException;
 
+  /** Resets this stream to the beginning. This is an
+   *  optional operation, so subclasses may or may not
+   *  implement this method. Reset() is not needed for
+   *  the standard indexing process. However, if the Tokens 
+   *  of a TokenStream are intended to be consumed more than 
+   *  once, it is neccessary to implement reset(). 
+   */
+  public void reset() throws IOException {}
+  
   /** Releases resources associated with this stream. */
   public void close() throws IOException {}
 }

Modified: lucene/java/trunk/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/document/Field.java?view=diff&rev=533549&r1=533548&r2=533549
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/document/Field.java Sun Apr 29 12:26:11 2007
@@ -17,6 +17,7 @@
  * limitations under the License.
  */
 
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.index.IndexWriter;   // for javadoc
 import org.apache.lucene.util.Parameter;
 
@@ -134,21 +135,26 @@
   }
   
   
-  /** The value of the field as a String, or null.  If null, the Reader value
-   * or binary value is used.  Exactly one of stringValue(), readerValue(), and
-   * binaryValue() must be set. */
+  /** The value of the field as a String, or null.  If null, the Reader value,
+   * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
+   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
   public String stringValue()   { return fieldsData instanceof String ? (String)fieldsData
: null; }
   
-  /** The value of the field as a Reader, or null.  If null, the String value
-   * or binary value is  used.  Exactly one of stringValue(), readerValue(),
-   * and binaryValue() must be set. */
+  /** The value of the field as a Reader, or null.  If null, the String value,
+   * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
+   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
   public Reader readerValue()   { return fieldsData instanceof Reader ? (Reader)fieldsData
: null; }
   
-  /** The value of the field in Binary, or null.  If null, the Reader or
-   * String value is used.  Exactly one of stringValue(), readerValue() and
-   * binaryValue() must be set. */
+  /** The value of the field in Binary, or null.  If null, the Reader value,
+   * String value, or TokenStream value is used. Exactly one of stringValue(), 
+   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
   public byte[] binaryValue()   { return fieldsData instanceof byte[] ? (byte[])fieldsData
: null; }
   
+  /** The value of the field as a TokesStream, or null.  If null, the Reader value,
+   * String value, or binary value is used. Exactly one of stringValue(), 
+   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
+  public TokenStream tokenStreamValue()   { return fieldsData instanceof TokenStream ? (TokenStream)fieldsData
: null; }
+  
   /**
    * Create a field by specifying its name, value and how it will
    * be saved in the index. Term vectors will not be stored in the index.
@@ -280,6 +286,54 @@
     
     setStoreTermVector(termVector);
   }
+
+  /**
+   * Create a tokenized and indexed field that is not stored. Term vectors will
+   * not be stored. This is useful for pre-analyzed fields.
+   * The TokenStream is read only when the Document is added to the index,
+   * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(Document)}
+   * has been called.
+   * 
+   * @param name The name of the field
+   * @param tokenStream The TokenStream with the content
+   * @throws NullPointerException if name or tokenStream is <code>null</code>
+   */ 
+  public Field(String name, TokenStream tokenStream) {
+    this(name, tokenStream, TermVector.NO);
+  }
+  
+  /**
+   * Create a tokenized and indexed field that is not stored, optionally with 
+   * storing term vectors.  This is useful for pre-analyzed fields.
+   * The TokenStream is read only when the Document is added to the index,
+   * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(Document)}
+   * has been called.
+   * 
+   * @param name The name of the field
+   * @param tokenStream The TokenStream with the content
+   * @param termVector Whether term vector should be stored
+   * @throws NullPointerException if name or tokenStream is <code>null</code>
+   */ 
+  public Field(String name, TokenStream tokenStream, TermVector termVector) {
+    if (name == null)
+      throw new NullPointerException("name cannot be null");
+    if (tokenStream == null)
+      throw new NullPointerException("tokenStream cannot be null");
+    
+    this.name = name.intern();        // field names are interned
+    this.fieldsData = tokenStream;
+    
+    this.isStored = false;
+    this.isCompressed = false;
+    
+    this.isIndexed = true;
+    this.isTokenized = true;
+    
+    this.isBinary = false;
+    
+    setStoreTermVector(termVector);
+  }
+
   
   /**
    * Create a stored field with binary value. Optionally the value may be compressed.

Modified: lucene/java/trunk/src/java/org/apache/lucene/document/Fieldable.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/document/Fieldable.java?view=diff&rev=533549&r1=533548&r2=533549
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/document/Fieldable.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/document/Fieldable.java Sun Apr 29 12:26:11
2007
@@ -19,6 +19,8 @@
 import java.io.Reader;
 import java.io.Serializable;
 
+import org.apache.lucene.analysis.TokenStream;
+
 /**
  * Synonymous with {@link Field}.
  *
@@ -60,20 +62,25 @@
    */
   String name();
 
-  /** The value of the field as a String, or null.  If null, the Reader value
-   * or binary value is used.  Exactly one of stringValue(), readerValue(), and
-   * binaryValue() must be set. */
-  String stringValue();
-
-  /** The value of the field as a Reader, or null.  If null, the String value
-   * or binary value is  used.  Exactly one of stringValue(), readerValue(),
-   * and binaryValue() must be set. */
-  Reader readerValue();
-
-  /** The value of the field in Binary, or null.  If null, the Reader or
-   * String value is used.  Exactly one of stringValue(), readerValue() and
-   * binaryValue() must be set. */
-  byte[] binaryValue();
+  /** The value of the field as a String, or null.  If null, the Reader value,
+   * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
+   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
+  public String stringValue();
+  
+  /** The value of the field as a Reader, or null.  If null, the String value,
+   * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
+   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
+  public Reader readerValue();
+  
+  /** The value of the field in Binary, or null.  If null, the Reader value,
+   * String value, or TokenStream value is used. Exactly one of stringValue(), 
+   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
+  public byte[] binaryValue();
+  
+  /** The value of the field as a TokesStream, or null.  If null, the Reader value,
+   * String value, or binary value is used. Exactly one of stringValue(), 
+   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
+  public TokenStream tokenStreamValue();
 
   /** True iff the value of the field is to be stored in the index for return
     with search hits.  It is an error for this to be true if a field is

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DocumentWriter.java?view=diff&rev=533549&r1=533548&r2=533549
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DocumentWriter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DocumentWriter.java Sun Apr 29 12:26:11
2007
@@ -162,18 +162,28 @@
           offset += stringValue.length();
           length++;
         } else 
-        {
-          Reader reader;			  // find or make Reader
-          if (field.readerValue() != null)
-            reader = field.readerValue();
-          else if (field.stringValue() != null)
-            reader = new StringReader(field.stringValue());
-          else
-            throw new IllegalArgumentException
-                    ("field must have either String or Reader value");
-
-          // Tokenize field and add to postingTable
-          TokenStream stream = analyzer.tokenStream(fieldName, reader);
+        { // tokenized field
+          TokenStream stream = field.tokenStreamValue();
+          
+          // the field does not have a TokenStream,
+          // so we have to obtain one from the analyzer
+          if (stream == null) {
+            Reader reader;			  // find or make Reader
+            if (field.readerValue() != null)
+              reader = field.readerValue();
+            else if (field.stringValue() != null)
+              reader = new StringReader(field.stringValue());
+            else
+              throw new IllegalArgumentException
+                      ("field must have either String or Reader value");
+  
+            // Tokenize field and add to postingTable
+            stream = analyzer.tokenStream(fieldName, reader);
+          }
+          
+          // reset the TokenStream to the first token
+          stream.reset();
+          
           try {
             Token lastToken = null;
             for (Token t = stream.next(); t != null; t = stream.next()) {

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/FieldsReader.java?view=diff&rev=533549&r1=533548&r2=533549
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/FieldsReader.java Sun Apr 29 12:26:11
2007
@@ -17,6 +17,7 @@
  * limitations under the License.
  */
 
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.*;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
@@ -331,11 +332,9 @@
       return localFieldsStream;
     }
 
-    /**
-     * The value of the field in Binary, or null.  If null, the Reader or
-     * String value is used.  Exactly one of stringValue(), readerValue() and
-     * binaryValue() must be set.
-     */
+    /** The value of the field in Binary, or null.  If null, the Reader value,
+     * String value, or TokenStream value is used. Exactly one of stringValue(), 
+     * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
     public byte[] binaryValue() {
       ensureOpen();
       if (fieldsData == null) {
@@ -358,21 +357,26 @@
       return fieldsData instanceof byte[] ? (byte[]) fieldsData : null;
     }
 
-    /**
-     * The value of the field as a Reader, or null.  If null, the String value
-     * or binary value is  used.  Exactly one of stringValue(), readerValue(),
-     * and binaryValue() must be set.
-     */
+    /** The value of the field as a Reader, or null.  If null, the String value,
+     * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
+     * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
     public Reader readerValue() {
       ensureOpen();
       return fieldsData instanceof Reader ? (Reader) fieldsData : null;
     }
 
-    /**
-     * The value of the field as a String, or null.  If null, the Reader value
-     * or binary value is used.  Exactly one of stringValue(), readerValue(), and
-     * binaryValue() must be set.
-     */
+    /** The value of the field as a TokesStream, or null.  If null, the Reader value,
+     * String value, or binary value is used. Exactly one of stringValue(), 
+     * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
+    public TokenStream tokenStreamValue() {
+      ensureOpen();
+      return fieldsData instanceof TokenStream ? (TokenStream) fieldsData : null;
+    }
+
+    
+    /** The value of the field as a String, or null.  If null, the Reader value,
+     * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
+     * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
     public String stringValue() {
       ensureOpen();
       if (fieldsData == null) {
@@ -461,6 +465,11 @@
 
     public byte[] binaryValue() {
       return (byte[]) this.fieldsData;
+    }
+
+    public TokenStream tokenStreamValue() {
+      // not needed for merge
+      return null;
     }
     
     public FieldForMerge(Object value, FieldInfo fi, boolean binary, boolean compressed,
boolean tokenize) {

Added: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java?view=auto&rev=533549
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Sun
Apr 29 12:26:11 2007
@@ -0,0 +1,103 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+
+public class TestCachingTokenFilter extends TestCase {
+  private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
+  
+  public void testCaching() throws IOException {
+    Directory dir = new RAMDirectory();
+    IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer());
+    Document doc = new Document();
+    TokenStream stream = new TokenStream() {
+      private int index = 0;
+      
+      public Token next() throws IOException {
+        if (index == tokens.length) {
+          return null;
+        } else {
+          return new Token(tokens[index++], 0, 0);
+        }        
+      }
+      
+    };
+    
+    stream = new CachingTokenFilter(stream);
+    
+    doc.add(new Field("preanalyzed", stream, TermVector.NO));
+    
+    // 1) we consume all tokens twice before we add the doc to the index
+    checkTokens(stream);
+    stream.reset();  
+    checkTokens(stream);
+    
+    // 2) now add the document to the index and verify if all tokens are indexed
+    //    don't reset the stream here, the DocumentWriter should do that implicitly
+    writer.addDocument(doc);
+    writer.close();
+    
+    IndexReader reader = IndexReader.open(dir);
+    TermPositions termPositions = reader.termPositions(new Term("preanalyzed", "term1"));
+    assertTrue(termPositions.next());
+    assertEquals(1, termPositions.freq());
+    assertEquals(0, termPositions.nextPosition());
+
+    termPositions.seek(new Term("preanalyzed", "term2"));
+    assertTrue(termPositions.next());
+    assertEquals(2, termPositions.freq());
+    assertEquals(1, termPositions.nextPosition());
+    assertEquals(3, termPositions.nextPosition());
+    
+    termPositions.seek(new Term("preanalyzed", "term3"));
+    assertTrue(termPositions.next());
+    assertEquals(1, termPositions.freq());
+    assertEquals(2, termPositions.nextPosition());
+    reader.close();
+    
+    // 3) reset stream and consume tokens again
+    stream.reset();
+    checkTokens(stream);
+  }
+  
+  private void checkTokens(TokenStream stream) throws IOException {
+    int count = 0;
+    Token token;
+    while ((token = stream.next()) != null) {
+      assertTrue(count < tokens.length);
+      assertEquals(tokens[count], token.termText);
+      count++;
+    }
+    
+    assertEquals(tokens.length, count);
+  }
+}

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java?view=diff&rev=533549&r1=533548&r2=533549
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java Sun Apr 29
12:26:11 2007
@@ -19,10 +19,13 @@
 
 import junit.framework.TestCase;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.document.*;
+import org.apache.lucene.document.Field.TermVector;
 import org.apache.lucene.search.Similarity;
 import org.apache.lucene.store.RAMDirectory;
 
@@ -123,5 +126,46 @@
     assertEquals(2, freq);
     assertEquals(0, termPositions.nextPosition());
     assertEquals(502, termPositions.nextPosition());
+  }
+  
+  public void testPreAnalyzedField() throws IOException {
+    Similarity similarity = Similarity.getDefault();
+    DocumentWriter writer = new DocumentWriter(dir, new SimpleAnalyzer(), similarity, 50);
+    Document doc = new Document();
+    
+    doc.add(new Field("preanalyzed", new TokenStream() {
+      private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
+      private int index = 0;
+      
+      public Token next() throws IOException {
+        if (index == tokens.length) {
+          return null;
+        } else {
+          return new Token(tokens[index++], 0, 0);
+        }        
+      }
+      
+    }, TermVector.NO));
+    
+    String segName = "test";
+    writer.addDocument(segName, doc);
+    SegmentReader reader = SegmentReader.get(new SegmentInfo(segName, 1, dir));
+
+    TermPositions termPositions = reader.termPositions(new Term("preanalyzed", "term1"));
+    assertTrue(termPositions.next());
+    assertEquals(1, termPositions.freq());
+    assertEquals(0, termPositions.nextPosition());
+
+    termPositions.seek(new Term("preanalyzed", "term2"));
+    assertTrue(termPositions.next());
+    assertEquals(2, termPositions.freq());
+    assertEquals(1, termPositions.nextPosition());
+    assertEquals(3, termPositions.nextPosition());
+    
+    termPositions.seek(new Term("preanalyzed", "term3"));
+    assertTrue(termPositions.next());
+    assertEquals(1, termPositions.freq());
+    assertEquals(2, termPositions.nextPosition());
+
   }
 }



Mime
View raw message