lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r712232 - in /lucene/java/trunk/src: java/org/apache/lucene/analysis/KeywordTokenizer.java test/org/apache/lucene/analysis/TestKeywordAnalyzer.java
Date Fri, 07 Nov 2008 19:36:35 GMT
Author: mikemccand
Date: Fri Nov  7 11:36:35 2008
New Revision: 712232

URL: http://svn.apache.org/viewvc?rev=712232&view=rev
Log:
LUCENE-1441: fix KeywordTokenizer to set start/end offset on its token

Modified:
    lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java?rev=712232&r1=712231&r2=712232&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java Fri Nov  7
11:36:35 2008
@@ -53,6 +53,9 @@
           buffer = reusableToken.resizeTermBuffer(1+buffer.length);
       }
       reusableToken.setTermLength(upto);
+      reusableToken.setStartOffset(0);
+      reusableToken.setEndOffset(upto);
+      
       return reusableToken;
     }
     return null;

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java?rev=712232&r1=712231&r2=712232&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java Fri Nov
 7 11:36:35 2008
@@ -17,6 +17,8 @@
  * limitations under the License.
  */
 
+import java.io.StringReader;
+
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
@@ -82,4 +84,13 @@
     td = reader.termDocs(new Term("partnum", "Q37"));
     assertTrue(td.next());
   }
+
+  // LUCENE-1441
+  public void testOffsets() throws Exception {
+    TokenStream stream = new KeywordAnalyzer().tokenStream("field", new StringReader("abcd"));
+    Token token = new Token();
+    assertTrue(stream.next(token) != null);
+    assertEquals(0, token.startOffset);
+    assertEquals(4, token.endOffset);
+  }
 }



Mime
View raw message