lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ehatc...@apache.org
Subject svn commit: r290943 - in /lucene/java/trunk: CHANGES.txt contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
Date Thu, 22 Sep 2005 13:39:03 GMT
Author: ehatcher
Date: Thu Sep 22 06:38:58 2005
New Revision: 290943

URL: http://svn.apache.org/viewcvs?rev=290943&view=rev
Log:
LUCENE-437 - Add position increment pass through on SnowballFilter tokens

Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
    lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/CHANGES.txt?rev=290943&r1=290942&r2=290943&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Thu Sep 22 06:38:58 2005
@@ -213,6 +213,10 @@
     inside a BooleanQuery.
     (Hans Hjelm and Scotty Allen via Daniel Naber, Bug #35626)
 
+11. Fixed SnowballFilter to pass through the position increment from
+    the original token.
+    (Yonik Seeley via Erik Hatcher, LUCENE-437)
+
 Optimizations
      
  1. Disk usage (peak requirements during indexing and optimization)

Modified: lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java?rev=290943&r1=290942&r2=290943&view=diff
==============================================================================
--- lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
(original)
+++ lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
Thu Sep 22 06:38:58 2005
@@ -69,7 +69,10 @@
     } catch (Exception e) {
       throw new RuntimeException(e.toString());
     }
-    return new Token(stemmer.getCurrent(),
-                     token.startOffset(), token.endOffset(), token.type());
+    
+    Token newToken = new Token(stemmer.getCurrent(),
+                      token.startOffset(), token.endOffset(), token.type());
+    newToken.setPositionIncrement(token.getPositionIncrement());
+    return newToken;
   }
 }

Modified: lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=290943&r1=290942&r2=290943&view=diff
==============================================================================
--- lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
(original)
+++ lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
Thu Sep 22 06:38:58 2005
@@ -55,6 +55,7 @@
  */
 
 import java.io.*;
+
 import junit.framework.*;
 
 import org.apache.lucene.analysis.*;
@@ -65,7 +66,7 @@
                                String input,
                                String[] output) throws Exception {
     TokenStream ts = a.tokenStream("dummy", new StringReader(input));
-    for (int i=0; i<output.length; i++) {
+    for (int i = 0; i < output.length; i++) {
       Token t = ts.next();
       assertNotNull(t);
       assertEquals(output[i], t.termText());
@@ -77,7 +78,30 @@
   public void testEnglish() throws Exception {
     Analyzer a = new SnowballAnalyzer("English");
     assertAnalyzesTo(a, "he abhorred accents",
-                     new String[] { "he", "abhor", "accent" });
+        new String[]{"he", "abhor", "accent"});
+  }
+
+
+  public void testFilterTokens() throws Exception {
+    final Token tok = new Token("accents", 2, 7, "wrd");
+    tok.setPositionIncrement(3);
+
+    SnowballFilter filter = new SnowballFilter(
+        new TokenStream() {
+          public Token next() {
+            return tok;
+          }
+        },
+        "English"
+    );
+
+    Token newtok = filter.next();
+
+    assertEquals("accent", newtok.termText());
+    assertEquals(2, newtok.startOffset());
+    assertEquals(7, newtok.endOffset());
+    assertEquals("wrd", newtok.type());
+    assertEquals(3, newtok.getPositionIncrement());
   }
 }
 



Mime
View raw message