lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dna...@apache.org
Subject svn commit: r331923 - in /lucene/java/trunk/src: java/org/apache/lucene/queryParser/QueryParser.java java/org/apache/lucene/queryParser/QueryParser.jj test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
Date Tue, 08 Nov 2005 23:05:53 GMT
Author: dnaber
Date: Tue Nov  8 15:05:44 2005
New Revision: 331923

URL: http://svn.apache.org/viewcvs?rev=331923&view=rev
Log:
Make QueryParser work with analyzers that return tokens with a position
increment > 1. Thanks to Ahmed El-dawy for the fix.

Modified:
    lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java
    lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.jj
    lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java

Modified: lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java?rev=331923&r1=331922&r2=331923&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java Tue Nov  8 15:05:44
2005
@@ -381,8 +381,8 @@
       if (t == null)
         break;
       v.addElement(t);
-      if (t.getPositionIncrement() == 1)
-        positionCount++;
+      if (t.getPositionIncrement() != 0)
+        positionCount += t.getPositionIncrement();
       else
         severalTokensAtSamePosition = true;
     }

Modified: lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.jj
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.jj?rev=331923&r1=331922&r2=331923&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.jj (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.jj Tue Nov  8 15:05:44
2005
@@ -404,8 +404,8 @@
       if (t == null)
         break;
       v.addElement(t);
-      if (t.getPositionIncrement() == 1)
-        positionCount++;
+      if (t.getPositionIncrement() != 0)
+        positionCount += t.getPositionIncrement();
       else
         severalTokensAtSamePosition = true;
     }

Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java?rev=331923&r1=331922&r2=331923&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java Tue Nov
 8 15:05:44 2005
@@ -22,6 +22,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
@@ -36,9 +37,9 @@
 
   private static int multiToken = 0;
 
-  public void testAnalyzer() throws ParseException {
+  public void testMultiAnalyzer() throws ParseException {
     
-    QueryParser qp = new QueryParser("", new TestAnalyzer());
+    QueryParser qp = new QueryParser("", new MultiAnalyzer());
 
     // trivial, no multiple tokens:
     assertEquals("foo", qp.parse("foo").toString());
@@ -79,14 +80,22 @@
     assertEquals("+(multi multi2) +foo", qp.parse("multi foo").toString());
 
   }
+
+  public void testPosIncrementAnalyzer() throws ParseException {
+    QueryParser qp = new QueryParser("", new PosIncrementAnalyzer());
+    assertEquals("quick brown", qp.parse("the quick brown").toString());
+    assertEquals("\"quick brown\"", qp.parse("\"the quick brown\"").toString());
+    assertEquals("quick brown fox", qp.parse("the quick brown fox").toString());
+    assertEquals("\"quick brown fox\"", qp.parse("\"the quick brown fox\"").toString());
+  }
   
   /**
    * Expands "multi" to "multi" and "multi2", both at the same position,
    * and expands "triplemulti" to "triplemulti", "multi3", and "multi2".  
    */
-  private class TestAnalyzer extends Analyzer {
+  private class MultiAnalyzer extends Analyzer {
 
-    public TestAnalyzer() {
+    public MultiAnalyzer() {
     }
 
     public TokenStream tokenStream(String fieldName, Reader reader) {
@@ -132,4 +141,49 @@
     }
   }
 
-}
\ No newline at end of file
+  /**
+   * Analyzes "the quick brown" as: quick(incr=2) brown(incr=1).
+   * Does not work correctly for input other than "the quick brown ...".
+   */
+  private class PosIncrementAnalyzer extends Analyzer {
+
+    public PosIncrementAnalyzer() {
+    }
+
+    public TokenStream tokenStream(String fieldName, Reader reader) {
+      TokenStream result = new StandardTokenizer(reader);
+      result = new TestPosIncrementFilter(result);
+      result = new LowerCaseFilter(result);
+      return result;
+    }
+  }
+
+  private final class TestPosIncrementFilter extends TokenFilter {
+    
+    public TestPosIncrementFilter(TokenStream in) {
+      super(in);
+    }
+
+    public final org.apache.lucene.analysis.Token next() throws java.io.IOException {
+      for (Token t = input.next(); t != null; t = input.next()) {
+        if (t.termText().equals("the")) {
+          // stopword, do nothing
+        } else if (t.termText().equals("quick")) {
+          org.apache.lucene.analysis.Token token = 
+            new org.apache.lucene.analysis.Token(t.termText(), t.startOffset(),
+                t.endOffset(), t.type());
+          token.setPositionIncrement(2);
+          return token;
+        } else {
+          org.apache.lucene.analysis.Token token = 
+            new org.apache.lucene.analysis.Token(t.termText(), t.startOffset(),
+                t.endOffset(), t.type());
+          token.setPositionIncrement(1);
+          return token;
+        }
+      }
+      return null;
+    }
+  }
+
+}



Mime
View raw message