lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r801856 - /lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
Date Fri, 07 Aug 2009 01:30:45 GMT
Author: markrmiller
Date: Fri Aug  7 01:30:45 2009
New Revision: 801856

URL: http://svn.apache.org/viewvc?rev=801856&view=rev
Log:
LUCENE-1788: cleanup Highlighter test class

Modified:
    lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java

Modified: lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=801856&r1=801855&r2=801856&view=diff
==============================================================================
--- lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
(original)
+++ lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
Fri Aug  7 01:30:45 2009
@@ -36,7 +36,6 @@
 import junit.framework.TestCase;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CachingTokenFilter;
 import org.apache.lucene.analysis.LowerCaseTokenizer;
 import org.apache.lucene.analysis.SimpleAnalyzer;
 import org.apache.lucene.analysis.Token;
@@ -112,7 +111,7 @@
     super(arg0);
   }
 
-  public void testHits() throws Exception {
+  public void testQueryScorerHits() throws Exception {
     Analyzer analyzer = new SimpleAnalyzer();
     QueryParser qp = new QueryParser(FIELD_NAME, analyzer);
     query = qp.parse("\"very long\"");
@@ -176,8 +175,7 @@
    */
   private static String highlightField(Query query, String fieldName, String text)
       throws IOException, InvalidTokenOffsetsException {
-    CachingTokenFilter tokenStream = new CachingTokenFilter(new StandardAnalyzer(Version.LUCENE_CURRENT).tokenStream(
-        fieldName, new StringReader(text)));
+    TokenStream tokenStream = new StandardAnalyzer(Version.LUCENE_CURRENT).tokenStream(fieldName,
new StringReader(text));
     // Assuming "<B>", "</B>" used to highlight
     SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
     QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
@@ -237,7 +235,7 @@
         numHighlights == 7);
   }
 
-  public void testSimpleSpanPhraseHighlighting() throws Exception {
+  public void testSimpleQueryScorerPhraseHighlighting() throws Exception {
     doSearching("\"very long and contains\"");
 
     int maxNumFragmentsRequired = 2;
@@ -260,7 +258,7 @@
         numHighlights == 3);
   }
 
-  public void testSimpleSpanPhraseHighlighting2() throws Exception {
+  public void testSimpleQueryScorerPhraseHighlighting2() throws Exception {
     doSearching("\"text piece long\"~5");
 
     int maxNumFragmentsRequired = 2;
@@ -282,15 +280,14 @@
         numHighlights == 6);
   }
 
-  public void testSimpleSpanPhraseHighlighting3() throws Exception {
+  public void testSimpleQueryScorerPhraseHighlighting3() throws Exception {
     doSearching("\"x y z\"");
 
     int maxNumFragmentsRequired = 2;
 
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
-      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
-          new StringReader(text)));
+      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
       QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
       Highlighter highlighter = new Highlighter(this, scorer);
 
@@ -315,8 +312,7 @@
     
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
-      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer
-          .tokenStream(FIELD_NAME, new StringReader(text)));
+      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
 
       highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));
 
@@ -335,8 +331,7 @@
 
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
-      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer
-          .tokenStream(FIELD_NAME, new StringReader(text)));
+      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
 
       highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 20));
 
@@ -358,8 +353,7 @@
     
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
-      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
-          new StringReader(text)));
+      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,new StringReader(text));
 
       highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
@@ -372,7 +366,7 @@
     }
   }
 
-  public void testSpanMultiPhraseQueryHighlighting() throws Exception {
+  public void testQueryScorerMultiPhraseQueryHighlighting() throws Exception {
     MultiPhraseQuery mpq = new MultiPhraseQuery();
 
     mpq.add(new Term[] { new Term(FIELD_NAME, "wordx"), new Term(FIELD_NAME, "wordb") });
@@ -384,7 +378,7 @@
     assertExpectedHighlightCount(maxNumFragmentsRequired, 6);
   }
 
-  public void testSpanMultiPhraseQueryHighlightingWithGap() throws Exception {
+  public void testQueryScorerMultiPhraseQueryHighlightingWithGap() throws Exception {
     MultiPhraseQuery mpq = new MultiPhraseQuery();
 
     /*
@@ -411,7 +405,7 @@
     TestHighlightRunner helper = new TestHighlightRunner() {
 
       public void run() throws Exception {
-        mode = SPAN;
+        mode = QUERY;
         doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
       }
     };
@@ -422,7 +416,7 @@
         numHighlights == 2);
   }
 
-  public void testSimpleHighlighter() throws Exception {
+  public void testSimpleQueryTermScorerHighlighter() throws Exception {
     doSearching("Kennedy");
     Highlighter highlighter = new Highlighter(new QueryTermScorer(query));
     highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -453,7 +447,7 @@
     TestHighlightRunner helper = new TestHighlightRunner() {
 
       public void run() throws Exception {
-        mode = SPAN;
+        mode = QUERY;
         doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
       }
     };
@@ -471,7 +465,7 @@
     TestHighlightRunner helper = new TestHighlightRunner() {
 
       public void run() throws Exception {
-        mode = SPAN;
+        mode = QUERY;
         doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
       }
     };
@@ -585,8 +579,7 @@
       QueryScorer scorer = null;
       TokenStream tokenStream = null;
 
-      tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
-          new StringReader(text)));
+      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
       
       scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
 
@@ -622,8 +615,7 @@
       QueryScorer scorer = null;
       TokenStream tokenStream = null;
 
-      tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
-          new StringReader(text)));
+      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
       
       scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
 
@@ -651,8 +643,7 @@
       QueryScorer scorer = null;
       TokenStream tokenStream = null;
 
-      tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
-          new StringReader(text)));
+      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
       
       scorer = new QueryScorer(query, null);
 
@@ -680,8 +671,7 @@
       QueryScorer scorer = null;
       TokenStream tokenStream = null;
 
-      tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
-          new StringReader(text)));
+      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
       
       scorer = new QueryScorer(query, "random_field", HighlighterTest.FIELD_NAME);
 
@@ -713,7 +703,7 @@
     helper.start();
   }
 
-  public void testGetBestFragmentsSpan() throws Exception {
+  public void testGetBestFragmentsQueryScorer() throws Exception {
     TestHighlightRunner helper = new TestHighlightRunner() {
 
       public void run() throws Exception {
@@ -1129,7 +1119,7 @@
 
         for (int i = 0; i < hits.length(); i++) {
           String text = hits.doc(i).get(FIELD_NAME);
-          TokenStream tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
new StringReader(text)));
+          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
           Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this,
false);
 
           highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -1285,12 +1275,9 @@
         // highlighting respects fieldnames used in query
 
         Scorer fieldSpecificScorer = null;
-        if (mode == this.SPAN) {
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(docMainText));
-          CachingTokenFilter ctf = new CachingTokenFilter(tokenStream);
+        if (mode == this.QUERY) {
           fieldSpecificScorer = new QueryScorer(query, FIELD_NAME);
-
-        } else if (mode == this.STANDARD) {
+        } else if (mode == this.QUERY_TERM) {
           fieldSpecificScorer = new QueryTermScorer(query, "contents");
         }
         Highlighter fieldSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),
@@ -1301,12 +1288,9 @@
 
         // highlighting does not respect fieldnames used in query
         Scorer fieldInSpecificScorer = null;
-        if (mode == this.SPAN) {
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(docMainText));
-          CachingTokenFilter ctf = new CachingTokenFilter(tokenStream);
+        if (mode == this.QUERY) {
           fieldInSpecificScorer = new QueryScorer(query, null);
-
-        } else if (mode == this.STANDARD) {
+        } else if (mode == this.QUERY_TERM) {
           fieldInSpecificScorer = new QueryTermScorer(query);
         }
 
@@ -1601,8 +1585,7 @@
       final int expectedHighlights) throws Exception {
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
-      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
-          new StringReader(text)));
+      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
       QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
       Highlighter highlighter = new Highlighter(this, scorer);
 
@@ -1756,9 +1739,10 @@
   }
 
   static abstract class TestHighlightRunner {
-    static final int STANDARD = 0;
-    static final int SPAN = 1;
-    int mode = STANDARD;
+    static final int QUERY = 0;
+    static final int QUERY_TERM = 1;
+
+    int mode = QUERY;
     Fragmenter frag = new SimpleFragmenter(20);
     
     public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream,
Formatter formatter) {
@@ -1767,13 +1751,13 @@
     
     public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream,
Formatter formatter, boolean expanMultiTerm) {
       Scorer scorer = null;
-      if (mode == STANDARD) {
-        scorer = new QueryTermScorer(query);
-      } else if (mode == SPAN) {
+      if (mode == QUERY) {
         scorer = new QueryScorer(query, fieldName);
         if(!expanMultiTerm) {
           ((QueryScorer)scorer).setExpandMultiTermQuery(false);
         }
+      } else if (mode == QUERY_TERM) {
+        scorer = new QueryTermScorer(query);
       } else {
         throw new RuntimeException("Unknown highlight mode");
       }
@@ -1782,14 +1766,11 @@
     }
 
     Highlighter getHighlighter(WeightedTerm[] weightedTerms, Formatter formatter) {
-      if (mode == STANDARD) {
+      if (mode == QUERY) {
+        return  new Highlighter(formatter, new QueryScorer((WeightedSpanTerm[]) weightedTerms));
+      } else if (mode == QUERY_TERM) {
         return new Highlighter(formatter, new QueryTermScorer(weightedTerms));
-      } else if (mode == SPAN) {
-        Highlighter highlighter;
 
-        highlighter = new Highlighter(formatter, new QueryScorer((WeightedSpanTerm[]) weightedTerms));
-
-        return highlighter;
       } else {
         throw new RuntimeException("Unknown highlight mode");
       }
@@ -1808,18 +1789,13 @@
         int maxNumFragmentsRequired = 2;
         String fragmentSeparator = "...";
         Scorer scorer = null;
-        TokenStream tokenStream = null;
-        if (mode == SPAN) {
-          tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
-              new StringReader(text)));
-          scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
-
-        } else if (mode == STANDARD) {
+        TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+        if (mode == QUERY) {
+          scorer = new QueryScorer(query);
+        } else if (mode == QUERY_TERM) {
           scorer = new QueryTermScorer(query);
-          tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
         }
         Highlighter highlighter = new Highlighter(formatter, scorer);
-
         highlighter.setTextFragmenter(frag);
 
         String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
@@ -1831,10 +1807,10 @@
     abstract void run() throws Exception;
 
     void start() throws Exception {
-      System.out.println("Run standard");
+      System.out.println("Run QueryScorer");
       run();
-      System.out.println("Run span");
-      mode = SPAN;
+      System.out.println("Run QueryTermScorer");
+      mode = QUERY_TERM;
       run();
     }
   }



Mime
View raw message