lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gsing...@apache.org
Subject svn commit: r605957 - /lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
Date Thu, 20 Dec 2007 15:08:00 GMT
Author: gsingers
Date: Thu Dec 20 07:07:59 2007
New Revision: 605957

URL: http://svn.apache.org/viewvc?rev=605957&view=rev
Log:
Checking in simple performance test

Modified:
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java?rev=605957&r1=605956&r2=605957&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java Thu Dec 20
07:07:59 2007
@@ -17,9 +17,14 @@
  */
 
 import junit.framework.TestCase;
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.English;
 
-import java.io.StringReader;
 import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * tests for the TeeTokenFilter and SinkTokenizer
@@ -56,9 +61,9 @@
 
   public void test() throws IOException {
 
-    SinkTokenizer sink1 = new SinkTokenizer(null){
+    SinkTokenizer sink1 = new SinkTokenizer(null) {
       public void add(Token t) {
-        if (t != null && t.termText().equalsIgnoreCase("The")){
+        if (t != null && t.termText().equalsIgnoreCase("The")) {
           super.add(t);
         }
       }
@@ -66,14 +71,14 @@
     TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())),
sink1);
     Token token = null;
     int i = 0;
-    while ((token = source.next()) != null){
+    while ((token = source.next()) != null) {
       assertTrue(token.termText() + " is not equal to " + tokens1[i], token.termText().equals(tokens1[i])
== true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
     assertTrue("sink1 Size: " + sink1.getTokens().size() + " is not: " + 2, sink1.getTokens().size()
== 2);
     i = 0;
-    while ((token = sink1.next()) != null){
+    while ((token = sink1.next()) != null) {
       assertTrue(token.termText() + " is not equal to " + "The", token.termText().equalsIgnoreCase("The")
== true);
       i++;
     }
@@ -81,16 +86,16 @@
   }
 
   public void testMultipleSources() throws Exception {
-    SinkTokenizer theDetector = new SinkTokenizer(null){
+    SinkTokenizer theDetector = new SinkTokenizer(null) {
       public void add(Token t) {
-        if (t != null && t.termText().equalsIgnoreCase("The")){
+        if (t != null && t.termText().equalsIgnoreCase("The")) {
           super.add(t);
         }
       }
     };
-    SinkTokenizer dogDetector = new SinkTokenizer(null){
+    SinkTokenizer dogDetector = new SinkTokenizer(null) {
       public void add(Token t) {
-        if (t != null && t.termText().equalsIgnoreCase("Dogs")){
+        if (t != null && t.termText().equalsIgnoreCase("Dogs")) {
           super.add(t);
         }
       }
@@ -99,7 +104,7 @@
     TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new
StringReader(buffer2.toString())), theDetector), dogDetector);
     Token token = null;
     int i = 0;
-    while ((token = source1.next()) != null){
+    while ((token = source1.next()) != null) {
       assertTrue(token.termText() + " is not equal to " + tokens1[i], token.termText().equals(tokens1[i])
== true);
       i++;
     }
@@ -107,7 +112,7 @@
     assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 2, theDetector.getTokens().size()
== 2);
     assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 1, dogDetector.getTokens().size()
== 1);
     i = 0;
-    while ((token = source2.next()) != null){
+    while ((token = source2.next()) != null) {
       assertTrue(token.termText() + " is not equal to " + tokens2[i], token.termText().equals(tokens2[i])
== true);
       i++;
     }
@@ -115,13 +120,13 @@
     assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 4, theDetector.getTokens().size()
== 4);
     assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 2, dogDetector.getTokens().size()
== 2);
     i = 0;
-    while ((token = theDetector.next()) != null){
+    while ((token = theDetector.next()) != null) {
       assertTrue(token.termText() + " is not equal to " + "The", token.termText().equalsIgnoreCase("The")
== true);
       i++;
     }
     assertTrue(i + " does not equal: " + theDetector.getTokens().size(), i == theDetector.getTokens().size());
     i = 0;
-    while ((token = dogDetector.next()) != null){
+    while ((token = dogDetector.next()) != null) {
       assertTrue(token.termText() + " is not equal to " + "Dogs", token.termText().equalsIgnoreCase("Dogs")
== true);
       i++;
     }
@@ -129,10 +134,130 @@
     source1.reset();
     TokenStream lowerCasing = new LowerCaseFilter(source1);
     i = 0;
-    while ((token = lowerCasing.next()) != null){
+    while ((token = lowerCasing.next()) != null) {
       assertTrue(token.termText() + " is not equal to " + tokens1[i].toLowerCase(), token.termText().equals(tokens1[i].toLowerCase())
== true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
   }
-}
\ No newline at end of file
+
+  /**
+   * Not an explicit test, just useful to print out some info on performance
+   *
+   * @throws Exception
+   */
+  public void testPerformance() throws Exception {
+    int[] tokCount = {100, 500, 1000, 2000, 5000, 10000};
+    int[] modCounts = {1, 2, 5, 10, 20, 50, 100, 200, 500};
+    for (int k = 0; k < tokCount.length; k++) {
+      StringBuffer buffer = new StringBuffer();
+      System.out.println("-----Tokens: " + tokCount[k] + "-----");
+      for (int i = 0; i < tokCount[k]; i++) {
+        buffer.append(English.intToEnglish(i).toUpperCase()).append(' ');
+      }
+      //make sure we produce the same tokens
+      ModuloSinkTokenizer sink = new ModuloSinkTokenizer(tokCount[k], 100);
+      Token next = new Token();
+      TokenStream result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new
StringReader(buffer.toString()))), sink);
+      while ((next = result.next(next)) != null) {
+      }
+      result = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))),
100);
+      next = new Token();
+      List tmp = new ArrayList();
+      while ((next = result.next(next)) != null) {
+        tmp.add(next.clone());
+      }
+      List sinkList = sink.getTokens();
+      assertTrue("tmp Size: " + tmp.size() + " is not: " + sinkList.size(), tmp.size() ==
sinkList.size());
+      for (int i = 0; i < tmp.size(); i++) {
+        Token tfTok = (Token) tmp.get(i);
+        Token sinkTok = (Token) sinkList.get(i);
+        assertTrue(tfTok.termText() + " is not equal to " + sinkTok.termText() + " at token:
" + i, tfTok.termText().equals(sinkTok.termText()) == true);
+      }
+      //simulate two fields, each being analyzed once, for 20 documents
+
+      for (int j = 0; j < modCounts.length; j++) {
+        int tfPos = 0;
+        long start = System.currentTimeMillis();
+        for (int i = 0; i < 20; i++) {
+          next = new Token();
+          result = new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString())));
+          while ((next = result.next(next)) != null) {
+            tfPos += next.getPositionIncrement();
+          }
+          next = new Token();
+          result = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))),
modCounts[j]);
+          while ((next = result.next(next)) != null) {
+            tfPos += next.getPositionIncrement();
+          }
+        }
+        long finish = System.currentTimeMillis();
+        System.out.println("ModCount: " + modCounts[j] + " Two fields took " + (finish -
start) + " ms");
+        int sinkPos = 0;
+        //simulate one field with one sink
+        start = System.currentTimeMillis();
+        for (int i = 0; i < 20; i++) {
+          sink = new ModuloSinkTokenizer(tokCount[k], modCounts[j]);
+          next = new Token();
+          result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))),
sink);
+          while ((next = result.next(next)) != null) {
+            sinkPos += next.getPositionIncrement();
+          }
+          //System.out.println("Modulo--------");
+          result = sink;
+          while ((next = result.next(next)) != null) {
+            sinkPos += next.getPositionIncrement();
+          }
+        }
+        finish = System.currentTimeMillis();
+        System.out.println("ModCount: " + modCounts[j] + " Tee fields took " + (finish -
start) + " ms");
+        assertTrue(sinkPos + " does not equal: " + tfPos, sinkPos == tfPos);
+
+      }
+      System.out.println("- End Tokens: " + tokCount[k] + "-----");
+    }
+
+  }
+
+
+  class ModuloTokenFilter extends TokenFilter {
+
+    int modCount;
+
+    ModuloTokenFilter(TokenStream input, int mc) {
+      super(input);
+      modCount = mc;
+    }
+
+    int count = 0;
+
+    //return every 100 tokens
+    public Token next(Token result) throws IOException {
+
+      while ((result = input.next(result)) != null && count % modCount != 0) {
+        count++;
+      }
+      count++;
+      return result;
+    }
+  }
+
+  class ModuloSinkTokenizer extends SinkTokenizer {
+    int count = 0;
+    int modCount;
+
+
+    ModuloSinkTokenizer(int numToks, int mc) {
+      modCount = mc;
+      lst = new ArrayList(numToks % mc);
+    }
+
+    public void add(Token t) {
+      if (t != null && count % modCount == 0) {
+        lst.add(t.clone());
+      }
+      count++;
+    }
+  }
+}
+



Mime
View raw message