lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject svn commit: r1087471 - /lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
Date Thu, 31 Mar 2011 22:44:21 GMT
Author: sarowe
Date: Thu Mar 31 22:44:20 2011
New Revision: 1087471

URL: http://svn.apache.org/viewvc?rev=1087471&view=rev
Log:
ReadTokensTask now converts tokens to their indexed forms (char[]->byte[]), just as the
indexer does.  This allows measurement of the conversion process, which is important for analysis
components that customize it, e.g. (ICU)CollationKeyFilter.

NB: as a result, benchmarks that incorporate this task will no longer be directly comparable
between 3.X and 4.0

Modified:
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java

Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java?rev=1087471&r1=1087470&r2=1087471&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
(original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
Thu Mar 31 22:44:20 2011
@@ -22,6 +22,7 @@ import java.util.List;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.benchmark.byTask.PerfRunData;
 import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
 import org.apache.lucene.document.Document;
@@ -98,8 +99,11 @@ public class ReadTokensTask extends Perf
       // reset the TokenStream to the first token
       stream.reset();
 
-      while(stream.incrementToken())
+      TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
+      while(stream.incrementToken()) {
+        termAtt.fillBytesRef();
         tokenCount++;
+      }
     }
     totalTokenCount += tokenCount;
     return tokenCount;



Mime
View raw message