lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dor...@apache.org
Subject svn commit: r635280 - in /lucene/java/trunk/contrib/benchmark: ./ src/java/org/apache/lucene/benchmark/byTask/feeds/ src/test/org/apache/lucene/benchmark/byTask/
Date Sun, 09 Mar 2008 16:43:38 GMT
Author: doronc
Date: Sun Mar  9 09:43:32 2008
New Revision: 635280

URL: http://svn.apache.org/viewvc?rev=635280&view=rev
Log:
LUCENE-1209: Fixed DocMaker settings by round. Prior to this fix, DocMaker settings of
first round were used in all rounds.  (E.g. term vectors.)

Modified:
    lucene/java/trunk/contrib/benchmark/CHANGES.txt
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/BasicDocMaker.java
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersDocMaker.java
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java
    lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java

Modified: lucene/java/trunk/contrib/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/CHANGES.txt?rev=635280&r1=635279&r2=635280&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/CHANGES.txt (original)
+++ lucene/java/trunk/contrib/benchmark/CHANGES.txt Sun Mar  9 09:43:32 2008
@@ -3,6 +3,11 @@
 The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
 
 $Id:$
+3/9/08
+  LUCENE-1209: Fixed DocMaker settings by round. Prior to this fix, DocMaker settings of

+  first round were used in all rounds.  (E.g. term vectors.)
+  (Mark Miller via Doron Cohen) 
+
 1/30/08
   LUCENE-1156: Fixed redirect problem in EnwikiDocMaker.  Refactored ExtractWikipedia to
use EnwikiDocMaker.  Added property to EnwikiDocMaker to allow
   for skipping image only documents.

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/BasicDocMaker.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/BasicDocMaker.java?rev=635280&r1=635279&r2=635280&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/BasicDocMaker.java
(original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/BasicDocMaker.java
Sun Mar  9 09:43:32 2008
@@ -219,6 +219,7 @@
    */
   public synchronized void resetInputs() {
     printDocStatistics();
+    setConfig(config); //re-initiate since properties by round may have changed.  
     numBytes = 0;
     numDocsCreated = 0;
     resetLeftovers();
@@ -252,6 +253,10 @@
     numUniqueBytes += n;
   }
   
+  protected void resetUniqueBytes () {
+    numUniqueBytes = 0;
+  }
+
   protected synchronized void addBytes (long n) {
     numBytes += n;
   }

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersDocMaker.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersDocMaker.java?rev=635280&r1=635279&r2=635280&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersDocMaker.java
(original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersDocMaker.java
Sun Mar  9 09:43:32 2008
@@ -56,7 +56,8 @@
     if (!dataDir.isAbsolute()) {
       dataDir = new File(workDir, d);
     }
-
+    resetUniqueBytes();
+    inputFiles.clear();
     collectFiles(dataDir,inputFiles);
     if (inputFiles.size()==0) {
       throw new RuntimeException("No txt files in dataDir: "+dataDir.getAbsolutePath());

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java?rev=635280&r1=635279&r2=635280&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java
(original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java
Sun Mar  9 09:43:32 2008
@@ -72,6 +72,8 @@
     if (!dataDir.isAbsolute()) {
       dataDir = new File(workDir, d);
     }
+    resetUniqueBytes();
+    inputFiles.clear();
     collectFiles(dataDir,inputFiles);
     if (inputFiles.size()==0) {
       throw new RuntimeException("No txt files in dataDir: "+dataDir.getAbsolutePath());

Modified: lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=635280&r1=635279&r2=635280&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
(original)
+++ lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
Sun Mar  9 09:43:32 2008
@@ -38,6 +38,8 @@
 import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.index.SerialMergeScheduler;
 import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.store.Directory;
 
 import junit.framework.TestCase;
 
@@ -165,7 +167,7 @@
 
     assertTrue("Index does not exist?...!", IndexReader.indexExists(benchmark.getRunData().getDirectory()));
     // now we should be able to open the index for write.
-    IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),null,false);
+    IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),null,false,IndexWriter.MaxFieldLength.UNLIMITED);
     iw.close();
     IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory());
     assertEquals("1000 docs were added to the index, this is what we expect to find!",1000,ir.numDocs());
@@ -237,7 +239,7 @@
     assertEquals("TestSearchTask was supposed to be called!",139,CountingSearchTestTask.numSearches);
     assertTrue("Index does not exist?...!", IndexReader.indexExists(benchmark.getRunData().getDirectory()));
     // now we should be able to open the index for write. 
-    IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),null,false);
+    IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),null,false,IndexWriter.MaxFieldLength.UNLIMITED);
     iw.close();
     IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory());
     assertEquals("1 docs were added to the index, this is what we expect to find!",1,ir.numDocs());
@@ -327,7 +329,7 @@
     benchmark = execBenchmark(algLines2);
 
     // now we should be able to open the index for write. 
-    IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),null,false);
+    IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),null,false,IndexWriter.MaxFieldLength.UNLIMITED);
     iw.close();
 
     IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory());
@@ -639,8 +641,8 @@
         "doc.add.log.step=3",
         "ram.flush.mb=-1",
         "max.buffered=2",
-        "compound=false",
-        "doc.term.vector=false",
+        "compound=cmpnd:true:false",
+        "doc.term.vector=vector:false:true",
         "doc.maker.forever=false",
         "directory=RAMDirectory",
         "doc.stored=false",
@@ -652,6 +654,7 @@
         "  ResetSystemErase",
         "  CreateIndex",
         "  { \"AddDocs\"  AddDoc > : * ",
+        "  NewRound",
         "} : 2",
     };
 
@@ -661,7 +664,14 @@
     assertEquals(2, writer.getMaxBufferedDocs());
     assertEquals(IndexWriter.DISABLE_AUTO_FLUSH, (int) writer.getRAMBufferSizeMB());
     assertEquals(3, writer.getMergeFactor());
-    assertEquals(false, writer.getUseCompoundFile());
+    assertFalse(writer.getUseCompoundFile());
+    writer.close();
+    Directory dir = benchmark.getRunData().getDirectory();
+    IndexReader reader = IndexReader.open(dir);
+    TermFreqVector [] tfv = reader.getTermFreqVectors(0);
+    assertNotNull(tfv);
+    assertTrue(tfv.length > 0);
+    reader.close();
   }
 
   /**



Mime
View raw message