Return-Path: 
 <java-commits-return-1594-apmail-lucene-java-commits-archive=lucene.apache.org@lucene.apache.org>
Delivered-To: apmail-lucene-java-commits-archive@www.apache.org
Received: (qmail 51584 invoked from network); 3 Jan 2008 01:48:45 -0000
Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2)
  by minotaur.apache.org with SMTP; 3 Jan 2008 01:48:45 -0000
Received: (qmail 89427 invoked by uid 500); 3 Jan 2008 01:48:33 -0000
Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org
Received: (qmail 89397 invoked by uid 500); 3 Jan 2008 01:48:33 -0000
Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:java-commits-help@lucene.apache.org>
List-Unsubscribe: <mailto:java-commits-unsubscribe@lucene.apache.org>
List-Post: <mailto:java-commits@lucene.apache.org>
List-Id: <java-commits.lucene.apache.org>
Reply-To: java-dev@lucene.apache.org
Delivered-To: mailing list java-commits@lucene.apache.org
Received: (qmail 89386 invoked by uid 99); 3 Jan 2008 01:48:33 -0000
Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230)
    by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Jan 2008 17:48:33 -0800
X-ASF-Spam-Status: No, hits=-100.0 required=10.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3)
    by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 03 Jan 2008 01:48:29 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
	id BB7581A9832; Wed,  2 Jan 2008 17:48:20 -0800 (PST)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r608306 - in /lucene/java/trunk/contrib/benchmark/src:
 java/org/apache/lucene/benchmark/byTask/feeds/
 java/org/apache/lucene/benchmark/byTask/tasks/
 test/org/apache/lucene/benchmark/byTask/
Date: Thu, 03 Jan 2008 01:48:20 -0000
To: java-commits@lucene.apache.org
From: mikemccand@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20080103014820.BB7581A9832@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Author: mikemccand
Date: Wed Jan  2 17:48:18 2008
New Revision: 608306

URL: http://svn.apache.org/viewvc?rev=608306&view=rev
Log:
LUCENE-1115: some small fixes to contrib/benchmark

Modified:
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocMaker.java
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
    lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocMaker.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocMaker.java?rev=608306&r1=608305&r2=608306&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocMaker.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocMaker.java Wed Jan  2 17:48:18 2008
@@ -85,10 +85,18 @@
     public Document setFields(String line) {
       // title <TAB> date <TAB> body <NEWLINE>
       int spot = line.indexOf(SEP);
-      titleField.setValue(line.substring(0, spot));
-      int spot2 = line.indexOf(SEP, 1+spot);
-      dateField.setValue(line.substring(1+spot, spot2));
-      bodyField.setValue(line.substring(1+spot2, line.length()));
+      if (spot != -1) {
+        titleField.setValue(line.substring(0, spot));
+        int spot2 = line.indexOf(SEP, 1+spot);
+        if (spot2 != -1) {
+          dateField.setValue(line.substring(1+spot, spot2));
+          bodyField.setValue(line.substring(1+spot2, line.length()));
+        } else {
+          dateField.setValue("");
+          bodyField.setValue("");
+        }
+      } else
+        titleField.setValue("");
       return doc;
     }
   }
@@ -121,12 +129,10 @@
       while(true) {
         line = fileIn.readLine();
         if (line == null) {
+          // Reset the file
+          openFile();
           if (!forever)
             throw new NoMoreDataException();
-          else {
-            // Reset the file
-            openFile();
-          }
         } else {
           break;
         }

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java?rev=608306&r1=608305&r2=608306&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java Wed Jan  2 17:48:18 2008
@@ -57,12 +57,8 @@
     iw.setUseCompoundFile(cmpnd);
     iw.setMergeFactor(mrgf);
     iw.setMaxFieldLength(mxfl);
-    if (flushAtRAMUsage > 0)
-      iw.setRAMBufferSizeMB(flushAtRAMUsage);
-    else if (mxbf != 0)
-      iw.setMaxBufferedDocs(mxbf);
-    else
-      throw new RuntimeException("either max.buffered or ram.flush.mb must be non-zero");
+    iw.setRAMBufferSizeMB(flushAtRAMUsage);
+    iw.setMaxBufferedDocs(mxbf);
     getRunData().setIndexWriter(iw);
     return 1;
   }

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java?rev=608306&r1=608305&r2=608306&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java Wed Jan  2 17:48:18 2008
@@ -61,12 +61,8 @@
     IndexWriter writer = new IndexWriter(dir, autoCommit, analyzer, false);
 
     // must update params for newly opened writer
-    if (flushAtRAMUsage > 0)
-      writer.setRAMBufferSizeMB(flushAtRAMUsage);
-    else if (mxbf != 0)
-      writer.setMaxBufferedDocs(mxbf);
-    else
-      throw new RuntimeException("either max.buffered or ram.flush.mb must be non-zero");
+    writer.setRAMBufferSizeMB(flushAtRAMUsage);
+    writer.setMaxBufferedDocs(mxbf);
     writer.setMaxFieldLength(mxfl);
     writer.setMergeFactor(mrgf);
     writer.setUseCompoundFile(cmpnd); // this one redundant?

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java?rev=608306&r1=608305&r2=608306&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java Wed Jan  2 17:48:18 2008
@@ -38,6 +38,8 @@
   private String seqName;
   private boolean exhausted = false;
   private boolean resetExhausted = false;
+  private PerfTask[] tasksArray;
+  private boolean anyExhaustableTasks;
   
   public TaskSequence (PerfRunData runData, String name, TaskSequence parent, boolean parallel) {
     super(runData);
@@ -49,6 +51,18 @@
     tasks = new ArrayList();
   }
 
+  private void initTasksArray() {
+    if (tasksArray == null) {
+      final int numTasks = tasks.size();
+      tasksArray = new PerfTask[numTasks];
+      for(int k=0;k<numTasks;k++) {
+        tasksArray[k] = (PerfTask) tasks.get(k);
+        anyExhaustableTasks |= tasksArray[k] instanceof ResetInputsTask;
+        anyExhaustableTasks |= tasksArray[k] instanceof TaskSequence;
+      }
+    }
+  }
+
   /**
    * @return Returns the parallel.
    */
@@ -92,7 +106,7 @@
    * @see org.apache.lucene.benchmark.byTask.tasks.PerfTask#doLogic()
    */
   public int doLogic() throws Exception {
-    resetExhausted = false;
+    exhausted = resetExhausted = false;
     return ( parallel ? doParallelTasks() : doSerialTasks());
   }
 
@@ -101,18 +115,16 @@
       return doSerialTasksWithRate();
     }
     
+    initTasksArray();
     int count = 0;
     
-    final int numTasks = tasks.size();
-    final PerfTask[] tasksArray = new PerfTask[numTasks];
-    for(int k=0;k<numTasks;k++)
-      tasksArray[k] = (PerfTask) tasks.get(k);
-
     for (int k=0; (repetitions==REPEAT_EXHAUST && !exhausted) || k<repetitions; k++) {
-      for(int l=0;l<numTasks;l++)
+      for(int l=0;l<tasksArray.length;l++)
         try {
-          count += tasksArray[l].runAndMaybeStats(letChildReport);
-          updateExhausted(tasksArray[l]);
+          final PerfTask task = tasksArray[l];
+          count += task.runAndMaybeStats(letChildReport);
+          if (anyExhaustableTasks)
+            updateExhausted(task);
         } catch (NoMoreDataException e) {
           exhausted = true;
         }
@@ -121,12 +133,13 @@
   }
 
   private int doSerialTasksWithRate() throws Exception {
+    initTasksArray();
     long delayStep = (perMin ? 60000 : 1000) /rate;
     long nextStartTime = System.currentTimeMillis();
     int count = 0;
     for (int k=0; (repetitions==REPEAT_EXHAUST && !exhausted) || k<repetitions; k++) {
-      for (Iterator it = tasks.iterator(); it.hasNext();) {
-        PerfTask task = (PerfTask) it.next();
+      for (int l=0;l<tasksArray.length;l++) {
+        final PerfTask task = tasksArray[l];
         long waitMore = nextStartTime - System.currentTimeMillis();
         if (waitMore > 0) {
           //System.out.println("wait: "+waitMore+" for rate: "+ratePerMin+" (delayStep="+delayStep+")");
@@ -135,7 +148,8 @@
         nextStartTime += delayStep; // this aims at avarage rate. 
         try {
           count += task.runAndMaybeStats(letChildReport);
-          updateExhausted(task);
+          if (anyExhaustableTasks)
+            updateExhausted(task);
         } catch (NoMoreDataException e) {
           exhausted = true;
         }
@@ -149,28 +163,27 @@
     if (task instanceof ResetInputsTask) {
       exhausted = false;
       resetExhausted = true;
-    } else {
-      if (task instanceof TaskSequence) {
-        TaskSequence t = (TaskSequence) task;
-        if (t.resetExhausted) {
-          exhausted = false;
-          resetExhausted = true;
-          t.resetExhausted = false;
-        } else {
-          exhausted |= t.exhausted;
-        }
+    } else if (task instanceof TaskSequence) {
+      TaskSequence t = (TaskSequence) task;
+      if (t.resetExhausted) {
+        exhausted = false;
+        resetExhausted = true;
+        t.resetExhausted = false;
+      } else {
+        exhausted |= t.exhausted;
       }
     }
   }
 
   private int doParallelTasks() throws Exception {
+    initTasksArray();
     final int count [] = {0};
     Thread t[] = new Thread [repetitions * tasks.size()];
     // prepare threads
     int indx = 0;
     for (int k=0; k<repetitions; k++) {
-      for (int i = 0; i < tasks.size(); i++) {
-        final PerfTask task = (PerfTask) ((PerfTask) tasks.get(i)).clone();
+      for (int i = 0; i < tasksArray.length; i++) {
+        final PerfTask task = (PerfTask) tasksArray[i].clone();
         t[indx++] = new Thread() {
           public void run() {
             int n;

Modified: lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=608306&r1=608305&r2=608306&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Wed Jan  2 17:48:18 2008
@@ -359,4 +359,38 @@
       nDocs = 0;
     }
   }
+  
+  /**
+   * Test that exhaust in loop works as expected (LUCENE-1115).
+   */
+  public void testExhaustedLooped() throws Exception {
+    // 1. alg definition (required in every "logic" test)
+    String algLines[] = {
+        "# ----- properties ",
+        "doc.maker="+Reuters20DocMaker.class.getName(),
+        "doc.add.log.step=3",
+        "doc.term.vector=false",
+        "doc.maker.forever=false",
+        "directory=RAMDirectory",
+        "doc.stored=false",
+        "doc.tokenized=false",
+        "debug.level=1",
+        "# ----- alg ",
+        "{ \"Rounds\"",
+        "  ResetSystemErase",
+        "  CreateIndex",
+        "  { \"AddDocs\"  AddDoc > : * ",
+        "  CloseIndex",
+        "} : 2",
+    };
+    
+    // 2. execute the algorithm  (required in every "logic" test)
+    Benchmark benchmark = execBenchmark(algLines);
+
+    // 3. test number of docs in the index
+    IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory());
+    int ndocsExpected = 20; // Reuters20DocMaker exhausts after 20 docs.
+    assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs());
+    ir.close();
+  }
 }