lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r830905 - in /lucene/java/branches/lucene_2_9/contrib/benchmark: ./ src/java/org/apache/lucene/benchmark/byTask/feeds/ src/test/org/apache/lucene/benchmark/byTask/
Date Thu, 29 Oct 2009 11:46:38 GMT
Author: mikemccand
Date: Thu Oct 29 11:46:37 2009
New Revision: 830905

URL: http://svn.apache.org/viewvc?rev=830905&view=rev
Log:
LUCENE-1994 (on 2.9.x branch): fix thread safety of EnwikiContentSource and DocMaker

Modified:
    lucene/java/branches/lucene_2_9/contrib/benchmark/CHANGES.txt
    lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java
    lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java
    lucene/java/branches/lucene_2_9/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java

Modified: lucene/java/branches/lucene_2_9/contrib/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/benchmark/CHANGES.txt?rev=830905&r1=830904&r2=830905&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/benchmark/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/contrib/benchmark/CHANGES.txt Thu Oct 29 11:46:37 2009
@@ -4,6 +4,11 @@
 
 $Id:$
 
+10/28/2009
+  LUCENE-1994: Fix thread safety of EnwikiContentSource and DocMaker
+  when doc.reuse.fields is false.  Also made docs.reuse.fields=true
+  thread safe.  (Mark Miller, Shai Erera, Mike McCandless)
+
 8/4/2009
   LUCENE-1770: Add EnwikiQueryMaker (Mark Miller)
 

Modified: lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java?rev=830905&r1=830904&r2=830905&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java
(original)
+++ lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java
Thu Oct 29 11:46:37 2009
@@ -147,7 +147,6 @@
   
   protected ContentSource source;
   protected boolean reuseFields;
-  protected DocState localDocState;
   protected boolean indexProperties;
   
   private int lastPrintedNumUniqueTexts = 0;
@@ -161,7 +160,7 @@
   // reset the docdata properties so they are not added more than once.
   private Document createDocument(DocData docData, int size, int cnt) throws UnsupportedEncodingException
{
 
-    final DocState ds = reuseFields ? getDocState() : localDocState;
+    final DocState ds = getDocState();
     final Document doc = reuseFields ? ds.doc : new Document();
     doc.getFields().clear();
     
@@ -289,7 +288,7 @@
    */
   public Document makeDocument() throws Exception {
     resetLeftovers();
-    DocData docData = source.getNextDocData(reuseFields ? getDocState().docData : localDocState.docData);
+    DocData docData = source.getNextDocData(getDocState().docData);
     Document doc = createDocument(docData, 0, -1);
     return doc;
   }
@@ -304,7 +303,7 @@
         || lvr.docdata.getBody().length() == 0) {
       resetLeftovers();
     }
-    DocData docData = reuseFields ? getDocState().docData : localDocState.docData;
+    DocData docData = getDocState().docData;
     DocData dd = (lvr == null ? source.getNextDocData(docData) : lvr.docdata);
     int cnt = (lvr == null ? 0 : lvr.cnt);
     while (dd.getBody() == null || dd.getBody().length() < size) {
@@ -407,15 +406,12 @@
     storeBytes = config.get("doc.store.body.bytes", false);
     
     reuseFields = config.get("doc.reuse.fields", true);
-    if (!reuseFields) {
-      localDocState = new DocState(false, storeVal, indexVal, bodyIndexVal, termVecVal);
-    } else {
-      // In a multi-rounds run, it is important to reset DocState since settings
-      // of fields may change between rounds, and this is the only way to reset
-      // the cache of all threads.
-      docState = new ThreadLocal();
-    }
-    
+
+    // In a multi-rounds run, it is important to reset DocState since settings
+    // of fields may change between rounds, and this is the only way to reset
+    // the cache of all threads.
+    docState = new ThreadLocal();
+
     indexProperties = config.get("doc.index.props", false);
 
     updateDocIDLimit = config.get("doc.random.id.limit", -1);

Modified: lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java?rev=830905&r1=830904&r2=830905&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java
(original)
+++ lucene/java/branches/lucene_2_9/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java
Thu Oct 29 11:46:37 2009
@@ -266,7 +266,7 @@
     }
   }
   
-  public DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException
{
+  public synchronized DocData getNextDocData(DocData docData) throws NoMoreDataException,
IOException {
     String[] tuple = parser.next();
     docData.clear();
     docData.setName(tuple[ID]);

Modified: lucene/java/branches/lucene_2_9/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=830905&r1=830904&r2=830905&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
(original)
+++ lucene/java/branches/lucene_2_9/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
Thu Oct 29 11:46:37 2009
@@ -40,6 +40,8 @@
 import org.apache.lucene.index.LogDocMergePolicy;
 import org.apache.lucene.index.TermFreqVector;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.search.FieldCache.StringIndex;
+import org.apache.lucene.search.FieldCache;
 
 import junit.framework.TestCase;
 
@@ -268,6 +270,42 @@
     ir.close();
   }
 
+  // LUCENE-1994: test thread safety of SortableSingleDocMaker
+  public void testDocMakerThreadSafety() throws Exception {
+    // 1. alg definition (required in every "logic" test)
+    String algLines[] = {
+        "# ----- properties ",
+        "content.source=org.apache.lucene.benchmark.byTask.feeds.SortableSingleDocSource",
+        "doc.term.vector=false",
+        "log.step.AddDoc=10000",
+        "content.source.forever=true",
+        "directory=RAMDirectory",
+        "doc.reuse.fields=false",
+        "doc.stored=false",
+        "doc.tokenized=false",
+        "doc.index.props=true",
+        "# ----- alg ",
+        "CreateIndex",
+        "[ { AddDoc > : 2500 ] : 4",
+        "CloseIndex",
+    };
+    
+    // 2. we test this value later
+    CountingSearchTestTask.numSearches = 0;
+    
+    // 3. execute the algorithm  (required in every "logic" test)
+    Benchmark benchmark = execBenchmark(algLines);
+
+    IndexReader r = IndexReader.open(benchmark.getRunData().getDirectory(), true);
+    StringIndex idx = FieldCache.DEFAULT.getStringIndex(r, "country");
+    final int maxDoc = r.maxDoc();
+    assertEquals(10000, maxDoc);
+    for(int i=0;i<10000;i++) {
+      assertNotNull("doc " + i + " has null country", idx.lookup[idx.order[i]]);
+    }
+    r.close();
+  }
+
   /**
    * Test Parallel Doc Maker logic (for LUCENE-940)
    */



Mime
View raw message