Return-Path: 
 <commits-return-24216-apmail-lucene-commits-archive=www.apache.org@lucene.apache.org>
X-Original-To: apmail-lucene-commits-archive@www.apache.org
Delivered-To: apmail-lucene-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id C0657E283
	for <apmail-lucene-commits-archive@www.apache.org>;
 Sat,  2 Feb 2013 15:21:31 +0000 (UTC)
Received: (qmail 29856 invoked by uid 500); 2 Feb 2013 15:21:31 -0000
Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@lucene.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@lucene.apache.org>
List-Post: <mailto:commits@lucene.apache.org>
List-Id: <commits.lucene.apache.org>
Reply-To: dev@lucene.apache.org
Delivered-To: mailing list commits@lucene.apache.org
Received: (qmail 29839 invoked by uid 99); 2 Feb 2013 15:21:30 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
    by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 02 Feb 2013 15:21:30 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=5.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 02 Feb 2013 15:21:29 +0000
Received: from eris.apache.org (localhost [127.0.0.1])
	by eris.apache.org (Postfix) with ESMTP id 4F3AB2388847;
	Sat,  2 Feb 2013 15:21:10 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1441761 -
 /lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java
Date: Sat, 02 Feb 2013 15:21:10 -0000
To: commits@lucene.apache.org
From: jpountz@apache.org
X-Mailer: svnmailer-1.0.8-patched
Message-Id: <20130202152110.4F3AB2388847@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Author: jpountz
Date: Sat Feb  2 15:21:09 2013
New Revision: 1441761

URL: http://svn.apache.org/viewvc?rev=1441761&view=rev
Log:
LUCENE-4733: More term vectors formats tests (TVReader.clone and TVWriter.merge).

Modified:
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java?rev=1441761&r1=1441760&r2=1441761&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java Sat Feb  2 15:21:09 2013
@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -388,9 +389,9 @@ public abstract class BaseTermVectorsFor
   }
 
   // to test reuse
-  private TermsEnum termsEnum = null;
-  private DocsEnum docsEnum = null;
-  private DocsAndPositionsEnum docsAndPositionsEnum = null;
+  private final ThreadLocal<TermsEnum> termsEnum = new ThreadLocal<TermsEnum>();
+  private final ThreadLocal<DocsEnum> docsEnum = new ThreadLocal<DocsEnum>();
+  private final ThreadLocal<DocsAndPositionsEnum> docsAndPositionsEnum = new ThreadLocal<DocsAndPositionsEnum>();
 
   protected void assertEquals(RandomTokenStream tk, FieldType ft, Terms terms) throws IOException {
     assertEquals(1, terms.getDocCount());
@@ -406,7 +407,8 @@ public abstract class BaseTermVectorsFor
     }
     final BytesRef[] sortedTerms = uniqueTerms.toArray(new BytesRef[0]);
     Arrays.sort(sortedTerms, terms.getComparator());
-    termsEnum = terms.iterator(random().nextBoolean() ? null : termsEnum);
+    final TermsEnum termsEnum = terms.iterator(random().nextBoolean() ? null : this.termsEnum.get());
+    this.termsEnum.set(termsEnum);
     for (int i = 0; i < sortedTerms.length; ++i) {
       final BytesRef nextTerm = termsEnum.next();
       assertEquals(sortedTerms[i], nextTerm);
@@ -414,7 +416,7 @@ public abstract class BaseTermVectorsFor
       assertEquals(1, termsEnum.docFreq());
 
       final FixedBitSet bits = new FixedBitSet(1);
-      docsEnum = termsEnum.docs(bits, random().nextBoolean() ? null : docsEnum);
+      DocsEnum docsEnum = termsEnum.docs(bits, random().nextBoolean() ? null : this.docsEnum.get());
       assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
       bits.set(0);
 
@@ -424,9 +426,10 @@ public abstract class BaseTermVectorsFor
       assertEquals(0, docsEnum.docID());
       assertEquals(tk.freqs.get(termsEnum.term().utf8ToString()), (Integer) docsEnum.freq());
       assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
+      this.docsEnum.set(docsEnum);
 
       bits.clear(0);
-      docsAndPositionsEnum = termsEnum.docsAndPositions(bits, random().nextBoolean() ? null : docsAndPositionsEnum);
+      DocsAndPositionsEnum docsAndPositionsEnum = termsEnum.docsAndPositions(bits, random().nextBoolean() ? null : this.docsAndPositionsEnum.get());
       assertEquals(ft.storeTermVectorOffsets() || ft.storeTermVectorPositions(), docsAndPositionsEnum != null);
       if (docsAndPositionsEnum != null) {
         assertEquals(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc());
@@ -492,6 +495,7 @@ public abstract class BaseTermVectorsFor
         }
         assertEquals(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc());
       }
+      this.docsAndPositionsEnum.set(docsAndPositionsEnum);
     }
     assertNull(termsEnum.next());
     for (int i = 0; i < 5; ++i) {
@@ -630,4 +634,97 @@ public abstract class BaseTermVectorsFor
     dir.close();
   }
 
+  public void testMerge() throws IOException {
+    final RandomDocumentFactory docFactory = new RandomDocumentFactory(5, 20);
+    final int numDocs = _TestUtil.nextInt(random(), 100, 500);
+    final int numDeletes = random().nextInt(numDocs);
+    final Set<Integer> deletes = new HashSet<Integer>();
+    while (deletes.size() < numDeletes) {
+      deletes.add(random().nextInt(numDocs));
+    }
+    for (Options options : validOptions()) {
+      final RandomDocument[] docs = new RandomDocument[numDocs];
+      for (int i = 0; i < numDocs; ++i) {
+        docs[i] = docFactory.newDocument(_TestUtil.nextInt(random(), 1, 3), _TestUtil.nextInt(random(), 10, 50), options);
+      }
+      final Directory dir = newDirectory();
+      final RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+      for (int i = 0; i < numDocs; ++i) {
+        writer.addDocument(addId(docs[i].toDocument(), ""+i));
+        if (rarely()) {
+          writer.commit();
+        }
+      }
+      for (int delete : deletes) {
+        writer.deleteDocuments(new Term("id", "" + delete));
+      }
+      // merge with deletes
+      writer.forceMerge(1);
+      final IndexReader reader = writer.getReader();
+      for (int i = 0; i < numDocs; ++i) {
+        if (!deletes.contains(i)) {
+          final int docID = docID(reader, ""+i);
+          assertEquals(docs[i], reader.getTermVectors(docID));
+        }
+      }
+      reader.close();
+      writer.close();
+      dir.close();
+    }
+  }
+
+  // run random tests from different threads to make sure the per-thread clones
+  // don't share mutable data
+  public void testClone() throws IOException, InterruptedException {
+    final RandomDocumentFactory docFactory = new RandomDocumentFactory(5, 20);
+    final int numDocs = _TestUtil.nextInt(random(), 100, 1000);
+    for (Options options : validOptions()) {
+      final RandomDocument[] docs = new RandomDocument[numDocs];
+      for (int i = 0; i < numDocs; ++i) {
+        docs[i] = docFactory.newDocument(_TestUtil.nextInt(random(), 1, 3), _TestUtil.nextInt(random(), 10, 50), options);
+      }
+      final Directory dir = newDirectory();
+      final RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+      for (int i = 0; i < numDocs; ++i) {
+        writer.addDocument(addId(docs[i].toDocument(), ""+i));
+      }
+      final IndexReader reader = writer.getReader();
+      for (int i = 0; i < numDocs; ++i) {
+        final int docID = docID(reader, ""+i);
+        assertEquals(docs[i], reader.getTermVectors(docID));
+      }
+
+      final AtomicReference<Throwable> exception = new AtomicReference<Throwable>();
+      final Thread[] threads = new Thread[2];
+      for (int i = 0; i < threads.length; ++i) {
+        threads[i] = new Thread() {
+          @Override
+          public void run() {
+            try {
+              for (int i = 0; i < atLeast(100); ++i) {
+                final int idx = random().nextInt(numDocs);
+                final int docID = docID(reader, ""+idx);
+                assertEquals(docs[idx], reader.getTermVectors(docID));
+              }
+            } catch (Throwable t) {
+              exception.set(t);
+            }
+          }
+        };
+      }
+      for (Thread thread : threads) {
+        thread.start();
+      }
+      for (Thread thread : threads) {
+        thread.join();
+      }
+      reader.close();
+      writer.close();
+      dir.close();
+      if (exception.get() != null) {
+        throw new AssertionError("One thread threw an exception", exception.get());
+      }
+    }
+  }
+
 }