lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject lucene-solr:master: LUCENE-7894: make sure IW finishes segment flush so we don't see a temporary drop in IW.maxDoc()
Date Sun, 02 Jul 2017 19:32:56 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/master 80ae5e29d -> ee1edd9d4


LUCENE-7894: make sure IW finishes segment flush so we don't see a temporary drop in IW.maxDoc()


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ee1edd9d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ee1edd9d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ee1edd9d

Branch: refs/heads/master
Commit: ee1edd9d467bbccfe7015ec5ebee93f8b5bd4b14
Parents: 80ae5e2
Author: Mike McCandless <mikemccand@apache.org>
Authored: Sun Jul 2 15:32:44 2017 -0400
Committer: Mike McCandless <mikemccand@apache.org>
Committed: Sun Jul 2 15:32:44 2017 -0400

----------------------------------------------------------------------
 .../apache/lucene/index/DocumentsWriter.java    | 20 +++---
 .../lucene/index/DocumentsWriterFlushQueue.java |  4 ++
 .../lucene/index/TestIndexManyDocuments.java    | 71 ++++++++++++++++++++
 3 files changed, 86 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee1edd9d/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
index 27e28c0..0e2a067 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
@@ -544,11 +544,11 @@ final class DocumentsWriter implements Closeable, Accountable {
             dwptSuccess = true;
           } finally {
             subtractFlushedNumDocs(flushingDocsInRam);
-            if (!flushingDWPT.pendingFilesToDelete().isEmpty()) {
+            if (flushingDWPT.pendingFilesToDelete().isEmpty() == false) {
               putEvent(new DeleteNewFilesEvent(flushingDWPT.pendingFilesToDelete()));
               hasEvents = true;
             }
-            if (!dwptSuccess) {
+            if (dwptSuccess == false) {
               putEvent(new FlushFailedEvent(flushingDWPT.getSegmentInfo()));
               hasEvents = true;
             }
@@ -582,6 +582,10 @@ final class DocumentsWriter implements Closeable, Accountable {
       flushingDWPT = flushControl.nextPendingFlush();
     }
 
+    if (hasEvents) {
+      writer.doAfterSegmentFlushed(false, false);
+    }
+
     // If deletes alone are consuming > 1/2 our RAM
     // buffer, force them all to apply now. This is to
     // prevent too-frequent flushing of a long tail of
@@ -605,7 +609,7 @@ final class DocumentsWriter implements Closeable, Accountable {
   
   void subtractFlushedNumDocs(int numFlushed) {
     int oldValue = numDocsInRAM.get();
-    while (!numDocsInRAM.compareAndSet(oldValue, oldValue - numFlushed)) {
+    while (numDocsInRAM.compareAndSet(oldValue, oldValue - numFlushed) == false) {
       oldValue = numDocsInRAM.get();
     }
     assert numDocsInRAM.get() >= 0;
@@ -726,10 +730,9 @@ final class DocumentsWriter implements Closeable, Accountable {
 
   static final class ApplyDeletesEvent implements Event {
     static final Event INSTANCE = new ApplyDeletesEvent();
-    private int instCount = 0;
+
     private ApplyDeletesEvent() {
-      assert instCount == 0;
-      instCount++;
+      // only one instance
     }
     
     @Override
@@ -740,10 +743,9 @@ final class DocumentsWriter implements Closeable, Accountable {
 
   static final class ForcedPurgeEvent implements Event {
     static final Event INSTANCE = new ForcedPurgeEvent();
-    private int instCount = 0;
+
     private ForcedPurgeEvent() {
-      assert instCount == 0;
-      instCount++;
+      // only one instance
     }
     
     @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee1edd9d/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
index df1b38c..b051545 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
@@ -119,6 +119,10 @@ class DocumentsWriterFlushQueue {
           synchronized (this) {
             // finally remove the published ticket from the queue
             final FlushTicket poll = queue.poll();
+
+            // we hold the purgeLock so no other thread should have polled:
+            assert poll == head;
+            
             ticketCount.decrementAndGet();
             assert poll == head;
           }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee1edd9d/lucene/core/src/test/org/apache/lucene/index/TestIndexManyDocuments.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexManyDocuments.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexManyDocuments.java
new file mode 100644
index 0000000..eb31e73
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexManyDocuments.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+public class TestIndexManyDocuments extends LuceneTestCase {
+
+  public void test() throws Exception {
+    Directory dir = newFSDirectory(createTempDir());
+    IndexWriterConfig iwc = new IndexWriterConfig();
+    iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 2000));
+
+    int numDocs = atLeast(10000);
+
+    final IndexWriter w = new IndexWriter(dir, iwc);
+    final AtomicInteger count = new AtomicInteger();
+    Thread[] threads = new Thread[2];
+    for(int i=0;i<threads.length;i++) {
+      threads[i] = new Thread() {
+          @Override
+          public void run() {
+           while (count.getAndIncrement() < numDocs) {
+              Document doc = new Document();
+              doc.add(newTextField("field", "text", Field.Store.NO));
+              try {
+                w.addDocument(doc);
+              } catch (IOException ioe) {
+                throw new RuntimeException(ioe);
+              }
+            }
+          }
+        };
+      threads[i].start();
+    }
+
+    for (Thread thread : threads) {
+      thread.join();
+    }
+
+    assertEquals("lost " + (numDocs - w.maxDoc()) + " documents; maxBufferedDocs=" + iwc.getMaxBufferedDocs(),
numDocs, w.maxDoc());
+    w.close();
+             
+    IndexReader r = DirectoryReader.open(dir);
+    assertEquals(numDocs, r.maxDoc());
+    IOUtils.close(r, dir);
+  }
+}


Mime
View raw message