lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sim...@apache.org
Subject [2/3] lucene-solr:branch_7x: LUCENE-8355: Prevent IW from opening an already dropped segment while DV updates are written
Date Wed, 13 Jun 2018 08:10:24 GMT
LUCENE-8355: Prevent IW from opening an already dropped segment while DV updates are written

This change fixes an isse where IW asks ReadersAndUpdates to write a DV updates for a
segment that has been dropped concurrently. The race only occurs if ram buffers are filled
up enough to trigger flushing DV to disk.

Co-authored-by: Nhat Nguyen <nhat.nguyen@elastic.co>


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/96833128
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/96833128
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/96833128

Branch: refs/heads/branch_7x
Commit: 96833128923b2bcded7e82e9ad9bc0b5c202c8c9
Parents: 2cbd23c
Author: Simon Willnauer <simonw@apache.org>
Authored: Wed Jun 13 09:42:05 2018 +0200
Committer: Simon Willnauer <simonw@apache.org>
Committed: Wed Jun 13 10:09:48 2018 +0200

----------------------------------------------------------------------
 lucene/CHANGES.txt                                   |  3 +++
 .../java/org/apache/lucene/index/IndexWriter.java    |  6 ++++++
 .../lucene/index/TestIndexWriterWithThreads.java     | 15 +++++++++++++--
 3 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96833128/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 25d9bf6..fabd444 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -166,6 +166,9 @@ Bug Fixes
 * LUCENE-8278: Some end-of-input no-scheme domain-only URL tokens are typed as
   <ALPHANUM> rather than <URL>.  (Junte Zhang, Steve Rowe)
 
+* LUCENE-8355: Prevent IW from opening an already dropped segment while DV updates
+  are written. (Nhat Nguyen via Simon Willnauer)
+
 Other
 
 * LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96833128/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 63952ec..beda294 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -607,6 +607,12 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
               // Only acquire IW lock on each write, since this is a time consuming operation.
 This way
               // other threads get a chance to run in between our writes.
               synchronized (this) {
+                // It's possible that the segment of a reader returned by readerPool#getReadersByRam
+                // is dropped before being processed here. If it happens, we need to skip
that reader.
+                if (readerPool.get(rld.info, false) == null) {
+                  assert segmentInfos.contains(rld.info) == false : "Segment [" + rld.info
+ "] is not dropped yet";
+                  continue;
+                }
                 if (rld.writeFieldUpdates(directory, globalFieldNumberMap, bufferedUpdatesStream.getCompletedDelGen(),
infoStream)) {
                   checkpointNoSIS();
                 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96833128/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
index 6e9645b..596df10 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
@@ -658,10 +658,18 @@ public class TestIndexWriterWithThreads extends LuceneTestCase {
     d.close();
   }
 
-  public void testUpdateSingleDocWithThreads() throws IOException, BrokenBarrierException,
InterruptedException {
+  public void testUpdateSingleDocWithThreads() throws Exception {
+    stressUpdateSingleDocWithThreads(false, rarely());
+  }
+
+  public void testSoftUpdateSingleDocWithThreads() throws Exception {
+    stressUpdateSingleDocWithThreads(true, rarely());
+  }
+
+  public void stressUpdateSingleDocWithThreads(boolean useSoftDeletes, boolean forceMerge)
throws Exception{
     try (Directory dir = newDirectory();
          RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
-             newIndexWriterConfig().setMaxBufferedDocs(-1).setRAMBufferSizeMB(0.00001)))
{
+             newIndexWriterConfig().setMaxBufferedDocs(-1).setRAMBufferSizeMB(0.00001), useSoftDeletes))
{
       Thread[] threads = new Thread[3 + random().nextInt(3)];
       AtomicInteger done = new AtomicInteger(0);
       CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
@@ -691,6 +699,9 @@ public class TestIndexWriterWithThreads extends LuceneTestCase {
       barrier.await();
       try {
         do {
+          if (forceMerge && random().nextBoolean()) {
+            writer.forceMerge(1);
+          }
           DirectoryReader newReader = DirectoryReader.openIfChanged(open);
           if (newReader != null) {
             open.close();


Mime
View raw message