lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dnh...@apache.org
Subject lucene-solr:branch_7_4: LUCENE-8384: Fix missing advance DVGen in PendingSoftDeletes
Date Thu, 05 Jul 2018 02:43:19 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_7_4 911f96b0f -> c4f3c2799


LUCENE-8384: Fix missing advance DVGen in PendingSoftDeletes

Today we only advance the docValues update generation in
PendingSoftDeletes for the soft-deletes field. If we update a
soft-deletes DV field, then update a non-soft-deletes DV field, then
onNewReader will consider that we never update DV field although we did.

This commit makes sure that we always advance the docValues update
generation when handling docValues update in PendingSoftDeletes.

Co-authored-by: Simon Willnauer <simonw@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c4f3c279
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c4f3c279
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c4f3c279

Branch: refs/heads/branch_7_4
Commit: c4f3c27998b7ae5f3247e81ea1b0a940cc3e834f
Parents: 911f96b
Author: Nhat Nguyen <nhat.nguyen@elastic.co>
Authored: Wed Jul 4 12:46:42 2018 -0400
Committer: Nhat Nguyen <nhat.nguyen@elastic.co>
Committed: Wed Jul 4 22:24:07 2018 -0400

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  3 +++
 .../apache/lucene/index/PendingSoftDeletes.java |  6 +++---
 .../apache/lucene/index/SegmentCommitInfo.java  |  4 ++--
 .../TestSoftDeletesRetentionMergePolicy.java    | 22 ++++++++++++++++++++
 4 files changed, 30 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c4f3c279/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2583bf9..d3435c1 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -15,6 +15,9 @@ Bug Fixes:
  * LUCENE-8381: Fix IndexWriter incorrectly interprets hard-deletes as soft-deletes
    while wrapping reader for merges. (Simon Willnauer, Nhat Nguyen)
 
+ * LUCENE-8384: Fix missing advance docValues generation while handling docValues
+   update in PendingSoftDeletes. (Simon Willnauer, Nhat Nguyen)
+
 ======================= Lucene 7.4.0 =======================
 
 Upgrading

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c4f3c279/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java b/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
index 4074903..eb6e4ff 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
@@ -146,12 +146,12 @@ final class PendingSoftDeletes extends PendingDeletes {
     if (this.field.equals(info.name)) {
       pendingDeleteCount += applySoftDeletes(iterator, getMutableBits());
       assert assertPendingDeletes();
-      assert dvGeneration < info.getDocValuesGen() : "we have seen this generation update
already: " + dvGeneration + " vs. " + info.getDocValuesGen();
-      assert dvGeneration != -2 : "docValues generation is still uninitialized";
-      dvGeneration = info.getDocValuesGen();
       this.info.setSoftDelCount(this.info.getSoftDelCount() + pendingDeleteCount);
       super.dropChanges();
     }
+    assert dvGeneration < info.getDocValuesGen() : "we have seen this generation update
already: " + dvGeneration + " vs. " + info.getDocValuesGen();
+    assert dvGeneration != -2 : "docValues generation is still uninitialized";
+    dvGeneration = info.getDocValuesGen();
   }
 
   private boolean assertPendingDeletes() {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c4f3c279/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
index 954a138..6911757 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
@@ -328,7 +328,7 @@ public class SegmentCommitInfo {
     if (delCount < 0 || delCount > info.maxDoc()) {
       throw new IllegalArgumentException("invalid delCount=" + delCount + " (maxDoc=" + info.maxDoc()
+ ")");
     }
-    assert softDelCount + delCount <= info.maxDoc();
+    assert softDelCount + delCount <= info.maxDoc() : "maxDoc=" + info.maxDoc() + ",delCount="
+ delCount + ",softDelCount=" + softDelCount;
     this.delCount = delCount;
   }
 
@@ -336,7 +336,7 @@ public class SegmentCommitInfo {
     if (softDelCount < 0 || softDelCount > info.maxDoc()) {
       throw new IllegalArgumentException("invalid softDelCount=" + softDelCount + " (maxDoc="
+ info.maxDoc() + ")");
     }
-    assert softDelCount + delCount <= info.maxDoc();
+    assert softDelCount + delCount <= info.maxDoc() : "maxDoc=" + info.maxDoc() + ",delCount="
+ delCount + ",softDelCount=" + softDelCount;
     this.softDelCount = softDelCount;
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c4f3c279/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
index bb36ba3..ee2b650 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
@@ -40,6 +40,8 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
@@ -607,6 +609,26 @@ public class TestSoftDeletesRetentionMergePolicy extends LuceneTestCase
{
     IOUtils.close(writer, dir);
   }
 
+  public void testSoftDeleteWithTryUpdateDocValue() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField("soft_delete")
+        .setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", MatchAllDocsQuery::new,
newLogMergePolicy()));
+    IndexWriter writer = new IndexWriter(dir, config);
+    SearcherManager sm = new SearcherManager(writer, new SearcherFactory());
+    Document d = new Document();
+    d.add(new StringField("id", "0", Field.Store.YES));
+    writer.addDocument(d);
+    sm.maybeRefreshBlocking();
+    doUpdate(new Term("id", "0"), writer,
+        new NumericDocValuesField("soft_delete", 1), new NumericDocValuesField("other-field",
1));
+    sm.maybeRefreshBlocking();
+    assertEquals(1, writer.segmentInfos.asList().size());
+    SegmentCommitInfo si = writer.segmentInfos.asList().get(0);
+    assertEquals(1, si.getSoftDelCount());
+    assertEquals(1, si.info.maxDoc());
+    IOUtils.close(sm, writer, dir);
+  }
+
   static void doUpdate(Term doc, IndexWriter writer, Field... fields) throws IOException
{
     long seqId = -1;
     do { // retry if we just committing a merge


Mime
View raw message