lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [03/50] lucene-solr:jira/solr-12181: LUCENE-8233: Add support for soft deletes to IndexWriter
Date Tue, 10 Apr 2018 14:11:42 GMT
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ecc17f90/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
new file mode 100644
index 0000000..3f4f405
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestSoftDeletesRetentionMergePolicy extends LuceneTestCase {
+
+  public void testKeepFullyDeletedSegments() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+    IndexWriter writer = new IndexWriter(dir, indexWriterConfig);
+
+    Document doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    doc.add(new NumericDocValuesField("soft_delete", 1));
+    writer.addDocument(doc);
+    DirectoryReader reader = writer.getReader();
+    assertEquals(1, reader.leaves().size());
+    SegmentReader segmentReader = (SegmentReader) reader.leaves().get(0).reader();
+    MergePolicy policy = new SoftDeletesRetentionMergePolicy("soft_delete",
+        () -> new DocValuesFieldExistsQuery("keep_around"), NoMergePolicy.INSTANCE);
+    assertFalse(policy.keepFullyDeletedSegment(segmentReader));
+    reader.close();
+
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    doc.add(new NumericDocValuesField("keep_around", 1));
+    doc.add(new NumericDocValuesField("soft_delete", 1));
+    writer.addDocument(doc);
+
+    reader = writer.getReader();
+    assertEquals(2, reader.leaves().size());
+    segmentReader = (SegmentReader) reader.leaves().get(0).reader();
+    assertFalse(policy.keepFullyDeletedSegment(segmentReader));
+
+    segmentReader = (SegmentReader) reader.leaves().get(1).reader();
+    assertTrue(policy.keepFullyDeletedSegment(segmentReader));
+
+    IOUtils.close(reader, writer, dir);
+  }
+
+  public void testFieldBasedRetention() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+    Instant now = Instant.now();
+    Instant time24HoursAgo = now.minus(Duration.ofDays(1));
+    String softDeletesField = "soft_delete";
+    Supplier<Query> docsOfLast24Hours = () -> LongPoint.newRangeQuery("creation_date",
time24HoursAgo.toEpochMilli(), now.toEpochMilli());
+    indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy(softDeletesField,
docsOfLast24Hours,
+        new LogDocMergePolicy()));
+    indexWriterConfig.setSoftDeletesField(softDeletesField);
+    IndexWriter writer = new IndexWriter(dir, indexWriterConfig);
+
+    long time28HoursAgo = now.minus(Duration.ofHours(28)).toEpochMilli();
+    Document doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    doc.add(new StringField("version", "1", Field.Store.YES));
+    doc.add(new LongPoint("creation_date", time28HoursAgo));
+    writer.addDocument(doc);
+
+    writer.flush();
+    long time26HoursAgo = now.minus(Duration.ofHours(26)).toEpochMilli();
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    doc.add(new StringField("version", "2", Field.Store.YES));
+    doc.add(new LongPoint("creation_date", time26HoursAgo));
+    writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField("soft_delete",
1));
+
+    if (random().nextBoolean()) {
+      writer.flush();
+    }
+    long time23HoursAgo = now.minus(Duration.ofHours(23)).toEpochMilli();
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    doc.add(new StringField("version", "3", Field.Store.YES));
+    doc.add(new LongPoint("creation_date", time23HoursAgo));
+    writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField("soft_delete",
1));
+
+    if (random().nextBoolean()) {
+      writer.flush();
+    }
+    long time12HoursAgo = now.minus(Duration.ofHours(12)).toEpochMilli();
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    doc.add(new StringField("version", "4", Field.Store.YES));
+    doc.add(new LongPoint("creation_date", time12HoursAgo));
+    writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField("soft_delete",
1));
+
+    if (random().nextBoolean()) {
+      writer.flush();
+    }
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    doc.add(new StringField("version", "5", Field.Store.YES));
+    doc.add(new LongPoint("creation_date", now.toEpochMilli()));
+    writer.softUpdateDocument(new Term("id", "1"), doc, new NumericDocValuesField("soft_delete",
1));
+
+    if (random().nextBoolean()) {
+      writer.flush();
+    }
+    writer.forceMerge(1);
+    DirectoryReader reader = writer.getReader();
+    assertEquals(1, reader.numDocs());
+    assertEquals(3, reader.maxDoc());
+    Set<String> versions = new HashSet<>();
+    versions.add(reader.document(0, Collections.singleton("version")).get("version"));
+    versions.add(reader.document(1, Collections.singleton("version")).get("version"));
+    versions.add(reader.document(2, Collections.singleton("version")).get("version"));
+    assertTrue(versions.contains("5"));
+    assertTrue(versions.contains("4"));
+    assertTrue(versions.contains("3"));
+    IOUtils.close(reader, writer, dir);
+  }
+
+  public void testKeepAllDocsAcrossMerges() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+    indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete",
+        () -> new MatchAllDocsQuery(),
+        indexWriterConfig.getMergePolicy()));
+    indexWriterConfig.setSoftDeletesField("soft_delete");
+    IndexWriter writer = new IndexWriter(dir, indexWriterConfig);
+
+    Document doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    writer.softUpdateDocument(new Term("id", "1"), doc,
+        new NumericDocValuesField("soft_delete", 1));
+
+    writer.commit();
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    writer.softUpdateDocument(new Term("id", "1"), doc,
+        new NumericDocValuesField("soft_delete", 1));
+
+    writer.commit();
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.YES));
+    doc.add(new NumericDocValuesField("soft_delete", 1)); // already deleted
+    writer.softUpdateDocument(new Term("id", "1"), doc,
+        new NumericDocValuesField("soft_delete", 1));
+    writer.commit();
+    DirectoryReader reader = writer.getReader();
+    assertEquals(0, reader.numDocs());
+    assertEquals(3, reader.maxDoc());
+    assertEquals(0, writer.numDocs());
+    assertEquals(3, writer.maxDoc());
+    assertEquals(3, reader.leaves().size());
+    reader.close();
+    writer.forceMerge(1);
+    reader = writer.getReader();
+    assertEquals(0, reader.numDocs());
+    assertEquals(3, reader.maxDoc());
+    assertEquals(0, writer.numDocs());
+    assertEquals(3, writer.maxDoc());
+    assertEquals(1, reader.leaves().size());
+    IOUtils.close(reader, writer, dir);
+  }
+
+  /**
+   * tests soft deletes that carry over deleted documents on merge for history rentention.
+   */
+  public void testSoftDeleteWithRetention() throws IOException, InterruptedException {
+    AtomicInteger seqIds = new AtomicInteger(0);
+    Directory dir = newDirectory();
+    IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+    indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete",
+        () -> IntPoint.newRangeQuery("seq_id", seqIds.intValue() - 50, Integer.MAX_VALUE),
+        indexWriterConfig.getMergePolicy()));
+    indexWriterConfig.setSoftDeletesField("soft_delete");
+    IndexWriter writer = new IndexWriter(dir, indexWriterConfig);
+    Thread[] threads = new Thread[2 + random().nextInt(3)];
+    CountDownLatch startLatch = new CountDownLatch(1);
+    CountDownLatch started = new CountDownLatch(threads.length);
+    boolean updateSeveralDocs = random().nextBoolean();
+    Set<String> ids = Collections.synchronizedSet(new HashSet<>());
+    for (int i = 0; i < threads.length; i++) {
+      threads[i] = new Thread(() -> {
+        try {
+          started.countDown();
+          startLatch.await();
+          for (int d = 0;  d < 100; d++) {
+            String id = String.valueOf(random().nextInt(10));
+            int seqId = seqIds.incrementAndGet();
+            if (updateSeveralDocs) {
+              Document doc = new Document();
+              doc.add(new StringField("id", id, Field.Store.YES));
+              doc.add(new IntPoint("seq_id", seqId));
+              writer.softUpdateDocuments(new Term("id", id), Arrays.asList(doc, doc),
+                  new NumericDocValuesField("soft_delete", 1));
+            } else {
+              Document doc = new Document();
+              doc.add(new StringField("id", id, Field.Store.YES));
+              doc.add(new IntPoint("seq_id", seqId));
+              writer.softUpdateDocument(new Term("id", id), doc,
+                  new NumericDocValuesField("soft_delete", 1));
+            }
+            ids.add(id);
+          }
+        } catch (IOException | InterruptedException e) {
+          throw new AssertionError(e);
+        }
+      });
+      threads[i].start();
+    }
+    started.await();
+    startLatch.countDown();
+
+    for (int i = 0; i < threads.length; i++) {
+      threads[i].join();
+    }
+    DirectoryReader reader = DirectoryReader.open(writer);
+    IndexSearcher searcher = new IndexSearcher(reader);
+    for (String id : ids) {
+      TopDocs topDocs = searcher.search(new TermQuery(new Term("id", id)), 10);
+      if (updateSeveralDocs) {
+        assertEquals(2, topDocs.totalHits);
+        assertEquals(Math.abs(topDocs.scoreDocs[0].doc - topDocs.scoreDocs[1].doc), 1);
+      } else {
+        assertEquals(1, topDocs.totalHits);
+      }
+    }
+    writer.addDocument(new Document()); // add a dummy doc to trigger a segment here
+    writer.flush();
+    writer.forceMerge(1);
+    DirectoryReader oldReader = reader;
+    reader = DirectoryReader.openIfChanged(reader, writer);
+    if (reader != null) {
+      oldReader.close();
+      assertNotSame(oldReader, reader);
+    } else {
+      reader = oldReader;
+    }
+    assertEquals(1, reader.leaves().size());
+    LeafReaderContext leafReaderContext = reader.leaves().get(0);
+    LeafReader leafReader = leafReaderContext.reader();
+    searcher = new IndexSearcher(new FilterLeafReader(leafReader) {
+      @Override
+      public CacheHelper getCoreCacheHelper() {
+        return leafReader.getCoreCacheHelper();
+      }
+
+      @Override
+      public CacheHelper getReaderCacheHelper() {
+        return leafReader.getReaderCacheHelper();
+      }
+
+      @Override
+      public Bits getLiveDocs() {
+        return null;
+      }
+
+      @Override
+      public int numDocs() {
+        return maxDoc();
+      }
+    });
+    TopDocs seq_id = searcher.search(IntPoint.newRangeQuery("seq_id", seqIds.intValue() -
50, Integer.MAX_VALUE), 10);
+    assertTrue(seq_id.totalHits + " hits", seq_id.totalHits >= 50);
+    searcher = new IndexSearcher(reader);
+    for (String id : ids) {
+      if (updateSeveralDocs) {
+        assertEquals(2, searcher.search(new TermQuery(new Term("id", id)), 10).totalHits);
+      } else {
+        assertEquals(1, searcher.search(new TermQuery(new Term("id", id)), 10).totalHits);
+      }
+    }
+    IOUtils.close(reader, writer, dir);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ecc17f90/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java b/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
index b08a85d..e6c91b8 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
@@ -73,6 +73,7 @@ public class TestStressNRT extends LuceneTestCase {
     final int ndocs = atLeast(50);
     final int nWriteThreads = TestUtil.nextInt(random(), 1, TEST_NIGHTLY ? 10 : 5);
     final int maxConcurrentCommits = TestUtil.nextInt(random(), 1, TEST_NIGHTLY ? 10 : 5);
  // number of committers at a time... needed if we want to avoid commit errors due to exceeding
the max
+    final boolean useSoftDeletes = random().nextInt(10) < 3;
     
     final boolean tombstones = random().nextBoolean();
 
@@ -106,10 +107,10 @@ public class TestStressNRT extends LuceneTestCase {
 
     Directory dir = newMaybeVirusCheckingDirectory();
 
-    final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new
MockAnalyzer(random())));
+    final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new
MockAnalyzer(random())), useSoftDeletes);
     writer.setDoRandomForceMergeAssert(false);
     writer.commit();
-    reader = DirectoryReader.open(dir);
+    reader = useSoftDeletes ? writer.getReader() : DirectoryReader.open(dir);
 
     for (int i=0; i<nWriteThreads; i++) {
       Thread thread = new Thread("WRITER"+i) {
@@ -135,7 +136,7 @@ public class TestStressNRT extends LuceneTestCase {
                   }
 
                   DirectoryReader newReader;
-                  if (rand.nextInt(100) < softCommitPercent) {
+                  if (rand.nextInt(100) < softCommitPercent || useSoftDeletes) {
                     // assertU(h.commit("softCommit","true"));
                     if (random().nextBoolean()) {
                       if (VERBOSE) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ecc17f90/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java
b/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java
index 0574e70..0318109 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java
@@ -67,7 +67,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(makeIDField("id0", 100));
     w.addDocument(doc);
@@ -192,7 +192,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     int minItemsInBlock = TestUtil.nextInt(random(), 2, 50);
     int maxItemsInBlock = 2*(minItemsInBlock-1) + random().nextInt(50);
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat(minItemsInBlock,
maxItemsInBlock)));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     //IndexWriter w = new IndexWriter(dir, iwc);
     int numDocs = atLeast(1000);
     Map<String,Long> idValues = new HashMap<String,Long>();
@@ -359,7 +359,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(makeIDField("id", 17));
     w.addDocument(doc);
@@ -415,7 +415,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(makeIDField("id", 17));
     w.addDocument(doc);
@@ -432,7 +432,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(makeIDField("id", 17));
     w.addDocument(doc);
@@ -460,7 +460,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
       };
     IndexWriterConfig iwc = newIndexWriterConfig(a);
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(newTextField("id", "id", Field.Store.NO));
     expectThrows(IllegalArgumentException.class, () -> {
@@ -476,7 +476,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(newStringField("id", "id", Field.Store.NO));
     expectThrows(IllegalArgumentException.class, () -> {
@@ -493,7 +493,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(new StringAndPayloadField("id", "id", new BytesRef("foo")));
     expectThrows(IllegalArgumentException.class, () -> {
@@ -509,7 +509,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(makeIDField("id", 17));
     w.addDocument(doc);
@@ -529,7 +529,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
 
     FieldType ft = new FieldType(StringAndPayloadField.TYPE);
@@ -555,7 +555,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     doc.add(makeIDField("id", 17));
     doc.add(makeIDField("id", 17));
@@ -572,7 +572,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     // -1
     doc.add(new StringAndPayloadField("id", "id", new BytesRef(new byte[] {(byte)0xff, (byte)0xff,
(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff})));
@@ -590,7 +590,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
     Document doc = new Document();
     // Long.MAX_VALUE:
     doc.add(new StringAndPayloadField("id", "id", new BytesRef(new byte[] {(byte)0x7f, (byte)0xff,
(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff})));
@@ -610,7 +610,7 @@ public class TestIDVersionPostingsFormat extends LuceneTestCase {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    final RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    final RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
 
     IDSource idsSource = getRandomIDs();
     int numIDs = atLeast(100);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ecc17f90/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
index f4abb54..e02164b 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingLiveDocsFormat.java
@@ -82,7 +82,7 @@ public class AssertingLiveDocsFormat extends LiveDocsFormat {
         deletedCount++;
       }
     }
-    assert deletedCount == expectedDeleteCount;
+    assert deletedCount == expectedDeleteCount : "deleted: " + deletedCount + " != expected:
" + expectedDeleteCount;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ecc17f90/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
index aa4da54..b82df68 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
@@ -18,12 +18,14 @@ package org.apache.lucene.index;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.Random;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
@@ -41,13 +43,14 @@ import org.apache.lucene.util.TestUtil;
 
 public class RandomIndexWriter implements Closeable {
 
-  public IndexWriter w;
+  public final IndexWriter w;
   private final Random r;
   int docCount;
   int flushAt;
   private double flushAtFactor = 1.0;
   private boolean getReaderCalled;
   private final Analyzer analyzer; // only if WE created it (then we close it)
+  private final double softDeletesRatio;
 
   /** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test
points) */
   public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random
r) throws IOException {
@@ -94,7 +97,7 @@ public class RandomIndexWriter implements Closeable {
 
   /** create a RandomIndexWriter with a random config: Uses MockAnalyzer */
   public RandomIndexWriter(Random r, Directory dir) throws IOException {
-    this(r, dir, LuceneTestCase.newIndexWriterConfig(r, new MockAnalyzer(r)), true);
+    this(r, dir, LuceneTestCase.newIndexWriterConfig(r, new MockAnalyzer(r)), true, r.nextBoolean());
   }
   
   /** create a RandomIndexWriter with a random config */
@@ -104,12 +107,23 @@ public class RandomIndexWriter implements Closeable {
   
   /** create a RandomIndexWriter with the provided config */
   public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException
{
-    this(r, dir, c, false);
+    this(r, dir, c, false, r.nextBoolean());
+  }
+
+  /** create a RandomIndexWriter with the provided config */
+  public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean useSoftDeletes)
throws IOException {
+    this(r, dir, c, false, useSoftDeletes);
   }
       
-  private RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean closeAnalyzer)
throws IOException {
+  private RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean closeAnalyzer,
boolean useSoftDeletes) throws IOException {
     // TODO: this should be solved in a different way; Random should not be shared (!).
     this.r = new Random(r.nextLong());
+    if (useSoftDeletes) {
+      c.setSoftDeletesField("___soft_deletes");
+      softDeletesRatio = 1.d / (double)1 + r.nextInt(10);
+    } else {
+      softDeletesRatio = 0d;
+    }
     w = mockIndexWriter(dir, c, r);
     flushAt = TestUtil.nextInt(r, 10, 1000);
     if (closeAnalyzer) {
@@ -218,49 +232,39 @@ public class RandomIndexWriter implements Closeable {
 
   public long updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>>
docs) throws IOException {
     LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
-    long seqNo = w.updateDocuments(delTerm, docs);
+    long seqNo;
+    if (useSoftDeletes()) {
+      seqNo = w.softUpdateDocuments(delTerm, docs, new NumericDocValuesField(w.getConfig().getSoftDeletesField(),
1));
+    } else {
+      seqNo = w.updateDocuments(delTerm, docs);
+    }
     maybeFlushOrCommit();
     return seqNo;
   }
 
+  private boolean useSoftDeletes() {
+    return r.nextDouble() < softDeletesRatio;
+  }
+
   /**
    * Updates a document.
    * @see IndexWriter#updateDocument(Term, Iterable)
    */
   public <T extends IndexableField> long updateDocument(Term t, final Iterable<T>
doc) throws IOException {
     LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
-    long seqNo;
-    if (r.nextInt(5) == 3) {
-      seqNo = w.updateDocuments(t, new Iterable<Iterable<T>>() {
-
-        @Override
-        public Iterator<Iterable<T>> iterator() {
-          return new Iterator<Iterable<T>>() {
-            boolean done;
-            
-            @Override
-            public boolean hasNext() {
-              return !done;
-            }
-
-            @Override
-            public void remove() {
-              throw new UnsupportedOperationException();
-            }
-
-            @Override
-            public Iterable<T> next() {
-              if (done) {
-                throw new IllegalStateException();
-              }
-              done = true;
-              return doc;
-            }
-          };
-        }
-        });
+    final long seqNo;
+    if (useSoftDeletes()) {
+      if (r.nextInt(5) == 3) {
+        seqNo = w.softUpdateDocuments(t, Arrays.asList(doc), new NumericDocValuesField(w.getConfig().getSoftDeletesField(),
1));
+      } else {
+        seqNo = w.softUpdateDocument(t, doc, new NumericDocValuesField(w.getConfig().getSoftDeletesField(),
1));
+      }
     } else {
-      seqNo = w.updateDocument(t, doc);
+      if (r.nextInt(5) == 3) {
+        seqNo = w.updateDocuments(t, Arrays.asList(doc));
+      } else {
+        seqNo = w.updateDocument(t, doc);
+      }
     }
     maybeFlushOrCommit();
 
@@ -377,7 +381,8 @@ public class RandomIndexWriter implements Closeable {
     if (r.nextInt(20) == 2) {
       doRandomForceMerge();
     }
-    if (!applyDeletions || r.nextBoolean()) {
+    if (!applyDeletions || r.nextBoolean() || w.getConfig().getSoftDeletesField() != null)
{
+      // if we have soft deletes we can't open from a directory
       if (LuceneTestCase.VERBOSE) {
         System.out.println("RIW.getReader: use NRT reader");
       }


Mime
View raw message