lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From romseyg...@apache.org
Subject [2/4] lucene-solr:branch_7x: LUCENE-8045: ParallelLeafReader should correctly report FieldInfo.dvGen
Date Thu, 09 Nov 2017 14:03:15 GMT
LUCENE-8045: ParallelLeafReader should correctly report FieldInfo.dvGen


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c0ed9199
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c0ed9199
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c0ed9199

Branch: refs/heads/branch_7x
Commit: c0ed9199fae03142b4fe6adebdccec728158eb5b
Parents: 87e0565
Author: Alan Woodward <romseygeek@apache.org>
Authored: Thu Nov 9 13:31:20 2017 +0000
Committer: Alan Woodward <romseygeek@apache.org>
Committed: Thu Nov 9 14:02:49 2017 +0000

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  3 +++
 .../org/apache/lucene/index/FieldInfos.java     | 12 ++++++---
 .../apache/lucene/index/ParallelLeafReader.java |  2 +-
 .../lucene/index/TestParallelLeafReader.java    | 26 ++++++++++++++++++++
 4 files changed, 39 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0ed9199/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4875f9f..fea28a9 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -30,6 +30,9 @@ Bug Fixes
   Previously bogus numbers were used, and many similarities would
   completely degrade. (Robert Muir, Adrien Grand)
 
+* LUCENE-8045: ParallelLeafReader did not correctly report FieldInfo.dvGen
+  (Alan Woodward)
+
 Optimizations
 
 * LUCENE-8018: Smaller FieldInfos memory footprint by not retaining unnecessary

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0ed9199/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index a7dc0b8..3e68693 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -403,7 +403,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
    
     private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber,
                                           boolean storeTermVector,
-                                          boolean omitNorms, boolean storePayloads, IndexOptions
indexOptions, DocValuesType docValues,
+                                          boolean omitNorms, boolean storePayloads, IndexOptions
indexOptions,
+                                          DocValuesType docValues, long dvGen,
                                           int dimensionCount, int dimensionNumBytes) {
       assert assertNotFinished();
       if (docValues == null) {
@@ -417,7 +418,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
         // before then we'll get the same name and number,
         // else we'll allocate a new one:
         final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues,
dimensionCount, dimensionNumBytes);
-        fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
indexOptions, docValues, -1, new HashMap<>(), dimensionCount, dimensionNumBytes);
+        fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes);
         assert !byName.containsKey(fi.name);
         globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
         byName.put(fi.name, fi);
@@ -435,16 +436,21 @@ public class FieldInfos implements Iterable<FieldInfo> {
           }
 
           fi.setDocValuesType(docValues); // this will also perform the consistency check.
+          fi.setDocValuesGen(dvGen);
         }
       }
       return fi;
     }
 
     public FieldInfo add(FieldInfo fi) {
+      return add(fi, -1);
+    }
+
+    public FieldInfo add(FieldInfo fi, long dvGen) {
       // IMPORTANT - reuse the field number if possible for consistent field numbers across
segments
       return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(),
                                  fi.omitsNorms(), fi.hasPayloads(),
-                                 fi.getIndexOptions(), fi.getDocValuesType(),
+                                 fi.getIndexOptions(), fi.getDocValuesType(), dvGen,
                                  fi.getPointDimensionCount(), fi.getPointNumBytes());
     }
     

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0ed9199/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
index 74b3962..492b6e7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
@@ -129,7 +129,7 @@ public class ParallelLeafReader extends LeafReader {
       for (FieldInfo fieldInfo : readerFieldInfos) {
         // NOTE: first reader having a given field "wins":
         if (!fieldToReader.containsKey(fieldInfo.name)) {
-          builder.add(fieldInfo);
+          builder.add(fieldInfo, fieldInfo.getDocValuesGen());
           fieldToReader.put(fieldInfo.name, reader);
           // only add these if the reader responsible for that field name is the current:
           // TODO consider populating 1st leaf with vectors even if the field name has been
seen on a previous leaf

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0ed9199/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
index 35523f3..71110d3 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
@@ -23,6 +23,7 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.search.*;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.store.AlreadyClosedException;
@@ -371,4 +372,29 @@ public class TestParallelLeafReader extends LuceneTestCase {
     new ParallelLeafReader(false, getOnlyLeafReader(r2), getOnlyLeafReader(r1)).close();
     IOUtils.close(r1, dir1, r2, dir2);
   }
+
+  public void testWithDocValuesUpdates() throws Exception {
+    Directory dir1 = newDirectory();
+    IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
+    IndexWriter w1 = new IndexWriter(dir1, iwc1);
+    Document d = new Document();
+    d.add(newTextField("name", "billy", Field.Store.NO));
+    d.add(new NumericDocValuesField("age", 21));
+    w1.addDocument(d);
+    w1.commit();
+    w1.updateNumericDocValue(new Term("name", "billy"), "age", 22);
+    w1.close();
+
+    IndexReader r1 = DirectoryReader.open(dir1);
+    LeafReader lr = new ParallelLeafReader(false, getOnlyLeafReader(r1));
+
+    NumericDocValues dv = lr.getNumericDocValues("age");
+    assertEquals(0, dv.nextDoc());
+    assertEquals(22, dv.longValue());
+
+    assertEquals(1, lr.getFieldInfos().fieldInfo("age").getDocValuesGen());
+
+    IOUtils.close(lr, r1, dir1);
+
+  }
 }


Mime
View raw message