lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jim...@apache.org
Subject lucene-solr:branch_6_5: LUCENE-7791: Fixed index sorting to work with sparse numeric and binary docvalues field.
Date Thu, 20 Apr 2017 21:46:18 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_6_5 c357151db -> 7477dcd11


LUCENE-7791: Fixed index sorting to work with sparse numeric and binary docvalues field.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7477dcd1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7477dcd1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7477dcd1

Branch: refs/heads/branch_6_5
Commit: 7477dcd111b6950d4105623ad2cfe60faea463da
Parents: c357151
Author: Jim Ferenczi <jimczi@apache.org>
Authored: Thu Apr 20 23:29:35 2017 +0200
Committer: Jim Ferenczi <jimczi@apache.org>
Committed: Thu Apr 20 23:46:00 2017 +0200

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  3 +
 .../lucene/index/BinaryDocValuesWriter.java     |  6 +-
 .../lucene/index/NumericDocValuesWriter.java    | 14 ++--
 .../apache/lucene/index/TestIndexSorting.java   | 76 ++++++++++++++++++++
 4 files changed, 88 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7477dcd1/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 98f2dfa..710d744 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -20,6 +20,9 @@ Bug Fixes
   ArrayIndexOutOfBoundsException when byte blocks larger than 32 KB
   were added (Mike McCandless)
 
+* LUCENE-7791: Fixed index sorting to work with sparse numeric and binary docvalues field.
+  (Przemyslaw Szeremiota via Jim Ferenczi)
+
 Other
 
 * LUCENE-7763: Remove outdated comment in IndexWriterConfig.setIndexSort javadocs.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7477dcd1/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
index 0a59278..aa21d32 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
@@ -227,15 +227,15 @@ class BinaryDocValuesWriter extends DocValuesWriter {
         throw new NoSuchElementException();
       }
       final BytesRef v;
-      if (upto < size) {
-        int oldID = sortMap.newToOld(upto);
+      int oldID = sortMap.newToOld(upto);
+      if (oldID < values.size()) {
         int length = (int) values.get(oldID);
         long pos = starts[oldID];
         bytesIterator.setPosition(pos);
         value.grow(length);
         value.setLength(length);
         bytesIterator.readBytes(value.bytes(), 0, value.length());
-        if (docsWithField.get(upto)) {
+        if (docsWithField.get(oldID)) {
           v = value.get();
         } else {
           v = null;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7477dcd1/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
index a307afd..15105a5e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
@@ -108,7 +108,9 @@ class NumericDocValuesWriter extends DocValuesWriter {
 
   @Override
   Sorter.DocComparator getDocComparator(int numDoc, SortField sortField) throws IOException
{
-    return getDocComparator(sortField, sortField.getType(), (docID) -> docsWithField.get(docID),
(docID) -> finalValues.get(docID));
+    return getDocComparator(sortField, sortField.getType(),
+        (docID) -> docID < docsWithField.length() ? docsWithField.get(docID) : false,
+        (docID) -> finalValues.get(docID));
   }
 
   static Sorter.DocComparator getDocComparator(SortField sortField, SortField.Type sortType,
IntPredicate docsWithField, IntToLongFunction docValueFunction) {
@@ -252,13 +254,9 @@ class NumericDocValuesWriter extends DocValuesWriter {
         throw new NoSuchElementException();
       }
       Long value;
-      if (upto < size) {
-        int old = sortMap.newToOld(upto);
-        if (docsWithField.get(old)) {
-          value = values.get(old);
-        } else {
-          value = null;
-        }
+      int old = sortMap.newToOld(upto);
+      if (old < size && docsWithField.get(old)) {
+        value = values.get(old);
       } else {
         value = null;
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7477dcd1/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 1287dd2..897ab6d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -2332,4 +2332,80 @@ public class TestIndexSorting extends LuceneTestCase {
     }
     IOUtils.close(r, w, dir);
   }
+
+  public void testIndexSortWithSparseField() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("dense_int", SortField.Type.INT, true);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    for (int i = 0; i < 128; i++) {
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("dense_int", i));
+      if (i < 64) {
+        doc.add(new NumericDocValuesField("sparse_int", i));
+        doc.add(new BinaryDocValuesField("sparse_binary", new BytesRef(Integer.toString(i))));
+      }
+      w.addDocument(doc);
+    }
+    w.commit();
+    w.forceMerge(1);
+    DirectoryReader r = DirectoryReader.open(w);
+    assertEquals(1, r.leaves().size());
+    LeafReader leafReader = r.leaves().get(0).reader();
+    NumericDocValues denseValues = leafReader.getNumericDocValues("dense_int");
+    NumericDocValues sparseValues = leafReader.getNumericDocValues("sparse_int");
+    BinaryDocValues sparseBinaryValues = leafReader.getBinaryDocValues("sparse_binary");
+
+    Bits docsWithField = r.leaves().get(0).reader().getDocsWithField("sparse_int");
+    Bits docsWithBinaryField = r.leaves().get(0).reader().getDocsWithField("sparse_binary");
+    for(int docID = 0; docID < 128; docID++) {
+      assertEquals(127-docID, denseValues.get(docID));
+      if (docID >= 64) {
+        assertTrue(docsWithField.get(docID));
+        assertTrue(docsWithBinaryField.get(docID));
+        assertEquals(127-docID, sparseValues.get(docID));
+        assertEquals(new BytesRef(Integer.toString(127-docID)), sparseBinaryValues.get(docID));
+      } else {
+        assertFalse(docsWithField.get(docID));
+        assertFalse(docsWithBinaryField.get(docID));
+      }
+    }
+    IOUtils.close(r, w, dir);
+  }
+
+  public void testIndexSortOnSparseField() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("sparse", SortField.Type.INT, false);
+    sortField.setMissingValue(Integer.MIN_VALUE);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    for (int i = 0; i < 128; i++) {
+      Document doc = new Document();
+      if (i < 64) {
+        doc.add(new NumericDocValuesField("sparse", i));
+      }
+      w.addDocument(doc);
+    }
+    w.commit();
+    w.forceMerge(1);
+    DirectoryReader r = DirectoryReader.open(w);
+    assertEquals(1, r.leaves().size());
+    LeafReader leafReader = r.leaves().get(0).reader();
+    NumericDocValues sparseValues = leafReader.getNumericDocValues("sparse");
+    Bits docsWithField = r.leaves().get(0).reader().getDocsWithField("sparse");
+    for(int docID = 0; docID < 128; docID++) {
+      if (docID >= 64) {
+        assertTrue(docsWithField.get(docID));
+        assertEquals(docID-64, sparseValues.get(docID));
+      } else {
+        assertFalse(docsWithField.get(docID));
+      }
+    }
+    IOUtils.close(r, w, dir);
+  }
+
 }


Mime
View raw message