Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@lucene.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: mikemccand@apache.org
To: commits@lucene.apache.org
Date: Wed, 16 Nov 2016 15:37:56 -0000
Message-Id: <dfdf88a43ddd41e58d6482c72e07b739@git.apache.org>
Subject: [1/3] lucene-solr:branch_6x: LUCENE-7537: fix some 6.x backport issues
archived-at: Wed, 16 Nov 2016 15:37:58 -0000

Repository: lucene-solr
Updated Branches:
  refs/heads/branch_6x 696238118 -> 64b9eefaa


LUCENE-7537: fix some 6.x backport issues


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/64b9eefa
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/64b9eefa
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/64b9eefa

Branch: refs/heads/branch_6x
Commit: 64b9eefaa931b4fc8b2345e2307eff4a317e3450
Parents: e357f95
Author: Mike McCandless <mikemccand@apache.org>
Authored: Wed Nov 16 10:35:50 2016 -0500
Committer: Mike McCandless <mikemccand@apache.org>
Committed: Wed Nov 16 10:37:02 2016 -0500

----------------------------------------------------------------------
 .../apache/lucene/codecs/DocValuesConsumer.java | 22 +++++++++++----
 .../org/apache/lucene/index/MultiDocValues.java | 29 +++++++++++++++++++-
 .../org/apache/lucene/index/MultiSorter.java    | 11 +++++---
 .../apache/lucene/index/TestIndexSorting.java   | 27 ++++++++++++++----
 .../asserting/AssertingDocValuesFormat.java     |  2 +-
 5 files changed, 74 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64b9eefa/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
index 427b520..defe438 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
@@ -221,21 +221,26 @@ public abstract class DocValuesConsumer implements Closeable {
           mergeSortedSetField(mergeFieldInfo, mergeState, toMerge);
         } else if (type == DocValuesType.SORTED_NUMERIC) {
           List<SortedNumericDocValues> toMerge = new ArrayList<>();
+          List<SortedNumericDocValues> toMerge2 = new ArrayList<>();
           for (int i=0;i<mergeState.docValuesProducers.length;i++) {
             SortedNumericDocValues values = null;
+            SortedNumericDocValues values2 = null;
             DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
             if (docValuesProducer != null) {
               FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
               if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED_NUMERIC) {
                 values = docValuesProducer.getSortedNumeric(fieldInfo);
+                values2 = docValuesProducer.getSortedNumeric(fieldInfo);
               }
             }
             if (values == null) {
               values = DocValues.emptySortedNumeric(mergeState.maxDocs[i]);
+              values2 = values;
             }
             toMerge.add(values);
+            toMerge2.add(values2);
           }
-          mergeSortedNumericField(mergeFieldInfo, mergeState, toMerge);
+          mergeSortedNumericField(mergeFieldInfo, mergeState, toMerge, toMerge2);
         } else {
           throw new AssertionError("type=" + type);
         }
@@ -445,6 +450,11 @@ public abstract class DocValuesConsumer implements Closeable {
         return docID;
       }
     }
+
+    @Override
+    public String toString() {
+      return "SortedNumericDocValuesSub values=" + values + " docID=" + docID + " mappedDocID=" + mappedDocID;
+    }
   }
 
   /**
@@ -452,8 +462,10 @@ public abstract class DocValuesConsumer implements Closeable {
    * <p>
    * The default implementation calls {@link #addSortedNumericField}, passing
    * iterables that filter deleted documents.
+   * <p>
+   * We require two <code>toMerge</code> lists because we need to separately iterate the values for each segment concurrently.
    */
-  public void mergeSortedNumericField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedNumericDocValues> toMerge) throws IOException {
+  public void mergeSortedNumericField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedNumericDocValues> toMerge, List<SortedNumericDocValues> toMerge2) throws IOException {
     
     addSortedNumericField(fieldInfo,
         // doc -> value count
@@ -514,9 +526,9 @@ public abstract class DocValuesConsumer implements Closeable {
           public Iterator<Number> iterator() {
             // We must make a new DocIDMerger for each iterator:
             List<SortedNumericDocValuesSub> subs = new ArrayList<>();
-            assert mergeState.docMaps.length == toMerge.size();
-            for(int i=0;i<toMerge.size();i++) {
-              subs.add(new SortedNumericDocValuesSub(mergeState.docMaps[i], toMerge.get(i), mergeState.maxDocs[i]));
+            assert mergeState.docMaps.length == toMerge2.size();
+            for(int i=0;i<toMerge2.size();i++) {
+              subs.add(new SortedNumericDocValuesSub(mergeState.docMaps[i], toMerge2.get(i), mergeState.maxDocs[i]));
             }
 
             final DocIDMerger<SortedNumericDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64b9eefa/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
index af4dcfc..8fa4a59 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
@@ -143,7 +143,7 @@ public class MultiDocValues {
       };
     }
   }
-  
+
   /** Returns a Bits for a reader's docsWithField (potentially merging on-the-fly) 
    * <p>
    * This is a slow way to access this bitset. Instead, access them per-segment
@@ -330,6 +330,33 @@ public class MultiDocValues {
       return new MultiSortedDocValues(values, starts, mapping);
     }
   }
+
+  /** Expert: returns a SortedDocValues from an array of leaf reader's sorted doc values (potentially doing extremely slow things).
+   * <p>
+   * This is an extremely slow way to access sorted values. Instead, access them per-segment
+   * with {@link LeafReader#getSortedDocValues(String)}
+   * </p>
+   */
+  public static SortedDocValues getSortedValues(IndexReader r, final SortedDocValues[] leafValues, final int[] docStarts) throws IOException {
+    final List<LeafReaderContext> leaves = r.leaves();
+    final int size = leaves.size();
+
+    if (leafValues.length != size) {
+      throw new IllegalArgumentException("leafValues must match the number of leaves; got leafValues.length=" + leafValues.length + " vs leaves.size()=" + leaves.size());
+    }
+    if (docStarts.length != size+1) {
+      throw new IllegalArgumentException("docStarts must match the number of leaves, plus one; got docStarts.length=" + docStarts.length + " vs leaves.size()=" + leaves.size());
+    }
+    
+    if (leafValues.length == 0) {
+      return null;
+    } else if (leafValues.length == 1) {
+      return leafValues[0];
+    }
+
+    OrdinalMap mapping = OrdinalMap.build(r.getCoreCacheKey(), leafValues, PackedInts.DEFAULT);
+    return new MultiSortedDocValues(leafValues, docStarts, mapping);
+  }
   
   /** Returns a SortedSetDocValues for a reader's docvalues (potentially doing extremely slow things).
    * <p>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64b9eefa/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index 7c2c3be..4e461e3 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -128,18 +128,21 @@ final class MultiSorter {
     final int reverseMul = sortField.getReverse() ? -1 : 1;
     final SortField.Type sortType = Sorter.getSortFieldType(sortField);
 
-    switch(sortField.getType()) {
+    switch(sortType) {
 
     case STRING:
       {
         // this uses the efficient segment-local ordinal map:
         MultiReader multiReader = new MultiReader(readers.toArray(new LeafReader[readers.size()]));
-        final SortedDocValues sorted = MultiDocValues.getSortedValues(multiReader, sortField.getField());
-        final int[] docStarts = new int[readers.size()];
+        final int[] docStarts = new int[readers.size()+1];
         List<LeafReaderContext> leaves = multiReader.leaves();
+        final SortedDocValues[] leafValues = new SortedDocValues[readers.size()];
         for(int i=0;i<readers.size();i++) {
+          leafValues[i] = Sorter.getOrWrapSorted(readers.get(i), sortField);
           docStarts[i] = leaves.get(i).docBase;
         }
+        docStarts[readers.size()] = multiReader.maxDoc();
+        final SortedDocValues sorted = MultiDocValues.getSortedValues(multiReader, leafValues, docStarts);
         final int missingOrd;
         if (sortField.getMissingValue() == SortField.STRING_LAST) {
           missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
@@ -176,7 +179,7 @@ final class MultiSorter {
         if (sortField.getMissingValue() != null) {
           missingValue = (Long) sortField.getMissingValue();
         } else {
-          missingValue = 0;
+          missingValue = 0l;
         }
 
         return new CrossReaderComparator() {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64b9eefa/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index e25558d..ae8756b 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -411,6 +411,9 @@ public class TestIndexSorting extends LuceneTestCase {
     assertFalse(docsWithField.get(0));
     assertEquals(7, values.get(1));
     assertEquals(18, values.get(2));
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingMultiValuedLongFirst() throws Exception {
@@ -1016,9 +1019,9 @@ public class TestIndexSorting extends LuceneTestCase {
     LeafReader leaf = getOnlyLeafReader(r);
     assertEquals(3, leaf.maxDoc());
     NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(-1.0, Double.longBitsToDouble(values.get(0)), 0.0);
-    assertEquals(7.0, Double.longBitsToDouble(values.get(1)), 0.0);
-    assertEquals(18.0, Double.longBitsToDouble(values.get(2)), 0.0);
+    assertEquals(-1.0, Float.intBitsToFloat((int) values.get(0)), 0.0);
+    assertEquals(7.0, Float.intBitsToFloat((int) values.get(1)), 0.0);
+    assertEquals(18.0, Float.intBitsToFloat((int) values.get(2)), 0.0);
     r.close();
     w.close();
     dir.close();
@@ -1298,11 +1301,21 @@ public class TestIndexSorting extends LuceneTestCase {
     IndexWriter w = new IndexWriter(dir, iwc);
     final int numDocs = atLeast(1000);
     final FixedBitSet deleted = new FixedBitSet(numDocs);
+    if (VERBOSE) {
+      System.out.println("TEST: " + numDocs + " docs");
+    }
     for (int i = 0; i < numDocs; ++i) {
       Document doc = new Document();
       int num = random().nextInt(10);
+      if (VERBOSE) {
+        System.out.println("doc id=" + i + " count=" + num);
+      }
       for (int j = 0; j < num; j++) {
-        doc.add(new SortedNumericDocValuesField("foo", random().nextInt(2000)));
+        int n = random().nextInt(2000);
+        if (VERBOSE) {
+          System.out.println("  " + n);
+        }
+        doc.add(new SortedNumericDocValuesField("foo", n));
       }
       doc.add(new StringField("id", Integer.toString(i), Store.YES));
       doc.add(new NumericDocValuesField("id", i));
@@ -1315,6 +1328,9 @@ public class TestIndexSorting extends LuceneTestCase {
         final int id = TestUtil.nextInt(random(), 0, i);
         deleted.set(id);
         w.deleteDocuments(new Term("id", Integer.toString(id)));
+        if (VERBOSE) {
+          System.out.println("  delete doc id=" + id);
+        }
       }
     }
 
@@ -1329,8 +1345,7 @@ public class TestIndexSorting extends LuceneTestCase {
       } else {
         assertEquals(1, topDocs.totalHits);
         NumericDocValues values = MultiDocValues.getNumericValues(reader, "id");
-        assertEquals(topDocs.scoreDocs[0].doc, values.advance(topDocs.scoreDocs[0].doc));
-        assertEquals(i, values.longValue());
+        assertEquals(i, MultiDocValues.getNumericValues(reader, "id").get(topDocs.scoreDocs[0].doc));
         Document document = reader.document(topDocs.scoreDocs[0].doc);
         assertEquals(Integer.toString(i), document.get("id"));
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64b9eefa/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
index e0af9a1..0a54e2e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
@@ -149,7 +149,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
           Number next = valueIterator.next();
           assert next != null;
           long nextValue = next.longValue();
-          assert nextValue >= previous;
+          assert nextValue >= previous: "nextValue=" + nextValue + " vs previous=" + previous;
           previous = nextValue;
         }
       }