lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cpoersc...@apache.org
Subject [10/21] lucene-solr git commit: SOLR-8285: Ensure the /export handler works with NULL field values
Date Tue, 02 Feb 2016 09:53:22 GMT
SOLR-8285: Ensure the /export handler works with NULL field values


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e20820a0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e20820a0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e20820a0

Branch: refs/heads/master-solr-8621
Commit: e20820a04d1aff5ccde64b7183308454bda62fef
Parents: a4d1586
Author: jbernste <jbernste@apache.org>
Authored: Sun Jan 31 13:18:48 2016 -0500
Committer: jbernste <jbernste@apache.org>
Committed: Sun Jan 31 13:21:10 2016 -0500

----------------------------------------------------------------------
 .../solr/response/SortingResponseWriter.java    | 100 +++++++++++++------
 .../response/TestSortingResponseWriter.java     |   5 +-
 .../solrj/io/stream/StreamExpressionTest.java   |  63 ++++++++++++
 3 files changed, 135 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e20820a0/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java b/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
index 4b9f89f..6b6d36c 100644
--- a/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
@@ -22,7 +22,9 @@ import java.io.PrintWriter;
 import java.io.Writer;
 import java.lang.invoke.MethodHandles;
 import java.util.List;
+import java.util.ArrayList;
 
+import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.MultiDocValues;
@@ -239,13 +241,11 @@ public class SortingResponseWriter implements QueryResponseWriter {
     FixedBitSet set = sets[ord];
     set.clear(sortDoc.docId);
     LeafReaderContext context = leaves.get(ord);
-    boolean needsComma = false;
+    int fieldIndex = 0;
     for(FieldWriter fieldWriter : fieldWriters) {
-      if(needsComma) {
-        out.write(',');
+      if(fieldWriter.write(sortDoc.docId, context.reader(), out, fieldIndex)){
+        ++fieldIndex;
       }
-      fieldWriter.write(sortDoc.docId, context.reader(), out);
-      needsComma = true;
     }
   }
 
@@ -827,7 +827,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
     }
 
     public void setNextReader(LeafReaderContext context) throws IOException {
-      this.vals = context.reader().getNumericDocValues(field);
+      this.vals = DocValues.getNumeric(context.reader(), field);
     }
 
     public void setCurrentValue(int docId) {
@@ -905,7 +905,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
     }
 
     public void setNextReader(LeafReaderContext context) throws IOException {
-      this.vals = context.reader().getNumericDocValues(field);
+      this.vals = DocValues.getNumeric(context.reader(), field);
     }
 
     public void setCurrentValue(int docId) {
@@ -984,7 +984,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
     }
 
     public void setNextReader(LeafReaderContext context) throws IOException {
-      this.vals = context.reader().getNumericDocValues(field);
+      this.vals = DocValues.getNumeric(context.reader(), field);
     }
 
     public void setCurrentValue(int docId) {
@@ -1061,7 +1061,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
     }
 
     public void setNextReader(LeafReaderContext context) throws IOException {
-      this.vals = context.reader().getNumericDocValues(field);
+      this.vals = DocValues.getNumeric(context.reader(), field);
     }
 
     public void setCurrentValue(int docId) {
@@ -1193,7 +1193,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
   }
 
   protected abstract class FieldWriter {
-    public abstract void write(int docId, LeafReader reader, Writer out) throws IOException;
+    public abstract boolean write(int docId, LeafReader reader, Writer out, int fieldIndex)
throws IOException;
   }
 
   class IntFieldWriter extends FieldWriter {
@@ -1203,14 +1203,18 @@ public class SortingResponseWriter implements QueryResponseWriter
{
       this.field = field;
     }
 
-    public void write(int docId, LeafReader reader, Writer out) throws IOException {
-      NumericDocValues vals = reader.getNumericDocValues(this.field);
+    public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws
IOException {
+      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
       int val = (int)vals.get(docId);
-       out.write('"');
-       out.write(this.field);
-       out.write('"');
-       out.write(':');
-       out.write(Integer.toString(val));
+      if(fieldIndex>0) {
+        out.write(',');
+      }
+      out.write('"');
+      out.write(this.field);
+      out.write('"');
+      out.write(':');
+      out.write(Integer.toString(val));
+      return true;
     }
   }
 
@@ -1225,18 +1229,30 @@ public class SortingResponseWriter implements QueryResponseWriter
{
       this.fieldType = fieldType;
       this.numeric = numeric;
     }
-
-    public void write(int docId, LeafReader reader, Writer out) throws IOException {
-      SortedSetDocValues vals = reader.getSortedSetDocValues(this.field);
+    public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws
IOException {
+      SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
       vals.setDocument(docId);
+      List<Long> ords = new ArrayList();
+      long o = -1;
+      while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+        ords.add(o);
+      }
+
+      if(ords.size()== 0) {
+        return false;
+      }
+
+
+      if(fieldIndex>0) {
+        out.write(',');
+      }
       out.write('"');
       out.write(this.field);
       out.write('"');
       out.write(':');
       out.write('[');
       int v = 0;
-      long ord = -1;
-      while((ord = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+      for(long ord : ords) {
         BytesRef ref = vals.lookupOrd(ord);
         fieldType.indexedToReadable(ref, cref);
         if(v > 0) {
@@ -1255,6 +1271,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
         ++v;
       }
       out.write("]");
+      return true;
     }
   }
 
@@ -1265,14 +1282,18 @@ public class SortingResponseWriter implements QueryResponseWriter
{
       this.field = field;
     }
 
-    public void write(int docId, LeafReader reader, Writer out) throws IOException {
-      NumericDocValues vals = reader.getNumericDocValues(this.field);
+    public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws
IOException {
+      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
       long val = vals.get(docId);
+      if(fieldIndex > 0) {
+        out.write(',');
+      }
       out.write('"');
       out.write(this.field);
       out.write('"');
       out.write(':');
       out.write(Long.toString(val));
+      return true;
     }
   }
 
@@ -1283,14 +1304,18 @@ public class SortingResponseWriter implements QueryResponseWriter
{
       this.field = field;
     }
 
-    public void write(int docId, LeafReader reader, Writer out) throws IOException {
-      NumericDocValues vals = reader.getNumericDocValues(this.field);
+    public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws
IOException {
+      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
       int val = (int)vals.get(docId);
+      if(fieldIndex > 0) {
+        out.write(',');
+      }
       out.write('"');
       out.write(this.field);
       out.write('"');
       out.write(':');
       out.write(Float.toString(Float.intBitsToFloat(val)));
+      return true;
     }
   }
 
@@ -1301,14 +1326,18 @@ public class SortingResponseWriter implements QueryResponseWriter
{
       this.field = field;
     }
 
-    public void write(int docId, LeafReader reader, Writer out) throws IOException {
-      NumericDocValues vals = reader.getNumericDocValues(this.field);
+    public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws
IOException {
+      NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+      if(fieldIndex > 0) {
+        out.write(',');
+      }
       long val = vals.get(docId);
       out.write('"');
       out.write(this.field);
       out.write('"');
       out.write(':');
       out.write(Double.toString(Double.longBitsToDouble(val)));
+      return true;
     }
   }
 
@@ -1322,10 +1351,18 @@ public class SortingResponseWriter implements QueryResponseWriter
{
       this.fieldType = fieldType;
     }
 
-    public void write(int docId, LeafReader reader, Writer out) throws IOException {
-      SortedDocValues vals = reader.getSortedDocValues(this.field);
-      BytesRef ref = vals.get(docId);
+    public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws
IOException {
+      SortedDocValues vals = DocValues.getSorted(reader, this.field);
+      int ord = vals.getOrd(docId);
+      if(ord == -1) {
+        return false;
+      }
+
+      BytesRef ref = vals.lookupOrd(ord);
       fieldType.indexedToReadable(ref, cref);
+      if(fieldIndex > 0) {
+        out.write(',');
+      }
       out.write('"');
       out.write(this.field);
       out.write('"');
@@ -1333,6 +1370,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
       out.write('"');
       writeStr(cref.toString(), out);
       out.write('"');
+      return true;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e20820a0/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
index 0e7f9b1..bb432ec 100644
--- a/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
@@ -114,7 +114,8 @@ public class TestSortingResponseWriter extends SolrTestCaseJ4 {
 
     //Test null value string:
     s = h.query(req("q", "id:7", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv",
"sort", "intdv asc"));
-    assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1,
\"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"stringdv\":\"\",\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
+
+    assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1,
\"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
 
     //Test multiValue docValues output
     s = h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m",
"sort", "intdv asc"));
@@ -122,7 +123,7 @@ public class TestSortingResponseWriter extends SolrTestCaseJ4 {
 
     //Test multiValues docValues output with nulls
     s =  h.query(req("q", "id:7", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m",
"sort", "intdv asc"));
-    assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1,
\"docs\":[{\"intdv_m\":[],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[]}]}}");
+    assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1,
\"docs\":[{\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434]}]}}");
 
     //Test single sort param is working
     s =  h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e20820a0/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 69ef6e8..33ee767 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -136,6 +136,7 @@ public class StreamExpressionTest extends AbstractFullDistribZkTestBase
{
     testUniqueStream();
     testRollupStream();
     testStatsStream();
+    testNulls();
     testDaemonStream();
     testParallelUniqueStream();
     testParallelReducerStream();
@@ -303,6 +304,68 @@ public class StreamExpressionTest extends AbstractFullDistribZkTestBase
{
     del("*:*");
     commit();
   }
+
+
+  private void testNulls() throws Exception {
+
+    indexr(id, "0",                  "a_i", "1", "a_f", "0", "s_multi", "aaa", "s_multi",
"bbb", "i_multi", "100", "i_multi", "200");
+    indexr(id, "2", "a_s", "hello2", "a_i", "3", "a_f", "0");
+    indexr(id, "3", "a_s", "hello3", "a_i", "4", "a_f", "3");
+    indexr(id, "4", "a_s", "hello4",             "a_f", "4");
+    indexr(id, "1", "a_s", "hello1", "a_i", "2", "a_f", "1");
+    commit();
+
+    StreamExpression expression;
+    TupleStream stream;
+    List<Tuple> tuples;
+    Tuple tuple;
+    StreamFactory factory = new StreamFactory()
+        .withCollectionZkHost("collection1", zkServer.getZkAddress())
+        .withFunctionName("search", CloudSolrStream.class);
+    // Basic test
+    expression = StreamExpressionParser.parse("search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f,
s_multi, i_multi\", qt=\"/export\", sort=\"a_i asc\")");
+    stream = new CloudSolrStream(expression, factory);
+    tuples = getTuples(stream);
+
+    assert(tuples.size() == 5);
+    assertOrder(tuples, 4, 0, 1, 2, 3);
+
+    tuple = tuples.get(0);
+    assertTrue("hello4".equals(tuple.getString("a_s")));
+    assertNull(tuple.get("s_multi"));
+    assertNull(tuple.get("i_multi"));
+    assertEquals(0L, (long)tuple.getLong("a_i"));
+
+
+    tuple = tuples.get(1);
+    assertNull(tuple.get("a_s"));
+    List<String> strings = tuple.getStrings("s_multi");
+    assertNotNull(strings);
+    assertEquals("aaa", strings.get(0));
+    assertEquals("bbb", strings.get(1));
+    List<Long> longs = tuple.getLongs("i_multi");
+    assertNotNull(longs);
+
+    //test sort (asc) with null string field. Null should sort to the top.
+    expression = StreamExpressionParser.parse("search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f,
s_multi, i_multi\", qt=\"/export\", sort=\"a_s asc\")");
+    stream = new CloudSolrStream(expression, factory);
+    tuples = getTuples(stream);
+
+    assert(tuples.size() == 5);
+    assertOrder(tuples, 0, 1, 2, 3, 4);
+
+    //test sort(desc) with null string field.  Null should sort to the bottom.
+    expression = StreamExpressionParser.parse("search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f,
s_multi, i_multi\", qt=\"/export\", sort=\"a_s desc\")");
+    stream = new CloudSolrStream(expression, factory);
+    tuples = getTuples(stream);
+
+    assert(tuples.size() == 5);
+    assertOrder(tuples, 4, 3, 2, 1, 0);
+
+    del("*:*");
+    commit();
+  }
+
   
   private void testMergeStream() throws Exception {
 


Mime
View raw message