Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@lucene.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1618224 - in /lucene/dev/trunk/solr: CHANGES.txt
 core/src/java/org/apache/solr/response/SortingResponseWriter.java
 core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
Date: Fri, 15 Aug 2014 16:23:36 -0000
To: commits@lucene.apache.org
From: jbernste@apache.org
Message-Id: <20140815162336.CBB9B2388868@eris.apache.org>

Author: jbernste
Date: Fri Aug 15 16:23:36 2014
New Revision: 1618224

URL: http://svn.apache.org/r1618224
Log:
 SOLR-5244: Exporting Full Sorted Result Sets

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1618224&r1=1618223&r2=1618224&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Fri Aug 15 16:23:36 2014
@@ -193,6 +193,8 @@ New Features
   
 * SOLR-5656: Add autoAddReplicas feature for shared file systems. (Mark Miller, Gregory Chanan)
 
+* SOLR-5244: Exporting Full Sorted Result Sets (Erik Hatcher, Joel Bernstein)
+
 Bug Fixes
 ----------------------
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java?rev=1618224&r1=1618223&r2=1618224&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java Fri Aug 15 16:23:36 2014
@@ -17,7 +17,6 @@
     
 package org.apache.solr.response;
 
-import com.carrotsearch.hppc.IntArrayList;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.MultiDocValues;
@@ -48,15 +47,19 @@ import org.apache.solr.schema.StrField;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.search.SortSpec;
 import org.apache.solr.search.SyntaxError;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.Writer;
 import java.io.PrintWriter;
+import java.net.SocketException;
 import java.util.List;
 
 
 public class SortingResponseWriter implements QueryResponseWriter {
 
+  private final static Logger logger = LoggerFactory.getLogger(SortingResponseWriter.class);
 
   public void init(NamedList args) {
     /* NOOP */
@@ -157,16 +160,33 @@ public class SortingResponseWriter imple
 
       count += (outDocsIndex+1);
 
-      for(int i=outDocsIndex; i>=0; --i) {
-        SortDoc s = outDocs[i];
-        if(commaNeeded){writer.write(',');}
-        writer.write('{');
-        writeDoc(s, leaves, fieldWriters, sets, writer);
-        writer.write('}');
-        commaNeeded = true;
-        s.reset();
+      try {
+        for(int i=outDocsIndex; i>=0; --i) {
+          SortDoc s = outDocs[i];
+          if(commaNeeded){writer.write(',');}
+          writer.write('{');
+          writeDoc(s, leaves, fieldWriters, sets, writer);
+          writer.write('}');
+          commaNeeded = true;
+          s.reset();
+        }
+      } catch(Throwable e) {
+        Throwable ex = e;
+        while(ex != null) {
+          String m = ex.getMessage();
+          if(m != null && m.contains("Broken pipe")) {
+            logger.info("Early client disconnect during export");
+            return;
+          }
+          ex = ex.getCause();
+        }
+
+        if(e instanceof IOException) {
+          throw ((IOException)e);
+        } else {
+          throw new IOException(e);
+        }
       }
-      //total+=end-begin;
     }
 
     //System.out.println("Sort Time 2:"+Long.toString(total/1000000));
@@ -240,6 +260,8 @@ public class SortingResponseWriter imple
         } else {
           writers[i] = new StringFieldWriter(field, fieldType);
         }
+      } else {
+        throw new IOException("Export fields must either be one of the following types: int,float,long,double,string");
       }
     }
     return writers;
@@ -252,7 +274,13 @@ public class SortingResponseWriter imple
       SortField sf = sortFields[i];
       String field = sf.getField();
       boolean reverse = sf.getReverse();
-      FieldType ft = schema.getField(field).getType();
+      SchemaField schemaField = schema.getField(field);
+      FieldType ft = schemaField.getType();
+
+      if(!schemaField.hasDocValues()) {
+        throw new IOException(field+" must have DocValues to use this feature.");
+      }
+
       if(ft instanceof TrieIntField) {
         if(reverse) {
           sortValues[i] = new IntValue(field, new IntDesc());
@@ -285,6 +313,8 @@ public class SortingResponseWriter imple
         } else {
           sortValues[i] = new StringValue(vals, field, new IntAsc());
         }
+      } else {
+        throw new IOException("Sort fields must be one of the following types: int,float,long,double,string");
       }
     }
 
@@ -1060,7 +1090,12 @@ public class SortingResponseWriter imple
     }
 
     public void setCurrentValue(int docId) {
-      currentOrd = (int)globalOrds.get(currentVals.getOrd(docId));
+      int ord = currentVals.getOrd(docId);
+      if(ord < 0) {
+        currentOrd = -1;
+      } else {
+        currentOrd = (int)globalOrds.get(ord);
+      }
     }
 
     public void setCurrentValue(SortValue sv) {

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java?rev=1618224&r1=1618223&r2=1618224&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java Fri Aug 15 16:23:36 2014
@@ -48,6 +48,18 @@ public class TestSortingResponseWriter e
                  "stringdv_m", "liverpool",
                  "stringdv_m", "Everton"));
 
+    assertU(adoc("id","7",
+        "floatdv","2.1",
+        "intdv", "7",
+        "longdv", "323223232323",
+        "doubledv","2344.345",
+        "floatdv_m", "123.321",
+        "floatdv_m", "345.123",
+        "doubledv_m", "3444.222",
+        "doubledv_m", "23232.2",
+        "longdv_m", "43434343434",
+        "longdv_m", "343332"));
+
     assertU(commit());
     assertU(adoc("id","2", "floatdv","2.1", "intdv", "2", "stringdv", "hello world", "longdv", "323223232323","doubledv","2344.344"));
     assertU(commit());
@@ -80,18 +92,34 @@ public class TestSortingResponseWriter e
     String s =  h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
     assertEquals(s, "{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":1,\"stringdv\":\"hello world\",\"longdv\":323223232323,\"doubledv\":2344.345}]}");
 
+    //Test null value string:
+    s =  h.query(req("q", "id:7", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
+    assertEquals(s, "{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"stringdv\":\"\",\"longdv\":323223232323,\"doubledv\":2344.345}]}");
+
     //Test multiValue docValues output
     s =  h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
-    System.out.println(s);
     assertEquals(s, "{\"numFound\":1, \"docs\":[{\"intdv_m\":[100,250],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[\"Everton\",\"liverpool\",\"manchester city\"]}]}");
+
+    //Test multiValues docValues output with nulls
+    s =  h.query(req("q", "id:7", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
+    assertEquals(s, "{\"numFound\":1, \"docs\":[{\"intdv_m\":[],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[]}]}");
+
     //Test single sort param is working
     s =  h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));
-    System.out.println("Output:"+s);
     assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}");
 
     s =  h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv asc"));
     assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}");
 
+    // Test sort on String will null value. Null value should sort last on desc and first on asc.
+
+    s =  h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv desc"));
+    assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":7}]}");
+
+    s =  h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv asc"));
+    assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":7},{\"intdv\":1}]}");
+
+
     //Test multi-sort params
     s =  h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc"));
     assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}");
@@ -112,7 +140,6 @@ public class TestSortingResponseWriter e
     assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}");
 
     s =  h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
-    System.out.println("Results:"+s);
     assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}");
 
   }