lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbern...@apache.org
Subject [1/3] lucene-solr:branch_7x: SOLR-12795: Introduce 'rows' and 'offset' parameter in FacetStream
Date Wed, 07 Nov 2018 14:09:14 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_7x 8af39cf05 -> 19f4ebd28


SOLR-12795: Introduce 'rows' and 'offset' parameter in FacetStream


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/68ed57f7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/68ed57f7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/68ed57f7

Branch: refs/heads/branch_7x
Commit: 68ed57f796704006cc4e9fc7ce3a9e124d8b88ea
Parents: 8af39cf
Author: Joel Bernstein <jbernste@apache.org>
Authored: Tue Nov 6 11:22:51 2018 -0500
Committer: Joel Bernstein <jbernste@apache.org>
Committed: Wed Nov 7 09:05:09 2018 -0500

----------------------------------------------------------------------
 .../client/solrj/io/stream/FacetStream.java     | 351 ++++++++++++++++---
 .../solrj/io/stream/StreamExpressionTest.java   | 141 ++++++++
 .../stream/StreamExpressionToExpessionTest.java | 203 ++++++++++-
 3 files changed, 631 insertions(+), 64 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/68ed57f7/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
index 4564ba0..6923a3b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
@@ -64,13 +64,21 @@ public class FacetStream extends TupleStream implements Expressible  {
 
   private Bucket[] buckets;
   private Metric[] metrics;
+  private int rows;
+  private int offset;
+  private int overfetch;
   private int bucketSizeLimit;
+  private boolean refine;
+  private String method;
   private FieldComparator[] bucketSorts;
   private List<Tuple> tuples = new ArrayList<Tuple>();
   private int index;
   private String zkHost;
   private ModifiableSolrParams params;
   private String collection;
+  private boolean resortNeeded;
+  private boolean serializeBucketSizeLimit;
+
   protected transient SolrClientCache cache;
   protected transient CloudSolrClient cloudSolrClient;
 
@@ -81,7 +89,11 @@ public class FacetStream extends TupleStream implements Expressible  {
                      Metric[] metrics,
                      FieldComparator[] bucketSorts,
                      int bucketSizeLimit) throws IOException {
-    init(collection, params, buckets, bucketSorts, metrics, bucketSizeLimit, zkHost);
+
+    if(bucketSizeLimit == -1) {
+      bucketSizeLimit = Integer.MAX_VALUE;
+    }
+    init(collection, params, buckets, bucketSorts, metrics, bucketSizeLimit,0, bucketSizeLimit,
false, null, true, 0, zkHost);
   }
   
   public FacetStream(StreamExpression expression, StreamFactory factory) throws IOException{
  
@@ -91,9 +103,14 @@ public class FacetStream extends TupleStream implements Expressible  {
     StreamExpressionNamedParameter bucketExpression = factory.getNamedOperand(expression,
"buckets");
     StreamExpressionNamedParameter bucketSortExpression = factory.getNamedOperand(expression,
"bucketSorts");
     List<StreamExpression> metricExpressions = factory.getExpressionOperandsRepresentingTypes(expression,
Expressible.class, Metric.class);
-    StreamExpressionNamedParameter limitExpression = factory.getNamedOperand(expression,
"bucketSizeLimit");
+    StreamExpressionNamedParameter bucketLimitExpression = factory.getNamedOperand(expression,
"bucketSizeLimit");
     StreamExpressionNamedParameter zkHostExpression = factory.getNamedOperand(expression,
"zkHost");
-    
+    StreamExpressionNamedParameter rowsExpression = factory.getNamedOperand(expression, "rows");
+    StreamExpressionNamedParameter offsetExpression = factory.getNamedOperand(expression,
"offset");
+    StreamExpressionNamedParameter overfetchExpression = factory.getNamedOperand(expression,
"overfetch");
+    StreamExpressionNamedParameter refineExpression = factory.getNamedOperand(expression,
"refine");
+    StreamExpressionNamedParameter methodExpression = factory.getNamedOperand(expression,
"method");
+
     // Validate there are no unknown parameters
     if(expression.getParameters().size() != 1 + namedParams.size() + metricExpressions.size()){
       throw new IOException(String.format(Locale.ROOT,"invalid expression %s - unknown operands
found",expression));
@@ -112,7 +129,15 @@ public class FacetStream extends TupleStream implements Expressible 
{
     // pull out known named params
     ModifiableSolrParams params = new ModifiableSolrParams();
     for(StreamExpressionNamedParameter namedParam : namedParams){
-      if(!namedParam.getName().equals("zkHost") && !namedParam.getName().equals("buckets")
&& !namedParam.getName().equals("bucketSorts") && !namedParam.getName().equals("limit")){
+      if(!namedParam.getName().equals("zkHost") &&
+          !namedParam.getName().equals("buckets") &&
+          !namedParam.getName().equals("bucketSorts") &&
+          !namedParam.getName().equals("bucketSizeLimit") &&
+          !namedParam.getName().equals("method") &&
+          !namedParam.getName().equals("offset") &&
+          !namedParam.getName().equals("rows") &&
+          !namedParam.getName().equals("refine") &&
+          !namedParam.getName().equals("overfetch")){
         params.add(namedParam.getName(), namedParam.getParameter().toString().trim());
       }
     }
@@ -130,45 +155,121 @@ public class FacetStream extends TupleStream implements Expressible
 {
         }
       }
     }
+
     if(null == buckets){      
       throw new IOException(String.format(Locale.ROOT,"invalid expression %s - at least one
bucket expected. eg. 'buckets=\"name\"'",expression,collectionName));
     }
-    
-    // bucketSorts, required
-    FieldComparator[] bucketSorts = null;
-    if(null != bucketSortExpression){
-      if(bucketSortExpression.getParameter() instanceof StreamExpressionValue){
-        bucketSorts = parseBucketSorts(((StreamExpressionValue)bucketSortExpression.getParameter()).getValue());
-      }
+
+    String bucketSortString = null;
+
+    if(bucketSortExpression == null) {
+      bucketSortString = "count(*) desc";
+    } else {
+      bucketSortString = ((StreamExpressionValue)bucketSortExpression.getParameter()).getValue();
     }
-    if(null == bucketSorts || 0 == bucketSorts.length){      
+
+    FieldComparator[] bucketSorts = parseBucketSorts(bucketSortString, buckets);
+
+    if(null == bucketSorts || 0 == bucketSorts.length) {
       throw new IOException(String.format(Locale.ROOT,"invalid expression %s - at least one
bucket sort expected. eg. 'bucketSorts=\"name asc\"'",expression,collectionName));
     }
-    
+
     // Construct the metrics
     Metric[] metrics = new Metric[metricExpressions.size()];
-    for(int idx = 0; idx < metricExpressions.size(); ++idx){
+    for(int idx = 0; idx < metricExpressions.size(); ++idx) {
       metrics[idx] = factory.constructMetric(metricExpressions.get(idx));
     }
-    if(0 == metrics.length){
+
+    if(0 == metrics.length) {
       throw new IOException(String.format(Locale.ROOT,"invalid expression %s - at least one
metric expected.",expression,collectionName));
     }
-    
-    if(null == limitExpression || null == limitExpression.getParameter() || !(limitExpression.getParameter()
instanceof StreamExpressionValue)){
-      throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting
a single 'limit' parameter of type positive integer but didn't find one",expression));
-    }
-    String limitStr = ((StreamExpressionValue)limitExpression.getParameter()).getValue();
-    int limitInt = 0;
-    try{
-      limitInt = Integer.parseInt(limitStr);
-      if(limitInt <= 0 && limitInt != -1){
-        throw new IOException(String.format(Locale.ROOT,"invalid expression %s - limit '%s'
must be greater than 0 or -1.",expression, limitStr));
+
+    boolean refine = false;
+
+    if(refineExpression != null) {
+      String refineStr = ((StreamExpressionValue) refineExpression.getParameter()).getValue();
+      if (refineStr != null) {
+        refine = Boolean.parseBoolean(refineStr);
       }
     }
-    catch(NumberFormatException e){
-      throw new IOException(String.format(Locale.ROOT,"invalid expression %s - limit '%s'
is not a valid integer.",expression, limitStr));
+
+    if(bucketLimitExpression != null && (rowsExpression != null ||
+                                         offsetExpression != null ||
+                                         overfetchExpression != null)) {
+      throw new IOException("bucketSizeLimit is incompatible with rows, offset and overfetch.");
     }
-    
+
+    String methodStr = null;
+    if(methodExpression != null) {
+      methodStr = ((StreamExpressionValue) methodExpression.getParameter()).getValue();
+    }
+
+    int overfetchInt = 150;
+    if(overfetchExpression != null) {
+      String overfetchStr = ((StreamExpressionValue) overfetchExpression.getParameter()).getValue();
+      overfetchInt = Integer.parseInt(overfetchStr);
+    }
+
+    int offsetInt = 0;
+    if(offsetExpression != null) {
+      String offsetStr = ((StreamExpressionValue) offsetExpression.getParameter()).getValue();
+      offsetInt = Integer.parseInt(offsetStr);
+    }
+
+    int rowsInt = Integer.MIN_VALUE;
+    int bucketLimit = Integer.MIN_VALUE;
+    boolean bucketLimitSet = false;
+
+    if(null != rowsExpression) {
+      String rowsStr = ((StreamExpressionValue)rowsExpression.getParameter()).getValue();
+      try {
+        rowsInt = Integer.parseInt(rowsStr);
+        if (rowsInt <= 0 && rowsInt != -1) {
+          throw new IOException(String.format(Locale.ROOT, "invalid expression %s - limit
'%s' must be greater than 0 or -1.", expression, rowsStr));
+        }
+        //Rows is set so configure the bucketLimitSize
+        if(rowsInt == -1) {
+          bucketLimit = rowsInt = Integer.MAX_VALUE;
+        } else if(overfetchInt == -1) {
+          bucketLimit = Integer.MAX_VALUE;
+        }else{
+          bucketLimit = offsetInt+overfetchInt+rowsInt;
+        }
+      } catch (NumberFormatException e) {
+        throw new IOException(String.format(Locale.ROOT, "invalid expression %s - limit '%s'
is not a valid integer.", expression, rowsStr));
+      }
+    }
+
+    if(bucketLimitExpression != null) {
+      String bucketLimitStr = ((StreamExpressionValue) bucketLimitExpression.getParameter()).getValue();
+      try {
+        bucketLimit = Integer.parseInt(bucketLimitStr);
+        bucketLimitSet = true;
+
+        if (bucketLimit <= 0 && bucketLimit != -1) {
+          throw new IOException(String.format(Locale.ROOT, "invalid expression %s - bucketSizeLimit
'%s' must be greater than 0 or -1.", expression, bucketLimitStr));
+        }
+
+        // Bucket limit is set. So set rows.
+        if(bucketLimit == -1) {
+         rowsInt = bucketLimit = Integer.MAX_VALUE;
+        } else {
+          rowsInt = bucketLimit;
+        }
+      }  catch (NumberFormatException e) {
+        throw new IOException(String.format(Locale.ROOT, "invalid expression %s - bucketSizeLimit
'%s' is not a valid integer.", expression, bucketLimitStr));
+      }
+    }
+
+    if(rowsExpression == null && bucketLimitExpression == null) {
+      rowsInt = 10;
+      if(overfetchInt == -1) {
+        bucketLimit = Integer.MAX_VALUE;
+      }else{
+        bucketLimit = offsetInt+overfetchInt+rowsInt;
+      }
+    }
+
     // zkHost, optional - if not provided then will look into factory list to get
     String zkHost = null;
     if(null == zkHostExpression){
@@ -176,16 +277,44 @@ public class FacetStream extends TupleStream implements Expressible
 {
       if(zkHost == null) {
         zkHost = factory.getDefaultZkHost();
       }
-    }
-    else if(zkHostExpression.getParameter() instanceof StreamExpressionValue){
+    } else if(zkHostExpression.getParameter() instanceof StreamExpressionValue) {
       zkHost = ((StreamExpressionValue)zkHostExpression.getParameter()).getValue();
     }
+
     if(null == zkHost){
       throw new IOException(String.format(Locale.ROOT,"invalid expression %s - zkHost not
found for collection '%s'",expression,collectionName));
     }
     
     // We've got all the required items
-    init(collectionName, params, buckets, bucketSorts, metrics, limitInt, zkHost);
+    init(collectionName,
+         params,
+         buckets,
+         bucketSorts,
+         metrics,
+         rowsInt,
+         offsetInt,
+         bucketLimit,
+         refine,
+         methodStr,
+         bucketLimitSet,
+         overfetchInt,
+         zkHost);
+  }
+
+  public int getBucketSizeLimit() {
+    return this.bucketSizeLimit;
+  }
+
+  public int getRows() {
+    return this.rows;
+  }
+
+  public int getOffset() {
+    return this.offset;
+  }
+
+  public int getOverfetch() {
+    return this.overfetch;
   }
 
   public Bucket[] getBuckets() {
@@ -196,7 +325,7 @@ public class FacetStream extends TupleStream implements Expressible  {
     return this.collection;
   }
 
-  private FieldComparator[] parseBucketSorts(String bucketSortString) throws IOException
{
+  private FieldComparator[] parseBucketSorts(String bucketSortString, Bucket[] buckets) throws
IOException {
 
     String[] sorts = bucketSortString.split(",");
     FieldComparator[] comps = new FieldComparator[sorts.length];
@@ -217,17 +346,20 @@ public class FacetStream extends TupleStream implements Expressible
 {
     return comps;
   }
 
-  private void init(String collection, SolrParams params, Bucket[] buckets, FieldComparator[]
bucketSorts, Metric[] metrics, int bucketSizeLimit, String zkHost) throws IOException {
+  private void init(String collection, SolrParams params, Bucket[] buckets, FieldComparator[]
bucketSorts, Metric[] metrics, int rows, int offset, int bucketSizeLimit, boolean refine,
String method, boolean serializeBucketSizeLimit, int overfetch, String zkHost) throws IOException
{
     this.zkHost  = zkHost;
     this.params = new ModifiableSolrParams(params);
     this.buckets = buckets;
     this.metrics = metrics;
+    this.rows = rows;
+    this.offset = offset;
+    this.refine = refine;
     this.bucketSizeLimit   = bucketSizeLimit;
-    if (this.bucketSizeLimit == -1) {
-      this.bucketSizeLimit = Integer.MAX_VALUE;
-    }
     this.collection = collection;
     this.bucketSorts = bucketSorts;
+    this.method = method;
+    this.serializeBucketSizeLimit = serializeBucketSizeLimit;
+    this.overfetch = overfetch;
     
     // In a facet world it only makes sense to have the same field name in all of the sorters
     // Because FieldComparator allows for left and right field names we will need to validate
@@ -280,8 +412,31 @@ public class FacetStream extends TupleStream implements Expressible 
{
       expression.addParameter(metric.toExpression(factory));
     }
     
-    // limit
-    expression.addParameter(new StreamExpressionNamedParameter("bucketSizeLimit", Integer.toString(bucketSizeLimit)));
+    if(serializeBucketSizeLimit) {
+      if(bucketSizeLimit == Integer.MAX_VALUE) {
+        expression.addParameter(new StreamExpressionNamedParameter("bucketSizeLimit", Integer.toString(-1)));
+      } else {
+        expression.addParameter(new StreamExpressionNamedParameter("bucketSizeLimit", Integer.toString(bucketSizeLimit)));
+      }
+    } else {
+      if (rows == Integer.MAX_VALUE) {
+        expression.addParameter(new StreamExpressionNamedParameter("rows", Integer.toString(-1)));
+      } else{
+        expression.addParameter(new StreamExpressionNamedParameter("rows", Integer.toString(rows)));
+      }
+
+      expression.addParameter(new StreamExpressionNamedParameter("offset", Integer.toString(offset)));
+
+      if(overfetch == Integer.MAX_VALUE) {
+        expression.addParameter(new StreamExpressionNamedParameter("overfetch", Integer.toString(-1)));
+      } else {
+        expression.addParameter(new StreamExpressionNamedParameter("overfetch", Integer.toString(overfetch)));
+      }
+    }
+
+    if(method != null) {
+      expression.addParameter(new StreamExpressionNamedParameter("method", this.method));
+    }
         
     // zkHost
     expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));
@@ -333,8 +488,10 @@ public class FacetStream extends TupleStream implements Expressible 
{
     }
 
     FieldComparator[] adjustedSorts = adjustSorts(buckets, bucketSorts);
-    String json = getJsonFacetString(buckets, metrics, adjustedSorts, bucketSizeLimit);
+    this.resortNeeded = resortNeeded(adjustedSorts);
 
+    String json = getJsonFacetString(buckets, metrics, adjustedSorts, method, refine, bucketSizeLimit);
+    assert expectedJson(json);
     ModifiableSolrParams paramsLoc = new ModifiableSolrParams(params);
     paramsLoc.set("json.facet", json);
     paramsLoc.set("rows", "0");
@@ -343,13 +500,58 @@ public class FacetStream extends TupleStream implements Expressible
 {
     try {
       NamedList response = cloudSolrClient.request(request, collection);
       getTuples(response, buckets, metrics);
-      Collections.sort(tuples, getStreamSort());
 
+      if(resortNeeded) {
+        Collections.sort(tuples, getStreamSort());
+      }
+
+      index=this.offset;
     } catch (Exception e) {
       throw new IOException(e);
     }
   }
 
+  private boolean expectedJson(String json) {
+    if(this.method != null) {
+      if(!json.contains("\"method\":\""+this.method+"\"")) {
+        return false;
+      }
+    }
+
+    if(this.refine) {
+      if(!json.contains("\"refine\":true")) {
+        return false;
+      }
+    }
+
+    if(serializeBucketSizeLimit) {
+      if(!json.contains("\"limit\":"+bucketSizeLimit)) {
+        return false;
+      }
+    } else {
+      if(!json.contains("\"limit\":"+(this.rows+this.offset+this.overfetch))) {
+        return false;
+      }
+    }
+
+    for(Bucket bucket : buckets) {
+      if(!json.contains("\""+bucket.toString()+"\":")) {
+        return false;
+      }
+    }
+
+    for(Metric metric: metrics) {
+      String func = metric.getFunctionName();
+      if(!func.equals("count")) {
+        if (!json.contains(metric.getIdentifier())) {
+          return false;
+        }
+      }
+    }
+
+    return true;
+  }
+
   public void close() throws IOException {
     if(cache == null) {
       if (cloudSolrClient != null) {
@@ -359,21 +561,32 @@ public class FacetStream extends TupleStream implements Expressible
 {
   }
 
   public Tuple read() throws IOException {
-    if(index < tuples.size() && index < bucketSizeLimit) {
+    if(index < tuples.size() && index < (offset+rows)) {
       Tuple tuple = tuples.get(index);
       ++index;
       return tuple;
     } else {
       Map fields = new HashMap();
+
+      if(bucketSizeLimit == Integer.MAX_VALUE) {
+        fields.put("totalRows", tuples.size());
+      }
+
       fields.put("EOF", true);
+
       Tuple tuple = new Tuple(fields);
       return tuple;
     }
   }
 
-  private String getJsonFacetString(Bucket[] _buckets, Metric[] _metrics, FieldComparator[]
_sorts, int _limit) {
+  private String getJsonFacetString(Bucket[] _buckets,
+                                    Metric[] _metrics,
+                                    FieldComparator[] _sorts,
+                                    String _method,
+                                    boolean _refine,
+                                    int _limit) {
     StringBuilder buf = new StringBuilder();
-    appendJson(buf, _buckets, _metrics, _sorts, _limit, 0);
+    appendJson(buf, _buckets, _metrics, _sorts, _limit, _method, _refine,0);
     return "{"+buf.toString()+"}";
   }
 
@@ -399,11 +612,22 @@ public class FacetStream extends TupleStream implements Expressible
 {
     }
   }
 
+  private boolean resortNeeded(FieldComparator[] fieldComparators) {
+    for(FieldComparator fieldComparator : fieldComparators) {
+      if(fieldComparator.getLeftFieldName().contains("(")) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   private void appendJson(StringBuilder buf,
                           Bucket[] _buckets,
                           Metric[] _metrics,
                           FieldComparator[] _sorts,
                           int _limit,
+                          String method,
+                          boolean refine,
                           int level) {
     buf.append('"');
     buf.append(_buckets[level].toString());
@@ -412,42 +636,63 @@ public class FacetStream extends TupleStream implements Expressible
 {
     buf.append("\"type\":\"terms\"");
     buf.append(",\"field\":\""+_buckets[level].toString()+"\"");
     buf.append(",\"limit\":"+_limit);
-    buf.append(",\"sort\":{\""+getFacetSort(_sorts[level].getLeftFieldName(), _metrics)+"\":\""+_sorts[level].getOrder()+"\"}");
+
+    if(refine) {
+      buf.append(",\"refine\":true");
+    }
+
+    if(method != null) {
+      buf.append(",\"method\":\""+method+"\"");
+    }
+
+    String fsort = getFacetSort(_sorts[level].getLeftFieldName(), _metrics);
+
+    buf.append(",\"sort\":{\""+fsort+"\":\""+_sorts[level].getOrder()+"\"}");
 
     buf.append(",\"facet\":{");
     int metricCount = 0;
+
+
+    ++level;
     for(Metric metric : _metrics) {
-      String identifier = metric.getIdentifier();
-      if(!identifier.startsWith("count(")) {
-        if(metricCount>0) {
-          buf.append(",");
+      //Only compute the metric if it's a leaf node or if the branch level sort equals is
the metric
+      String facetKey = "facet_"+metricCount;
+      if(level == _buckets.length || fsort.equals(facetKey) ) {
+        String identifier = metric.getIdentifier();
+        if (!identifier.startsWith("count(")) {
+          if (metricCount > 0) {
+            buf.append(",");
+          }
+          buf.append("\""+ facetKey + "\":\"" + identifier + "\"");
+          ++metricCount;
         }
-        buf.append("\"facet_" + metricCount + "\":\"" +identifier+"\"");
-        ++metricCount;
       }
     }
-    ++level;
+
     if(level < _buckets.length) {
       if(metricCount>0) {
         buf.append(",");
       }
-      appendJson(buf, _buckets, _metrics, _sorts, _limit, level);
+      appendJson(buf, _buckets, _metrics, _sorts, _limit, method, refine, level);
     }
     buf.append("}}");
   }
 
   private String getFacetSort(String id, Metric[] _metrics) {
     int index = 0;
+    int metricCount=0;
     for(Metric metric : _metrics) {
       if(metric.getIdentifier().startsWith("count(")) {
         if(id.startsWith("count(")) {
           return "count";
         }
+        ++index;
       } else {
         if (id.equals(_metrics[index].getIdentifier())) {
-          return "facet_" + index;
+          return "facet_" + metricCount;
         }
         ++index;
+        ++metricCount;
       }
     }
     return "index";

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/68ed57f7/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 24264b6..7e2451e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -986,6 +986,147 @@ public class StreamExpressionTest extends SolrCloudTestCase {
     assertTrue(count.doubleValue() == 2);
 
 
+    clause = "facet("
+        +   "collection1, "
+        +   "q=\"*:*\", "
+        +   "fl=\"a_s,a_i,a_f\", "
+        +   "sort=\"a_s asc\", "
+        +   "buckets=\"a_s\", "
+        +   "bucketSorts=\"sum(a_i) desc\", "
+        +   "rows=2, "
+        +   "sum(a_i), sum(a_f), "
+        +   "min(a_i), min(a_f), "
+        +   "max(a_i), max(a_f), "
+        +   "avg(a_i), avg(a_f), "
+        +   "count(*)"
+        + ")";
+
+    stream = factory.constructStream(clause);
+    tuples = getTuples(stream);
+
+
+    //Test rows
+
+    tuple = tuples.get(0);
+    assertEquals(tuples.size(), 2);
+
+    bucket = tuple.getString("a_s");
+    sumi = tuple.getDouble("sum(a_i)");
+    sumf = tuple.getDouble("sum(a_f)");
+    mini = tuple.getDouble("min(a_i)");
+    minf = tuple.getDouble("min(a_f)");
+    maxi = tuple.getDouble("max(a_i)");
+    maxf = tuple.getDouble("max(a_f)");
+    avgi = tuple.getDouble("avg(a_i)");
+    avgf = tuple.getDouble("avg(a_f)");
+    count = tuple.getDouble("count(*)");
+
+    assertTrue(bucket.equals("hello3"));
+    assertTrue(sumi.doubleValue() == 38.0D);
+    assertTrue(sumf.doubleValue() == 26.0D);
+    assertTrue(mini.doubleValue() == 3.0D);
+    assertTrue(minf.doubleValue() == 3.0D);
+    assertTrue(maxi.doubleValue() == 13.0D);
+    assertTrue(maxf.doubleValue() == 9.0D);
+    assertTrue(avgi.doubleValue() == 9.5D);
+    assertTrue(avgf.doubleValue() == 6.5D);
+    assertTrue(count.doubleValue() == 4);
+
+    tuple = tuples.get(1);
+    bucket = tuple.getString("a_s");
+    sumi = tuple.getDouble("sum(a_i)");
+    sumf = tuple.getDouble("sum(a_f)");
+    mini = tuple.getDouble("min(a_i)");
+    minf = tuple.getDouble("min(a_f)");
+    maxi = tuple.getDouble("max(a_i)");
+    maxf = tuple.getDouble("max(a_f)");
+    avgi = tuple.getDouble("avg(a_i)");
+    avgf = tuple.getDouble("avg(a_f)");
+    count = tuple.getDouble("count(*)");
+
+    assertTrue(bucket.equals("hello0"));
+    assertTrue(sumi.doubleValue() == 17.0D);
+    assertTrue(sumf.doubleValue() == 18.0D);
+    assertTrue(mini.doubleValue() == 0.0D);
+    assertTrue(minf.doubleValue() == 1.0D);
+    assertTrue(maxi.doubleValue() == 14.0D);
+    assertTrue(maxf.doubleValue() == 10.0D);
+    assertTrue(avgi.doubleValue() == 4.25D);
+    assertTrue(avgf.doubleValue() == 4.5D);
+    assertTrue(count.doubleValue() == 4);
+
+
+    clause = "facet("
+        +   "collection1, "
+        +   "q=\"*:*\", "
+        +   "fl=\"a_s,a_i,a_f\", "
+        +   "sort=\"a_s asc\", "
+        +   "buckets=\"a_s\", "
+        +   "bucketSorts=\"sum(a_i) desc\", "
+        +   "rows=2, offset=1, method=dvhash, refine=true,"
+        +   "sum(a_i), sum(a_f), "
+        +   "min(a_i), min(a_f), "
+        +   "max(a_i), max(a_f), "
+        +   "avg(a_i), avg(a_f), "
+        +   "count(*)"
+        + ")";
+
+    stream = factory.constructStream(clause);
+    tuples = getTuples(stream);
+
+
+    //Test offset
+
+    tuple = tuples.get(0);
+    assertEquals(tuples.size(), 2);
+
+    tuple = tuples.get(0);
+    bucket = tuple.getString("a_s");
+    sumi = tuple.getDouble("sum(a_i)");
+    sumf = tuple.getDouble("sum(a_f)");
+    mini = tuple.getDouble("min(a_i)");
+    minf = tuple.getDouble("min(a_f)");
+    maxi = tuple.getDouble("max(a_i)");
+    maxf = tuple.getDouble("max(a_f)");
+    avgi = tuple.getDouble("avg(a_i)");
+    avgf = tuple.getDouble("avg(a_f)");
+    count = tuple.getDouble("count(*)");
+
+    assertTrue(bucket.equals("hello0"));
+    assertTrue(sumi.doubleValue() == 17.0D);
+    assertTrue(sumf.doubleValue() == 18.0D);
+    assertTrue(mini.doubleValue() == 0.0D);
+    assertTrue(minf.doubleValue() == 1.0D);
+    assertTrue(maxi.doubleValue() == 14.0D);
+    assertTrue(maxf.doubleValue() == 10.0D);
+    assertTrue(avgi.doubleValue() == 4.25D);
+    assertTrue(avgf.doubleValue() == 4.5D);
+    assertTrue(count.doubleValue() == 4);
+
+    tuple = tuples.get(1);
+    bucket = tuple.getString("a_s");
+    sumi = tuple.getDouble("sum(a_i)");
+    sumf = tuple.getDouble("sum(a_f)");
+    mini = tuple.getDouble("min(a_i)");
+    minf = tuple.getDouble("min(a_f)");
+    maxi = tuple.getDouble("max(a_i)");
+    maxf = tuple.getDouble("max(a_f)");
+    avgi = tuple.getDouble("avg(a_i)");
+    avgf = tuple.getDouble("avg(a_f)");
+    count = tuple.getDouble("count(*)");
+
+    assertTrue(bucket.equals("hello4"));
+    assertTrue(sumi.longValue() == 15);
+    assertTrue(sumf.doubleValue() == 11.0D);
+    assertTrue(mini.doubleValue() == 4.0D);
+    assertTrue(minf.doubleValue() == 4.0D);
+    assertTrue(maxi.doubleValue() == 11.0D);
+    assertTrue(maxf.doubleValue() == 7.0D);
+    assertTrue(avgi.doubleValue() == 7.5D);
+    assertTrue(avgf.doubleValue() == 5.5D);
+    assertTrue(count.doubleValue() == 2);
+
+
     //Test index sort
     clause = "facet("
         +   "collection1, "

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/68ed57f7/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
index e43176a..289b925 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.client.solrj.io.stream;
 
+import java.io.IOException;
+
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.client.solrj.io.ops.GroupOperation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
@@ -247,17 +249,17 @@ public class StreamExpressionToExpessionTest extends LuceneTestCase
{
     
     // Basic test
     try (FacetStream stream = new FacetStream(StreamExpressionParser.parse("facet("
-                                                        +   "collection1, "
-                                                        +   "q=\"*:*\", "
-                                                        +   "buckets=\"a_s\", "
-                                                        +   "bucketSorts=\"sum(a_i) asc\",
"
-                                                        +   "bucketSizeLimit=100, "
-                                                        +   "sum(a_i), sum(a_f), "
-                                                        +   "min(a_i), min(a_f), "
-                                                        +   "max(a_i), max(a_f), "
-                                                        +   "avg(a_i), avg(a_f), "
-                                                        +   "count(*)"
-                                                        + ")"), factory)){
+        +   "collection1, "
+        +   "q=\"*:*\", "
+        +   "buckets=\"a_s\", "
+        +   "bucketSorts=\"sum(a_i) asc\", "
+        +   "bucketSizeLimit=100, "
+        +   "sum(a_i), sum(a_f), "
+        +   "min(a_i), min(a_f), "
+        +   "max(a_i), max(a_f), "
+        +   "avg(a_i), avg(a_f), "
+        +   "count(*)"
+        + ")"), factory)){
       expressionString = stream.toExpression(factory).toString();
       assertTrue(expressionString.contains("facet(collection1"));
       assertTrue(expressionString.contains("q=\"*:*\""));
@@ -273,7 +275,186 @@ public class StreamExpressionToExpessionTest extends LuceneTestCase
{
       assertTrue(expressionString.contains("avg(a_i,false)"));
       assertTrue(expressionString.contains("avg(a_f,false)"));
       assertTrue(expressionString.contains("count(*)"));
+      assertEquals(stream.getBucketSizeLimit(), 100);
+      assertEquals(stream.getRows(), 100);
+      assertEquals(stream.getOffset(), 0);
     }
+
+    try (FacetStream stream = new FacetStream(StreamExpressionParser.parse("facet("
+        +   "collection1, "
+        +   "q=\"*:*\", "
+        +   "buckets=\"a_s\", "
+        +   "bucketSorts=\"sum(a_i) asc\", "
+        +   "sum(a_i), sum(a_f), "
+        +   "min(a_i), min(a_f), "
+        +   "max(a_i), max(a_f), "
+        +   "avg(a_i), avg(a_f), "
+        +   "count(*)"
+        + ")"), factory)){
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("facet(collection1"));
+      assertTrue(expressionString.contains("q=\"*:*\""));
+      assertTrue(expressionString.contains("buckets=a_s"));
+      assertTrue(expressionString.contains("bucketSorts=\"sum(a_i) asc\""));
+      assertTrue(expressionString.contains("rows=10"));
+      assertTrue(expressionString.contains("offset=0"));
+      assertTrue(expressionString.contains("overfetch=150"));
+      assertTrue(expressionString.contains("sum(a_i)"));
+      assertTrue(expressionString.contains("sum(a_f)"));
+      assertTrue(expressionString.contains("min(a_i)"));
+      assertTrue(expressionString.contains("min(a_f)"));
+      assertTrue(expressionString.contains("max(a_i)"));
+      assertTrue(expressionString.contains("max(a_f)"));
+      assertTrue(expressionString.contains("avg(a_i,false)"));
+      assertTrue(expressionString.contains("avg(a_f,false)"));
+      assertTrue(expressionString.contains("count(*)"));
+      assertEquals(stream.getOverfetch(), 150);
+      assertEquals(stream.getBucketSizeLimit(), 160);
+      assertEquals(stream.getRows(), 10);
+      assertEquals(stream.getOffset(), 0);
+    }
+
+    try (FacetStream stream = new FacetStream(StreamExpressionParser.parse("facet("
+        +   "collection1, "
+        +   "q=\"*:*\", "
+        +   "buckets=\"a_s\", "
+        +   "bucketSizeLimit=100, "
+        +   "sum(a_i), sum(a_f), "
+        +   "min(a_i), min(a_f), "
+        +   "max(a_i), max(a_f), "
+        +   "avg(a_i), avg(a_f), "
+        +   "count(*)"
+        + ")"), factory)){
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("facet(collection1"));
+      assertTrue(expressionString.contains("q=\"*:*\""));
+      assertTrue(expressionString.contains("buckets=a_s"));
+      assertTrue(expressionString.contains("bucketSorts=\"count(*) desc\""));
+      assertTrue(expressionString.contains("bucketSizeLimit=100"));
+      assertTrue(expressionString.contains("sum(a_i)"));
+      assertTrue(expressionString.contains("sum(a_f)"));
+      assertTrue(expressionString.contains("min(a_i)"));
+      assertTrue(expressionString.contains("min(a_f)"));
+      assertTrue(expressionString.contains("max(a_i)"));
+      assertTrue(expressionString.contains("max(a_f)"));
+      assertTrue(expressionString.contains("avg(a_i,false)"));
+      assertTrue(expressionString.contains("avg(a_f,false)"));
+      assertTrue(expressionString.contains("count(*)"));
+    }
+
+    try (FacetStream stream = new FacetStream(StreamExpressionParser.parse("facet("
+        +   "collection1, "
+        +   "q=\"*:*\", "
+        +   "buckets=\"a_s\", "
+        +   "bucketSorts=\"sum(a_i) asc\", "
+        +   "rows=10, method=dvhash, "
+        +   "sum(a_i), sum(a_f), "
+        +   "min(a_i), min(a_f), "
+        +   "max(a_i), max(a_f), "
+        +   "avg(a_i), avg(a_f), "
+        +   "count(*)"
+        + ")"), factory)){
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("facet(collection1"));
+      assertTrue(expressionString.contains("q=\"*:*\""));
+      assertTrue(expressionString.contains("buckets=a_s"));
+      assertTrue(expressionString.contains("bucketSorts=\"sum(a_i) asc\""));
+      assertTrue(!expressionString.contains("bucketSizeLimit"));
+      assertTrue(expressionString.contains("rows=10"));
+      assertTrue(expressionString.contains("offset=0"));
+      assertTrue(expressionString.contains("overfetch=150"));
+      assertTrue(expressionString.contains("method=dvhash"));
+      assertTrue(expressionString.contains("sum(a_i)"));
+      assertTrue(expressionString.contains("sum(a_f)"));
+      assertTrue(expressionString.contains("min(a_i)"));
+      assertTrue(expressionString.contains("min(a_f)"));
+      assertTrue(expressionString.contains("max(a_i)"));
+      assertTrue(expressionString.contains("max(a_f)"));
+      assertTrue(expressionString.contains("avg(a_i,false)"));
+      assertTrue(expressionString.contains("avg(a_f,false)"));
+      assertTrue(expressionString.contains("count(*)"));
+      assertEquals(stream.getBucketSizeLimit(), 160);
+      assertEquals(stream.getRows(), 10);
+      assertEquals(stream.getOffset(), 0);
+      assertEquals(stream.getOverfetch(), 150);
+
+    }
+
+    try (FacetStream stream = new FacetStream(StreamExpressionParser.parse("facet("
+        +   "collection1, "
+        +   "q=\"*:*\", "
+        +   "buckets=\"a_s\", "
+        +   "bucketSorts=\"sum(a_i) asc\", "
+        +   "rows=10, offset=100, overfetch=30, method=dvhash, "
+        +   "sum(a_i), sum(a_f), "
+        +   "min(a_i), min(a_f), "
+        +   "max(a_i), max(a_f), "
+        +   "avg(a_i), avg(a_f), "
+        +   "count(*)"
+        + ")"), factory)){
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("facet(collection1"));
+      assertTrue(expressionString.contains("q=\"*:*\""));
+      assertTrue(expressionString.contains("buckets=a_s"));
+      assertTrue(expressionString.contains("bucketSorts=\"sum(a_i) asc\""));
+      assertTrue(!expressionString.contains("bucketSizeLimit"));
+      assertTrue(expressionString.contains("rows=10"));
+      assertTrue(expressionString.contains("offset=100"));
+      assertTrue(expressionString.contains("overfetch=30"));
+      assertTrue(expressionString.contains("method=dvhash"));
+      assertTrue(expressionString.contains("sum(a_i)"));
+      assertTrue(expressionString.contains("sum(a_f)"));
+      assertTrue(expressionString.contains("min(a_i)"));
+      assertTrue(expressionString.contains("min(a_f)"));
+      assertTrue(expressionString.contains("max(a_i)"));
+      assertTrue(expressionString.contains("max(a_f)"));
+      assertTrue(expressionString.contains("avg(a_i,false)"));
+      assertTrue(expressionString.contains("avg(a_f,false)"));
+      assertTrue(expressionString.contains("count(*)"));
+      assertEquals(stream.getBucketSizeLimit(), 140);
+      assertEquals(stream.getRows(), 10);
+      assertEquals(stream.getOffset(), 100);
+      assertEquals(stream.getOverfetch(), 30);
+
+    }
+
+    try (FacetStream stream = new FacetStream(StreamExpressionParser.parse("facet("
+        +   "collection1, "
+        +   "q=\"*:*\", "
+        +   "buckets=\"a_s\", "
+        +   "bucketSorts=\"sum(a_i) asc\", "
+        +   "rows=-1, offset=100, overfetch=-1, method=dvhash, "
+        +   "sum(a_i), sum(a_f), "
+        +   "min(a_i), min(a_f), "
+        +   "max(a_i), max(a_f), "
+        +   "avg(a_i), avg(a_f), "
+        +   "count(*)"
+        + ")"), factory)){
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("facet(collection1"));
+      assertTrue(expressionString.contains("q=\"*:*\""));
+      assertTrue(expressionString.contains("buckets=a_s"));
+      assertTrue(expressionString.contains("bucketSorts=\"sum(a_i) asc\""));
+      assertTrue(!expressionString.contains("bucketSizeLimit"));
+      assertTrue(expressionString.contains("rows=-1"));
+      assertTrue(expressionString.contains("offset=100"));
+      assertTrue(expressionString.contains("overfetch=-1"));
+      assertTrue(expressionString.contains("method=dvhash"));
+      assertTrue(expressionString.contains("sum(a_i)"));
+      assertTrue(expressionString.contains("sum(a_f)"));
+      assertTrue(expressionString.contains("min(a_i)"));
+      assertTrue(expressionString.contains("min(a_f)"));
+      assertTrue(expressionString.contains("max(a_i)"));
+      assertTrue(expressionString.contains("max(a_f)"));
+      assertTrue(expressionString.contains("avg(a_i,false)"));
+      assertTrue(expressionString.contains("avg(a_f,false)"));
+      assertTrue(expressionString.contains("count(*)"));
+      assertEquals(stream.getBucketSizeLimit(), Integer.MAX_VALUE);
+      assertEquals(stream.getRows(), Integer.MAX_VALUE);
+      assertEquals(stream.getOffset(), 100);
+      assertEquals(stream.getOverfetch(), -1);
+    }
+
   }
   
   @Test


Mime
View raw message