lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbern...@apache.org
Subject lucene-solr:branch_7x: SOLR-12971: Add pivot Stream Evaluator to pivot facet co-occurrence counts into a matrix
Date Wed, 07 Nov 2018 13:34:21 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_7x decfd6707 -> 8af39cf05


SOLR-12971: Add pivot Stream Evaluator to pivot facet co-occurrence counts into a matrix


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8af39cf0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8af39cf0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8af39cf0

Branch: refs/heads/branch_7x
Commit: 8af39cf055245f68e5bcb5cd89d064333c6b1db4
Parents: decfd67
Author: Joel Bernstein <jbernste@apache.org>
Authored: Wed Nov 7 08:12:53 2018 -0500
Committer: Joel Bernstein <jbernste@apache.org>
Committed: Wed Nov 7 08:30:22 2018 -0500

----------------------------------------------------------------------
 .../org/apache/solr/client/solrj/io/Lang.java   |   2 +
 .../client/solrj/io/eval/PivotEvaluator.java    | 111 +++++++++++++++++++
 .../apache/solr/client/solrj/io/TestLang.java   |   2 +-
 .../solrj/io/stream/MathExpressionTest.java     |  44 ++++++++
 4 files changed, 158 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8af39cf0/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index 1777467..200f07b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -273,6 +273,8 @@ public class Lang {
         .withFunctionName("getSupportPoints", GetSupportPointsEvaluator.class)
         .withFunctionName("pairSort", PairSortEvaluator.class)
         .withFunctionName("recip", RecipEvaluator.class)
+        .withFunctionName("pivot", PivotEvaluator.class)
+
         // Boolean Stream Evaluators
 
         .withFunctionName("and", AndEvaluator.class)

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8af39cf0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PivotEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PivotEvaluator.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PivotEvaluator.java
new file mode 100644
index 0000000..919db14
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PivotEvaluator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.Locale;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.io.Tuple;
+
+public class PivotEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+  protected static final long serialVersionUID = 1L;
+
+  public  PivotEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+    super(expression, factory);
+
+    if(4 != containedEvaluators.size()){
+      throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting
exactly 4 values but found %d",expression,containedEvaluators.size()));
+    }
+  }
+
+  @Override
+  public Object doWork(Object[] values) throws IOException {
+    if(values.length != 4) {
+      throw new IOException("The pivot function requires four parameters.");
+    }
+
+    Object value1 = values[0];
+    Object value2 = values[1];
+    Object value3 = values[2];
+    Object value4 = values[3];
+
+    if(value1 instanceof List) {
+      List<Tuple> tuples = (List<Tuple>)value1;
+      String x = (String)value2;
+      x = x.replace("\"", "");
+      String y = (String)value3;
+      y = y.replace("\"", "");
+
+      String vlabel = (String)value4;
+      vlabel = vlabel.replace("\"", "");
+
+      Set<String> xset = new TreeSet();
+      Set<String> yset = new TreeSet();
+
+      for(int i=0; i<tuples.size(); i++) {
+        Tuple tuple = tuples.get(i);
+        xset.add(tuple.getString(x));
+        yset.add(tuple.getString(y));
+      }
+
+      double[][] data = new double[xset.size()][yset.size()];
+
+      List<String> xlabels = new ArrayList(xset.size());
+      Map<String, Integer> xindexes = new HashMap();
+      int xindex = 0;
+      for (String xlabel :xset) {
+        xlabels.add(xlabel);
+        xindexes.put(xlabel, xindex);
+        ++xindex;
+      }
+
+      List<String> ylabels = new ArrayList(yset.size());
+      Map<String, Integer> yindexes = new HashMap();
+      int yindex = 0;
+      for (String ylabel : yset) {
+        ylabels.add(ylabel);
+        yindexes.put(ylabel, yindex);
+        ++yindex;
+      }
+
+      for(Tuple tuple : tuples) {
+        String xlabel = tuple.getString(x);
+        String ylabel = tuple.getString(y);
+        int xi = xindexes.get(xlabel);
+        int yi = yindexes.get(ylabel);
+        double val = tuple.getDouble(vlabel);
+        data[xi][yi] = val;
+      }
+
+      Matrix matrix = new Matrix(data);
+      matrix.setRowLabels(xlabels);
+      matrix.setColumnLabels(ylabels);
+      return matrix;
+    } else {
+      throw new IOException("The getValue function expects a list of tuples as the first
parameter");
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8af39cf0/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index 960eb50..98a6d45 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -73,7 +73,7 @@ public class TestLang extends LuceneTestCase {
       "outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores",
"latlonVectors",
       "convexHull", "getVertices", "getBaryCenter", "getArea", "getBoundarySize","oscillate",
       "getAmplitude", "getPhase", "getAngularFrequency", "enclosingDisk", "getCenter", "getRadius",
-      "getSupportPoints", "pairSort", "log10", "plist", "recip"};
+      "getSupportPoints", "pairSort", "log10", "plist", "recip", "pivot"};
 
   @Test
   public void testLang() {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8af39cf0/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 4642388..9c93ced 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -2164,6 +2164,50 @@ public class MathExpressionTest extends SolrCloudTestCase {
   }
 
   @Test
+  public void testPivot() throws Exception {
+    String cexpr = "let(echo=true," +
+        "               a=list(tuple(fx=x1, fy=f1, fv=add(1,1)), " +
+        "                      tuple(fx=x1, fy=f2, fv=add(1,3)), " +
+        "                      tuple(fx=x2, fy=f1, fv=add(1,7)), " +
+        "                      tuple(fx=x3, fy=f1, fv=add(1,4))," +
+        "                      tuple(fx=x3, fy=f3, fv=add(1,7)))," +
+                   "    b=pivot(a, fx, fy, fv)," +
+        "               c=getRowLabels(b)," +
+        "               d=getColumnLabels(b))";
+    ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+    paramsLoc.set("expr", cexpr);
+    paramsLoc.set("qt", "/stream");
+    String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+    TupleStream solrStream = new SolrStream(url, paramsLoc);
+    StreamContext context = new StreamContext();
+    solrStream.setStreamContext(context);
+    List<Tuple> tuples = getTuples(solrStream);
+    assertEquals(tuples.size(), 1);
+    List<List<Number>> matrix = (List<List<Number>>)tuples.get(0).get("b");
+    List<Number> row1 = matrix.get(0);
+    assertEquals(row1.get(0).doubleValue(), 2.0,0);
+    assertEquals(row1.get(1).doubleValue(), 4.0,0);
+    assertEquals(row1.get(2).doubleValue(), 0,0);
+    List<Number> row2 = matrix.get(1);
+    assertEquals(row2.get(0).doubleValue(), 8.0,0);
+    assertEquals(row2.get(1).doubleValue(), 0,0);
+    assertEquals(row2.get(2).doubleValue(), 0,0);
+    List<Number> row3 = matrix.get(2);
+    assertEquals(row3.get(0).doubleValue(), 5.0,0);
+    assertEquals(row3.get(1).doubleValue(), 0,0);
+    assertEquals(row3.get(2).doubleValue(), 8.0,0);
+
+    List<String> rowLabels = (List<String>)tuples.get(0).get("c");
+    assertEquals(rowLabels.get(0), "x1");
+    assertEquals(rowLabels.get(1), "x2");
+    assertEquals(rowLabels.get(2), "x3");
+    List<String> columnLabels = (List<String>)tuples.get(0).get("d");
+    assertEquals(columnLabels.get(0), "f1");
+    assertEquals(columnLabels.get(1), "f2");
+    assertEquals(columnLabels.get(2), "f3");
+  }
+
+  @Test
   public void testEbeSubtract() throws Exception {
     String cexpr = "let(echo=true," +
         "               a=array(2, 4, 6, 8, 10, 12)," +


Mime
View raw message