lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject [28/50] [abbrv] lucene-solr:jira/solr-11990: SOLR-12567: JSON Facet "functions" now support an extended "type:func" syntax, similar to other types of facets
Date Sat, 28 Jul 2018 04:49:51 GMT
SOLR-12567: JSON Facet "functions" now support an extended "type:func" syntax, similar to other
types of facets

This also allows additional local params to be specified for if the aggregation function can
take advantage of them.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/98d463ae
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/98d463ae
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/98d463ae

Branch: refs/heads/jira/solr-11990
Commit: 98d463ae48d5feac61daea918ed29480109f37b1
Parents: d66c05c
Author: Chris Hostetter <hossman@apache.org>
Authored: Tue Jul 24 10:43:49 2018 -0700
Committer: Chris Hostetter <hossman@apache.org>
Committed: Tue Jul 24 10:43:49 2018 -0700

----------------------------------------------------------------------
 solr/CHANGES.txt                                |   4 +
 .../apache/solr/search/facet/FacetRequest.java  |  56 +++++++---
 .../org/apache/solr/search/facet/DebugAgg.java  |  16 +--
 .../facet/TestJsonFacetsStatsParsing.java       | 106 +++++++++++++++++++
 solr/solr-ref-guide/src/json-facet-api.adoc     |  29 ++++-
 5 files changed, 185 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98d463ae/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4f6a75e..efcb8c4 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -114,6 +114,10 @@ New Features
 
 * SOLR-12522: Support a runtime function `#ALL` for 'replica' in autoscaling policies (noble)
 
+* SOLR-12567: JSON Facet "functions" now support an extended "type:func" syntax, similar
to other types
+  of facets.  This also allows additional local params to be specified for if the aggregation
function
+  can take advantage of them.  (hossman)
+
 Bug Fixes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98d463ae/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
index a71eb40..658e0fc 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
@@ -33,7 +33,6 @@ import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.search.DocSet;
 import org.apache.solr.search.FunctionQParser;
-import org.apache.solr.search.FunctionQParserPlugin;
 import org.apache.solr.search.JoinQParserPlugin;
 import org.apache.solr.search.QParser;
 import org.apache.solr.search.QueryContext;
@@ -224,10 +223,7 @@ public abstract class FacetRequest {
                                     "'graph' domain change requires non-null 'from' and 'to'
field names");
           }
 
-          NamedList<String> graphParams = new NamedList<>();
-          graphParams.addAll(graph);
-          SolrParams localParams = SolrParams.toSolrParams(graphParams);
-          domain.graphField = new GraphField(localParams);
+          domain.graphField = new GraphField(FacetParser.jsonToSolrParams(graph));
         }
       }
 
@@ -553,30 +549,46 @@ abstract class FacetParser<FacetRequestT extends FacetRequest>
{
         return new FacetRangeParser(this, key).parse(args);
       case "heatmap":
         return new FacetHeatmap.Parser(this, key).parse(args);
+      case "func":
+        return parseStat(key, args);
     }
 
-    AggValueSource stat = parseStat(key, type, args);
-    if (stat == null) {
-      throw err("Unknown facet or stat. key=" + key + " type=" + type + " args=" + args);
-    }
-    return stat;
+    throw err("Unknown facet or stat. key=" + key + " type=" + type + " args=" + args);
   }
 
   public Object parseStringFacetOrStat(String key, String s) throws SyntaxError {
     // "avg(myfield)"
-    return parseStringStat(key, s);
+    return parseStat(key, s);
     // TODO - simple string representation of facets
   }
 
-  // parses avg(x)
-  private AggValueSource parseStringStat(String key, String stat) throws SyntaxError {
-    FunctionQParser parser = (FunctionQParser)QParser.getParser(stat, FunctionQParserPlugin.NAME,
getSolrRequest());
+  /** Parses simple strings like "avg(x)" in the context of optional local params (may be
null) */
+  private AggValueSource parseStatWithParams(String key, SolrParams localparams, String stat)
throws SyntaxError {
+    SolrQueryRequest req = getSolrRequest();
+    FunctionQParser parser = new FunctionQParser(stat, localparams, req.getParams(), req);
     AggValueSource agg = parser.parseAgg(FunctionQParser.FLAG_DEFAULT);
     return agg;
   }
-
-  public AggValueSource parseStat(String key, String type, Object args) throws SyntaxError
{
-    return null;
+  
+  /** Parses simple strings like "avg(x)" or robust Maps that may contain local params */
+  private AggValueSource parseStat(String key, Object args) throws SyntaxError {
+    assert null != args;
+
+    if (args instanceof CharSequence) {
+      // Both of these variants are already unpacked for us in this case, and use no local
params...
+      // 1) x:{func:'min(foo)'}
+      // 2) x:'min(foo)'
+      return parseStatWithParams(key, null, args.toString());
+    }
+    
+    if (args instanceof Map) {
+      final Map<String,Object> statMap = (Map<String,Object>)args;
+      return parseStatWithParams(key, jsonToSolrParams(statMap), statMap.get("func").toString());
+    }
+      
+    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+                            "Stats must be specified as either a simple string, or a json
Map");
+      
   }
 
 
@@ -764,6 +776,16 @@ abstract class FacetParser<FacetRequestT extends FacetRequest>
{
     return parent.getSolrRequest();
   }
 
+  /** 
+   * Helper that handles the possibility of map values being lists 
+   * NOTE: does *NOT* fail on map values that are sub-maps (ie: nested json objects)
+   */
+  public static SolrParams jsonToSolrParams(Map jsonObject) {
+    // HACK, but NamedList already handles the list processing for us...
+    NamedList<String> nl = new NamedList<>();
+    nl.addAll(jsonObject);
+    return SolrParams.toSolrParams(nl);
+  }
 }
 
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98d463ae/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
index aeb1f7e..9de0d12 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
@@ -23,6 +23,7 @@ import java.util.function.IntFunction;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.ValueSource;
+import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.search.DocSet;
@@ -41,7 +42,7 @@ public class DebugAgg extends AggValueSource {
       final String what = fp.hasMoreArguments() ? fp.parseId() : "debug";
 
       switch (what) {
-        case "debug": return new DebugAgg();
+        case "debug": return new DebugAgg(fp.getLocalParams());
         case "numShards": return new DebugAggNumShards();
         default: /* No-Op */
       }
@@ -53,13 +54,14 @@ public class DebugAgg extends AggValueSource {
     }
   }
 
-
-  public DebugAgg() {
+  /**
+   * This exposes the raw localparams used by the FunctionQParser, it does <b>NOT</b>
+   * wrap them in defaults from the request
+   */
+  public final SolrParams localParams;
+  public DebugAgg(SolrParams localParams) {
     super("debug");
-  }
-
-  public DebugAgg(String name) {
-    super(name);
+    this.localParams = localParams;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98d463ae/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsStatsParsing.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsStatsParsing.java
b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsStatsParsing.java
new file mode 100644
index 0000000..438c545
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsStatsParsing.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.facet;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.queries.function.valuesource.IntFieldSource;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.BeforeClass;
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
+
+import org.noggit.ObjectBuilder;
+
+/** Whitebox test of the various syntaxes for specifying stats in JSON Facets */
+public class TestJsonFacetsStatsParsing extends SolrTestCaseJ4 {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig-tlog.xml","schema15.xml");
+  }
+
+  public void testEquality() throws IOException {
+    try (SolrQueryRequest req = req("custom_req_param","foo_i",
+                                    "overridden_param","xxxxx_i")) {
+      
+      // NOTE: we don't bother trying to test 'min(foo_i)' because of SOLR-12559
+      // ...once that bug is fixed, several assertions below will need to change
+      final FacetRequest fr = FacetRequest.parse
+        (req, (Map<String,Object>) ObjectBuilder.fromJSON
+         ("{ " +
+          "  s1:'min(field(\"foo_i\"))', " +
+          "  s2:'min($custom_req_param)', " +
+          "  s3:'min(field($custom_req_param))', " +
+          "  s4:{ func:'min($custom_req_param)' }, " +
+          "  s5:{ type:func, func:'min($custom_req_param)' }, " +
+          "  s6:{ type:func, func:'min($custom_local_param)', custom_local_param:foo_i },
" +
+          "  s7:{ type:func, func:'min($overridden_param)', overridden_param:foo_i }, " +
+          // test the test...
+          "  diff:'min(field(\"bar_i\"))'," +
+          "}"));
+         
+      final Map<String, AggValueSource> stats = fr.getFacetStats();
+      assertEquals(8, stats.size());
+      
+      for (Map.Entry<String,AggValueSource> entry : stats.entrySet()) {
+        final String key = entry.getKey();
+        final AggValueSource agg = entry.getValue();
+        
+        assertEquals("name of " + key, "min", agg.name());
+        assertThat("type of " + key, agg, instanceOf(SimpleAggValueSource.class));
+        SimpleAggValueSource sagg = (SimpleAggValueSource) agg;
+        assertThat("vs of " + key, sagg.getArg(), instanceOf(IntFieldSource.class));
+        
+        if ("diff".equals(key)) {
+          assertEquals("field of " + key, "bar_i", ((IntFieldSource)sagg.getArg()).getField());
+          assertFalse("diff.equals(s1) ?!?!", agg.equals(stats.get("s1")));
+          
+        } else {
+          assertEquals("field of " + key, "foo_i", ((IntFieldSource)sagg.getArg()).getField());
+          
+          assertEquals(key + ".equals(s1)", agg, stats.get("s1"));
+          assertEquals("s1.equals("+key+")", stats.get("s1"), agg);
+        }
+      }
+    }
+  }
+
+  public void testVerboseSyntaxWithLocalParams() throws IOException {
+    // some parsers may choose to use "global" req params as defaults/shadows for
+    // local params, but DebugAgg does not -- so use these to test that the
+    // JSON Parsing doesn't pollute the local params the ValueSourceParser gets...
+    try (SolrQueryRequest req = req("foo", "zzzz", "yaz", "zzzzz")) { 
+      final FacetRequest fr = FacetRequest.parse
+        (req, (Map<String,Object>) ObjectBuilder.fromJSON
+         ("{ x:{type:func, func:'debug()', foo:['abc','xyz'], bar:4.2 } }"));
+
+      final Map<String, AggValueSource> stats = fr.getFacetStats();
+      assertEquals(1, stats.size());
+      AggValueSource agg = stats.get("x");
+      assertNotNull(agg);
+      assertThat(agg, instanceOf(DebugAgg.class));
+      
+      DebugAgg x = (DebugAgg)agg;
+      assertEquals(new String[] {"abc", "xyz"}, x.localParams.getParams("foo"));
+      assertEquals((Float)4.2F, x.localParams.getFloat("bar"));
+      assertNull(x.localParams.get("yaz"));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98d463ae/solr/solr-ref-guide/src/json-facet-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/json-facet-api.adoc b/solr/solr-ref-guide/src/json-facet-api.adoc
index 12628dc..b9a8a09 100644
--- a/solr/solr-ref-guide/src/json-facet-api.adoc
+++ b/solr/solr-ref-guide/src/json-facet-api.adoc
@@ -373,10 +373,35 @@ Unlike all the facets discussed so far, Aggregation functions (also
called *face
 |sumsq |`sumsq(rent)` |sum of squares of field or function
 |variance |`variance(rent)` |variance of numeric field or function
 |stddev |`stddev(rent)` |standard deviation of field or function
-|relatedness |`relatedness('popularity:[100 TO \*]','inStock:true')`|A function for computing
a relatedness score of the documents in the domain to a Foreground set, relative to a Background
set (both defined as queries).  This is primarily for use when building <<Semantic Knowledge
Graphs>>.
+|relatedness |`relatedness('popularity:[100 TO *]','inStock:true')`|A function for computing
a relatedness score of the documents in the domain to a Foreground set, relative to a Background
set (both defined as queries).  This is primarily for use when building <<Semantic Knowledge
Graphs>>.
 |===
 
-Numeric aggregation functions such as `avg` can be on any numeric field, or on another function
of multiple numeric fields such as `avg(mul(price,popularity))`.
+Numeric aggregation functions such as `avg` can be on any numeric field, or on a <<function-queries.adoc#function-queries,nested
function>> of multiple numeric fields such as `avg(div(popularity,price))`.
+
+The most common way of requesting an aggregation function is as a simple containing the expression
you wish to compute:
+
+[source,javascript]
+----
+{
+  "average_roi": "avg(div(popularity,price))"
+}
+----
+
+An expanded form allows for <<local-parameters-in-queries.adoc#local-parameters-in-queries,Local
Parameters>> to be specified.  These may be used explicitly by some custom aggregations,
but can more commonly be used as parameter references to make aggregation expressions more
readable, with out needing to use (global) request parameters:
+
+[source,javascript]
+----
+{
+  "average_roi" : {
+    "type": "func",
+    "func": "avg(div($numer,$denom))",
+    "numer": "mul(popularity,rating)",
+    "denom": "mul(price,size)"
+  }
+}
+----
+
+
 
 == Nested Facets
 


Mime
View raw message