lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject lucene-solr:master: SOLR-7452: json facet API, refine/skip through buckets already visited
Date Tue, 21 Mar 2017 12:42:52 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/master 4171ef79b -> 6786089b0


SOLR-7452: json facet API, refine/skip through buckets already visited


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6786089b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6786089b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6786089b

Branch: refs/heads/master
Commit: 6786089b0bc8be50287c2527874ca4503114addd
Parents: 4171ef7
Author: yonik <yonik@apache.org>
Authored: Tue Mar 21 08:42:33 2017 -0400
Committer: yonik <yonik@apache.org>
Committed: Tue Mar 21 08:42:47 2017 -0400

----------------------------------------------------------------------
 .../solr/search/facet/FacetFieldProcessor.java  | 52 +++++++++++++++-----
 .../FacetFieldProcessorByEnumTermsStream.java   |  2 +-
 .../apache/solr/search/facet/FacetModule.java   |  1 +
 .../solr/search/facet/FacetProcessor.java       | 10 ++--
 .../apache/solr/search/facet/FacetQuery.java    |  2 +-
 .../apache/solr/search/facet/FacetRange.java    |  4 +-
 .../search/facet/TestJsonFacetRefinement.java   | 22 +++++----
 7 files changed, 62 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6786089b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
index fb44f62..1ba252e 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
@@ -19,6 +19,7 @@ package org.apache.solr.search.facet;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -311,7 +312,7 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField>
{
     if (freq.missing) {
       // TODO: it would be more efficient to build up a missing DocSet if we need it here
anyway.
       SimpleOrderedMap<Object> missingBucket = new SimpleOrderedMap<>();
-      fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null,
false);
+      fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null,
false, null);
       res.add("missing", missingBucket);
     }
 
@@ -379,7 +380,7 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField>
{
       }
     }
 
-    processSubs(target, filter, subDomain, false);
+    processSubs(target, filter, subDomain, false, null);
   }
 
   @Override
@@ -513,31 +514,43 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField>
{
   }
 
 
+  /*
+   "qfacet":{"cat2":{"_l":["A"]}},
+   "all":{"_s":[[
+     "all",
+     {"cat3":{"_l":["A"]}}]]},
+   "cat1":{"_l":["A"]}}}
+
+   */
+
+  static <T> List<T> asList(Object list) {
+    return list != null ? (List<T>)list : Collections.EMPTY_LIST;
+  }
 
   protected SimpleOrderedMap<Object> refineFacets() throws IOException {
-    List leaves = (List)fcontext.facetInfo.get("_l");
+    List leaves = asList(fcontext.facetInfo.get("_l"));
+    List<List> skip = asList(fcontext.facetInfo.get("_s"));
+    List<List> missing = asList(fcontext.facetInfo.get("_m"));
 
     // For leaf refinements, we do full faceting for each leaf bucket.  Any sub-facets of
these buckets will be fully evaluated.  Because of this, we should never
     // encounter leaf refinements that have sub-facets that return partial results.
 
     SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();
-    List<SimpleOrderedMap> bucketList = new ArrayList<>(leaves.size());
+    List<SimpleOrderedMap> bucketList = new ArrayList<>( leaves.size() + skip.size()
+ missing.size() );
     res.add("buckets", bucketList);
 
     // TODO: an alternate implementations can fill all accs at once
     createAccs(-1, 1);
 
-    FieldType ft = sf.getType();
     for (Object bucketVal : leaves) {
-      SimpleOrderedMap<Object> bucket = new SimpleOrderedMap<>();
-      bucketList.add(bucket);
-      bucket.add("val", bucketVal);
-
-      // String internal = ft.toInternal( tobj.toString() );  // TODO - we need a better
way to get from object to query...
-
-      Query domainQ = ft.getFieldQuery(null, sf, bucketVal.toString());
+      bucketList.add( refineBucket(bucketVal, false, null) );
+    }
+    for (List bucketAndFacetInfo : skip) {
+      assert bucketAndFacetInfo.size() == 2;
+      Object bucketVal = bucketAndFacetInfo.get(0);
+      Map<String,Object> facetInfo = (Map<String, Object>) bucketAndFacetInfo.get(1);
 
-      fillBucket(bucket, domainQ, null, false);
+      bucketList.add( refineBucket(bucketVal, true, facetInfo ) );
     }
 
     // If there are just a couple of leaves, and if the domain is large, then
@@ -548,4 +561,17 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField>
{
     return res;
   }
 
+  private SimpleOrderedMap<Object> refineBucket(Object bucketVal, boolean skip, Map<String,Object>
facetInfo) throws IOException {
+    SimpleOrderedMap<Object> bucket = new SimpleOrderedMap<>();
+    FieldType ft = sf.getType();
+    bucket.add("val", bucketVal);
+    // String internal = ft.toInternal( tobj.toString() );  // TODO - we need a better way
to get from object to query...
+
+    Query domainQ = ft.getFieldQuery(null, sf, bucketVal.toString());
+
+    fillBucket(bucket, domainQ, null, skip, facetInfo);
+
+    return bucket;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6786089b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java
b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java
index 94f3b2d..d28e024 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByEnumTermsStream.java
@@ -333,7 +333,7 @@ class FacetFieldProcessorByEnumTermsStream extends FacetFieldProcessor
implement
         bucket.add("val", bucketVal);
         addStats(bucket, 0);
         if (hasSubFacets) {
-          processSubs(bucket, bucketQuery, termSet, false);
+          processSubs(bucket, bucketQuery, termSet, false, null);
         }
 
         // TODO... termSet needs to stick around for streaming sub-facets?

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6786089b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
index 630e968..bf13791 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
@@ -235,6 +235,7 @@ public class FacetModule extends SearchComponent {
       Map<String,Object> finfo = new HashMap<>(1);
       finfo.put(FACET_REFINE, refinement);
       String finfoStr = JSONUtil.toJSON(finfo);
+      // System.err.println("##################### REFINE=" + finfoStr);
       shardsRefineRequest.params.add(FACET_INFO, finfoStr);
 
       if (newRequest) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6786089b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java
index de6dd72..cf4d0fe 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java
@@ -367,7 +367,7 @@ public abstract class FacetProcessor<FacetRequestT extends FacetRequest>
 {
   }
 
   // TODO: rather than just have a raw "response", perhaps we should model as a bucket object
that contains the response plus extra info?
-  void fillBucket(SimpleOrderedMap<Object> bucket, Query q, DocSet result, boolean
skip) throws IOException {
+  void fillBucket(SimpleOrderedMap<Object> bucket, Query q, DocSet result, boolean
skip, Map<String,Object> facetInfo) throws IOException {
 
     // TODO: we don't need the DocSet if we've already calculated everything during the first
phase
     boolean needDocSet = freq.getFacetStats().size() > 0 || freq.getSubFacets().size()
> 0;
@@ -398,7 +398,7 @@ public abstract class FacetProcessor<FacetRequestT extends FacetRequest>
 {
       if (!skip) {
         processStats(bucket, result, count);
       }
-      processSubs(bucket, q, result, skip);
+      processSubs(bucket, q, result, skip, facetInfo);
     } finally {
       if (result != null) {
         // result.decref(); // OFF-HEAP
@@ -407,7 +407,7 @@ public abstract class FacetProcessor<FacetRequestT extends FacetRequest>
 {
     }
   }
 
-  void processSubs(SimpleOrderedMap<Object> response, Query filter, DocSet domain,
boolean skip) throws IOException {
+  void processSubs(SimpleOrderedMap<Object> response, Query filter, DocSet domain,
boolean skip, Map<String,Object> facetInfo) throws IOException {
 
     boolean emptyDomain = domain == null || domain.size() == 0;
 
@@ -423,8 +423,8 @@ public abstract class FacetProcessor<FacetRequestT extends FacetRequest>
 {
       }
 
       Map<String,Object>facetInfoSub = null;
-      if (fcontext.facetInfo != null) {
-        facetInfoSub = (Map<String,Object>)fcontext.facetInfo.get(sub.getKey());
+      if (facetInfo != null) {
+        facetInfoSub = (Map<String,Object>)facetInfo.get(sub.getKey());
       }
 
       // If we're skipping this node, then we only need to process sub-facets that have facet
info specified.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6786089b/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java b/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java
index 584bec3..a6782bf7 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java
@@ -61,7 +61,7 @@ class FacetQueryProcessor extends FacetProcessor<FacetQuery> {
       // FIXME - what needs to be done here?
     }
     response = new SimpleOrderedMap<>();
-    fillBucket(response, freq.q, null, (fcontext.flags & FacetContext.SKIP_FACET)!=0);
+    fillBucket(response, freq.q, null, (fcontext.flags & FacetContext.SKIP_FACET)!=0,
fcontext.facetInfo);
   }
 
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6786089b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java
index 5d0989b..682dc19 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java
@@ -350,7 +350,7 @@ class FacetRangeProcessor extends FacetProcessor<FacetRange> {
     if (freq.getSubFacets().size() > 0) {
       DocSet subBase = intersections[slot];
       try {
-        processSubs(bucket, filters[slot], subBase, false);
+        processSubs(bucket, filters[slot], subBase, false, null);
       } finally {
         // subContext.base.decref();  // OFF-HEAP
         // subContext.base = null;  // do not modify context after creation... there may
be deferred execution (i.e. streaming)
@@ -367,7 +367,7 @@ class FacetRangeProcessor extends FacetProcessor<FacetRange> {
     }
 
     Query rangeQ = sf.getType().getRangeQuery(null, sf, range.low == null ? null : calc.formatValue(range.low),
range.high==null ? null : calc.formatValue(range.high), range.includeLower, range.includeUpper);
-    fillBucket(bucket, rangeQ, null, false);
+    fillBucket(bucket, rangeQ, null, false, null);
 
     return bucket;
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6786089b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
index f23ae8c..869c90b 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
@@ -227,16 +227,16 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
     String cat_s = p.get("cat_s");
     String num_d = p.get("num_d");
 
-    clients.get(0).add( sdoc("id", "01", cat_s, "A", num_d, -1) ); // A wins count tie
-    clients.get(0).add( sdoc("id", "02", cat_s, "B", num_d, 3) );
+    clients.get(0).add( sdoc("id", "01", "all_s","all", cat_s, "A", num_d, -1) ); // A wins
count tie
+    clients.get(0).add( sdoc("id", "02", "all_s","all", cat_s, "B", num_d, 3) );
 
-    clients.get(1).add( sdoc("id", "11", cat_s, "B", num_d, -5) ); // B highest count
-    clients.get(1).add( sdoc("id", "12", cat_s, "B", num_d, -11) );
-    clients.get(1).add( sdoc("id", "13", cat_s, "A", num_d, 7) );
+    clients.get(1).add( sdoc("id", "11", "all_s","all", cat_s, "B", num_d, -5) ); // B highest
count
+    clients.get(1).add( sdoc("id", "12", "all_s","all", cat_s, "B", num_d, -11) );
+    clients.get(1).add( sdoc("id", "13", "all_s","all", cat_s, "A", num_d, 7) );
 
-    clients.get(2).add( sdoc("id", "21", cat_s, "A", num_d, 17) ); // A highest count
-    clients.get(2).add( sdoc("id", "22", cat_s, "A", num_d, -19) );
-    clients.get(2).add( sdoc("id", "23", cat_s, "B", num_d, 11) );
+    clients.get(2).add( sdoc("id", "21", "all_s","all", cat_s, "A", num_d, 17) ); // A highest
count
+    clients.get(2).add( sdoc("id", "22", "all_s","all", cat_s, "A", num_d, -19) );
+    clients.get(2).add( sdoc("id", "23", "all_s","all", cat_s, "B", num_d, 11) );
 
     client.commit();
 
@@ -291,12 +291,16 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
         "json.facet", "{" +
             " cat0:{type:terms, field:${cat_s}, sort:'min1 asc', limit:1, overrequest:0,
refine:false, facet:{ min1:'min(${num_d})'}   }" +
             ",cat1:{type:terms, field:${cat_s}, sort:'min1 asc', limit:1, overrequest:0,
refine:true,  facet:{ min1:'min(${num_d})'}   }" +
+            ",qfacet:{type:query, q:'*:*', facet:{  cat2:{type:terms, field:${cat_s}, sort:'min1
asc', limit:1, overrequest:0, refine:true,  facet:{ min1:'min(${num_d})'}   }  }}" +  // refinement
needed through a query facet
+            ",allf:{type:terms, field:all_s,  facet:{  cat3:{type:terms, field:${cat_s},
sort:'min1 asc', limit:1, overrequest:0, refine:true,  facet:{ min1:'min(${num_d})'}   } 
}}" +  // refinement needed through field facet
             ",sum1:'sum(num_d)'" +  // make sure that root bucket stats aren't affected by
refinement
             "}"
         )
         , "facets=={ count:8" +
-            ", cat0:{ buckets:[ {val:A,count:3, min1:-19.0} ] }" +  // B wins in shard2,
so we're missing the "A" count for that shar w/o refinement.
+            ", cat0:{ buckets:[ {val:A,count:3, min1:-19.0} ] }" +  // B wins in shard2,
so we're missing the "A" count for that shard w/o refinement.
             ", cat1:{ buckets:[ {val:A,count:4, min1:-19.0} ] }" +  // with refinement, we
get the right count
+            ", qfacet:{ count:8,  cat2:{ buckets:[ {val:A,count:4, min1:-19.0} ] }    }"
+  // just like the previous response, just nested under a query facet
+            ", allf:{ buckets:[  {cat3:{ buckets:[ {val:A,count:4, min1:-19.0} ] }  ,count:8,val:all
  }]  }" +  // just like the previous response, just nested under a field facet
             ", sum1:2.0" +
             "}"
     );


Mime
View raw message