lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject svn commit: r1694211 - in /lucene/dev/branches/branch_5x: ./ solr/ solr/core/ solr/core/src/java/org/apache/solr/search/ solr/core/src/java/org/apache/solr/search/stats/ solr/core/src/test/org/apache/solr/search/stats/
Date Wed, 05 Aug 2015 13:35:40 GMT
Author: varun
Date: Wed Aug  5 13:35:40 2015
New Revision: 1694211

URL: http://svn.apache.org/r1694211
Log:
SOLR-7818: Distributed stats is only calculated with the terms that are present in the last
shard of a distributed request (merged from trunk r1694210)

Modified:
    lucene/dev/branches/branch_5x/   (props changed)
    lucene/dev/branches/branch_5x/solr/   (props changed)
    lucene/dev/branches/branch_5x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_5x/solr/core/   (props changed)
    lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/   (props changed)
    lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
    lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java

Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1694211&r1=1694210&r2=1694211&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Wed Aug  5 13:35:40 2015
@@ -208,6 +208,9 @@ Bug Fixes
 * SOLR-7756: ExactStatsCache and LRUStatsCache will throw an NPE when a term is not present
on a shard.
   (Varun Thacker, Anshum Gupta)
 
+* SOLR-7818: Distributed stats is only calculated with the terms that are present in the
last shard
+  of a distributed request. (Varun Thacker, Anshum Gupta)
+
 Optimizations
 ----------------------
 

Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java?rev=1694211&r1=1694210&r2=1694211&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
(original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
Wed Aug  5 13:35:40 2015
@@ -17,6 +17,7 @@ package org.apache.solr.search.stats;
  * limitations under the License.
  */
 
+import com.google.common.collect.Lists;
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
@@ -91,6 +92,7 @@ public class ExactStatsCache extends Sta
 
   @Override
   public void mergeToGlobalStats(SolrQueryRequest req, List<ShardResponse> responses)
{
+    Set<Object> allTerms = new HashSet<>();
     for (ShardResponse r : responses) {
       LOG.debug("Merging to global stats, shard={}, response={}", r.getShard(), r.getSolrResponse().getResponse());
       String shard = r.getShard();
@@ -103,16 +105,16 @@ public class ExactStatsCache extends Sta
         addToPerShardTermStats(req, shard, termStatsString);
       }
       List<Object> terms = nl.getAll(TERMS_KEY);
-      if (terms != null) {
-        req.getContext().put(TERMS_KEY, terms);
-      }
-
+      allTerms.addAll(terms);
       String colStatsString = (String) nl.get(COL_STATS_KEY);
       Map<String,CollectionStats> colStats = StatsUtil.colStatsMapFromString(colStatsString);
       if (colStats != null) {
         addToPerShardColStats(req, shard, colStats);
       }
     }
+    if (allTerms.size() > 0) {
+      req.getContext().put(TERMS_KEY, Lists.newArrayList(allTerms));
+    }
     if (LOG.isDebugEnabled()) printStats(req);
   }
 

Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java?rev=1694211&r1=1694210&r2=1694211&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
(original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
Wed Aug  5 13:35:40 2015
@@ -138,6 +138,50 @@ public class TestDistribIDF extends Solr
     }
   }
 
+  @Test
+  public void testMultiCollectionQuery() throws Exception {
+    // collection1 and collection2 are collections which have distributed idf enabled
+    // collection1_local and collection2_local don't have distributed idf available
+    // Only one doc has cat:football in each collection
+    // When doing queries across collections we want to test that the query takes into account
+    // distributed idf for the collection=collection1,collection2 query.
+    // The way we verify is that score should be the same when querying across collection1
and collection2
+    // But should be different when querying across collection1_local and collection2_local
+    // since the idf is calculated per shard
+
+    createCollection("collection1", "conf1");
+    createCollection("collection1_local", "conf2");
+    createCollection("collection2", "conf1");
+    createCollection("collection2_local", "conf2");
+
+    addDocsRandomly();
+
+    //Test against all nodes
+    for (JettySolrRunner jettySolrRunner : solrCluster.getJettySolrRunners()) {
+      SolrClient solrClient = new HttpSolrClient(jettySolrRunner.getBaseUrl().toString());
+      SolrClient solrClient_local = new HttpSolrClient(jettySolrRunner.getBaseUrl().toString());
+
+      SolrQuery query = new SolrQuery("cat:football");
+      query.setFields("*,score").add("collection", "collection1,collection2");
+      QueryResponse queryResponse = solrClient.query("collection1", query);
+      assertEquals(2, queryResponse.getResults().getNumFound());
+      float score1 = (float) queryResponse.getResults().get(0).get("score");
+      float score2 = (float) queryResponse.getResults().get(1).get("score");
+      assertEquals("Doc1 score=" + score1 + " Doc2 score=" + score2, 0, Float.compare(score1,
score2));
+
+
+      query = new SolrQuery("cat:football");
+      query.setFields("*,score").add("collection", "collection1_local,collection2_local");
+      queryResponse = solrClient_local.query("collection1_local", query);
+      assertEquals(2, queryResponse.getResults().getNumFound());
+      assertEquals(2, queryResponse.getResults().get(0).get("id"));
+      assertEquals(1, queryResponse.getResults().get(1).get("id"));
+      float score1_local = (float) queryResponse.getResults().get(0).get("score");
+      float score2_local = (float) queryResponse.getResults().get(1).get("score");
+      assertEquals("Doc1 score=" + score1_local + " Doc2 score=" + score2_local, 1, Float.compare(score1_local,
score2_local));
+    }
+  }
+
   private void createCollection(String name, String config) throws Exception {
     createCollection(name, config, CompositeIdRouter.NAME);
   }



Mime
View raw message