lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r1607360 - in /lucene/dev/trunk/solr: ./ core/src/java/org/apache/solr/handler/component/ core/src/test/org/apache/solr/handler/component/ solrj/src/java/org/apache/solr/common/params/ test-framework/src/java/org/apache/solr/
Date Wed, 02 Jul 2014 13:22:14 GMT
Author: shalin
Date: Wed Jul  2 13:22:14 2014
New Revision: 1607360

URL: http://svn.apache.org/r1607360
Log:
SOLR-5768: Add a distrib.singlePass parameter to make EXECUTE_QUERY phase fetch all fields
and skip GET_FIELDS

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
    lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1607360&r1=1607359&r2=1607360&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Wed Jul  2 13:22:14 2014
@@ -127,6 +127,9 @@ New Features
 
 * SOLR-6044: The 'clusterstatus' API should return live_nodes as well. (shalin)
 
+* SOLR-5768: Add a distrib.singlePass parameter to make EXECUTE_QUERY phase fetch all fields
+  and skip GET_FIELDS. (Gregg Donovan, shalin)
+
 Bug Fixes
 ----------------------
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1607360&r1=1607359&r2=1607360&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
(original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
Wed Jul  2 13:22:14 2014
@@ -30,6 +30,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.lucene.index.AtomicReaderContext;
@@ -111,7 +112,7 @@ import java.util.Comparator;
 public class QueryComponent extends SearchComponent
 {
   public static final String COMPONENT_NAME = "query";
-  
+
   @Override
   public void prepare(ResponseBuilder rb) throws IOException
   {
@@ -148,7 +149,7 @@ public class QueryComponent extends Sear
       Query q = parser.getQuery();
       if (q == null) {
         // normalize a null query to a query that matches nothing
-        q = new BooleanQuery();        
+        q = new BooleanQuery();
       }
 
       rb.setQuery( q );
@@ -174,7 +175,7 @@ public class QueryComponent extends Sear
 
       rb.setSortSpec( parser.getSort(true) );
       rb.setQparser(parser);
-      
+
       final String cursorStr = rb.req.getParams().get(CursorMarkParams.CURSOR_MARK_PARAM);
       if (null != cursorStr) {
         final CursorMark cursorMark = new CursorMark(rb.req.getSchema(),
@@ -218,10 +219,10 @@ public class QueryComponent extends Sear
     if (null != rb.getCursorMark()) {
       // It's hard to imagine, conceptually, what it would mean to combine
       // grouping with a cursor - so for now we just don't allow the combination at all
-      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can not use Grouping
with " + 
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can not use Grouping
with " +
                               CursorMarkParams.CURSOR_MARK_PARAM);
     }
- 
+
     SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
     SolrIndexSearcher searcher = rb.req.getSearcher();
     GroupingSpecification groupingSpec = new GroupingSpecification();
@@ -290,7 +291,7 @@ public class QueryComponent extends Sear
     long timeAllowed = (long)params.getInt( CommonParams.TIME_ALLOWED, -1 );
     if (null != rb.getCursorMark() && 0 < timeAllowed) {
       // fundementally incompatible
-      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can not search using
both " + 
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can not search using
both " +
                               CursorMarkParams.CURSOR_MARK_PARAM + " and " + CommonParams.TIME_ALLOWED);
     }
 
@@ -322,7 +323,7 @@ public class QueryComponent extends Sear
         res.docSet = searcher.getDocSet(queries);
       }
       rb.setResults(res);
-      
+
       ResultContext ctx = new ResultContext();
       ctx.docs = rb.getResults().docList;
       ctx.query = null; // anything?
@@ -493,7 +494,7 @@ public class QueryComponent extends Sear
 
     if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
       if (null != rb.getNextCursorMark()) {
-        rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, 
+        rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
                    rb.getNextCursorMark().getSerializedTotem());
       }
     }
@@ -603,7 +604,7 @@ public class QueryComponent extends Sear
           comparator.setScorer(new FakeScorer(doc, score));
           comparator.copy(0, doc);
           Object val = comparator.value(0);
-          if (null != ft) val = ft.marshalSortValue(val); 
+          if (null != ft) val = ft.marshalSortValue(val);
           vals[position] = val;
         }
 
@@ -624,7 +625,7 @@ public class QueryComponent extends Sear
     }
   }
 
-  @Override  
+  @Override
   public int distributedProcess(ResponseBuilder rb) throws IOException {
     if (rb.grouping()) {
       return groupedDistributedProcess(rb);
@@ -784,7 +785,7 @@ public class QueryComponent extends Sear
 
     rb.rsp.add("response", rb._responseDocs);
     if (null != rb.getNextCursorMark()) {
-      rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, 
+      rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
                  rb.getNextCursorMark().getSerializedTotem());
     }
   }
@@ -802,8 +803,11 @@ public class QueryComponent extends Sear
     // one-pass algorithm if only id and score fields are requested, but not if fl=score
since that's the same as fl=*,score
     ReturnFields fields = rb.rsp.getReturnFields();
 
-    if(fields != null && fields.wantsField(keyFieldName)
-        && fields.getRequestedFieldNames() != null && Arrays.asList(keyFieldName,
"score").containsAll(fields.getRequestedFieldNames())) {
+    // distrib.singlePass=true forces a one-pass query regardless of requested fields
+    boolean distribSinglePass = rb.req.getParams().getBool(ShardParams.DISTRIB_SINGLE_PASS,
false);
+
+    if(distribSinglePass || (fields != null && fields.wantsField(keyFieldName)
+        && fields.getRequestedFieldNames() != null && Arrays.asList(keyFieldName,
"score").containsAll(fields.getRequestedFieldNames()))) {
       sreq.purpose |= ShardRequest.PURPOSE_GET_FIELDS;
       rb.onePassDistributedQuery = true;
     }
@@ -833,21 +837,53 @@ public class QueryComponent extends Sear
       sreq.params.set(CommonParams.ROWS, rb.getSortSpec().getOffset() + rb.getSortSpec().getCount());
     }
 
-    // in this first phase, request only the unique key field
-    // and any fields needed for merging.
     sreq.params.set(ResponseBuilder.FIELD_SORT_VALUES,"true");
 
-    if ( (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES)!=0 || rb.getSortSpec().includesScore())
{
-      sreq.params.set(CommonParams.FL, keyFieldName + ",score");
+    boolean shardQueryIncludeScore = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES)
!= 0 || rb.getSortSpec().includesScore();
+    if (distribSinglePass) {
+      String fl = rb.req.getParams().get(CommonParams.FL);
+      if (fl == null) {
+        if (fields.getRequestedFieldNames() == null && fields.wantsAllFields()) {
+          fl = "*";
+        } else  {
+          fl = "";
+          for (String s : fields.getRequestedFieldNames()) {
+            fl += s + ",";
+          }
+        }
+      }
+      if (!fields.wantsField(keyFieldName))  {
+        // the user has not requested the unique key but
+        // we still need to add it otherwise mergeIds can't work
+        if (fl.endsWith(",")) {
+          fl += keyFieldName;
+        } else  {
+          fl += "," + keyFieldName;
+        }
+      }
+      sreq.params.set(CommonParams.FL, updateFl(fl, shardQueryIncludeScore));
     } else {
-      sreq.params.set(CommonParams.FL, keyFieldName);
+      // in this first phase, request only the unique key field and any fields needed for
merging.
+      if (shardQueryIncludeScore) {
+        sreq.params.set(CommonParams.FL, keyFieldName + ",score");
+      } else {
+        sreq.params.set(CommonParams.FL, keyFieldName);
+      }
     }
 
     rb.addRequest(this, sreq);
   }
 
 
+  String updateFl(String originalFields, boolean includeScoreIfMissing) {
+    if (includeScoreIfMissing && !scorePattern.matcher(originalFields).find()) {
+      return originalFields + ",score";
+    } else {
+      return originalFields;
+    }
+  }
 
+  private static final Pattern scorePattern = Pattern.compile("\\bscore\\b");
 
 
   private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java?rev=1607360&r1=1607359&r2=1607360&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java
(original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java
Wed Jul  2 13:22:14 2014
@@ -18,10 +18,14 @@ package org.apache.solr.handler.componen
  */
 
 import org.apache.solr.BaseDistributedSearchTestCase;
+import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.SimpleOrderedMap;
 import org.junit.BeforeClass;
 
 import java.nio.ByteBuffer;
+import java.util.Map;
 
 /**
  * Test for QueryComponent's distributed querying optimization.
@@ -48,32 +52,67 @@ public class DistributedQueryComponentOp
   public void doTest() throws Exception {
     del("*:*");
 
-    index(id, "1", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x12, 0x62, 0x15
}),                     //  2
+    index(id, "1", "text", "a", "test_sS", "21", "payload", ByteBuffer.wrap(new byte[] {
0x12, 0x62, 0x15 }),                     //  2
           // quick check to prove "*" dynamicField hasn't been broken by somebody mucking
with schema
           "asdfasdf_field_should_match_catchall_dynamic_field_adsfasdf", "value");
-    index(id, "2", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x16
}));                    //  5
-    index(id, "3", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x32, 0x58
}));                    //  8
-    index(id, "4", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15
}));                    //  4
-    index(id, "5", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x35, 0x10,
0x00 }));              //  9
-    index(id, "6", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x1a, 0x2b, 0x3c,
0x00, 0x00, 0x03 }));  //  3
-    index(id, "7", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x00, 0x3c, 0x73
}));                    //  1
-    index(id, "8", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x59, 0x2d, 0x4d
}));                    // 11
-    index(id, "9", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x39, 0x79, 0x7a
}));                    // 10
-    index(id, "10", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c
}));                   //  6
-    index(id, "11", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0xff, (byte)0xaf,
(byte)0x9c })); // 13
-    index(id, "12", "text", "d", "payload", ByteBuffer.wrap(new byte[] { 0x34, (byte)0xdd,
0x4d }));             //  7
-    index(id, "13", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0x80, 0x11,
0x33 }));             // 12
+    index(id, "2", "text", "b", "test_sS", "22", "payload", ByteBuffer.wrap(new byte[] {
0x25, 0x21, 0x16 }));                    //  5
+    index(id, "3", "text", "a", "test_sS", "23", "payload", ByteBuffer.wrap(new byte[] {
0x35, 0x32, 0x58 }));                    //  8
+    index(id, "4", "text", "b", "test_sS", "24", "payload", ByteBuffer.wrap(new byte[] {
0x25, 0x21, 0x15 }));                    //  4
+    index(id, "5", "text", "a", "test_sS", "25", "payload", ByteBuffer.wrap(new byte[] {
0x35, 0x35, 0x10, 0x00 }));              //  9
+    index(id, "6", "text", "c", "test_sS", "26", "payload", ByteBuffer.wrap(new byte[] {
0x1a, 0x2b, 0x3c, 0x00, 0x00, 0x03 }));  //  3
+    index(id, "7", "text", "c", "test_sS", "27", "payload", ByteBuffer.wrap(new byte[] {
0x00, 0x3c, 0x73 }));                    //  1
+    index(id, "8", "text", "c", "test_sS", "28", "payload", ByteBuffer.wrap(new byte[] {
0x59, 0x2d, 0x4d }));                    // 11
+    index(id, "9", "text", "a", "test_sS", "29", "payload", ByteBuffer.wrap(new byte[] {
0x39, 0x79, 0x7a }));                    // 10
+    index(id, "10", "text", "b", "test_sS", "30", "payload", ByteBuffer.wrap(new byte[] {
0x31, 0x39, 0x7c }));                   //  6
+    index(id, "11", "text", "d", "test_sS", "31", "payload", ByteBuffer.wrap(new byte[] {
(byte)0xff, (byte)0xaf, (byte)0x9c })); // 13
+    index(id, "12", "text", "d", "test_sS", "32", "payload", ByteBuffer.wrap(new byte[] {
0x34, (byte)0xdd, 0x4d }));             //  7
+    index(id, "13", "text", "d", "test_sS", "33", "payload", ByteBuffer.wrap(new byte[] {
(byte)0x80, 0x11, 0x33 }));             // 12
     commit();
 
     handle.put("QTime", SKIPVAL);
 
     QueryResponse rsp;
-    rsp = query("q", "*:*", "fl", "id,score", "sort", "payload asc", "rows", "20");
+    rsp = query("q", "*:*", "fl", "id,test_sS,score", "sort", "payload asc", "rows", "20");
     assertFieldValues(rsp.getResults(), id, 7, 1, 6, 4, 2, 10, 12, 3, 5, 9, 8, 13, 11);
+    assertFieldValues(rsp.getResults(), "test_sS", "27", "21", "26", "24", "22", "30", "32",
"23", "25", "29", "28", "33", "31");
     rsp = query("q", "*:*", "fl", "id,score", "sort", "payload desc", "rows", "20");
     assertFieldValues(rsp.getResults(), id, 11, 13, 8, 9, 5, 3, 12, 10, 2, 4, 6, 1, 7);
     // works with just fl=id as well
     rsp = query("q", "*:*", "fl", "id", "sort", "payload desc", "rows", "20");
     assertFieldValues(rsp.getResults(), id, 11, 13, 8, 9, 5, 3, 12, 10, 2, 4, 6, 1, 7);
+
+    rsp = query("q", "*:*", "fl", "id,score", "sort", "payload asc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 7, 1, 6, 4, 2, 10, 12, 3, 5, 9, 8, 13, 11);
+
+    rsp = query("q", "*:*", "fl", "id,test_sS,score", "sort", "payload asc", "rows", "20",
"distrib.singlePass", "true");
+    assertFieldValues(rsp.getResults(), id, 7, 1, 6, 4, 2, 10, 12, 3, 5, 9, 8, 13, 11);
+    assertFieldValues(rsp.getResults(), "test_sS", "27", "21", "26", "24", "22", "30", "32",
"23", "25", "29", "28", "33", "31");
+
+    QueryResponse nonDistribRsp = query("q", "*:*", "fl", "id,test_sS,score", "sort", "payload
asc", "rows", "20");
+    compareResponses(rsp, nonDistribRsp); // make sure distrib and distrib.singlePass return
the same thing
+
+    nonDistribRsp = query("q", "*:*", "fl", "score", "sort", "payload asc", "rows", "20");
+    rsp = query("q", "*:*", "fl", "score", "sort", "payload asc", "rows", "20", "distrib.singlePass",
"true");
+    compareResponses(rsp, nonDistribRsp); // make sure distrib and distrib.singlePass return
the same thing
+
+    // verify that the optimization actually works
+    verifySinglePass("q", "*:*", "fl", "id", "sort", "payload desc", "rows", "20"); // id
only is optimized by default
+    verifySinglePass("q", "*:*", "fl", "id,score", "sort", "payload desc", "rows", "20");
// id,score only is optimized by default
+    verifySinglePass("q", "*:*", "fl", "score", "sort", "payload asc", "rows", "20", "distrib.singlePass",
"true");
+  }
+
+  private void verifySinglePass(String... q) throws SolrServerException {
+    QueryResponse rsp;ModifiableSolrParams params = new ModifiableSolrParams();
+    for (int i = 0; i < q.length; i += 2) {
+      params.add(q[i].toString(), q[i + 1].toString());
+    }
+    params.add("shards", getShardsString());
+    params.add("debug", "track");
+    rsp = queryServer(new ModifiableSolrParams(params));
+    Map<String, Object> debugMap = rsp.getDebugMap();
+    SimpleOrderedMap<Object> track = (SimpleOrderedMap<Object>) debugMap.get("track");
+    assertNotNull(track);
+    assertNotNull(track.get("EXECUTE_QUERY"));
+    assertNull("A single pass request should not have a GET_FIELDS phase", track.get("GET_FIELDS"));
   }
 }

Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java?rev=1607360&r1=1607359&r2=1607360&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java Wed
Jul  2 13:22:14 2014
@@ -52,5 +52,6 @@ public interface ShardParams {
 
   public static final String _ROUTE_ = "_route_";
 
-
+  /** Force a single-pass distributed query? (true/false) */
+  public static final String DISTRIB_SINGLE_PASS = "distrib.singlePass";
 }

Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java?rev=1607360&r1=1607359&r2=1607360&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java Wed
Jul  2 13:22:14 2014
@@ -1753,11 +1753,11 @@ public abstract class SolrTestCaseJ4 ext
       SolrDocument doc = documents.get(docNum - 1);
       Object expected = expectedValues[docNum - 1];
       Object actual = doc.get(fieldName);
-      if (null != expected && null != actual) {
-        if ( ! expected.equals(actual)) {
-          fail( "Unexpected " + fieldName + " field value in document #" + docNum
-              + ": expected=[" + expected + "], actual=[" + actual + "]");
-        }
+      if ((null == expected && null != actual) ||
+          (null != expected && null == actual) ||
+          (null != expected && null != actual && !expected.equals(actual)))
{
+        fail("Unexpected " + fieldName + " field value in document #" + docNum
+            + ": expected=[" + expected + "], actual=[" + actual + "]");
       }
     }
   }



Mime
View raw message