Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D502E18CCB for ; Mon, 1 Feb 2016 03:34:47 +0000 (UTC) Received: (qmail 28218 invoked by uid 500); 1 Feb 2016 03:34:47 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 28204 invoked by uid 99); 1 Feb 2016 03:34:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Feb 2016 03:34:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 42FFEDFDC7; Mon, 1 Feb 2016 03:34:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dsmiley@apache.org To: commits@lucene.apache.org Message-Id: <847f9e2e034d49f2b8a3ab7dd5761c47@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: lucene-solr git commit: SOLR-7968: Make QueryComponent extensible Date: Mon, 1 Feb 2016 03:34:47 +0000 (UTC) Repository: lucene-solr Updated Branches: refs/heads/branch_5x 6b62eb7cb -> ef2fda90b SOLR-7968: Make QueryComponent extensible Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ef2fda90 Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ef2fda90 Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ef2fda90 Branch: refs/heads/branch_5x Commit: ef2fda90b6f3aea131c83f3ca49b47abb00b920f Parents: 6b62eb7 Author: David Smiley Authored: Sun Jan 31 22:25:37 2016 -0500 Committer: David Smiley Committed: Sun Jan 31 22:28:20 2016 -0500 ---------------------------------------------------------------------- solr/CHANGES.txt | 2 + .../solr/handler/component/QueryComponent.java | 48 ++--- .../solr/handler/component/ResponseBuilder.java | 4 + .../apache/solr/handler/component/ShardDoc.java | 166 +---------------- .../component/ShardFieldSortedHitQueue.java | 179 +++++++++++++++++++ 5 files changed, 211 insertions(+), 188 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef2fda90/solr/CHANGES.txt ---------------------------------------------------------------------- diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 276a9b1..ba47838 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -366,6 +366,8 @@ Other Changes * SOLR-8597: add default, no-op QParserPlugin.init(NamedList) method (Christine Poerschke) +* SOLR-7968: Make QueryComponent more extensible. (Markus Jelsma via David Smiley) + ================== 5.4.1 ================== Bug Fixes http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef2fda90/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 9643a23..0fbeab7 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -232,7 +232,7 @@ public class QueryComponent extends SearchComponent } } - private void prepareGrouping(ResponseBuilder rb) throws IOException { + protected void prepareGrouping(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; SolrParams params = req.getParams(); @@ -674,7 +674,7 @@ public class QueryComponent extends SearchComponent } } - private int groupedDistributedProcess(ResponseBuilder rb) { + protected int groupedDistributedProcess(ResponseBuilder rb) { int nextStage = ResponseBuilder.STAGE_DONE; ShardRequestFactory shardRequestFactory = null; @@ -708,7 +708,7 @@ public class QueryComponent extends SearchComponent return nextStage; } - private int regularDistributedProcess(ResponseBuilder rb) { + protected int regularDistributedProcess(ResponseBuilder rb) { if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY) return ResponseBuilder.STAGE_PARSE_QUERY; if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) { @@ -737,7 +737,7 @@ public class QueryComponent extends SearchComponent } } - private void handleGroupedResponses(ResponseBuilder rb, ShardRequest sreq) { + protected void handleGroupedResponses(ResponseBuilder rb, ShardRequest sreq) { ShardResponseProcessor responseProcessor = null; if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_GROUPS) != 0) { responseProcessor = new SearchGroupShardResponseProcessor(); @@ -752,7 +752,7 @@ public class QueryComponent extends SearchComponent } } - private void handleRegularResponses(ResponseBuilder rb, ShardRequest sreq) { + protected void handleRegularResponses(ResponseBuilder rb, ShardRequest sreq) { if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) { mergeIds(rb, sreq); } @@ -778,11 +778,11 @@ public class QueryComponent extends SearchComponent } } - private static final EndResultTransformer MAIN_END_RESULT_TRANSFORMER = new MainEndResultTransformer(); - private static final EndResultTransformer SIMPLE_END_RESULT_TRANSFORMER = new SimpleEndResultTransformer(); + protected static final EndResultTransformer MAIN_END_RESULT_TRANSFORMER = new MainEndResultTransformer(); + protected static final EndResultTransformer SIMPLE_END_RESULT_TRANSFORMER = new SimpleEndResultTransformer(); @SuppressWarnings("unchecked") - private void groupedFinishStage(final ResponseBuilder rb) { + protected void groupedFinishStage(final ResponseBuilder rb) { // To have same response as non-distributed request. GroupingSpecification groupSpec = rb.getGroupingSpec(); if (rb.mergedTopGroups.isEmpty()) { @@ -817,24 +817,24 @@ public class QueryComponent extends SearchComponent endResultTransformer.transform(combinedMap, rb, solrDocumentSource); } - private void regularFinishStage(ResponseBuilder rb) { + protected void regularFinishStage(ResponseBuilder rb) { // We may not have been able to retrieve all the docs due to an // index change. Remove any null documents. - for (Iterator iter = rb._responseDocs.iterator(); iter.hasNext();) { + for (Iterator iter = rb.getResponseDocs().iterator(); iter.hasNext();) { if (iter.next() == null) { iter.remove(); - rb._responseDocs.setNumFound(rb._responseDocs.getNumFound()-1); + rb.getResponseDocs().setNumFound(rb.getResponseDocs().getNumFound()-1); } } - rb.rsp.addResponse(rb._responseDocs); + rb.rsp.addResponse(rb.getResponseDocs()); if (null != rb.getNextCursorMark()) { rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, rb.getNextCursorMark().getSerializedTotem()); } } - private void createDistributedStats(ResponseBuilder rb) { + protected void createDistributedStats(ResponseBuilder rb) { StatsCache cache = rb.req.getCore().getStatsCache(); if ( (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES)!=0 || rb.getSortSpec().includesScore()) { ShardRequest sreq = cache.retrieveStatsRequest(rb); @@ -844,12 +844,12 @@ public class QueryComponent extends SearchComponent } } - private void updateStats(ResponseBuilder rb, ShardRequest sreq) { + protected void updateStats(ResponseBuilder rb, ShardRequest sreq) { StatsCache cache = rb.req.getCore().getStatsCache(); cache.mergeToGlobalStats(rb.req, sreq.responses); } - private void createMainQuery(ResponseBuilder rb) { + protected void createMainQuery(ResponseBuilder rb) { ShardRequest sreq = new ShardRequest(); sreq.purpose = ShardRequest.PURPOSE_GET_TOP_IDS; @@ -934,13 +934,13 @@ public class QueryComponent extends SearchComponent rb.addRequest(this, sreq); } - private boolean addFL(StringBuilder fl, String field, boolean additionalAdded) { + protected boolean addFL(StringBuilder fl, String field, boolean additionalAdded) { if (additionalAdded) fl.append(","); fl.append(field); return true; } - private void mergeIds(ResponseBuilder rb, ShardRequest sreq) { + protected void mergeIds(ResponseBuilder rb, ShardRequest sreq) { List mergeStrategies = rb.getMergeStrategies(); if(mergeStrategies != null) { Collections.sort(mergeStrategies, MergeStrategy.MERGE_COMP); @@ -1113,7 +1113,7 @@ public class QueryComponent extends SearchComponent // again when retrieving stored fields. // TODO: use ResponseBuilder (w/ comments) or the request context? rb.resultIds = resultIds; - rb._responseDocs = responseDocs; + rb.setResponseDocs(responseDocs); populateNextCursorMarkFromMergedShards(rb); @@ -1133,7 +1133,7 @@ public class QueryComponent extends SearchComponent * ShardDocs in resultIds, may or may not be * part of a Cursor based request (method will NOOP if not needed) */ - private void populateNextCursorMarkFromMergedShards(ResponseBuilder rb) { + protected void populateNextCursorMarkFromMergedShards(ResponseBuilder rb) { final CursorMark lastCursorMark = rb.getCursorMark(); if (null == lastCursorMark) { @@ -1175,7 +1175,7 @@ public class QueryComponent extends SearchComponent rb.setNextCursorMark(nextCursorMark); } - private NamedList unmarshalSortValues(SortSpec sortSpec, + protected NamedList unmarshalSortValues(SortSpec sortSpec, NamedList sortFieldValues, IndexSchema schema) { NamedList unmarshalledSortValsPerField = new NamedList(); @@ -1216,7 +1216,7 @@ public class QueryComponent extends SearchComponent return unmarshalledSortValsPerField; } - private void createRetrieveDocs(ResponseBuilder rb) { + protected void createRetrieveDocs(ResponseBuilder rb) { // TODO: in a system with nTiers > 2, we could be passed "ids" here // unless those requests always go to the final destination shard @@ -1270,7 +1270,7 @@ public class QueryComponent extends SearchComponent } - private void returnFields(ResponseBuilder rb, ShardRequest sreq) { + protected void returnFields(ResponseBuilder rb, ShardRequest sreq) { // Keep in mind that this could also be a shard in a multi-tiered system. // TODO: if a multi-tiered system, it seems like some requests // could/should bypass middlemen (like retrieving stored fields) @@ -1321,7 +1321,7 @@ public class QueryComponent extends SearchComponent if (removeKeyField) { doc.removeFields(keyFieldName); } - rb._responseDocs.set(sdoc.positionInResponse, doc); + rb.getResponseDocs().set(sdoc.positionInResponse, doc); } } } @@ -1347,7 +1347,7 @@ public class QueryComponent extends SearchComponent * * TODO: when SOLR-5595 is fixed, this wont be needed, as we dont need to recompute sort values here from the comparator */ - private static class FakeScorer extends Scorer { + protected static class FakeScorer extends Scorer { final int docid; final float score; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef2fda90/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java index 601d8a6..bdb07fb 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java @@ -262,6 +262,10 @@ public class ResponseBuilder public void setResponseDocs(SolrDocumentList _responseDocs) { this._responseDocs = _responseDocs; } + + public SolrDocumentList getResponseDocs() { + return this._responseDocs; + } public boolean isDebugTrack() { return debugTrack; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef2fda90/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java index 97b831b..2935aa1 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java @@ -16,21 +16,9 @@ */ package org.apache.solr.handler.component; -import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.SortField; -import org.apache.lucene.util.PriorityQueue; -import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; - public class ShardDoc extends FieldDoc { public String shard; public String shardAddress; // TODO @@ -44,7 +32,7 @@ public class ShardDoc extends FieldDoc { // this is currently the uniqueKeyField but // may be replaced with internal docid in a future release. - NamedList sortFieldValues; + public NamedList sortFieldValues; // sort field values for *all* docs in a particular shard. // this doc's values are in position orderInShard @@ -93,154 +81,4 @@ public class ShardDoc extends FieldDoc { +" ,positionInResponse="+positionInResponse +" ,sortFieldValues="+sortFieldValues; } -} - - - -// used by distributed search to merge results. -class ShardFieldSortedHitQueue extends PriorityQueue { - - /** Stores a comparator corresponding to each field being sorted by */ - protected Comparator[] comparators; - - /** Stores the sort criteria being used. */ - protected SortField[] fields; - - /** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */ - protected List fieldNames = new ArrayList<>(); - - public ShardFieldSortedHitQueue(SortField[] fields, int size, IndexSearcher searcher) { - super(size); - final int n = fields.length; - //noinspection unchecked - comparators = new Comparator[n]; - this.fields = new SortField[n]; - for (int i = 0; i < n; ++i) { - - // keep track of the named fields - SortField.Type type = fields[i].getType(); - if (type!=SortField.Type.SCORE && type!=SortField.Type.DOC) { - fieldNames.add(fields[i].getField()); - } - - String fieldname = fields[i].getField(); - comparators[i] = getCachedComparator(fields[i], searcher); - - if (fields[i].getType() == SortField.Type.STRING) { - this.fields[i] = new SortField(fieldname, SortField.Type.STRING, - fields[i].getReverse()); - } else { - this.fields[i] = new SortField(fieldname, fields[i].getType(), - fields[i].getReverse()); - } - - //System.out.println("%%%%%%%%%%%%%%%%%% got "+fields[i].getType() +" for "+ fieldname +" fields[i].getReverse(): "+fields[i].getReverse()); - } - } - - @Override - protected boolean lessThan(ShardDoc docA, ShardDoc docB) { - // If these docs are from the same shard, then the relative order - // is how they appeared in the response from that shard. - if (docA.shard == docB.shard) { - // if docA has a smaller position, it should be "larger" so it - // comes before docB. - // This will handle sorting by docid within the same shard - - // comment this out to test comparators. - return !(docA.orderInShard < docB.orderInShard); - } - - - // run comparators - final int n = comparators.length; - int c = 0; - for (int i = 0; i < n && c == 0; i++) { - c = (fields[i].getReverse()) ? comparators[i].compare(docB, docA) - : comparators[i].compare(docA, docB); - } - - // solve tiebreaks by comparing shards (similar to using docid) - // smaller docid's beat larger ids, so reverse the natural ordering - if (c == 0) { - c = -docA.shard.compareTo(docB.shard); - } - - return c < 0; - } - - Comparator getCachedComparator(SortField sortField, IndexSearcher searcher) { - SortField.Type type = sortField.getType(); - if (type == SortField.Type.SCORE) { - return comparatorScore(); - } else if (type == SortField.Type.REWRITEABLE) { - try { - sortField = sortField.rewrite(searcher); - } catch (IOException e) { - throw new SolrException(SERVER_ERROR, "Exception rewriting sort field " + sortField, e); - } - } - return comparatorFieldComparator(sortField); - } - - abstract class ShardComparator implements Comparator { - final SortField sortField; - final String fieldName; - final int fieldNum; - - public ShardComparator(SortField sortField) { - this.sortField = sortField; - this.fieldName = sortField.getField(); - int fieldNum = 0; - for (int i=0; i comparatorScore() { - return new Comparator() { - @Override - public final int compare(final ShardDoc o1, final ShardDoc o2) { - final float f1 = o1.score; - final float f2 = o2.score; - if (f1 < f2) - return -1; - if (f1 > f2) - return 1; - return 0; - } - }; - } - - Comparator comparatorFieldComparator(SortField sortField) { - final FieldComparator fieldComparator; - try { - fieldComparator = sortField.getComparator(0, 0); - } catch (IOException e) { - throw new RuntimeException("Unable to get FieldComparator for sortField " + sortField); - } - - return new ShardComparator(sortField) { - // Since the PriorityQueue keeps the biggest elements by default, - // we need to reverse the field compare ordering so that the - // smallest elements are kept instead of the largest... hence - // the negative sign. - @Override - public int compare(final ShardDoc o1, final ShardDoc o2) { - //noinspection unchecked - return -fieldComparator.compareValues(sortVal(o1), sortVal(o2)); - } - }; - } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef2fda90/solr/core/src/java/org/apache/solr/handler/component/ShardFieldSortedHitQueue.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardFieldSortedHitQueue.java b/solr/core/src/java/org/apache/solr/handler/component/ShardFieldSortedHitQueue.java new file mode 100644 index 0000000..fd0603d --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardFieldSortedHitQueue.java @@ -0,0 +1,179 @@ +package org.apache.solr.handler.component; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.PriorityQueue; +import org.apache.solr.common.SolrException; + +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; + +// used by distributed search to merge results. +public class ShardFieldSortedHitQueue extends PriorityQueue { + + /** Stores a comparator corresponding to each field being sorted by */ + protected Comparator[] comparators; + + /** Stores the sort criteria being used. */ + protected SortField[] fields; + + /** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */ + protected List fieldNames = new ArrayList<>(); + + public ShardFieldSortedHitQueue(SortField[] fields, int size, IndexSearcher searcher) { + super(size); + final int n = fields.length; + //noinspection unchecked + comparators = new Comparator[n]; + this.fields = new SortField[n]; + for (int i = 0; i < n; ++i) { + + // keep track of the named fields + SortField.Type type = fields[i].getType(); + if (type!=SortField.Type.SCORE && type!=SortField.Type.DOC) { + fieldNames.add(fields[i].getField()); + } + + String fieldname = fields[i].getField(); + comparators[i] = getCachedComparator(fields[i], searcher); + + if (fields[i].getType() == SortField.Type.STRING) { + this.fields[i] = new SortField(fieldname, SortField.Type.STRING, + fields[i].getReverse()); + } else { + this.fields[i] = new SortField(fieldname, fields[i].getType(), + fields[i].getReverse()); + } + + //System.out.println("%%%%%%%%%%%%%%%%%% got "+fields[i].getType() +" for "+ fieldname +" fields[i].getReverse(): "+fields[i].getReverse()); + } + } + + @Override + protected boolean lessThan(ShardDoc docA, ShardDoc docB) { + // If these docs are from the same shard, then the relative order + // is how they appeared in the response from that shard. + if (docA.shard == docB.shard) { + // if docA has a smaller position, it should be "larger" so it + // comes before docB. + // This will handle sorting by docid within the same shard + + // comment this out to test comparators. + return !(docA.orderInShard < docB.orderInShard); + } + + + // run comparators + final int n = comparators.length; + int c = 0; + for (int i = 0; i < n && c == 0; i++) { + c = (fields[i].getReverse()) ? comparators[i].compare(docB, docA) + : comparators[i].compare(docA, docB); + } + + // solve tiebreaks by comparing shards (similar to using docid) + // smaller docid's beat larger ids, so reverse the natural ordering + if (c == 0) { + c = -docA.shard.compareTo(docB.shard); + } + + return c < 0; + } + + Comparator getCachedComparator(SortField sortField, IndexSearcher searcher) { + SortField.Type type = sortField.getType(); + if (type == SortField.Type.SCORE) { + return comparatorScore(); + } else if (type == SortField.Type.REWRITEABLE) { + try { + sortField = sortField.rewrite(searcher); + } catch (IOException e) { + throw new SolrException(SERVER_ERROR, "Exception rewriting sort field " + sortField, e); + } + } + return comparatorFieldComparator(sortField); + } + + abstract class ShardComparator implements Comparator { + final SortField sortField; + final String fieldName; + final int fieldNum; + + public ShardComparator(SortField sortField) { + this.sortField = sortField; + this.fieldName = sortField.getField(); + int fieldNum = 0; + for (int i=0; i comparatorScore() { + return new Comparator() { + @Override + public final int compare(final ShardDoc o1, final ShardDoc o2) { + final float f1 = o1.score; + final float f2 = o2.score; + if (f1 < f2) + return -1; + if (f1 > f2) + return 1; + return 0; + } + }; + } + + Comparator comparatorFieldComparator(SortField sortField) { + final FieldComparator fieldComparator; + try { + fieldComparator = sortField.getComparator(0, 0); + } catch (IOException e) { + throw new RuntimeException("Unable to get FieldComparator for sortField " + sortField); + } + + return new ShardComparator(sortField) { + // Since the PriorityQueue keeps the biggest elements by default, + // we need to reverse the field compare ordering so that the + // smallest elements are kept instead of the largest... hence + // the negative sign. + @Override + public int compare(final ShardDoc o1, final ShardDoc o2) { + //noinspection unchecked + return -fieldComparator.compareValues(sortVal(o1), sortVal(o2)); + } + }; + } +} \ No newline at end of file