Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D66DD200CFE for ; Fri, 8 Sep 2017 17:43:18 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D3D97160CC7; Fri, 8 Sep 2017 15:43:18 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A2C5E160CB2 for ; Fri, 8 Sep 2017 17:43:17 +0200 (CEST) Received: (qmail 3459 invoked by uid 500); 8 Sep 2017 15:43:16 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 3450 invoked by uid 99); 8 Sep 2017 15:43:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 08 Sep 2017 15:43:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5A4B3F55C9; Fri, 8 Sep 2017 15:43:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cpoerschke@apache.org To: commits@lucene.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: lucene-solr:master: SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke) Date: Fri, 8 Sep 2017 15:43:16 +0000 (UTC) archived-at: Fri, 08 Sep 2017 15:43:19 -0000 Repository: lucene-solr Updated Branches: refs/heads/master a5402f686 -> 244ebd1aa SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke) Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/244ebd1a Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/244ebd1a Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/244ebd1a Branch: refs/heads/master Commit: 244ebd1aa4c2311dcce4a52d8fd425e47acaf9ad Parents: a5402f6 Author: Christine Poerschke Authored: Fri Sep 8 16:26:56 2017 +0100 Committer: Christine Poerschke Committed: Fri Sep 8 16:42:46 2017 +0100 ---------------------------------------------------------------------- solr/CHANGES.txt | 1 + .../solr/handler/component/QueryComponent.java | 467 +++++++++++-------- 2 files changed, 273 insertions(+), 195 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/244ebd1a/solr/CHANGES.txt ---------------------------------------------------------------------- diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index f652fb2..d278821 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -197,6 +197,7 @@ Other Changes * SOLR-11322: JSON Facet API: instead of returning NaN, min & max aggregations omit the value for any bucket with no values in the numeric field. (yonik) +* SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke) ================== 7.0.0 ================== http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/244ebd1a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 1d88adc..417d583 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -306,12 +306,12 @@ public class QueryComponent extends SearchComponent if (!params.getBool(COMPONENT_NAME, true)) { return; } - SolrIndexSearcher searcher = req.getSearcher(); StatsCache statsCache = req.getCore().getStatsCache(); int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS); if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) { + SolrIndexSearcher searcher = req.getSearcher(); statsCache.returnLocalStats(rb, searcher); return; } @@ -321,50 +321,11 @@ public class QueryComponent extends SearchComponent statsCache.receiveGlobalStats(req); } - SolrQueryResponse rsp = rb.rsp; - IndexSchema schema = searcher.getSchema(); - // Optional: This could also be implemented by the top-level searcher sending // a filter that lists the ids... that would be transparent to // the request handler, but would be more expensive (and would preserve score // too if desired). - String ids = params.get(ShardParams.IDS); - if (ids != null) { - SchemaField idField = schema.getUniqueKeyField(); - List idArr = StrUtils.splitSmart(ids, ",", true); - int[] luceneIds = new int[idArr.size()]; - int docs = 0; - if (idField.getType().isPointField()) { - for (int i=0; i= 0) { - luceneIds[docs++] = id; - } - } - } else { - for (int i=0; i= 0) - luceneIds[docs++] = id; - } - } - - DocListAndSet res = new DocListAndSet(); - res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0); - if (rb.isNeedDocSet()) { - // TODO: create a cache for this! - List queries = new ArrayList<>(); - queries.add(rb.getQuery()); - List filters = rb.getFilters(); - if (filters != null) queries.addAll(filters); - res.docSet = searcher.getDocSet(queries); - } - rb.setResults(res); - - ResultContext ctx = new BasicResultContext(rb); - rsp.addResponse(ctx); + if (doProcessSearchByIds(rb)) { return; } @@ -395,145 +356,15 @@ public class QueryComponent extends SearchComponent if (groupingSpec != null) { cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag try { - boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0; if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) { - CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder() - .setQueryCommand(cmd) - .setNeedDocSet(false) // Order matters here - .setIncludeHitCount(true) - .setSearcher(searcher); - - for (String field : groupingSpec.getFields()) { - topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder() - .setField(schema.getField(field)) - .setGroupSort(groupingSpec.getGroupSort()) - .setTopNGroups(cmd.getOffset() + cmd.getLen()) - .setIncludeGroupCount(groupingSpec.isIncludeGroupCount()) - .build() - ); - } - - CommandHandler commandHandler = topsGroupsActionBuilder.build(); - commandHandler.execute(); - SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher); - rsp.add("firstPhase", commandHandler.processResult(result, serializer)); - rsp.add("totalHitCount", commandHandler.getTotalHitCount()); - rb.setResult(result); + doProcessGroupedDistributedSearchFirstPhase(rb, result); return; } else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) { - CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder() - .setQueryCommand(cmd) - .setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0) - .setSearcher(searcher); - - int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc()); - docsToCollect = Math.max(docsToCollect, 1); - - for (String field : groupingSpec.getFields()) { - SchemaField schemaField = schema.getField(field); - String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field); - if (topGroupsParam == null) { - topGroupsParam = new String[0]; - } - - List> topGroups = new ArrayList<>(topGroupsParam.length); - for (String topGroup : topGroupsParam) { - SearchGroup searchGroup = new SearchGroup<>(); - if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) { - BytesRefBuilder builder = new BytesRefBuilder(); - schemaField.getType().readableToIndexed(topGroup, builder); - searchGroup.groupValue = builder.get(); - } - topGroups.add(searchGroup); - } - - secondPhaseBuilder.addCommandField( - new TopGroupsFieldCommand.Builder() - .setField(schemaField) - .setGroupSort(groupingSpec.getGroupSort()) - .setSortWithinGroup(groupingSpec.getSortWithinGroup()) - .setFirstPhaseGroups(topGroups) - .setMaxDocPerGroup(docsToCollect) - .setNeedScores(needScores) - .setNeedMaxScore(needScores) - .build() - ); - } - - for (String query : groupingSpec.getQueries()) { - secondPhaseBuilder.addCommandField(new Builder() - .setDocsToCollect(docsToCollect) - .setSort(groupingSpec.getGroupSort()) - .setQuery(query, rb.req) - .setDocSet(searcher) - .build() - ); - } - - CommandHandler commandHandler = secondPhaseBuilder.build(); - commandHandler.execute(); - TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb); - rsp.add("secondPhase", commandHandler.processResult(result, serializer)); - rb.setResult(result); + doProcessGroupedDistributedSearchSecondPhase(rb, result); return; } - int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0); - boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100; - Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ? - Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped; - int limitDefault = cmd.getLen(); // this is normally from "rows" - Grouping grouping = - new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain()); - grouping.setGroupSort(groupingSpec.getGroupSort()) - .setWithinGroupSort(groupingSpec.getSortWithinGroup()) - .setDefaultFormat(groupingSpec.getResponseFormat()) - .setLimitDefault(limitDefault) - .setDefaultTotalCount(defaultTotalCount) - .setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit()) - .setGroupOffsetDefault(groupingSpec.getWithinGroupOffset()) - .setGetGroupedDocSet(groupingSpec.isTruncateGroups()); - - if (groupingSpec.getFields() != null) { - for (String field : groupingSpec.getFields()) { - grouping.addFieldCommand(field, rb.req); - } - } - - if (groupingSpec.getFunctions() != null) { - for (String groupByStr : groupingSpec.getFunctions()) { - grouping.addFunctionCommand(groupByStr, rb.req); - } - } - - if (groupingSpec.getQueries() != null) { - for (String groupByStr : groupingSpec.getQueries()) { - grouping.addQueryCommand(groupByStr, rb.req); - } - } - - if( rb.isNeedDocList() || rb.isDebug() ){ - // we need a single list of the returned docs - cmd.setFlags(SolrIndexSearcher.GET_DOCLIST); - } - - grouping.execute(); - if (grouping.isSignalCacheWarning()) { - rsp.add( - "cacheWarning", - String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache) - ); - } - rb.setResult(result); - - if (grouping.mainResult != null) { - ResultContext ctx = new BasicResultContext(rb, grouping.mainResult); - rsp.addResponse(ctx); - rsp.getToLog().add("hits", grouping.mainResult.matches()); - } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this. - rsp.add("grouped", result.groupedResults); - rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches()); - } + doProcessGroupedSearch(rb, result); return; } catch (SyntaxError e) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); @@ -541,27 +372,7 @@ public class QueryComponent extends SearchComponent } // normal search result - searcher.search(result, cmd); - rb.setResult(result); - - ResultContext ctx = new BasicResultContext(rb); - rsp.addResponse(ctx); - rsp.getToLog().add("hits", rb.getResults().docList.matches()); - - if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) { - if (null != rb.getNextCursorMark()) { - rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, - rb.getNextCursorMark().getSerializedTotem()); - } - } - - if(rb.mergeFieldHandler != null) { - rb.mergeFieldHandler.handleMergeFields(rb, searcher); - } else { - doFieldSortValues(rb, searcher); - } - - doPrefetch(rb); + doProcessUngroupedSearch(rb, result); } protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException @@ -1385,6 +1196,272 @@ public class QueryComponent extends SearchComponent return Category.QUERY; } + private boolean doProcessSearchByIds(ResponseBuilder rb) throws IOException { + + SolrQueryRequest req = rb.req; + SolrQueryResponse rsp = rb.rsp; + + SolrParams params = req.getParams(); + + String ids = params.get(ShardParams.IDS); + if (ids == null) { + return false; + } + + SolrIndexSearcher searcher = req.getSearcher(); + IndexSchema schema = searcher.getSchema(); + SchemaField idField = schema.getUniqueKeyField(); + List idArr = StrUtils.splitSmart(ids, ",", true); + int[] luceneIds = new int[idArr.size()]; + int docs = 0; + if (idField.getType().isPointField()) { + for (int i=0; i= 0) { + luceneIds[docs++] = id; + } + } + } else { + for (int i=0; i= 0) + luceneIds[docs++] = id; + } + } + + DocListAndSet res = new DocListAndSet(); + res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0); + if (rb.isNeedDocSet()) { + // TODO: create a cache for this! + List queries = new ArrayList<>(); + queries.add(rb.getQuery()); + List filters = rb.getFilters(); + if (filters != null) queries.addAll(filters); + res.docSet = searcher.getDocSet(queries); + } + rb.setResults(res); + + ResultContext ctx = new BasicResultContext(rb); + rsp.addResponse(ctx); + return true; + } + + private void doProcessGroupedDistributedSearchFirstPhase(ResponseBuilder rb, QueryResult result) throws IOException { + + GroupingSpecification groupingSpec = rb.getGroupingSpec(); + assert null != groupingSpec : "GroupingSpecification is null"; + + SolrQueryRequest req = rb.req; + SolrQueryResponse rsp = rb.rsp; + + SolrIndexSearcher searcher = req.getSearcher(); + IndexSchema schema = searcher.getSchema(); + + QueryCommand cmd = rb.getQueryCommand(); + + CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder() + .setQueryCommand(cmd) + .setNeedDocSet(false) // Order matters here + .setIncludeHitCount(true) + .setSearcher(searcher); + + for (String field : groupingSpec.getFields()) { + topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder() + .setField(schema.getField(field)) + .setGroupSort(groupingSpec.getGroupSort()) + .setTopNGroups(cmd.getOffset() + cmd.getLen()) + .setIncludeGroupCount(groupingSpec.isIncludeGroupCount()) + .build() + ); + } + + CommandHandler commandHandler = topsGroupsActionBuilder.build(); + commandHandler.execute(); + SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher); + + rsp.add("firstPhase", commandHandler.processResult(result, serializer)); + rsp.add("totalHitCount", commandHandler.getTotalHitCount()); + rb.setResult(result); + } + + private void doProcessGroupedDistributedSearchSecondPhase(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError { + + GroupingSpecification groupingSpec = rb.getGroupingSpec(); + assert null != groupingSpec : "GroupingSpecification is null"; + + SolrQueryRequest req = rb.req; + SolrQueryResponse rsp = rb.rsp; + + SolrParams params = req.getParams(); + + SolrIndexSearcher searcher = req.getSearcher(); + IndexSchema schema = searcher.getSchema(); + + QueryCommand cmd = rb.getQueryCommand(); + boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0; + + CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder() + .setQueryCommand(cmd) + .setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0) + .setSearcher(searcher); + + int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc()); + docsToCollect = Math.max(docsToCollect, 1); + + for (String field : groupingSpec.getFields()) { + SchemaField schemaField = schema.getField(field); + String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field); + if (topGroupsParam == null) { + topGroupsParam = new String[0]; + } + + List> topGroups = new ArrayList<>(topGroupsParam.length); + for (String topGroup : topGroupsParam) { + SearchGroup searchGroup = new SearchGroup<>(); + if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) { + BytesRefBuilder builder = new BytesRefBuilder(); + schemaField.getType().readableToIndexed(topGroup, builder); + searchGroup.groupValue = builder.get(); + } + topGroups.add(searchGroup); + } + + secondPhaseBuilder.addCommandField( + new TopGroupsFieldCommand.Builder() + .setField(schemaField) + .setGroupSort(groupingSpec.getGroupSort()) + .setSortWithinGroup(groupingSpec.getSortWithinGroup()) + .setFirstPhaseGroups(topGroups) + .setMaxDocPerGroup(docsToCollect) + .setNeedScores(needScores) + .setNeedMaxScore(needScores) + .build() + ); + } + + for (String query : groupingSpec.getQueries()) { + secondPhaseBuilder.addCommandField(new Builder() + .setDocsToCollect(docsToCollect) + .setSort(groupingSpec.getGroupSort()) + .setQuery(query, rb.req) + .setDocSet(searcher) + .build() + ); + } + + CommandHandler commandHandler = secondPhaseBuilder.build(); + commandHandler.execute(); + TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb); + rsp.add("secondPhase", commandHandler.processResult(result, serializer)); + rb.setResult(result); + } + + private void doProcessGroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError { + + GroupingSpecification groupingSpec = rb.getGroupingSpec(); + assert null != groupingSpec : "GroupingSpecification is null"; + + SolrQueryRequest req = rb.req; + SolrQueryResponse rsp = rb.rsp; + + SolrParams params = req.getParams(); + + SolrIndexSearcher searcher = req.getSearcher(); + + QueryCommand cmd = rb.getQueryCommand(); + + int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0); + boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100; + Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ? + Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped; + int limitDefault = cmd.getLen(); // this is normally from "rows" + Grouping grouping = + new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain()); + grouping.setGroupSort(groupingSpec.getGroupSort()) + .setWithinGroupSort(groupingSpec.getSortWithinGroup()) + .setDefaultFormat(groupingSpec.getResponseFormat()) + .setLimitDefault(limitDefault) + .setDefaultTotalCount(defaultTotalCount) + .setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit()) + .setGroupOffsetDefault(groupingSpec.getWithinGroupOffset()) + .setGetGroupedDocSet(groupingSpec.isTruncateGroups()); + + if (groupingSpec.getFields() != null) { + for (String field : groupingSpec.getFields()) { + grouping.addFieldCommand(field, rb.req); + } + } + + if (groupingSpec.getFunctions() != null) { + for (String groupByStr : groupingSpec.getFunctions()) { + grouping.addFunctionCommand(groupByStr, rb.req); + } + } + + if (groupingSpec.getQueries() != null) { + for (String groupByStr : groupingSpec.getQueries()) { + grouping.addQueryCommand(groupByStr, rb.req); + } + } + + if( rb.isNeedDocList() || rb.isDebug() ){ + // we need a single list of the returned docs + cmd.setFlags(SolrIndexSearcher.GET_DOCLIST); + } + + grouping.execute(); + if (grouping.isSignalCacheWarning()) { + rsp.add( + "cacheWarning", + String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache) + ); + } + rb.setResult(result); + + if (grouping.mainResult != null) { + ResultContext ctx = new BasicResultContext(rb, grouping.mainResult); + rsp.addResponse(ctx); + rsp.getToLog().add("hits", grouping.mainResult.matches()); + } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this. + rsp.add("grouped", result.groupedResults); + rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches()); + } + } + + private void doProcessUngroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException { + + SolrQueryRequest req = rb.req; + SolrQueryResponse rsp = rb.rsp; + + SolrIndexSearcher searcher = req.getSearcher(); + + QueryCommand cmd = rb.getQueryCommand(); + + searcher.search(result, cmd); + rb.setResult(result); + + ResultContext ctx = new BasicResultContext(rb); + rsp.addResponse(ctx); + rsp.getToLog().add("hits", rb.getResults().docList.matches()); + + if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) { + if (null != rb.getNextCursorMark()) { + rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, + rb.getNextCursorMark().getSerializedTotem()); + } + } + + if(rb.mergeFieldHandler != null) { + rb.mergeFieldHandler.handleMergeFields(rb, searcher); + } else { + doFieldSortValues(rb, searcher); + } + + doPrefetch(rb); + } + /** * Fake scorer for a single document *