Return-Path: X-Original-To: apmail-phoenix-commits-archive@minotaur.apache.org Delivered-To: apmail-phoenix-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5AE8018000 for ; Sun, 5 Apr 2015 21:09:02 +0000 (UTC) Received: (qmail 97421 invoked by uid 500); 5 Apr 2015 21:09:02 -0000 Delivered-To: apmail-phoenix-commits-archive@phoenix.apache.org Received: (qmail 97381 invoked by uid 500); 5 Apr 2015 21:09:02 -0000 Mailing-List: contact commits-help@phoenix.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@phoenix.apache.org Delivered-To: mailing list commits@phoenix.apache.org Received: (qmail 97372 invoked by uid 99); 5 Apr 2015 21:09:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 05 Apr 2015 21:09:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 01191E2F69; Sun, 5 Apr 2015 21:09:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jamestaylor@apache.org To: commits@phoenix.apache.org Message-Id: <2c558a78afd449a0a32395db8c6f246f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: phoenix git commit: PHOENIX-1809 Improve explain plan Date: Sun, 5 Apr 2015 21:09:02 +0000 (UTC) Repository: phoenix Updated Branches: refs/heads/master 9bbd5ead5 -> c823be992 PHOENIX-1809 Improve explain plan Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c823be99 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c823be99 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c823be99 Branch: refs/heads/master Commit: c823be992460c4211bce713d7e24a125b108c2b8 Parents: 9bbd5ea Author: James Taylor Authored: Sat Apr 4 13:20:31 2015 -0700 Committer: James Taylor Committed: Sun Apr 5 14:07:51 2015 -0700 ---------------------------------------------------------------------- .../org/apache/phoenix/end2end/HashJoinIT.java | 3 - .../org/apache/phoenix/end2end/KeyOnlyIT.java | 2 +- .../phoenix/end2end/QueryWithLimitIT.java | 2 +- .../phoenix/iterate/BaseResultIterators.java | 2 +- .../apache/phoenix/iterate/ExplainTable.java | 82 +++++++------------- .../iterate/MergeSortTopNResultIterator.java | 1 - .../java/org/apache/phoenix/util/ScanUtil.java | 19 ++++- .../org/apache/phoenix/query/QueryPlanTest.java | 2 - 8 files changed, 50 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/c823be99/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java index 596e5e9..1a2a1d0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java @@ -395,7 +395,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { * LEFT JOIN joinOrderTable o ON o.item_id = i.item_id LIMIT 4 */ "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_DISPLAY_NAME + "\n" + - " SERVER FILTER BY PageFilter 4\n" + " SERVER 4 ROW LIMIT\n" + "CLIENT 4 ROW LIMIT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + @@ -777,7 +776,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { * LEFT JOIN joinOrderTable o ON o.item_id = i.item_id LIMIT 4 */ "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_DISPLAY_NAME + "\n" + - " SERVER FILTER BY PageFilter 4\n" + " SERVER 4 ROW LIMIT\n" + "CLIENT 4 ROW LIMIT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + @@ -1179,7 +1177,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { * LEFT JOIN joinOrderTable o ON o.item_id = i.item_id LIMIT 4 */ "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_DISPLAY_NAME + "\n" + - " SERVER FILTER BY PageFilter 4\n" + " SERVER 4 ROW LIMIT\n" + "CLIENT 4 ROW LIMIT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/c823be99/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java index 7470598..dca57b4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java @@ -180,7 +180,7 @@ public class KeyOnlyIT extends BaseOwnClusterClientManagedTimeIT { rs = conn.createStatement().executeQuery("EXPLAIN " + query); assertEquals("CLIENT SERIAL 1-WAY FULL SCAN OVER KEYONLY\n" + - " SERVER FILTER BY FIRST KEY ONLY AND PageFilter 1\n" + + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER 1 ROW LIMIT\n" + "CLIENT 1 ROW LIMIT", QueryUtil.getExplainPlan(rs)); conn.close(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c823be99/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java index 437bf37..c05c92d 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java @@ -74,7 +74,7 @@ public class QueryWithLimitIT extends BaseOwnClusterHBaseManagedTimeIT { rs = conn.createStatement().executeQuery("EXPLAIN " + query); assertEquals("CLIENT SERIAL 1-WAY FULL SCAN OVER KEYONLY\n" + - " SERVER FILTER BY FIRST KEY ONLY AND PageFilter 1\n" + + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER 1 ROW LIMIT\n" + "CLIENT 1 ROW LIMIT", QueryUtil.getExplainPlan(rs)); } finally { http://git-wip-us.apache.org/repos/asf/phoenix/blob/c823be99/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 9ac6a29..8d602b5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -132,7 +132,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result } public BaseResultIterators(QueryPlan plan, Integer perScanLimit) throws SQLException { - super(plan.getContext(), plan.getTableRef(), plan.getGroupBy(), plan.getOrderBy(), plan.getStatement().getHint()); + super(plan.getContext(), plan.getTableRef(), plan.getGroupBy(), plan.getOrderBy(), plan.getStatement().getHint(), plan.getLimit()); this.plan = plan; StatementContext context = plan.getContext(); TableRef tableRef = plan.getTableRef(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c823be99/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index 2fcc2fb..3fe42fa 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -26,7 +26,6 @@ import java.util.NoSuchElementException; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; -import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -36,15 +35,17 @@ import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.ScanRanges; import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; +import org.apache.phoenix.filter.BooleanExpressionFilter; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.KeyRange.Bound; -import org.apache.phoenix.schema.types.PInteger; -import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.StringUtil; import com.google.common.collect.Iterators; @@ -57,17 +58,19 @@ public abstract class ExplainTable { protected final GroupBy groupBy; protected final OrderBy orderBy; protected final HintNode hint; + protected final Integer limit; public ExplainTable(StatementContext context, TableRef table) { - this(context,table,GroupBy.EMPTY_GROUP_BY, OrderBy.EMPTY_ORDER_BY, HintNode.EMPTY_HINT_NODE); + this(context,table,GroupBy.EMPTY_GROUP_BY, OrderBy.EMPTY_ORDER_BY, HintNode.EMPTY_HINT_NODE, null); } - public ExplainTable(StatementContext context, TableRef table, GroupBy groupBy, OrderBy orderBy, HintNode hintNode) { + public ExplainTable(StatementContext context, TableRef table, GroupBy groupBy, OrderBy orderBy, HintNode hintNode, Integer limit) { this.context = context; this.tableRef = table; this.groupBy = groupBy; this.orderBy = orderBy; this.hint = hintNode; + this.limit = limit; } private boolean explainSkipScan(StringBuilder buf) { @@ -98,7 +101,6 @@ public abstract class ExplainTable { protected void explain(String prefix, List planSteps) { StringBuilder buf = new StringBuilder(prefix); ScanRanges scanRanges = context.getScanRanges(); - boolean hasSkipScanFilter = false; Scan scan = context.getScan(); if (scan.getConsistency() != Consistency.STRONG){ @@ -113,7 +115,7 @@ public abstract class ExplainTable { if (scanRanges.isEverything()) { buf.append("FULL SCAN "); } else { - hasSkipScanFilter = explainSkipScan(buf); + explainSkipScan(buf); } buf.append("OVER " + tableRef.getTable().getPhysicalName().getString()); if (!scanRanges.isPointLookup()) { @@ -121,48 +123,31 @@ public abstract class ExplainTable { } planSteps.add(buf.toString()); - Filter filter = scan.getFilter(); - PageFilter pageFilter = null; - if (filter != null) { - int offset = 0; - boolean hasFirstKeyOnlyFilter = false; - String filterDesc = ""; - if (hasSkipScanFilter) { - if (filter instanceof FilterList) { - List filterList = ((FilterList) filter).getFilters(); - if (filterList.get(0) instanceof FirstKeyOnlyFilter) { - hasFirstKeyOnlyFilter = true; - offset = 1; - } - if (filterList.size() > offset+1) { - filterDesc = filterList.get(offset+1).toString(); - pageFilter = getPageFilter(filterList); - } - } - } else if (filter instanceof FilterList) { - List filterList = ((FilterList) filter).getFilters(); - if (filterList.get(0) instanceof FirstKeyOnlyFilter) { - hasFirstKeyOnlyFilter = true; - offset = 1; - } - if (filterList.size() > offset) { - filterDesc = filterList.get(offset).toString(); - pageFilter = getPageFilter(filterList); - } - } else { + Iterator filterIterator = ScanUtil.getFilterIterator(scan); + if (filterIterator.hasNext()) { + PageFilter pageFilter = null; + FirstKeyOnlyFilter firstKeyOnlyFilter = null; + BooleanExpressionFilter whereFilter = null; + do { + Filter filter = filterIterator.next(); if (filter instanceof FirstKeyOnlyFilter) { - hasFirstKeyOnlyFilter = true; - } else { - filterDesc = filter.toString(); + firstKeyOnlyFilter = (FirstKeyOnlyFilter)filter; + } else if (filter instanceof PageFilter) { + pageFilter = (PageFilter)filter; + } else if (filter instanceof BooleanExpressionFilter) { + whereFilter = (BooleanExpressionFilter)filter; } - } - if (filterDesc.length() > 0) { - planSteps.add(" SERVER FILTER BY " + (hasFirstKeyOnlyFilter ? "FIRST KEY ONLY AND " : "") + filterDesc); - } else if (hasFirstKeyOnlyFilter) { + } while (filterIterator.hasNext()); + if (whereFilter != null) { + planSteps.add(" SERVER FILTER BY " + (firstKeyOnlyFilter == null ? "" : "FIRST KEY ONLY AND ") + whereFilter.toString()); + } else if (firstKeyOnlyFilter != null) { planSteps.add(" SERVER FILTER BY FIRST KEY ONLY"); } - if (pageFilter != null) { - planSteps.add(" SERVER " + pageFilter.getPageSize() + " ROW LIMIT"); + if (!orderBy.getOrderByExpressions().isEmpty() && groupBy.isEmpty()) { // with GROUP BY, sort happens client-side + planSteps.add(" SERVER" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S")) + + " SORTED BY " + orderBy.getOrderByExpressions().toString()); + } else if (pageFilter != null) { + planSteps.add(" SERVER " + pageFilter.getPageSize() + " ROW LIMIT"); } } Integer groupByLimit = null; @@ -173,13 +158,6 @@ public abstract class ExplainTable { groupBy.explain(planSteps, groupByLimit); } - private PageFilter getPageFilter(List filterList) { - for (Filter filter : filterList) { - if (filter instanceof PageFilter) return (PageFilter)filter; - } - return null; - } - private void appendPKColumnValue(StringBuilder buf, byte[] range, Boolean isNull, int slotIndex) { if (Boolean.TRUE.equals(isNull)) { buf.append("null"); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c823be99/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortTopNResultIterator.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortTopNResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortTopNResultIterator.java index 64ededa..71259e0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortTopNResultIterator.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortTopNResultIterator.java @@ -91,7 +91,6 @@ public class MergeSortTopNResultIterator extends MergeSortResultIterator { @Override public void explain(List planSteps) { resultIterators.explain(planSteps); - planSteps.add(" SERVER" + (limit == -1 ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S")) + " SORTED BY " + orderByColumns.toString()); planSteps.add("CLIENT MERGE SORT"); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c823be99/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java index 2dfa573..2268866 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java @@ -24,6 +24,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NavigableSet; @@ -45,13 +46,14 @@ import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.KeyRange.Bound; import org.apache.phoenix.query.QueryConstants; -import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; -import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.ValueSchema.Field; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarbinary; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; /** @@ -643,4 +645,17 @@ public class ScanUtil { } return tenantId; } + + public static Iterator getFilterIterator(Scan scan) { + Iterator filterIterator; + Filter topLevelFilter = scan.getFilter(); + if (topLevelFilter == null) { + filterIterator = Iterators.emptyIterator(); + } else if (topLevelFilter instanceof FilterList) { + filterIterator = ((FilterList) topLevelFilter).getFilters().iterator(); + } else { + filterIterator = Iterators.singletonIterator(topLevelFilter); + } + return filterIterator; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/c823be99/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java index 7ad3e25..2f8088d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java @@ -207,13 +207,11 @@ public class QueryPlanTest extends BaseConnectionlessQueryTest { String query = "EXPLAIN SELECT * FROM TENANT_VIEW LIMIT 1"; ResultSet rs = conn.createStatement().executeQuery(query); assertEquals("CLIENT SERIAL 1-WAY RANGE SCAN OVER BASE_MULTI_TENANT_TABLE ['tenantId']\n" + - " SERVER FILTER BY PageFilter 1\n" + " SERVER 1 ROW LIMIT\n" + "CLIENT 1 ROW LIMIT", QueryUtil.getExplainPlan(rs)); query = "EXPLAIN SELECT * FROM TENANT_VIEW LIMIT " + Integer.MAX_VALUE; rs = conn.createStatement().executeQuery(query); assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER BASE_MULTI_TENANT_TABLE ['tenantId']\n" + - " SERVER FILTER BY PageFilter " + Integer.MAX_VALUE + "\n" + " SERVER " + Integer.MAX_VALUE + " ROW LIMIT\n" + "CLIENT " + Integer.MAX_VALUE + " ROW LIMIT", QueryUtil.getExplainPlan(rs)); query = "EXPLAIN SELECT * FROM TENANT_VIEW WHERE username = 'Joe' LIMIT 1";