Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2F6FA181AA for ; Thu, 16 Jul 2015 07:52:22 +0000 (UTC) Received: (qmail 52167 invoked by uid 500); 16 Jul 2015 07:52:20 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 52009 invoked by uid 500); 16 Jul 2015 07:52:20 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 51154 invoked by uid 99); 16 Jul 2015 07:52:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 16 Jul 2015 07:52:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E3F87E3AA1; Thu, 16 Jul 2015 07:52:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xuf@apache.org To: commits@hive.apache.org Date: Thu, 16 Jul 2015 07:52:31 -0000 Message-Id: In-Reply-To: <7775a3a3ac844aafa099916a6c3e2fab@git.apache.org> References: <7775a3a3ac844aafa099916a6c3e2fab@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [13/50] [abbrv] hive git commit: HIVE-11193 : ConstantPropagateProcCtx should use a Set instead of a List to hold operators to be deleted (Wei Zheng via Ashutosh Chauhan) HIVE-11193 : ConstantPropagateProcCtx should use a Set instead of a List to hold operators to be deleted (Wei Zheng via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3144256 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3144256 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3144256 Branch: refs/heads/parquet Commit: d3144256d90d78392c897a63e58b512b46ab1608 Parents: 8207489 Author: Wei Zheng Authored: Thu Jul 9 13:26:00 2015 -0700 Committer: Ashutosh Chauhan Committed: Thu Jul 9 16:52:54 2015 -0700 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../hive/ql/optimizer/ConstantPropagate.java | 1 + .../ql/optimizer/ConstantPropagateProcCtx.java | 10 +- .../test/queries/clientpositive/constprog_dpp.q | 17 +++ .../clientpositive/tez/constprog_dpp.q.out | 113 +++++++++++++++++++ 5 files changed, 137 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 441b278..8773bd3 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -318,6 +318,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ minitez.query.files=bucket_map_join_tez1.q,\ bucket_map_join_tez2.q,\ + constprog_dpp.q,\ dynamic_partition_pruning.q,\ dynamic_partition_pruning_2.q,\ explainuser_1.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java index b5ee4ef..dd53ced 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java @@ -125,6 +125,7 @@ public class ConstantPropagate implements Transform { } opToDelete.getParentOperators().get(0).removeChildAndAdoptItsChildren(opToDelete); } + cppCtx.getOpToDelete().clear(); return pGraphContext; } http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java index f30e330..d0b10c3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java @@ -20,12 +20,12 @@ package org.apache.hadoop.hive.ql.optimizer; import java.io.Serializable; -import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -53,7 +53,7 @@ public class ConstantPropagateProcCtx implements NodeProcessorCtx { .getLog(ConstantPropagateProcCtx.class); private final Map, Map> opToConstantExprs; - private final List> opToDelete; + private final Set> opToDelete; private ConstantPropagateOption constantPropagateOption = ConstantPropagateOption.FULL; public ConstantPropagateProcCtx() { @@ -63,7 +63,7 @@ public class ConstantPropagateProcCtx implements NodeProcessorCtx { public ConstantPropagateProcCtx(ConstantPropagateOption option) { opToConstantExprs = new HashMap, Map>(); - opToDelete = new ArrayList>(); + opToDelete = new HashSet>(); this.constantPropagateOption = option; } @@ -193,7 +193,7 @@ public class ConstantPropagateProcCtx implements NodeProcessorCtx { opToDelete.add(op); } - public List> getOpToDelete() { + public Set> getOpToDelete() { return opToDelete; } http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/ql/src/test/queries/clientpositive/constprog_dpp.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/constprog_dpp.q b/ql/src/test/queries/clientpositive/constprog_dpp.q new file mode 100644 index 0000000..f1c9b13 --- /dev/null +++ b/ql/src/test/queries/clientpositive/constprog_dpp.q @@ -0,0 +1,17 @@ +set hive.execution.engine=tez; +set hive.optimize.constant.propagation=true; +set hive.tez.dynamic.partition.pruning=true; + +drop table if exists tb1; +create table tb1 (id int); + +drop table if exists tb2; +create table tb2 (id smallint); + +explain +select a.id from tb1 a +left outer join +(select id from tb2 +union all +select 2 as id from tb2 limit 1) b +on a.id=b.id; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/ql/src/test/results/clientpositive/tez/constprog_dpp.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/constprog_dpp.q.out b/ql/src/test/results/clientpositive/tez/constprog_dpp.q.out new file mode 100644 index 0000000..b2b2371 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/constprog_dpp.q.out @@ -0,0 +1,113 @@ +PREHOOK: query: drop table if exists tb1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists tb1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tb1 (id int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tb1 +POSTHOOK: query: create table tb1 (id int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tb1 +PREHOOK: query: drop table if exists tb2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists tb2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tb2 (id smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tb2 +POSTHOOK: query: create table tb2 (id smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tb2 +PREHOOK: query: explain +select a.id from tb1 a +left outer join +(select id from tb2 +union all +select 2 as id from tb2 limit 1) b +on a.id=b.id +PREHOOK: type: QUERY +POSTHOOK: query: explain +select a.id from tb1 a +left outer join +(select id from tb2 +union all +select 2 as id from tb2 limit 1) b +on a.id=b.id +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing statistics. Please check log for more details. + +Vertex dependency in root stage +Map 1 <- Union 2 (CONTAINS) +Map 5 <- Union 2 (CONTAINS) +Reducer 3 <- Union 2 (SIMPLE_EDGE) +Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 4 + File Output Operator [FS_16] + compressed:false + Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} + Merge Join Operator [MERGEJOIN_20] + | condition map:[{"":"Left Outer Join0 to 1"}] + | keys:{"1":"_col0 (type: int)","0":"id (type: int)"} + | outputColumnNames:["_col0"] + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + |<-Map 6 [SIMPLE_EDGE] + | Reduce Output Operator [RS_12] + | key expressions:id (type: int) + | Map-reduce partition columns:id (type: int) + | sort order:+ + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + | TableScan [TS_11] + | alias:a + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + |<-Reducer 3 [SIMPLE_EDGE] + Reduce Output Operator [RS_13] + key expressions:_col0 (type: int) + Map-reduce partition columns:_col0 (type: int) + sort order:+ + Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Limit [LIM_10] + Number of rows:1 + Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator [SEL_9] + | outputColumnNames:["_col0"] + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + |<-Union 2 [SIMPLE_EDGE] + |<-Map 1 [CONTAINS] + | Reduce Output Operator [RS_8] + | sort order: + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + | value expressions:_col0 (type: int) + | Limit [LIM_7] + | Number of rows:1 + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + | Select Operator [SEL_1] + | outputColumnNames:["_col0"] + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + | TableScan [TS_0] + | alias:tb2 + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + |<-Map 5 [CONTAINS] + Reduce Output Operator [RS_8] + sort order: + Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + value expressions:_col0 (type: int) + Limit [LIM_7] + Number of rows:1 + Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator [SEL_3] + outputColumnNames:["_col0"] + Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + TableScan [TS_2] + alias:tb2 + Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE +