hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject hive git commit: HIVE-11193 : ConstantPropagateProcCtx should use a Set instead of a List to hold operators to be deleted (Wei Zheng via Ashutosh Chauhan)
Date Thu, 09 Jul 2015 23:53:23 GMT
Repository: hive
Updated Branches:
  refs/heads/master 820748945 -> d3144256d


HIVE-11193 : ConstantPropagateProcCtx should use a Set instead of a List to hold operators
to be deleted (Wei Zheng via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3144256
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3144256
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3144256

Branch: refs/heads/master
Commit: d3144256d90d78392c897a63e58b512b46ab1608
Parents: 8207489
Author: Wei Zheng <wzheng@hortonworks.com>
Authored: Thu Jul 9 13:26:00 2015 -0700
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Thu Jul 9 16:52:54 2015 -0700

----------------------------------------------------------------------
 .../test/resources/testconfiguration.properties |   1 +
 .../hive/ql/optimizer/ConstantPropagate.java    |   1 +
 .../ql/optimizer/ConstantPropagateProcCtx.java  |  10 +-
 .../test/queries/clientpositive/constprog_dpp.q |  17 +++
 .../clientpositive/tez/constprog_dpp.q.out      | 113 +++++++++++++++++++
 5 files changed, 137 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 441b278..8773bd3 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -318,6 +318,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
 
 minitez.query.files=bucket_map_join_tez1.q,\
   bucket_map_join_tez2.q,\
+  constprog_dpp.q,\
   dynamic_partition_pruning.q,\
   dynamic_partition_pruning_2.q,\
   explainuser_1.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
index b5ee4ef..dd53ced 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
@@ -125,6 +125,7 @@ public class ConstantPropagate implements Transform {
       }
       opToDelete.getParentOperators().get(0).removeChildAndAdoptItsChildren(opToDelete);
     }
+    cppCtx.getOpToDelete().clear();
     return pGraphContext;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
index f30e330..d0b10c3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.hive.ql.optimizer;
 
 
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
@@ -53,7 +53,7 @@ public class ConstantPropagateProcCtx implements NodeProcessorCtx {
       .getLog(ConstantPropagateProcCtx.class);
 
   private final Map<Operator<? extends Serializable>, Map<ColumnInfo, ExprNodeDesc>>
opToConstantExprs;
-  private final List<Operator<? extends Serializable>> opToDelete;
+  private final Set<Operator<? extends Serializable>> opToDelete;
   private ConstantPropagateOption constantPropagateOption = ConstantPropagateOption.FULL;
 
   public ConstantPropagateProcCtx() {
@@ -63,7 +63,7 @@ public class ConstantPropagateProcCtx implements NodeProcessorCtx {
   public ConstantPropagateProcCtx(ConstantPropagateOption option) {
     opToConstantExprs =
         new HashMap<Operator<? extends Serializable>, Map<ColumnInfo, ExprNodeDesc>>();
-    opToDelete = new ArrayList<Operator<? extends Serializable>>();
+    opToDelete = new HashSet<Operator<? extends Serializable>>();
     this.constantPropagateOption = option;
   }
 
@@ -193,7 +193,7 @@ public class ConstantPropagateProcCtx implements NodeProcessorCtx {
     opToDelete.add(op);
   }
 
-  public List<Operator<? extends Serializable>> getOpToDelete() {
+  public Set<Operator<? extends Serializable>> getOpToDelete() {
     return opToDelete;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/ql/src/test/queries/clientpositive/constprog_dpp.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/constprog_dpp.q b/ql/src/test/queries/clientpositive/constprog_dpp.q
new file mode 100644
index 0000000..f1c9b13
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/constprog_dpp.q
@@ -0,0 +1,17 @@
+set hive.execution.engine=tez;
+set hive.optimize.constant.propagation=true;
+set hive.tez.dynamic.partition.pruning=true;
+
+drop table if exists tb1;
+create table tb1 (id int);
+
+drop table if exists tb2;
+create table tb2 (id smallint);
+
+explain
+select a.id from tb1 a
+left outer join
+(select id from tb2
+union all
+select 2 as id from tb2 limit 1) b
+on a.id=b.id;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d3144256/ql/src/test/results/clientpositive/tez/constprog_dpp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/constprog_dpp.q.out b/ql/src/test/results/clientpositive/tez/constprog_dpp.q.out
new file mode 100644
index 0000000..b2b2371
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/constprog_dpp.q.out
@@ -0,0 +1,113 @@
+PREHOOK: query: drop table if exists tb1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists tb1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tb1 (id int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tb1
+POSTHOOK: query: create table tb1 (id int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tb1
+PREHOOK: query: drop table if exists tb2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists tb2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tb2 (id smallint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tb2
+POSTHOOK: query: create table tb2 (id smallint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tb2
+PREHOOK: query: explain
+select a.id from tb1 a
+left outer join
+(select id from tb2
+union all
+select 2 as id from tb2 limit 1) b
+on a.id=b.id
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select a.id from tb1 a
+left outer join
+(select id from tb2
+union all
+select 2 as id from tb2 limit 1) b
+on a.id=b.id
+POSTHOOK: type: QUERY
+Plan not optimized by CBO due to missing statistics. Please check log for more details.
+
+Vertex dependency in root stage
+Map 1 <- Union 2 (CONTAINS)
+Map 5 <- Union 2 (CONTAINS)
+Reducer 3 <- Union 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+
+Stage-0
+   Fetch Operator
+      limit:-1
+      Stage-1
+         Reducer 4
+         File Output Operator [FS_16]
+            compressed:false
+            Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+            table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output
format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
+            Merge Join Operator [MERGEJOIN_20]
+            |  condition map:[{"":"Left Outer Join0 to 1"}]
+            |  keys:{"1":"_col0 (type: int)","0":"id (type: int)"}
+            |  outputColumnNames:["_col0"]
+            |  Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+            |<-Map 6 [SIMPLE_EDGE]
+            |  Reduce Output Operator [RS_12]
+            |     key expressions:id (type: int)
+            |     Map-reduce partition columns:id (type: int)
+            |     sort order:+
+            |     Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
+            |     TableScan [TS_11]
+            |        alias:a
+            |        Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
+            |<-Reducer 3 [SIMPLE_EDGE]
+               Reduce Output Operator [RS_13]
+                  key expressions:_col0 (type: int)
+                  Map-reduce partition columns:_col0 (type: int)
+                  sort order:+
+                  Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
COMPLETE
+                  Limit [LIM_10]
+                     Number of rows:1
+                     Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
COMPLETE
+                     Select Operator [SEL_9]
+                     |  outputColumnNames:["_col0"]
+                     |  Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
COMPLETE
+                     |<-Union 2 [SIMPLE_EDGE]
+                        |<-Map 1 [CONTAINS]
+                        |  Reduce Output Operator [RS_8]
+                        |     sort order:
+                        |     Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column
stats: COMPLETE
+                        |     value expressions:_col0 (type: int)
+                        |     Limit [LIM_7]
+                        |        Number of rows:1
+                        |        Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL
Column stats: COMPLETE
+                        |        Select Operator [SEL_1]
+                        |           outputColumnNames:["_col0"]
+                        |           Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL
Column stats: NONE
+                        |           TableScan [TS_0]
+                        |              alias:tb2
+                        |              Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL
Column stats: NONE
+                        |<-Map 5 [CONTAINS]
+                           Reduce Output Operator [RS_8]
+                              sort order:
+                              Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column
stats: COMPLETE
+                              value expressions:_col0 (type: int)
+                              Limit [LIM_7]
+                                 Number of rows:1
+                                 Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL
Column stats: COMPLETE
+                                 Select Operator [SEL_3]
+                                    outputColumnNames:["_col0"]
+                                    Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL
Column stats: COMPLETE
+                                    TableScan [TS_2]
+                                       alias:tb2
+                                       Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL
Column stats: COMPLETE
+


Mime
View raw message