hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pxi...@apache.org
Subject hive git commit: CBO: Calcite Operator To Hive Operator (Calcite Return Path) : columnPruner prunes everything when union is the last operator before FS (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Date Thu, 29 Oct 2015 17:48:25 GMT
Repository: hive
Updated Branches:
  refs/heads/master 63dc1fa61 -> 034280ce0


CBO: Calcite Operator To Hive Operator (Calcite Return Path) : columnPruner prunes everything
when union is the last operator before FS (Pengcheng Xiong, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/034280ce
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/034280ce
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/034280ce

Branch: refs/heads/master
Commit: 034280ce070d812f1eb312567a974a8720943647
Parents: 63dc1fa
Author: pengchengxiong <pxiong@hortonworks.com>
Authored: Thu Oct 29 10:27:28 2015 -0700
Committer: pengchengxiong <pxiong@hortonworks.com>
Committed: Thu Oct 29 10:27:28 2015 -0700

----------------------------------------------------------------------
 .../hive/ql/optimizer/ColumnPrunerProcCtx.java  |   7 +
 .../clientpositive/cbo_rp_unionDistinct_2.q     | 128 +++++
 .../clientpositive/cbo_rp_unionDistinct_2.q.out | 545 +++++++++++++++++++
 3 files changed, 680 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/034280ce/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
index 2207cfb..b18a034 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
@@ -25,6 +25,7 @@ import java.util.Map;
 
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
@@ -115,6 +116,12 @@ public class ColumnPrunerProcCtx implements NodeProcessorCtx {
             prunList.add(colInfo.getInternalName());
           }
         }
+      } else if (child instanceof FileSinkOperator) {
+        prunList = new ArrayList<>();
+        RowSchema oldRS = curOp.getSchema();
+        for (ColumnInfo colInfo : oldRS.getSignature()) {
+          prunList.add(colInfo.getInternalName());
+        }
       } else {
         prunList = prunedColLists.get(child);
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/034280ce/ql/src/test/queries/clientpositive/cbo_rp_unionDistinct_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cbo_rp_unionDistinct_2.q b/ql/src/test/queries/clientpositive/cbo_rp_unionDistinct_2.q
new file mode 100644
index 0000000..ea98bd2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cbo_rp_unionDistinct_2.q
@@ -0,0 +1,128 @@
+set hive.cbo.returnpath.hiveop=true;
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5;
+
+CREATE TABLE u2 as select key, value from src order by key limit 3;
+
+CREATE TABLE u3 as select key, value from src order by key desc limit 5;
+
+select * from u1;
+
+select * from u2;
+
+select * from u3;
+
+select key, value from 
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+select key, value from 
+(
+select key, value from u1
+union 
+select key, value from u2
+union all
+select key, value from u3
+) tab;
+
+select key, value from 
+(
+select key, value from u1
+union distinct
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+select key, value from 
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key, value from u3
+) tab;
+
+select key, value from 
+(
+select key, value from u1
+union 
+select key, value from u2
+union
+select key as key, value from u3
+) tab;
+
+select distinct * from 
+(
+select key, value from u1
+union all 
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+select distinct * from 
+(
+select distinct * from u1
+union  
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+drop view if exists v;
+
+create view v as select distinct * from 
+(
+select distinct * from u1
+union  
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+describe extended v;
+
+select * from v;
+
+drop view if exists v;
+
+create view v as select tab.* from 
+(
+select distinct * from u1
+union  
+select distinct * from u2
+) tab;
+
+describe extended v;
+
+select * from v;
+
+drop view if exists v;
+
+create view v as select * from 
+(
+select distinct u1.* from u1
+union all 
+select distinct * from u2
+) tab;
+
+describe extended v;
+
+select * from v;
+
+select distinct * from 
+(
+select key, value from u1
+union all 
+select key, value from u2
+union 
+select key as key, value from u3
+) tab;
+

http://git-wip-us.apache.org/repos/asf/hive/blob/034280ce/ql/src/test/results/clientpositive/cbo_rp_unionDistinct_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_unionDistinct_2.q.out b/ql/src/test/results/clientpositive/cbo_rp_unionDistinct_2.q.out
new file mode 100644
index 0000000..6d59369
--- /dev/null
+++ b/ql/src/test/results/clientpositive/cbo_rp_unionDistinct_2.q.out
@@ -0,0 +1,545 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@u1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@u1
+PREHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@u2
+POSTHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@u2
+PREHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@u3
+POSTHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@u3
+PREHOOK: query: select * from u1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from u1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+#### A masked pattern was here ####
+0	val_0
+0	val_0
+0	val_0
+10	val_10
+100	val_100
+PREHOOK: query: select * from u2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from u2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u2
+#### A masked pattern was here ####
+0	val_0
+0	val_0
+0	val_0
+PREHOOK: query: select * from u3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from u3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+96	val_96
+97	val_97
+97	val_97
+98	val_98
+98	val_98
+PREHOOK: query: select key, value from 
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from 
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+97	val_97
+98	val_98
+98	val_98
+PREHOOK: query: select key, value from 
+(
+select key, value from u1
+union 
+select key, value from u2
+union all
+select key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from 
+(
+select key, value from u1
+union 
+select key, value from u2
+union all
+select key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+97	val_97
+98	val_98
+98	val_98
+PREHOOK: query: select key, value from 
+(
+select key, value from u1
+union distinct
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from 
+(
+select key, value from u1
+union distinct
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+97	val_97
+98	val_98
+98	val_98
+PREHOOK: query: select key, value from 
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from 
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+98	val_98
+PREHOOK: query: select key, value from 
+(
+select key, value from u1
+union 
+select key, value from u2
+union
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from 
+(
+select key, value from u1
+union 
+select key, value from u2
+union
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+98	val_98
+PREHOOK: query: select distinct * from 
+(
+select key, value from u1
+union all 
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct * from 
+(
+select key, value from u1
+union all 
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+98	val_98
+PREHOOK: query: select distinct * from 
+(
+select distinct * from u1
+union  
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct * from 
+(
+select distinct * from u1
+union  
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+98	val_98
+PREHOOK: query: drop view if exists v
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: drop view if exists v
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: create view v as select distinct * from 
+(
+select distinct * from u1
+union  
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as select distinct * from 
+(
+select distinct * from u1
+union  
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key                 	string              	                    
+value               	string              	                    
+	 	 
+#### A masked pattern was here ####
+(	 	 
+select distinct * from u1	 	 
+union  	 	 
+select key, value from u2	 	 
+union all	 	 
+select key as key, value from u3	 	 
+) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from 	 	 
+(	 	 
+select distinct `u1`.`key`, `u1`.`value` from `default`.`u1`	 	 
+union  	 	 
+select `u2`.`key`, `u2`.`value` from `default`.`u2`	 	 
+union all	 	 
+select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3`	 	 
+) `tab`, tableType:VIRTUAL_VIEW)		 
+PREHOOK: query: select * from v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+98	val_98
+PREHOOK: query: drop view if exists v
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@v
+PREHOOK: Output: default@v
+POSTHOOK: query: drop view if exists v
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@v
+POSTHOOK: Output: default@v
+PREHOOK: query: create view v as select tab.* from 
+(
+select distinct * from u1
+union  
+select distinct * from u2
+) tab
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as select tab.* from 
+(
+select distinct * from u1
+union  
+select distinct * from u2
+) tab
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key                 	string              	                    
+value               	string              	                    
+	 	 
+#### A masked pattern was here ####
+(	 	 
+select distinct * from u1	 	 
+union  	 	 
+select distinct * from u2	 	 
+) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from 	 	 
+(	 	 
+select distinct `u1`.`key`, `u1`.`value` from `default`.`u1`	 	 
+union  	 	 
+select distinct `u2`.`key`, `u2`.`value` from `default`.`u2`	 	 
+) `tab`, tableType:VIRTUAL_VIEW)		 
+PREHOOK: query: select * from v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+PREHOOK: query: drop view if exists v
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@v
+PREHOOK: Output: default@v
+POSTHOOK: query: drop view if exists v
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@v
+POSTHOOK: Output: default@v
+PREHOOK: query: create view v as select * from 
+(
+select distinct u1.* from u1
+union all 
+select distinct * from u2
+) tab
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as select * from 
+(
+select distinct u1.* from u1
+union all 
+select distinct * from u2
+) tab
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key                 	string              	                    
+value               	string              	                    
+	 	 
+#### A masked pattern was here ####
+(	 	 
+select distinct u1.* from u1	 	 
+union all 	 	 
+select distinct * from u2	 	 
+) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from 	 	 
+(	 	 
+select distinct `u1`.`key`, `u1`.`value` from `default`.`u1`	 	 
+union all 	 	 
+select distinct `u2`.`key`, `u2`.`value` from `default`.`u2`	 	 
+) `tab`, tableType:VIRTUAL_VIEW)		 
+PREHOOK: query: select * from v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0	val_0
+0	val_0
+10	val_10
+100	val_100
+PREHOOK: query: select distinct * from 
+(
+select key, value from u1
+union all 
+select key, value from u2
+union 
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct * from 
+(
+select key, value from u1
+union all 
+select key, value from u2
+union 
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0	val_0
+10	val_10
+100	val_100
+96	val_96
+97	val_97
+98	val_98


Mime
View raw message