hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmccl...@apache.org
Subject [17/51] [partial] hive git commit: Revert "Revert "HIVE-11394: Enhance EXPLAIN display for vectorization (Matt McCline, reviewed by Gopal Vijayaraghavan)""
Date Fri, 14 Oct 2016 22:16:28 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/16d28b34/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out
index 9369661..9a95606 100644
--- a/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out
@@ -126,113 +126,15 @@ POSTHOOK: Output: default@TJOIN4
 POSTHOOK: Lineage: tjoin4.c1 SIMPLE [(tjoin4_txt)tjoin4_txt.FieldSchema(name:c1, type:int,
comment:null), ]
 POSTHOOK: Lineage: tjoin4.c2 SIMPLE [(tjoin4_txt)tjoin4_txt.FieldSchema(name:c2, type:char(2),
comment:null), ]
 POSTHOOK: Lineage: tjoin4.rnum SIMPLE [(tjoin4_txt)tjoin4_txt.FieldSchema(name:rnum, type:int,
comment:null), ]
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization detail formatted
 select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from
    (select tjoin1.rnum tj1rnum, tjoin2.rnum tj2rnum, tjoin2.c1 tj2c1 from tjoin1 left outer
join tjoin2 on tjoin1.c1 = tjoin2.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1
 PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization detail formatted
 select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from
    (select tjoin1.rnum tj1rnum, tjoin2.rnum tj2rnum, tjoin2.c1 tj2c1 from tjoin1 left outer
join tjoin2 on tjoin1.c1 = tjoin2.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1
 POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
-  Stage: Stage-1
-    Tez
-#### A masked pattern was here ####
-      Edges:
-        Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
 #### A masked pattern was here ####
-      Vertices:
-        Map 1 
-            Map Operator Tree:
-                TableScan
-                  alias: tjoin1
-                  Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats:
NONE
-                  Select Operator
-                    expressions: rnum (type: int), c1 (type: int)
-                    outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats:
NONE
-                    Map Join Operator
-                      condition map:
-                           Left Outer Join0 to 1
-                      keys:
-                        0 _col1 (type: int)
-                        1 _col1 (type: int)
-                      outputColumnNames: _col0, _col2, _col3
-                      input vertices:
-                        1 Map 2
-                      Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column
stats: NONE
-                      Select Operator
-                        expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int)
-                        outputColumnNames: _col0, _col1, _col2
-                        Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column
stats: NONE
-                        Map Join Operator
-                          condition map:
-                               Left Outer Join0 to 1
-                          keys:
-                            0 _col2 (type: int)
-                            1 _col1 (type: int)
-                          outputColumnNames: _col0, _col1, _col3
-                          input vertices:
-                            1 Map 3
-                          Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE Column
stats: NONE
-                          Select Operator
-                            expressions: _col0 (type: int), _col1 (type: int), _col3 (type:
int)
-                            outputColumnNames: _col0, _col1, _col2
-                            Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE
Column stats: NONE
-                            File Output Operator
-                              compressed: false
-                              Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE
Column stats: NONE
-                              table:
-                                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: vectorized, llap
-            LLAP IO: all inputs
-        Map 2 
-            Map Operator Tree:
-                TableScan
-                  alias: tjoin2
-                  Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats:
NONE
-                  Select Operator
-                    expressions: rnum (type: int), c1 (type: int)
-                    outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats:
NONE
-                    Reduce Output Operator
-                      key expressions: _col1 (type: int)
-                      sort order: +
-                      Map-reduce partition columns: _col1 (type: int)
-                      Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column
stats: NONE
-                      value expressions: _col0 (type: int)
-            Execution mode: vectorized, llap
-            LLAP IO: all inputs
-        Map 3 
-            Map Operator Tree:
-                TableScan
-                  alias: tjoin3
-                  Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats:
NONE
-                  Select Operator
-                    expressions: rnum (type: int), c1 (type: int)
-                    outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats:
NONE
-                    Reduce Output Operator
-                      key expressions: _col1 (type: int)
-                      sort order: +
-                      Map-reduce partition columns: _col1 (type: int)
-                      Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column
stats: NONE
-                      value expressions: _col0 (type: int)
-            Execution mode: vectorized, llap
-            LLAP IO: all inputs
-
-  Stage: Stage-0
-    Fetch Operator
-      limit: -1
-      Processor Tree:
-        ListSink
-
 PREHOOK: query: select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from
    (select tjoin1.rnum tj1rnum, tjoin2.rnum tj2rnum, tjoin2.c1 tj2c1 from tjoin1 left outer
join tjoin2 on tjoin1.c1 = tjoin2.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1
 PREHOOK: type: QUERY
@@ -251,108 +153,15 @@ POSTHOOK: Input: default@tjoin3
 0	3	0
 1	NULL	NULL
 2	NULL	NULL
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization detail formatted
 select tj1rnum, tj2rnum as rnumt3 from
    (select tjoin1.rnum tj1rnum, tjoin2.rnum tj2rnum, tjoin2.c1 tj2c1 from tjoin1 left outer
join tjoin2 on tjoin1.c1 = tjoin2.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1
 PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization detail formatted
 select tj1rnum, tj2rnum as rnumt3 from
    (select tjoin1.rnum tj1rnum, tjoin2.rnum tj2rnum, tjoin2.c1 tj2c1 from tjoin1 left outer
join tjoin2 on tjoin1.c1 = tjoin2.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1
 POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
-  Stage: Stage-1
-    Tez
-#### A masked pattern was here ####
-      Edges:
-        Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
 #### A masked pattern was here ####
-      Vertices:
-        Map 1 
-            Map Operator Tree:
-                TableScan
-                  alias: tjoin1
-                  Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats:
NONE
-                  Select Operator
-                    expressions: rnum (type: int), c1 (type: int)
-                    outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats:
NONE
-                    Map Join Operator
-                      condition map:
-                           Left Outer Join0 to 1
-                      keys:
-                        0 _col1 (type: int)
-                        1 _col1 (type: int)
-                      outputColumnNames: _col0, _col2, _col3
-                      input vertices:
-                        1 Map 2
-                      Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column
stats: NONE
-                      Select Operator
-                        expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int)
-                        outputColumnNames: _col0, _col1, _col2
-                        Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column
stats: NONE
-                        Map Join Operator
-                          condition map:
-                               Left Outer Join0 to 1
-                          keys:
-                            0 _col2 (type: int)
-                            1 _col0 (type: int)
-                          outputColumnNames: _col0, _col1
-                          input vertices:
-                            1 Map 3
-                          Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE Column
stats: NONE
-                          File Output Operator
-                            compressed: false
-                            Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE
Column stats: NONE
-                            table:
-                                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: vectorized, llap
-            LLAP IO: all inputs
-        Map 2 
-            Map Operator Tree:
-                TableScan
-                  alias: tjoin2
-                  Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats:
NONE
-                  Select Operator
-                    expressions: rnum (type: int), c1 (type: int)
-                    outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats:
NONE
-                    Reduce Output Operator
-                      key expressions: _col1 (type: int)
-                      sort order: +
-                      Map-reduce partition columns: _col1 (type: int)
-                      Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column
stats: NONE
-                      value expressions: _col0 (type: int)
-            Execution mode: vectorized, llap
-            LLAP IO: all inputs
-        Map 3 
-            Map Operator Tree:
-                TableScan
-                  alias: tjoin3
-                  Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats:
NONE
-                  Select Operator
-                    expressions: c1 (type: int)
-                    outputColumnNames: _col0
-                    Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats:
NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: int)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: int)
-                      Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column
stats: NONE
-            Execution mode: vectorized, llap
-            LLAP IO: all inputs
-
-  Stage: Stage-0
-    Fetch Operator
-      limit: -1
-      Processor Tree:
-        ListSink
-
 PREHOOK: query: select tj1rnum, tj2rnum as rnumt3 from
    (select tjoin1.rnum tj1rnum, tjoin2.rnum tj2rnum, tjoin2.c1 tj2c1 from tjoin1 left outer
join tjoin2 on tjoin1.c1 = tjoin2.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1
 PREHOOK: type: QUERY

http://git-wip-us.apache.org/repos/asf/hive/blob/16d28b34/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out
b/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out
index ebe895f..38c55d6 100644
--- a/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out
@@ -76,12 +76,16 @@ POSTHOOK: Lineage: inventory_part_0 PARTITION(par=2).inv_date_sk SIMPLE
[(invent
 POSTHOOK: Lineage: inventory_part_0 PARTITION(par=2).inv_item_sk SIMPLE [(inventory_txt)inventory_txt.FieldSchema(name:inv_item_sk,
type:int, comment:null), ]
 POSTHOOK: Lineage: inventory_part_0 PARTITION(par=2).inv_quantity_on_hand SIMPLE [(inventory_txt)inventory_txt.FieldSchema(name:inv_quantity_on_hand,
type:int, comment:null), ]
 POSTHOOK: Lineage: inventory_part_0 PARTITION(par=2).inv_warehouse_sk SIMPLE [(inventory_txt)inventory_txt.FieldSchema(name:inv_warehouse_sk,
type:int, comment:null), ]
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_0
 PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_0
 POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
@@ -99,31 +103,73 @@ STAGE PLANS:
                 TableScan
                   alias: inventory_part_0
                   Statistics: Num rows: 200 Data size: 4776 Basic stats: COMPLETE Column
stats: NONE
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3, 4]
                   Select Operator
                     expressions: inv_quantity_on_hand (type: int)
                     outputColumnNames: inv_quantity_on_hand
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [3]
                     Statistics: Num rows: 200 Data size: 4776 Basic stats: COMPLETE Column
stats: NONE
                     Group By Operator
                       aggregations: sum(inv_quantity_on_hand)
+                      Group By Vectorization:
+                          aggregators: VectorUDAFSumLong(col 3) -> bigint
+                          className: VectorGroupByOperator
+                          vectorOutput: true
+                          native: false
+                          projectedOutputColumns: [0]
                       mode: hash
                       outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                       Reduce Output Operator
                         sort order: 
+                        Reduce Sink Vectorization:
+                            className: VectorReduceSinkOperator
+                            native: false
+                            nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true,
No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for
keys IS true, LazyBinarySerDe for values IS true
+                            nativeConditionsNotMet: Uniform Hash IS false
                         Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
                         value expressions: _col0 (type: bigint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
+            Map Vectorization:
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+                groupByVectorOutput: true
+                inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
         Reducer 2 
             Execution mode: vectorized, llap
+            Reduce Vectorization:
+                enabled: true
+                enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true
+                groupByVectorOutput: true
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
             Reduce Operator Tree:
               Group By Operator
                 aggregations: sum(VALUE._col0)
+                Group By Vectorization:
+                    aggregators: VectorUDAFSumLong(col 0) -> bigint
+                    className: VectorGroupByOperator
+                    vectorOutput: true
+                    native: false
+                    projectedOutputColumns: [0]
                 mode: mergepartial
                 outputColumnNames: _col0
                 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
                   compressed: false
+                  File Sink Vectorization:
+                      className: VectorFileSinkOperator
+                      native: false
                   Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                   table:
                       input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -204,12 +250,16 @@ POSTHOOK: Lineage: inventory_part_1 PARTITION(par=5cols).inv_date_sk
SIMPLE [(in
 POSTHOOK: Lineage: inventory_part_1 PARTITION(par=5cols).inv_item_sk SIMPLE [(inventory_txt)inventory_txt.FieldSchema(name:inv_item_sk,
type:int, comment:null), ]
 POSTHOOK: Lineage: inventory_part_1 PARTITION(par=5cols).inv_quantity_on_hand SIMPLE [(inventory_txt)inventory_txt.FieldSchema(name:inv_quantity_on_hand,
type:int, comment:null), ]
 POSTHOOK: Lineage: inventory_part_1 PARTITION(par=5cols).inv_warehouse_sk SIMPLE [(inventory_txt)inventory_txt.FieldSchema(name:inv_warehouse_sk,
type:int, comment:null), ]
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_1
 PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_1
 POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
@@ -227,31 +277,73 @@ STAGE PLANS:
                 TableScan
                   alias: inventory_part_1
                   Statistics: Num rows: 200 Data size: 13476 Basic stats: COMPLETE Column
stats: NONE
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3, 4, 5]
                   Select Operator
                     expressions: inv_quantity_on_hand (type: int)
                     outputColumnNames: inv_quantity_on_hand
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [3]
                     Statistics: Num rows: 200 Data size: 13476 Basic stats: COMPLETE Column
stats: NONE
                     Group By Operator
                       aggregations: sum(inv_quantity_on_hand)
+                      Group By Vectorization:
+                          aggregators: VectorUDAFSumLong(col 3) -> bigint
+                          className: VectorGroupByOperator
+                          vectorOutput: true
+                          native: false
+                          projectedOutputColumns: [0]
                       mode: hash
                       outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                       Reduce Output Operator
                         sort order: 
+                        Reduce Sink Vectorization:
+                            className: VectorReduceSinkOperator
+                            native: false
+                            nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true,
No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for
keys IS true, LazyBinarySerDe for values IS true
+                            nativeConditionsNotMet: Uniform Hash IS false
                         Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
                         value expressions: _col0 (type: bigint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
+            Map Vectorization:
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+                groupByVectorOutput: true
+                inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
         Reducer 2 
             Execution mode: vectorized, llap
+            Reduce Vectorization:
+                enabled: true
+                enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true
+                groupByVectorOutput: true
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
             Reduce Operator Tree:
               Group By Operator
                 aggregations: sum(VALUE._col0)
+                Group By Vectorization:
+                    aggregators: VectorUDAFSumLong(col 0) -> bigint
+                    className: VectorGroupByOperator
+                    vectorOutput: true
+                    native: false
+                    projectedOutputColumns: [0]
                 mode: mergepartial
                 outputColumnNames: _col0
                 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
                   compressed: false
+                  File Sink Vectorization:
+                      className: VectorFileSinkOperator
+                      native: false
                   Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                   table:
                       input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -334,12 +426,16 @@ POSTHOOK: type: ALTERTABLE_RENAMECOL
 POSTHOOK: Input: default@inventory_part_2a
 POSTHOOK: Input: default@inventory_part_2a@par=2
 POSTHOOK: Output: default@inventory_part_2a@par=2
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_2a
 PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_2a
 POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
@@ -357,31 +453,73 @@ STAGE PLANS:
                 TableScan
                   alias: inventory_part_2a
                   Statistics: Num rows: 200 Data size: 4776 Basic stats: COMPLETE Column
stats: NONE
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3, 4]
                   Select Operator
                     expressions: inv_quantity_on_hand (type: int)
                     outputColumnNames: inv_quantity_on_hand
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [3]
                     Statistics: Num rows: 200 Data size: 4776 Basic stats: COMPLETE Column
stats: NONE
                     Group By Operator
                       aggregations: sum(inv_quantity_on_hand)
+                      Group By Vectorization:
+                          aggregators: VectorUDAFSumLong(col 3) -> bigint
+                          className: VectorGroupByOperator
+                          vectorOutput: true
+                          native: false
+                          projectedOutputColumns: [0]
                       mode: hash
                       outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                       Reduce Output Operator
                         sort order: 
+                        Reduce Sink Vectorization:
+                            className: VectorReduceSinkOperator
+                            native: false
+                            nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true,
No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for
keys IS true, LazyBinarySerDe for values IS true
+                            nativeConditionsNotMet: Uniform Hash IS false
                         Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
                         value expressions: _col0 (type: bigint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
+            Map Vectorization:
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+                groupByVectorOutput: true
+                inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
         Reducer 2 
             Execution mode: vectorized, llap
+            Reduce Vectorization:
+                enabled: true
+                enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true
+                groupByVectorOutput: true
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
             Reduce Operator Tree:
               Group By Operator
                 aggregations: sum(VALUE._col0)
+                Group By Vectorization:
+                    aggregators: VectorUDAFSumLong(col 0) -> bigint
+                    className: VectorGroupByOperator
+                    vectorOutput: true
+                    native: false
+                    projectedOutputColumns: [0]
                 mode: mergepartial
                 outputColumnNames: _col0
                 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
                   compressed: false
+                  File Sink Vectorization:
+                      className: VectorFileSinkOperator
+                      native: false
                   Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                   table:
                       input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -445,12 +583,16 @@ POSTHOOK: type: ALTERTABLE_RENAMECOL
 POSTHOOK: Input: default@inventory_part_2b
 POSTHOOK: Input: default@inventory_part_2b@par1=2/par2=3
 POSTHOOK: Output: default@inventory_part_2b@par1=2/par2=3
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_2b
 PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_2b
 POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
@@ -468,31 +610,73 @@ STAGE PLANS:
                 TableScan
                   alias: inventory_part_2b
                   Statistics: Num rows: 200 Data size: 4776 Basic stats: COMPLETE Column
stats: NONE
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3, 4, 5]
                   Select Operator
                     expressions: inv_quantity_on_hand (type: int)
                     outputColumnNames: inv_quantity_on_hand
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [3]
                     Statistics: Num rows: 200 Data size: 4776 Basic stats: COMPLETE Column
stats: NONE
                     Group By Operator
                       aggregations: sum(inv_quantity_on_hand)
+                      Group By Vectorization:
+                          aggregators: VectorUDAFSumLong(col 3) -> bigint
+                          className: VectorGroupByOperator
+                          vectorOutput: true
+                          native: false
+                          projectedOutputColumns: [0]
                       mode: hash
                       outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                       Reduce Output Operator
                         sort order: 
+                        Reduce Sink Vectorization:
+                            className: VectorReduceSinkOperator
+                            native: false
+                            nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true,
No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for
keys IS true, LazyBinarySerDe for values IS true
+                            nativeConditionsNotMet: Uniform Hash IS false
                         Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
                         value expressions: _col0 (type: bigint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
+            Map Vectorization:
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+                groupByVectorOutput: true
+                inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
         Reducer 2 
             Execution mode: vectorized, llap
+            Reduce Vectorization:
+                enabled: true
+                enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true
+                groupByVectorOutput: true
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
             Reduce Operator Tree:
               Group By Operator
                 aggregations: sum(VALUE._col0)
+                Group By Vectorization:
+                    aggregators: VectorUDAFSumLong(col 0) -> bigint
+                    className: VectorGroupByOperator
+                    vectorOutput: true
+                    native: false
+                    projectedOutputColumns: [0]
                 mode: mergepartial
                 outputColumnNames: _col0
                 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
                   compressed: false
+                  File Sink Vectorization:
+                      className: VectorFileSinkOperator
+                      native: false
                   Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                   table:
                       input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -562,12 +746,16 @@ POSTHOOK: type: ALTERTABLE_RENAMECOL
 POSTHOOK: Input: default@inventory_part_3
 POSTHOOK: Input: default@inventory_part_3@par=2
 POSTHOOK: Output: default@inventory_part_3@par=2
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_3
 PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
 select sum(inv_quantity_on_hand) from inventory_part_3
 POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
@@ -585,31 +773,73 @@ STAGE PLANS:
                 TableScan
                   alias: inventory_part_3
                   Statistics: Num rows: 200 Data size: 4776 Basic stats: COMPLETE Column
stats: NONE
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1, 2, 3, 4]
                   Select Operator
                     expressions: inv_quantity_on_hand (type: int)
                     outputColumnNames: inv_quantity_on_hand
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [3]
                     Statistics: Num rows: 200 Data size: 4776 Basic stats: COMPLETE Column
stats: NONE
                     Group By Operator
                       aggregations: sum(inv_quantity_on_hand)
+                      Group By Vectorization:
+                          aggregators: VectorUDAFSumLong(col 3) -> bigint
+                          className: VectorGroupByOperator
+                          vectorOutput: true
+                          native: false
+                          projectedOutputColumns: [0]
                       mode: hash
                       outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                       Reduce Output Operator
                         sort order: 
+                        Reduce Sink Vectorization:
+                            className: VectorReduceSinkOperator
+                            native: false
+                            nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true,
No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for
keys IS true, LazyBinarySerDe for values IS true
+                            nativeConditionsNotMet: Uniform Hash IS false
                         Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
                         value expressions: _col0 (type: bigint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
+            Map Vectorization:
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+                groupByVectorOutput: true
+                inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
         Reducer 2 
             Execution mode: vectorized, llap
+            Reduce Vectorization:
+                enabled: true
+                enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine
tez IN [tez, spark] IS true
+                groupByVectorOutput: true
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
             Reduce Operator Tree:
               Group By Operator
                 aggregations: sum(VALUE._col0)
+                Group By Vectorization:
+                    aggregators: VectorUDAFSumLong(col 0) -> bigint
+                    className: VectorGroupByOperator
+                    vectorOutput: true
+                    native: false
+                    projectedOutputColumns: [0]
                 mode: mergepartial
                 outputColumnNames: _col0
                 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
                   compressed: false
+                  File Sink Vectorization:
+                      className: VectorFileSinkOperator
+                      native: false
                   Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
                   table:
                       input format: org.apache.hadoop.mapred.SequenceFileInputFormat


Mime
View raw message