hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-16222 : add a setting to disable row.serde for specific formats; enable for others (Sergey Shelukhin, reviewed by Matt McCline)
Date Mon, 24 Jul 2017 23:25:29 GMT
Repository: hive
Updated Branches:
  refs/heads/master 549ac12dd -> 88da23829


HIVE-16222 : add a setting to disable row.serde for specific formats; enable for others (Sergey
Shelukhin, reviewed by Matt McCline)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/88da2382
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/88da2382
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/88da2382

Branch: refs/heads/master
Commit: 88da2382953ca8b7a608331772ccaba3f53ef1d5
Parents: 549ac12
Author: sergey <sershe@apache.org>
Authored: Fri Jun 30 13:10:24 2017 -0700
Committer: sergey <sershe@apache.org>
Committed: Mon Jul 24 16:25:20 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   6 +-
 .../hive/ql/optimizer/physical/Vectorizer.java  |  67 +++++--
 .../clientpositive/parquet_no_row_serde.q       |  25 +++
 .../llap/vector_decimal_round.q.out             |  44 ++++-
 .../llap/vectorized_bucketmapjoin1.q.out        |  21 +-
 .../clientpositive/parquet_no_row_serde.q.out   | 190 +++++++++++++++++++
 .../clientpositive/vector_decimal_round.q.out   |  44 ++++-
 7 files changed, 370 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/88da2382/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index df45f2c..3ebe503 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2834,9 +2834,13 @@ public class HiveConf extends Configuration {
     HIVE_VECTORIZATION_USE_VECTOR_DESERIALIZE("hive.vectorized.use.vector.serde.deserialize",
true,
         "This flag should be set to true to enable vectorizing rows using vector deserialize.\n"
+
         "The default value is true."),
-    HIVE_VECTORIZATION_USE_ROW_DESERIALIZE("hive.vectorized.use.row.serde.deserialize", false,
+    HIVE_VECTORIZATION_USE_ROW_DESERIALIZE("hive.vectorized.use.row.serde.deserialize", true,
         "This flag should be set to true to enable vectorizing using row deserialize.\n"
+
         "The default value is false."),
+    HIVE_VECTORIZATION_ROW_DESERIALIZE_INPUTFORMAT_EXCLUDES(
+        "hive.vectorized.row.serde.inputformat.excludes",
+        "org.apache.parquet.hadoop.ParquetInputFormat,org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
+        "The input formats not supported by row deserialize vectorization."),
     HIVE_VECTOR_ADAPTOR_USAGE_MODE("hive.vectorized.adaptor.usage.mode", "all", new StringSet("none",
"chosen", "all"),
         "Specifies the extent to which the VectorUDFAdaptor will be used for UDFs that do
not have a cooresponding vectorized class.\n" +
         "0. none   : disable any usage of VectorUDFAdaptor\n" +

http://git-wip-us.apache.org/repos/asf/hive/blob/88da2382/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 0913f40..e9b0a26 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -24,6 +24,7 @@ import java.io.Serializable;
 import java.lang.annotation.Annotation;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -43,6 +44,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.exec.*;
 import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
@@ -229,6 +231,7 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hive.common.util.AnnotationUtils;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.base.Preconditions;
@@ -237,7 +240,7 @@ public class Vectorizer implements PhysicalPlanResolver {
 
   protected static transient final Logger LOG = LoggerFactory.getLogger(Vectorizer.class);
 
-  static Pattern supportedDataTypesPattern;
+  private static final Pattern supportedDataTypesPattern;
 
   static {
     StringBuilder patternBuilder = new StringBuilder();
@@ -290,6 +293,7 @@ public class Vectorizer implements PhysicalPlanResolver {
   private boolean isVectorizationComplexTypesEnabled;
   private boolean isVectorizationGroupByComplexTypesEnabled;
   private boolean isVectorizedRowIdentifierEnabled;
+  private Collection<Class<?>> rowDeserializeInputFormatExcludes;
 
   private boolean isSchemaEvolution;
 
@@ -877,7 +881,7 @@ public class Vectorizer implements PhysicalPlanResolver {
           if (lastColumnTakesRest) {
 
             // If row mode will not catch this input file format, then not enabled.
-            if (useRowDeserialize) {
+            if (useRowDeserialize && canUseRowDeserializeFor(inputFileFormatClassName))
{
               enabledConditionsNotMetList.add(
                   inputFileFormatClassName + " " +
                   serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST + " must be disabled
");
@@ -907,18 +911,21 @@ public class Vectorizer implements PhysicalPlanResolver {
       // inspect-able Object[] row to a VectorizedRowBatch in the VectorMapOperator.
 
       if (useRowDeserialize) {
-
-        pd.setVectorPartitionDesc(
-            VectorPartitionDesc.createRowDeserialize(
-                inputFileFormatClassName,
-                Utilities.isInputFileFormatSelfDescribing(pd),
-                deserializerClassName));
-
-        enabledConditionsMetSet.add(HiveConf.ConfVars.HIVE_VECTORIZATION_USE_ROW_DESERIALIZE.varname);
-        return true;
-
+        if (canUseRowDeserializeFor(inputFileFormatClassName)) {
+          pd.setVectorPartitionDesc(
+              VectorPartitionDesc.createRowDeserialize(
+                  inputFileFormatClassName,
+                  Utilities.isInputFileFormatSelfDescribing(pd),
+                  deserializerClassName));
+  
+          enabledConditionsMetSet.add(HiveConf.ConfVars.HIVE_VECTORIZATION_USE_ROW_DESERIALIZE.varname);
+          return true;
+        } else {
+          enabledConditionsNotMetList.add(ConfVars.HIVE_VECTORIZATION_USE_ROW_DESERIALIZE.varname
+              + " IS true AND " + ConfVars.HIVE_VECTORIZATION_ROW_DESERIALIZE_INPUTFORMAT_EXCLUDES.varname
+              + " NOT CONTAINS " + inputFileFormatClassName);
+        }
       }
-
       if (isInputFileFormatVectorized) {
         Preconditions.checkState(!useVectorizedInputFileFormat);
         enabledConditionsNotMetList.add(HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT.varname);
@@ -936,6 +943,21 @@ public class Vectorizer implements PhysicalPlanResolver {
       return false;
     }
 
+    private boolean canUseRowDeserializeFor(String inputFileFormatClassName) {
+      Class<?> ifClass = null;
+      try {
+        ifClass = Class.forName(inputFileFormatClassName);
+      } catch (ClassNotFoundException e) {
+        LOG.warn("Cannot verify class for " + inputFileFormatClassName
+            + ", not using row deserialize", e);
+        return false;
+      }
+      for (Class<?> badClass : rowDeserializeInputFormatExcludes) {
+        if (badClass.isAssignableFrom(ifClass)) return false;
+      }
+      return true;
+    }
+
     private ImmutablePair<Boolean, Boolean> validateInputFormatAndSchemaEvolution(MapWork
mapWork, String alias,
         TableScanOperator tableScanOperator, VectorTaskColumnInfo vectorTaskColumnInfo)
             throws SemanticException {
@@ -1801,6 +1823,10 @@ public class Vectorizer implements PhysicalPlanResolver {
     useRowDeserialize =
         HiveConf.getBoolVar(hiveConf,
             HiveConf.ConfVars.HIVE_VECTORIZATION_USE_ROW_DESERIALIZE);
+    if (useRowDeserialize) {
+      initRowDeserializeExcludeClasses();
+    }
+
     // TODO: we could also vectorize some formats based on hive.llap.io.encode.formats if
LLAP IO
     //       is enabled and we are going to run in LLAP. However, we don't know if we end
up in
     //       LLAP or not at this stage, so don't do this now. We may need to add a 'force'
option.
@@ -1842,6 +1868,21 @@ public class Vectorizer implements PhysicalPlanResolver {
     return physicalContext;
   }
 
+  private void initRowDeserializeExcludeClasses() {
+    rowDeserializeInputFormatExcludes = new ArrayList<>();
+    String[] classNames = StringUtils.getStrings(HiveConf.getVar(hiveConf,
+        ConfVars.HIVE_VECTORIZATION_ROW_DESERIALIZE_INPUTFORMAT_EXCLUDES));
+    if (classNames == null) return;
+    for (String className : classNames) {
+      if (className == null || className.isEmpty()) continue;
+      try {
+        rowDeserializeInputFormatExcludes.add(Class.forName(className));
+      } catch (Exception ex) {
+        LOG.warn("Cannot create class " + className + " for row.deserialize checks");
+      }
+    }
+  }
+
   private void setOperatorNotSupported(Operator<? extends OperatorDesc> op) {
     OperatorDesc desc = op.getConf();
     Annotation note = AnnotationUtils.getAnnotation(desc.getClass(), Explain.class);

http://git-wip-us.apache.org/repos/asf/hive/blob/88da2382/ql/src/test/queries/clientpositive/parquet_no_row_serde.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_no_row_serde.q b/ql/src/test/queries/clientpositive/parquet_no_row_serde.q
new file mode 100644
index 0000000..c33a650
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/parquet_no_row_serde.q
@@ -0,0 +1,25 @@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+set hive.vectorized.use.vectorized.input.format=false;
+set hive.vectorized.use.row.serde.deserialize=true;
+
+drop table tbl_rc;
+drop table tbl_parquet;
+
+create table tbl_rc (val decimal(10,0))
+row format serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' stored
as rcfile;
+create table tbl_parquet (val decimal(10,0)) 
+STORED AS PARQUET;
+
+insert into table tbl_rc values(101);
+insert into table tbl_parquet values(101);
+
+explain vectorization expression
+select val, round(val, -1) from tbl_rc order by val;
+explain vectorization expression
+select val, round(val, -1) from tbl_parquet order by val;
+
+drop table tbl_rc;
+drop table tbl_parquet;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/88da2382/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out
index b617564..d7a4e87 100644
--- a/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out
@@ -285,21 +285,37 @@ STAGE PLANS:
                 TableScan
                   alias: decimal_tbl_rc
                   Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0]
                   Select Operator
                     expressions: dec (type: decimal(10,0)), round(dec, -1) (type: decimal(11,0))
                     outputColumnNames: _col0, _col1
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [0, 1]
+                        selectExpressions: FuncRoundWithNumDigitsDecimalToDecimal(col 0,
decimalPlaces -1) -> 1:decimal(11,0)
                     Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
                     Reduce Output Operator
                       key expressions: _col0 (type: decimal(10,0))
                       sort order: +
+                      Reduce Sink Vectorization:
+                          className: VectorReduceSinkObjectHashOperator
+                          native: true
+                          nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT
columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
                       Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
                       value expressions: _col1 (type: decimal(11,0))
-            Execution mode: llap
+            Execution mode: vectorized, llap
             LLAP IO: no inputs
             Map Vectorization:
-                enabled: false
-                enabledConditionsNotMet: hive.vectorized.use.row.serde.deserialize IS false
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+                allNative: true
+                usesVectorUDFAdaptor: false
+                vectorized: true
         Reducer 2 
             Execution mode: vectorized, llap
             Reduce Vectorization:
@@ -371,21 +387,37 @@ STAGE PLANS:
                 TableScan
                   alias: decimal_tbl_rc
                   Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0]
                   Select Operator
                     expressions: dec (type: decimal(10,0)), round(dec, -1) (type: decimal(11,0))
                     outputColumnNames: _col0, _col2
+                    Select Vectorization:
+                        className: VectorSelectOperator
+                        native: true
+                        projectedOutputColumns: [0, 1]
+                        selectExpressions: FuncRoundWithNumDigitsDecimalToDecimal(col 0,
decimalPlaces -1) -> 1:decimal(11,0)
                     Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
                     Reduce Output Operator
                       key expressions: _col2 (type: decimal(11,0))
                       sort order: +
+                      Reduce Sink Vectorization:
+                          className: VectorReduceSinkObjectHashOperator
+                          native: true
+                          nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT
columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
                       Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
                       value expressions: _col0 (type: decimal(10,0))
-            Execution mode: llap
+            Execution mode: vectorized, llap
             LLAP IO: no inputs
             Map Vectorization:
-                enabled: false
-                enabledConditionsNotMet: hive.vectorized.use.row.serde.deserialize IS false
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+                allNative: true
+                usesVectorUDFAdaptor: false
+                vectorized: true
         Reducer 2 
             Execution mode: vectorized, llap
             Reduce Vectorization:

http://git-wip-us.apache.org/repos/asf/hive/blob/88da2382/ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out
index 855d2e8..0ba9481 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out
@@ -291,21 +291,36 @@ STAGE PLANS:
                 TableScan
                   alias: b
                   Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column stats:
NONE
+                  TableScan Vectorization:
+                      native: true
+                      projectedOutputColumns: [0, 1]
                   Filter Operator
+                    Filter Vectorization:
+                        className: VectorFilterOperator
+                        native: true
+                        predicateExpression: SelectColumnIsNotNull(col 0) -> boolean
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column stats:
NONE
                     Reduce Output Operator
                       key expressions: key (type: int)
                       sort order: +
                       Map-reduce partition columns: key (type: int)
+                      Reduce Sink Vectorization:
+                          className: VectorReduceSinkLongOperator
+                          native: true
+                          nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT
columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
                       Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column
stats: NONE
                       value expressions: value (type: string)
-            Execution mode: llap
+            Execution mode: vectorized, llap
             LLAP IO: no inputs
             Map Vectorization:
-                enabled: false
-                enabledConditionsNotMet: hive.vectorized.use.row.serde.deserialize IS false
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+                groupByVectorOutput: true
                 inputFileFormats: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+                allNative: true
+                usesVectorUDFAdaptor: false
+                vectorized: true
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:

http://git-wip-us.apache.org/repos/asf/hive/blob/88da2382/ql/src/test/results/clientpositive/parquet_no_row_serde.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_no_row_serde.q.out b/ql/src/test/results/clientpositive/parquet_no_row_serde.q.out
new file mode 100644
index 0000000..25e2625
--- /dev/null
+++ b/ql/src/test/results/clientpositive/parquet_no_row_serde.q.out
@@ -0,0 +1,190 @@
+PREHOOK: query: drop table tbl_rc
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tbl_rc
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table tbl_parquet
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tbl_parquet
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tbl_rc (val decimal(10,0))
+row format serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' stored
as rcfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl_rc
+POSTHOOK: query: create table tbl_rc (val decimal(10,0))
+row format serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' stored
as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl_rc
+PREHOOK: query: create table tbl_parquet (val decimal(10,0)) 
+STORED AS PARQUET
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl_parquet
+POSTHOOK: query: create table tbl_parquet (val decimal(10,0)) 
+STORED AS PARQUET
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl_parquet
+PREHOOK: query: insert into table tbl_rc values(101)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@tbl_rc
+POSTHOOK: query: insert into table tbl_rc values(101)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@tbl_rc
+POSTHOOK: Lineage: tbl_rc.val EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1,
type:string, comment:), ]
+PREHOOK: query: insert into table tbl_parquet values(101)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@tbl_parquet
+POSTHOOK: query: insert into table tbl_parquet values(101)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@tbl_parquet
+POSTHOOK: Lineage: tbl_parquet.val EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1,
type:string, comment:), ]
+PREHOOK: query: explain vectorization expression
+select val, round(val, -1) from tbl_rc order by val
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select val, round(val, -1) from tbl_rc order by val
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: tbl_rc
+            Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+            TableScan Vectorization:
+                native: true
+                projectedOutputColumns: [0]
+            Select Operator
+              expressions: val (type: decimal(10,0)), round(val, -1) (type: decimal(11,0))
+              outputColumnNames: _col0, _col1
+              Select Vectorization:
+                  className: VectorSelectOperator
+                  native: true
+                  projectedOutputColumns: [0, 1]
+                  selectExpressions: FuncRoundWithNumDigitsDecimalToDecimal(col 0, decimalPlaces
-1) -> 1:decimal(11,0)
+              Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+              Reduce Output Operator
+                key expressions: _col0 (type: decimal(10,0))
+                sort order: +
+                Reduce Sink Vectorization:
+                    className: VectorReduceSinkOperator
+                    native: false
+                    nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS
true, LazyBinarySerDe for values IS true
+                    nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+                Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
+                value expressions: _col1 (type: decimal(11,0))
+      Execution mode: vectorized
+      Map Vectorization:
+          enabled: true
+          enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+          groupByVectorOutput: true
+          inputFileFormats: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+          allNative: false
+          usesVectorUDFAdaptor: false
+          vectorized: true
+      Reduce Vectorization:
+          enabled: false
+          enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+          enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+      Reduce Operator Tree:
+        Select Operator
+          expressions: KEY.reducesinkkey0 (type: decimal(10,0)), VALUE._col0 (type: decimal(11,0))
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: explain vectorization expression
+select val, round(val, -1) from tbl_parquet order by val
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select val, round(val, -1) from tbl_parquet order by val
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: tbl_parquet
+            Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+            Select Operator
+              expressions: val (type: decimal(10,0)), round(val, -1) (type: decimal(11,0))
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+              Reduce Output Operator
+                key expressions: _col0 (type: decimal(10,0))
+                sort order: +
+                Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats:
NONE
+                value expressions: _col1 (type: decimal(11,0))
+      Map Vectorization:
+          enabled: false
+          enabledConditionsNotMet: hive.vectorized.use.row.serde.deserialize IS true AND
hive.vectorized.row.serde.inputformat.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
IS false, hive.vectorized.use.vectorized.input.format IS false
+          inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
+      Reduce Vectorization:
+          enabled: false
+          enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+          enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+      Reduce Operator Tree:
+        Select Operator
+          expressions: KEY.reducesinkkey0 (type: decimal(10,0)), VALUE._col0 (type: decimal(11,0))
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: drop table tbl_rc
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tbl_rc
+PREHOOK: Output: default@tbl_rc
+POSTHOOK: query: drop table tbl_rc
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tbl_rc
+POSTHOOK: Output: default@tbl_rc
+PREHOOK: query: drop table tbl_parquet
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tbl_parquet
+PREHOOK: Output: default@tbl_parquet
+POSTHOOK: query: drop table tbl_parquet
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tbl_parquet
+POSTHOOK: Output: default@tbl_parquet

http://git-wip-us.apache.org/repos/asf/hive/blob/88da2382/ql/src/test/results/clientpositive/vector_decimal_round.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_decimal_round.q.out b/ql/src/test/results/clientpositive/vector_decimal_round.q.out
index be7b509..4c28d05 100644
--- a/ql/src/test/results/clientpositive/vector_decimal_round.q.out
+++ b/ql/src/test/results/clientpositive/vector_decimal_round.q.out
@@ -243,19 +243,37 @@ STAGE PLANS:
           TableScan
             alias: decimal_tbl_rc
             Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+            TableScan Vectorization:
+                native: true
+                projectedOutputColumns: [0]
             Select Operator
               expressions: dec (type: decimal(10,0)), round(dec, -1) (type: decimal(11,0))
               outputColumnNames: _col0, _col1
+              Select Vectorization:
+                  className: VectorSelectOperator
+                  native: true
+                  projectedOutputColumns: [0, 1]
+                  selectExpressions: FuncRoundWithNumDigitsDecimalToDecimal(col 0, decimalPlaces
-1) -> 1:decimal(11,0)
               Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
                 key expressions: _col0 (type: decimal(10,0))
                 sort order: +
+                Reduce Sink Vectorization:
+                    className: VectorReduceSinkOperator
+                    native: false
+                    nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS
true, LazyBinarySerDe for values IS true
+                    nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
                 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
                 value expressions: _col1 (type: decimal(11,0))
+      Execution mode: vectorized
       Map Vectorization:
-          enabled: false
-          enabledConditionsNotMet: hive.vectorized.use.row.serde.deserialize IS false
+          enabled: true
+          enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+          groupByVectorOutput: true
           inputFileFormats: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+          allNative: false
+          usesVectorUDFAdaptor: false
+          vectorized: true
       Reduce Vectorization:
           enabled: false
           enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
@@ -309,19 +327,37 @@ STAGE PLANS:
           TableScan
             alias: decimal_tbl_rc
             Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+            TableScan Vectorization:
+                native: true
+                projectedOutputColumns: [0]
             Select Operator
               expressions: dec (type: decimal(10,0)), round(dec, -1) (type: decimal(11,0))
               outputColumnNames: _col0, _col2
+              Select Vectorization:
+                  className: VectorSelectOperator
+                  native: true
+                  projectedOutputColumns: [0, 1]
+                  selectExpressions: FuncRoundWithNumDigitsDecimalToDecimal(col 0, decimalPlaces
-1) -> 1:decimal(11,0)
               Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
                 key expressions: _col2 (type: decimal(11,0))
                 sort order: +
+                Reduce Sink Vectorization:
+                    className: VectorReduceSinkOperator
+                    native: false
+                    nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled
IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS
true, LazyBinarySerDe for values IS true
+                    nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
                 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats:
NONE
                 value expressions: _col0 (type: decimal(10,0))
+      Execution mode: vectorized
       Map Vectorization:
-          enabled: false
-          enabledConditionsNotMet: hive.vectorized.use.row.serde.deserialize IS false
+          enabled: true
+          enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+          groupByVectorOutput: true
           inputFileFormats: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+          allNative: false
+          usesVectorUDFAdaptor: false
+          vectorized: true
       Reduce Vectorization:
           enabled: false
           enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true


Mime
View raw message