Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D60A6200D44 for ; Fri, 13 Oct 2017 02:06:57 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D4B72160BE9; Fri, 13 Oct 2017 00:06:57 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 52952160BE4 for ; Fri, 13 Oct 2017 02:06:56 +0200 (CEST) Received: (qmail 42309 invoked by uid 500); 13 Oct 2017 00:06:54 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 41834 invoked by uid 99); 13 Oct 2017 00:06:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 13 Oct 2017 00:06:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D2D22DFCB7; Fri, 13 Oct 2017 00:06:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sershe@apache.org To: commits@hive.apache.org Date: Fri, 13 Oct 2017 00:06:53 -0000 Message-Id: <3c3f15c91fc0452688d69b0c9a875d35@git.apache.org> In-Reply-To: <8249c490c15f44cea41b948e362b0fef@git.apache.org> References: <8249c490c15f44cea41b948e362b0fef@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [02/23] hive git commit: HIVE-16511: CBO looses inner casts on constants of complex type (Vineet Garg, reviewed by Ashutosh Chauhan) archived-at: Fri, 13 Oct 2017 00:06:58 -0000 HIVE-16511: CBO looses inner casts on constants of complex type (Vineet Garg, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/71b2a267 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/71b2a267 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/71b2a267 Branch: refs/heads/hive-14535 Commit: 71b2a267f90d33d4d27b4190abca3d8ff1eddd29 Parents: 7463d67 Author: Vineet Garg Authored: Tue Oct 10 16:50:19 2017 -0700 Committer: Vineet Garg Committed: Tue Oct 10 16:50:19 2017 -0700 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 - .../ql/exec/vector/VectorizationContext.java | 7 ++- .../calcite/translator/ASTConverter.java | 57 +++++++++++++++----- .../clientpositive/llap/vector_coalesce.q.out | 2 +- .../vector_groupby_grouping_sets_grouping.q.out | 2 +- .../llap/vector_ptf_part_simple.q.out | 16 +++--- .../clientpositive/min_structvalue.q.out | 45 ++++++++++++++++ .../clientpositive/perf/spark/query36.q.out | 6 +-- .../clientpositive/perf/spark/query70.q.out | 6 +-- .../clientpositive/perf/spark/query86.q.out | 6 +-- .../clientpositive/perf/tez/query36.q.out | 4 +- .../clientpositive/perf/tez/query70.q.out | 4 +- .../clientpositive/perf/tez/query86.q.out | 4 +- .../clientpositive/vector_coalesce.q.out | 2 +- 14 files changed, 118 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 264c2b2..5190f04 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -30,7 +30,6 @@ disabled.query.files=ql_rewrite_gbtoidx.q,\ cbo_rp_subq_not_in.q,\ cbo_rp_subq_exists.q,\ orc_llap.q,\ - min_structvalue.q,\ ql_rewrite_gbtoidx_cbo_2.q,\ rcfile_merge1.q,\ smb_mapjoin_8.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index fcebb6f..13d78e2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -2351,11 +2351,10 @@ public class VectorizationContext { } private boolean isNullConst(ExprNodeDesc exprNodeDesc) { - if (exprNodeDesc instanceof ExprNodeConstantDesc) { - String typeString = exprNodeDesc.getTypeString(); - if (typeString.equalsIgnoreCase("void")) { + //null constant could be typed so we need to check the value + if (exprNodeDesc instanceof ExprNodeConstantDesc && + ((ExprNodeConstantDesc) exprNodeDesc).getValue() == null) { return true; - } } return false; } http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index 165f8c4..d6b978f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -19,9 +19,12 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.translator; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Comparator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.TreeMap; + import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.rel.RelFieldCollation; @@ -40,6 +43,7 @@ import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexFieldCollation; @@ -205,12 +209,8 @@ public class ASTConverter { int i = 0; for (RexNode r : select.getChildExps()) { - if (RexUtil.isNull(r) && r.getType().getSqlTypeName() != SqlTypeName.NULL) { - // It is NULL value with different type, we need to introduce a CAST - // to keep it - r = select.getCluster().getRexBuilder().makeAbstractCast(r.getType(), r); - } - ASTNode expr = r.accept(new RexVisitor(schema, r instanceof RexLiteral)); + ASTNode expr = r.accept(new RexVisitor(schema, r instanceof RexLiteral, + select.getCluster().getRexBuilder())); String alias = select.getRowType().getFieldNames().get(i++); ASTNode selectExpr = ASTBuilder.selectExpr(expr, alias); b.add(selectExpr); @@ -223,12 +223,8 @@ public class ASTConverter { List children = new ArrayList<>(); RexCall call = (RexCall) udtf.getCall(); for (RexNode r : call.getOperands()) { - if (RexUtil.isNull(r) && r.getType().getSqlTypeName() != SqlTypeName.NULL) { - // It is NULL value with different type, we need to introduce a CAST - // to keep it - r = select.getCluster().getRexBuilder().makeAbstractCast(r.getType(), r); - } - ASTNode expr = r.accept(new RexVisitor(schema, r instanceof RexLiteral)); + ASTNode expr = r.accept(new RexVisitor(schema, r instanceof RexLiteral, + select.getCluster().getRexBuilder())); children.add(expr); } ASTBuilder sel = ASTBuilder.construct(HiveParser.TOK_SELEXPR, "TOK_SELEXPR"); @@ -460,19 +456,41 @@ public class ASTConverter { } + static class RexVisitor extends RexVisitorImpl { private final Schema schema; private final boolean useTypeQualInLiteral; + private final RexBuilder rexBuilder; + // this is to keep track of null literal which already has been visited + private Map nullLiteralMap ; + + + protected RexVisitor(Schema schema, boolean useTypeQualInLiteral) { + this(schema, useTypeQualInLiteral, null); + } protected RexVisitor(Schema schema) { this(schema, false); } - protected RexVisitor(Schema schema, boolean useTypeQualInLiteral) { + protected RexVisitor(Schema schema, boolean useTypeQualInLiteral, RexBuilder rexBuilder) { super(true); this.schema = schema; this.useTypeQualInLiteral = useTypeQualInLiteral; + this.rexBuilder = rexBuilder; + + this.nullLiteralMap = + new TreeMap<>(new Comparator(){ + // RexLiteral's equal only consider value and type which isn't sufficient + // so providing custom comparator which distinguishes b/w objects irrespective + // of value/type + @Override + public int compare(RexLiteral o1, RexLiteral o2) { + if(o1 == o2) return 0; + else return 1; + } + }); } @Override @@ -497,6 +515,19 @@ public class ASTConverter { @Override public ASTNode visitLiteral(RexLiteral literal) { + + if (RexUtil.isNull(literal) && literal.getType().getSqlTypeName() != SqlTypeName.NULL + && rexBuilder != null) { + // It is NULL value with different type, we need to introduce a CAST + // to keep it + if(nullLiteralMap.containsKey(literal)) { + return ASTBuilder.literal(literal, useTypeQualInLiteral); + } + nullLiteralMap.put(literal, true); + RexNode r = rexBuilder.makeAbstractCast(literal.getType(), literal); + + return r.accept(this); + } return ASTBuilder.literal(literal, useTypeQualInLiteral); } http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out index d6865dd..eb8ec44 100644 --- a/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out @@ -523,7 +523,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [12, 0, 14] - selectExpressions: ConstantVectorExpression(val null) -> 12:bigint, VectorCoalesce(columns [13, 0])(children: ConstantVectorExpression(val null) -> 13:tinyint, col 0) -> 14:tinyint + selectExpressions: ConstantVectorExpression(val null) -> 12:bigint, VectorCoalesce(columns [13, 0])(children: ConstantVectorExpression(val null) -> 13:bigint, col 0) -> 14:bigint Limit Vectorization: className: VectorLimitOperator native: true http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out index ddc1d7b..0b5b053 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out @@ -772,7 +772,7 @@ STAGE PLANS: Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int) http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out b/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out index b63d1e4..614cec3 100644 --- a/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out @@ -5613,9 +5613,9 @@ STAGE PLANS: native: true projectedOutputColumns: [0, 1, 2] Reduce Output Operator - key expressions: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END (type: timestamp) + key expressions: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (CAST( null AS TIMESTAMP)) END (type: timestamp) sort order: ++ - Map-reduce partition columns: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END (type: timestamp) + Map-reduce partition columns: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (CAST( null AS TIMESTAMP)) END (type: timestamp) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator keyColumns: [0, 5] @@ -5662,13 +5662,13 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col0 ASC NULLS FIRST, CASE WHEN ((_col0 = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END ASC NULLS FIRST - partition by: _col0, CASE WHEN ((_col0 = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END + order by: _col0 ASC NULLS FIRST, CASE WHEN ((_col0 = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (CAST( null AS TIMESTAMP)) END ASC NULLS FIRST + partition by: _col0, CASE WHEN ((_col0 = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (CAST( null AS TIMESTAMP)) END raw input shape: window functions: window function definition alias: rank_window_0 - arguments: _col0, CASE WHEN ((_col0 = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END + arguments: _col0, CASE WHEN ((_col0 = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (CAST( null AS TIMESTAMP)) END name: rank window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) @@ -5781,9 +5781,9 @@ STAGE PLANS: native: true projectedOutputColumns: [0, 1, 2] Reduce Output Operator - key expressions: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END (type: timestamp), p_name (type: string) + key expressions: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (CAST( null AS TIMESTAMP)) END (type: timestamp), p_name (type: string) sort order: +++ - Map-reduce partition columns: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END (type: timestamp) + Map-reduce partition columns: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (CAST( null AS TIMESTAMP)) END (type: timestamp) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator keyColumns: [0, 5, 1] @@ -5845,7 +5845,7 @@ STAGE PLANS: input alias: ptf_1 name: windowingtablefunction order by: _col1 ASC NULLS FIRST - partition by: _col0, CASE WHEN ((_col0 = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END + partition by: _col0, CASE WHEN ((_col0 = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (CAST( null AS TIMESTAMP)) END raw input shape: window functions: window function definition http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/min_structvalue.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/min_structvalue.q.out b/ql/src/test/results/clientpositive/min_structvalue.q.out new file mode 100644 index 0000000..3582837 --- /dev/null +++ b/ql/src/test/results/clientpositive/min_structvalue.q.out @@ -0,0 +1,45 @@ +PREHOOK: query: select max(a), min(a) FROM (select named_struct("field",1) as a union all select named_struct("field",2) as a union all select named_struct("field",cast(null as int)) as a) tmp +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select max(a), min(a) FROM (select named_struct("field",1) as a union all select named_struct("field",2) as a union all select named_struct("field",cast(null as int)) as a) tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +{"field":2} {"field":1} +PREHOOK: query: select min(a) FROM (select named_struct("field",1) as a union all select named_struct("field",-2) as a union all select named_struct("field",cast(null as int)) as a) tmp +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select min(a) FROM (select named_struct("field",1) as a union all select named_struct("field",-2) as a union all select named_struct("field",cast(null as int)) as a) tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +{"field":-2} +PREHOOK: query: select min(a) FROM (select named_struct("field",1) as a union all select named_struct("field",2) as a union all select named_struct("field",cast(5 as int)) as a) tmp +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select min(a) FROM (select named_struct("field",1) as a union all select named_struct("field",2) as a union all select named_struct("field",cast(5 as int)) as a) tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +{"field":1} +PREHOOK: query: select min(a) FROM (select named_struct("field",1, "secf", cast(null as int) ) as a union all select named_struct("field",2, "secf", 3) as a union all select named_struct("field",cast(5 as int), "secf", 4) as a) tmp +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select min(a) FROM (select named_struct("field",1, "secf", cast(null as int) ) as a union all select named_struct("field",2, "secf", 3) as a union all select named_struct("field",cast(5 as int), "secf", 4) as a) tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +{"field":1,"secf":null} +PREHOOK: query: select min(a) FROM (select named_struct("field",1, "secf", 2) as a union all select named_struct("field",-2, "secf", 3) as a union all select named_struct("field",cast(null as int), "secf", 1) as a) tmp +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select min(a) FROM (select named_struct("field",1, "secf", 2) as a union all select named_struct("field",-2, "secf", 3) as a union all select named_struct("field",cast(null as int), "secf", 1) as a) tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +{"field":-2,"secf":3} http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/perf/spark/query36.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/spark/query36.q.out b/ql/src/test/results/clientpositive/perf/spark/query36.q.out index c072728..c0d1cc7 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query36.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query36.q.out @@ -215,9 +215,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col4, _col5, _col6 Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (grouping(_col6, 1) + grouping(_col6, 0)) (type: int), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (null) END (type: string), (_col4 / _col5) (type: decimal(37,20)) + key expressions: (grouping(_col6, 1) + grouping(_col6, 0)) (type: int), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string), (_col4 / _col5) (type: decimal(37,20)) sort order: +++ - Map-reduce partition columns: (grouping(_col6, 1) + grouping(_col6, 0)) (type: int), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (null) END (type: string) + Map-reduce partition columns: (grouping(_col6, 1) + grouping(_col6, 0)) (type: int), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string) Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2)), _col6 (type: int) Reducer 5 @@ -236,7 +236,7 @@ STAGE PLANS: input alias: ptf_1 name: windowingtablefunction order by: (_col4 / _col5) ASC NULLS FIRST - partition by: (grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (null) END + partition by: (grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END raw input shape: window functions: window function definition http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/perf/spark/query70.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/spark/query70.q.out b/ql/src/test/results/clientpositive/perf/spark/query70.q.out index 716665b..9f3b8a0 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query70.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query70.q.out @@ -362,9 +362,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col4, _col5 Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END (type: string), _col4 (type: decimal(17,2)) + key expressions: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string), _col4 (type: decimal(17,2)) sort order: ++- - Map-reduce partition columns: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END (type: string) + Map-reduce partition columns: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string) Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: int) Reducer 5 @@ -383,7 +383,7 @@ STAGE PLANS: input alias: ptf_1 name: windowingtablefunction order by: _col4 DESC NULLS LAST - partition by: (grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END + partition by: (grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END raw input shape: window functions: window function definition http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/perf/spark/query86.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/spark/query86.q.out b/ql/src/test/results/clientpositive/perf/spark/query86.q.out index c5b25e8..3642e65 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query86.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query86.q.out @@ -173,9 +173,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col4, _col5 Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END (type: string), _col4 (type: decimal(17,2)) + key expressions: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string), _col4 (type: decimal(17,2)) sort order: ++- - Map-reduce partition columns: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END (type: string) + Map-reduce partition columns: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string) Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: int) Reducer 4 @@ -194,7 +194,7 @@ STAGE PLANS: input alias: ptf_1 name: windowingtablefunction order by: _col4 DESC NULLS LAST - partition by: (grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END + partition by: (grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END raw input shape: window functions: window function definition http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/perf/tez/query36.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/tez/query36.q.out b/ql/src/test/results/clientpositive/perf/tez/query36.q.out index dee62b5..9a4349d 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query36.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query36.q.out @@ -81,12 +81,12 @@ Stage-0 Select Operator [SEL_30] (rows=1149975358 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] PTF Operator [PTF_29] (rows=1149975358 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"(_col4 / _col5) ASC NULLS FIRST","partition by:":"(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (null) END"}] + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"(_col4 / _col5) ASC NULLS FIRST","partition by:":"(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END"}] Select Operator [SEL_28] (rows=1149975358 width=88) Output:["_col0","_col1","_col4","_col5","_col6"] <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_27] - PartitionCols:(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (null) END + PartitionCols:(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END Select Operator [SEL_26] (rows=1149975358 width=88) Output:["_col0","_col1","_col4","_col5","_col6"] Group By Operator [GBY_25] (rows=1149975358 width=88) http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/perf/tez/query70.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/tez/query70.q.out b/ql/src/test/results/clientpositive/perf/tez/query70.q.out index d700d60..69f08c0 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query70.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query70.q.out @@ -101,12 +101,12 @@ Stage-0 Select Operator [SEL_55] (rows=1149975358 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] PTF Operator [PTF_54] (rows=1149975358 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END"}] + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END"}] Select Operator [SEL_53] (rows=1149975358 width=88) Output:["_col0","_col1","_col4","_col5"] <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_52] - PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END + PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END Select Operator [SEL_51] (rows=1149975358 width=88) Output:["_col0","_col1","_col4","_col5"] Group By Operator [GBY_50] (rows=1149975358 width=88) http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/perf/tez/query86.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/tez/query86.q.out b/ql/src/test/results/clientpositive/perf/tez/query86.q.out index 6b59dd2..c37b384 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query86.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query86.q.out @@ -72,12 +72,12 @@ Stage-0 Select Operator [SEL_24] (rows=261364852 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] PTF Operator [PTF_23] (rows=261364852 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END"}] + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END"}] Select Operator [SEL_22] (rows=261364852 width=135) Output:["_col0","_col1","_col4","_col5"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_21] - PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (null) END + PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END Select Operator [SEL_20] (rows=261364852 width=135) Output:["_col0","_col1","_col4","_col5"] Group By Operator [GBY_19] (rows=261364852 width=135) http://git-wip-us.apache.org/repos/asf/hive/blob/71b2a267/ql/src/test/results/clientpositive/vector_coalesce.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_coalesce.q.out b/ql/src/test/results/clientpositive/vector_coalesce.q.out index 445a246..4bfdac9 100644 --- a/ql/src/test/results/clientpositive/vector_coalesce.q.out +++ b/ql/src/test/results/clientpositive/vector_coalesce.q.out @@ -448,7 +448,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [12, 0, 14] - selectExpressions: ConstantVectorExpression(val null) -> 12:bigint, VectorCoalesce(columns [13, 0])(children: ConstantVectorExpression(val null) -> 13:tinyint, col 0) -> 14:tinyint + selectExpressions: ConstantVectorExpression(val null) -> 12:bigint, VectorCoalesce(columns [13, 0])(children: ConstantVectorExpression(val null) -> 13:bigint, col 0) -> 14:bigint Limit Vectorization: className: VectorLimitOperator native: true