Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 2FE9B2009F2 for ; Thu, 5 May 2016 23:19:30 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 2E80E160A04; Thu, 5 May 2016 21:19:30 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2552F1609F9 for ; Thu, 5 May 2016 23:19:28 +0200 (CEST) Received: (qmail 76512 invoked by uid 500); 5 May 2016 21:19:28 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 76501 invoked by uid 99); 5 May 2016 21:19:28 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 May 2016 21:19:28 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 31BE6DFB14; Thu, 5 May 2016 21:19:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jcamacho@apache.org To: commits@hive.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-13637: Fold CASE into NVL when CBO optimized the plan (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) Date: Thu, 5 May 2016 21:19:28 +0000 (UTC) archived-at: Thu, 05 May 2016 21:19:30 -0000 Repository: hive Updated Branches: refs/heads/master 10d054913 -> da82819bc HIVE-13637: Fold CASE into NVL when CBO optimized the plan (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da82819b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da82819b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da82819b Branch: refs/heads/master Commit: da82819bc112589e0d96874947c942e834681ed2 Parents: 10d0549 Author: Jesus Camacho Rodriguez Authored: Wed May 4 01:27:30 2016 +0100 Committer: Jesus Camacho Rodriguez Committed: Thu May 5 22:13:10 2016 +0100 ---------------------------------------------------------------------- .../calcite/translator/JoinTypeCheckCtx.java | 2 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 17 ++++++++----- .../hadoop/hive/ql/parse/TypeCheckCtx.java | 19 +++++++++----- .../hive/ql/parse/TypeCheckProcFactory.java | 26 ++++++++++++++++++++ .../queries/clientpositive/constantPropWhen.q | 2 ++ 5 files changed, 53 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/da82819b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java index dccd1d9..f166bb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java @@ -53,7 +53,7 @@ public class JoinTypeCheckCtx extends TypeCheckCtx { public JoinTypeCheckCtx(RowResolver leftRR, RowResolver rightRR, JoinType hiveJoinType) throws SemanticException { - super(RowResolver.getCombinedRR(leftRR, rightRR), true, false, false, false, false, false, false, + super(RowResolver.getCombinedRR(leftRR, rightRR), true, false, false, false, false, false, false, false, false, false); this.inputRRLst = ImmutableList.of(leftRR, rightRR); this.outerJoin = (hiveJoinType == JoinType.LEFTOUTER) || (hiveJoinType == JoinType.RIGHTOUTER) http://git-wip-us.apache.org/repos/asf/hive/blob/da82819b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 2983d38..f79a525 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -3143,8 +3143,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( - new FilterDesc(genExprNodeDesc(condn, inputRR, useCaching), false), new RowSchema( - inputRR.getColumnInfos()), input), inputRR); + new FilterDesc(genExprNodeDesc(condn, inputRR, useCaching, isCBOExecuted()), false), + new RowSchema(inputRR.getColumnInfos()), input), inputRR); if (LOG.isDebugEnabled()) { LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: " @@ -4146,7 +4146,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } else { // Case when this is an expression - TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); + TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR, true, isCBOExecuted()); // We allow stateful functions in the SELECT list (but nowhere else) tcCtx.setAllowStatefulFunctions(true); tcCtx.setAllowDistinctFunctions(false); @@ -7777,7 +7777,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { List expressions = joinTree.getExpressions().get(i); joinKeys[i] = new ExprNodeDesc[expressions.size()]; for (int j = 0; j < joinKeys[i].length; j++) { - joinKeys[i][j] = genExprNodeDesc(expressions.get(j), inputRR); + joinKeys[i][j] = genExprNodeDesc(expressions.get(j), inputRR, true, isCBOExecuted()); } } // Type checking and implicit type conversion for join keys @@ -10999,12 +10999,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { throws SemanticException { // Since the user didn't supply a customized type-checking context, // use default settings. - return genExprNodeDesc(expr, input, true); + return genExprNodeDesc(expr, input, true, false); } public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean useCaching) throws SemanticException { - TypeCheckCtx tcCtx = new TypeCheckCtx(input, useCaching); + return genExprNodeDesc(expr, input, useCaching, false); + } + + public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean useCaching, + boolean foldExpr) throws SemanticException { + TypeCheckCtx tcCtx = new TypeCheckCtx(input, useCaching, foldExpr); return genExprNodeDesc(expr, input, tcCtx); } http://git-wip-us.apache.org/repos/asf/hive/blob/da82819b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java index de1c043..02896ff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java @@ -37,6 +37,8 @@ public class TypeCheckCtx implements NodeProcessorCtx { private final boolean useCaching; + private final boolean foldExpr; + /** * Receives translations which will need to be applied during unparse. */ @@ -79,20 +81,21 @@ public class TypeCheckCtx implements NodeProcessorCtx { * The input row resolver of the previous operator. */ public TypeCheckCtx(RowResolver inputRR) { - this(inputRR, true); + this(inputRR, true, false); } - public TypeCheckCtx(RowResolver inputRR, boolean useCaching) { - this(inputRR, useCaching, false, true, true, true, true, true, true, true); + public TypeCheckCtx(RowResolver inputRR, boolean useCaching, boolean foldExpr) { + this(inputRR, useCaching, foldExpr, false, true, true, true, true, true, true, true); } - public TypeCheckCtx(RowResolver inputRR, boolean useCaching, boolean allowStatefulFunctions, - boolean allowDistinctFunctions, boolean allowGBExprElimination, boolean allowAllColRef, - boolean allowFunctionStar, boolean allowWindowing, + public TypeCheckCtx(RowResolver inputRR, boolean useCaching, boolean foldExpr, + boolean allowStatefulFunctions, boolean allowDistinctFunctions, boolean allowGBExprElimination, + boolean allowAllColRef, boolean allowFunctionStar, boolean allowWindowing, boolean allowIndexExpr, boolean allowSubQueryExpr) { setInputRR(inputRR); error = null; this.useCaching = useCaching; + this.foldExpr = foldExpr; this.allowStatefulFunctions = allowStatefulFunctions; this.allowDistinctFunctions = allowDistinctFunctions; this.allowGBExprElimination = allowGBExprElimination; @@ -209,4 +212,8 @@ public class TypeCheckCtx implements NodeProcessorCtx { public boolean isUseCaching() { return useCaching; } + + public boolean isFoldExpr() { + return foldExpr; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/da82819b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index da236d5..ceeb9b4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -61,9 +61,12 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.SettableUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFNvl; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -1055,6 +1058,14 @@ public class TypeCheckProcFactory { } desc = ExprNodeGenericFuncDesc.newInstance(genericUDF, funcText, childrenList); + } else if (ctx.isFoldExpr() && canConvertIntoNvl(genericUDF, children)) { + // Rewrite CASE into NVL + desc = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(), + Lists.newArrayList(children.get(0), new ExprNodeConstantDesc(false))); + if (Boolean.FALSE.equals(((ExprNodeConstantDesc) children.get(1)).getValue())) { + desc = ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), + Lists.newArrayList(desc)); + } } else { desc = ExprNodeGenericFuncDesc.newInstance(genericUDF, funcText, children); @@ -1072,6 +1083,21 @@ public class TypeCheckProcFactory { return desc; } + private boolean canConvertIntoNvl(GenericUDF genericUDF, ArrayList children) { + if (genericUDF instanceof GenericUDFWhen && children.size() == 3 && + children.get(1) instanceof ExprNodeConstantDesc && + children.get(2) instanceof ExprNodeConstantDesc) { + ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) children.get(1); + ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) children.get(2); + Object thenVal = constThen.getValue(); + Object elseVal = constElse.getValue(); + if (thenVal instanceof Boolean && elseVal instanceof Boolean) { + return true; + } + } + return false; + } + /** * Returns true if des is a descendant of ans (ancestor) */ http://git-wip-us.apache.org/repos/asf/hive/blob/da82819b/ql/src/test/queries/clientpositive/constantPropWhen.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/constantPropWhen.q b/ql/src/test/queries/clientpositive/constantPropWhen.q index c1d4885..03bfd54 100644 --- a/ql/src/test/queries/clientpositive/constantPropWhen.q +++ b/ql/src/test/queries/clientpositive/constantPropWhen.q @@ -1,4 +1,5 @@ set hive.mapred.mode=nonstrict; +set hive.optimize.constant.propagation=false; drop table test_1; @@ -24,6 +25,7 @@ SELECT cast(CASE id when id2 THEN TRUE ELSE FALSE END AS BOOLEAN) AS b FROM test set hive.cbo.enable=false; +set hive.optimize.constant.propagation=true; explain SELECT cast(CASE WHEN id = id2 THEN FALSE ELSE TRUE END AS BOOLEAN) AS b FROM test_1;