Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D82F9179BA for ; Tue, 12 May 2015 01:23:49 +0000 (UTC) Received: (qmail 53413 invoked by uid 500); 12 May 2015 01:23:48 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 53289 invoked by uid 500); 12 May 2015 01:23:48 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 52524 invoked by uid 99); 12 May 2015 01:23:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 May 2015 01:23:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D8AD0DFB8E; Tue, 12 May 2015 01:23:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: prasanthj@apache.org To: commits@hive.apache.org Date: Tue, 12 May 2015 01:23:54 -0000 Message-Id: <60e87a1ff6094103b448c31c83447570@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [08/39] hive git commit: HIVE-10506: CBO (Calcite Return Path): Disallow return path to be enable if CBO is off (Jesus Camacho Rodriguez via Laljo John Pullokkaran) HIVE-10506: CBO (Calcite Return Path): Disallow return path to be enable if CBO is off (Jesus Camacho Rodriguez via Laljo John Pullokkaran) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/93995c8b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/93995c8b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/93995c8b Branch: refs/heads/llap Commit: 93995c8be3dedc8785ced64939c608ae2433d4af Parents: ecde4ae Author: John Pullokkaran Authored: Wed May 6 18:15:33 2015 -0700 Committer: jpullokk Committed: Wed May 6 18:21:19 2015 -0700 ---------------------------------------------------------------------- ql/src/java/org/apache/hadoop/hive/ql/Context.java | 10 +++++++++- .../hive/ql/optimizer/IdentityProjectRemover.java | 12 ++++++++++++ .../hive/ql/optimizer/NonBlockingOpDeDupProc.java | 11 +++++++++++ .../org/apache/hadoop/hive/ql/optimizer/Optimizer.java | 8 +++----- .../calcite/translator/HiveOpConverterPostProc.java | 10 ++++++++++ .../org/apache/hadoop/hive/ql/parse/CalcitePlanner.java | 1 + 6 files changed, 46 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/Context.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 9692738..a74bbbe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -23,7 +23,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -86,6 +85,7 @@ public class Context { protected int pathid = 10000; protected boolean explain = false; protected String cboInfo; + protected boolean cboSucceeded; protected boolean explainLogical = false; protected String cmd = ""; // number of previous attempts @@ -706,4 +706,12 @@ public class Context { this.cboInfo = cboInfo; } + public boolean isCboSucceeded() { + return cboSucceeded; + } + + public void setCboSucceeded(boolean cboSucceeded) { + this.cboSucceeded = cboSucceeded; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java index 433699b..e3d3ce6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java @@ -26,8 +26,10 @@ import java.util.Stack; import com.google.common.base.Predicates; import com.google.common.collect.Iterators; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; @@ -69,6 +71,16 @@ public class IdentityProjectRemover implements Transform { private static final Log LOG = LogFactory.getLog(IdentityProjectRemover.class); @Override public ParseContext transform(ParseContext pctx) throws SemanticException { + // 0. We check the conditions to apply this transformation, + // if we do not meet them we bail out + final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); + final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); + if(cboEnabled && returnPathEnabled && cboSucceeded) { + return pctx; + } + + // 1. We apply the transformation Map opRules = new LinkedHashMap(); opRules.put(new RuleRegExp("R1", "(" + SelectOperator.getOperatorName() + "%)"), new ProjectRemover()); http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java index 95c2b0b..3006a6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Set; import java.util.Stack; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; @@ -57,6 +58,16 @@ public class NonBlockingOpDeDupProc implements Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { + // 0. We check the conditions to apply this transformation, + // if we do not meet them we bail out + final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); + final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); + if(cboEnabled && returnPathEnabled && cboSucceeded) { + return pctx; + } + + // 1. We apply the transformation String SEL = SelectOperator.getOperatorName(); String FIL = FilterOperator.getOperatorName(); Map opRules = new LinkedHashMap(); http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 58f8afe..a7cf8b7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -62,11 +62,9 @@ public class Optimizer { transformations = new ArrayList(); - // If we are translating Calcite operators into Hive operators, we need - // additional postprocessing - if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { - transformations.add(new HiveOpConverterPostProc()); - } + // Add the additional postprocessing transformations needed if + // we are translating Calcite operators into Hive operators. + transformations.add(new HiveOpConverterPostProc()); // Add the transformation that computes the lineage information. transformations.add(new Generator()); http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java index cdd7c7e..e7c8342 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java @@ -26,6 +26,7 @@ import java.util.Stack; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; @@ -55,6 +56,15 @@ public class HiveOpConverterPostProc implements Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { + // 0. We check the conditions to apply this transformation, + // if we do not meet them we bail out + final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); + final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); + if(!(cboEnabled && returnPathEnabled && cboSucceeded)) { + return pctx; + } + // 1. Initialize aux data structures this.pctx = pctx; this.aliasToOpInfo = new HashMap>(); http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 49ad6ad..48f488f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -265,6 +265,7 @@ public class CalcitePlanner extends SemanticAnalyzer { sinkOp = genPlan(getQB()); LOG.info("CBO Succeeded; optimized logical plan."); this.ctx.setCboInfo("Plan optimized by CBO."); + this.ctx.setCboSucceeded(true); LOG.debug(newAST.dump()); } } catch (Exception e) {