hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject [5/5] hive git commit: HIVE-16423: Add hint to enforce semi join optimization (Deepak Jaiswal, reviewed by Jason Dere)
Date Thu, 20 Apr 2017 17:18:56 GMT
HIVE-16423: Add hint to enforce semi join optimization (Deepak Jaiswal, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9d5d737d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9d5d737d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9d5d737d

Branch: refs/heads/master
Commit: 9d5d737db4f715a880f0d544d548a5ce370f602b
Parents: fa24d4b
Author: Gunther Hagleitner <gunther@apache.org>
Authored: Thu Apr 20 10:07:52 2017 -0700
Committer: Gunther Hagleitner <gunther@apache.org>
Committed: Thu Apr 20 10:07:52 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |     2 +
 .../test/resources/testconfiguration.properties |     1 +
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |     4 +-
 .../DynamicPartitionPruningOptimization.java    |   102 +-
 .../calcite/translator/HiveOpConverter.java     |    24 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |    35 +
 .../hive/ql/parse/CalcitePlanner.java.orig      |  4188 +++++
 .../hadoop/hive/ql/parse/GenTezUtils.java       |    25 +-
 .../apache/hadoop/hive/ql/parse/HintParser.g    |     3 +
 .../hadoop/hive/ql/parse/ParseContext.java      |    25 +-
 .../apache/hadoop/hive/ql/parse/QBJoinTree.java |    16 +
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |    63 +
 .../hive/ql/parse/SemanticAnalyzer.java.orig    | 13508 +++++++++++++++++
 .../hive/ql/parse/SemiJoinBranchInfo.java       |    45 +
 .../hadoop/hive/ql/parse/SemiJoinHint.java      |    43 +
 .../hadoop/hive/ql/parse/TaskCompiler.java      |     2 +-
 .../hadoop/hive/ql/parse/TezCompiler.java       |   137 +-
 .../hive/ql/plan/ExprNodeDynamicListDesc.java   |    15 +-
 .../apache/hadoop/hive/ql/plan/JoinDesc.java    |    18 +
 .../hive/ql/ppd/SyntheticJoinPredicate.java     |     6 +-
 .../ql/udf/generic/GenericUDAFBloomFilter.java  |    13 +
 .../test/queries/clientpositive/semijoin_hint.q |    54 +
 .../clientpositive/llap/semijoin_hint.q.out     |   899 ++
 23 files changed, 19107 insertions(+), 121 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/9d5d737d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 420d35e..b10b08e 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2892,6 +2892,8 @@ public class HiveConf extends Configuration {
             "Big table for runtime filteting should be of atleast this size"),
     TEZ_DYNAMIC_SEMIJOIN_REDUCTION_THRESHOLD("hive.tez.dynamic.semijoin.reduction.threshold",
(float) 0.50,
             "Only perform semijoin optimization if the estimated benefit at or above this
fraction of the target table"),
+    TEZ_DYNAMIC_SEMIJOIN_REDUCTION_HINT_ONLY("hive.tez.dynamic.semijoin.reduction.hint.only",
false,
+            "When true, only enforce semijoin when a hint is provided"),
     TEZ_SMB_NUMBER_WAVES(
         "hive.tez.smb.number.waves",
         (float) 0.5,

http://git-wip-us.apache.org/repos/asf/hive/blob/9d5d737d/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index ed5ce9d..116d0eb 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -572,6 +572,7 @@ minillaplocal.query.files=acid_globallimit.q,\
   schema_evol_text_vecrow_table.q,\
   selectDistinctStar.q,\
   semijoin.q,\
+  semijoin_hint.q,\
   smb_cache.q,\
   special_character_in_tabnames_1.q,\
   sqlmerge.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/9d5d737d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index db6b05b..637bc54 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -794,7 +794,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
       // The semijoin branch can potentially create a task level cycle
       // with the hashjoin except when it is dynamically partitioned hash
       // join which takes place in a separate task.
-      if (context.parseContext.getRsOpToTsOpMap().size() > 0
+      if (context.parseContext.getRsToSemiJoinBranchInfo().size() > 0
               && removeReduceSink) {
         removeCycleCreatingSemiJoinOps(mapJoinOp, parentSelectOpOfBigTableOp,
                 context.parseContext);
@@ -826,7 +826,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
       }
 
       ReduceSinkOperator rs = (ReduceSinkOperator) op;
-      TableScanOperator ts = parseContext.getRsOpToTsOpMap().get(rs);
+      TableScanOperator ts = parseContext.getRsToSemiJoinBranchInfo().get(rs).getTsOp();
       if (ts == null) {
         // skip, no semijoin branch
         continue;

http://git-wip-us.apache.org/repos/asf/hive/blob/9d5d737d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index 838cc69..eb3eba5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -43,12 +43,7 @@ import org.apache.hadoop.hive.ql.lib.RuleRegExp;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.optimizer.spark.SparkPartitionPruningSinkDesc;
-import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext;
-import org.apache.hadoop.hive.ql.parse.ParseContext;
-import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
-import org.apache.hadoop.hive.ql.parse.RuntimeValuesInfo;
-import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.*;
 import org.apache.hadoop.hive.ql.parse.spark.OptimizeSparkProcContext;
 import org.apache.hadoop.hive.ql.plan.*;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBloomFilter.GenericUDAFBloomFilterEvaluator;
@@ -215,16 +210,25 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor
{
         } else {
           LOG.debug("Column " + column + " is not a partition column");
           if (semiJoin && ts.getConf().getFilterExpr() != null) {
-            LOG.debug("Initiate semijoin reduction for " + column);
-            // Get the table name from which the min-max values will come.
+            LOG.debug("Initiate semijoin reduction for " + column + " ("
+                + ts.getConf().getFilterExpr().getExprString());
+            // Get the table name from which the min-max values and bloom filter will come.
             Operator<?> op = ctx.generator;
+
             while (!(op == null || op instanceof TableScanOperator)) {
               op = op.getParentOperators().get(0);
             }
             String tableAlias = (op == null ? "" : ((TableScanOperator) op).getConf().getAlias());
+
+            Map<String, SemiJoinHint> hints = ctx.desc.getHints();
+            SemiJoinHint sjHint = (hints != null) ? hints.get(tableAlias) : null;
             keyBaseAlias = ctx.generator.getOperatorId() + "_" + tableAlias + "_" + column;
 
-            semiJoinAttempted = generateSemiJoinOperatorPlan(ctx, parseContext, ts, keyBaseAlias);
+            semiJoinAttempted = generateSemiJoinOperatorPlan(
+                ctx, parseContext, ts, keyBaseAlias, sjHint);
+            if (!semiJoinAttempted && sjHint != null) {
+              throw new SemanticException("The user hint to enforce semijoin failed required
conditions");
+            }
           }
         }
 
@@ -387,7 +391,13 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor
{
 
   // Generates plan for min/max when dynamic partition pruning is ruled out.
   private boolean generateSemiJoinOperatorPlan(DynamicListContext ctx, ParseContext parseContext,
-      TableScanOperator ts, String keyBaseAlias) throws SemanticException {
+      TableScanOperator ts, String keyBaseAlias, SemiJoinHint sjHint) throws SemanticException
{
+
+    // If semijoin hint is enforced, make sure hint is provided
+    if (parseContext.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION_HINT_ONLY)
+            && sjHint == null) {
+        return false;
+    }
 
     // we will put a fork in the plan at the source of the reduce sink
     Operator<? extends OperatorDesc> parentOfRS = ctx.generator.getParentOperators().get(0);
@@ -441,6 +451,14 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor
{
       }
     }
 
+    // If hint is provided and only hinted semijoin optimizations should be
+    // created, then skip other columns on the table
+    if (parseContext.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION_HINT_ONLY)
+            && sjHint.getColName() != null &&
+            !internalColName.equals(sjHint.getColName())) {
+      return false;
+    }
+
     List<ExprNodeDesc> keyExprs = new ArrayList<ExprNodeDesc>();
     keyExprs.add(key);
 
@@ -484,8 +502,6 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor
{
             HiveConf.getFloatVar(parseContext.getConf(),
                     HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
 
-    ArrayList<ExprNodeDesc> groupByExprs = new ArrayList<ExprNodeDesc>();
-
     // Add min/max and bloom filter aggregations
     List<ObjectInspector> aggFnOIs = new ArrayList<ObjectInspector>();
     aggFnOIs.add(key.getWritableObjectInspector());
@@ -505,8 +521,14 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor
{
       AggregationDesc bloomFilter = new AggregationDesc("bloom_filter",
               FunctionRegistry.getGenericUDAFEvaluator("bloom_filter", aggFnOIs, false, false),
               params, false, Mode.PARTIAL1);
-      GenericUDAFBloomFilterEvaluator bloomFilterEval = (GenericUDAFBloomFilterEvaluator)
bloomFilter.getGenericUDAFEvaluator();
+      GenericUDAFBloomFilterEvaluator bloomFilterEval =
+          (GenericUDAFBloomFilterEvaluator) bloomFilter.getGenericUDAFEvaluator();
       bloomFilterEval.setSourceOperator(selectOp);
+
+      if (sjHint != null && sjHint.getNumEntries() > 0) {
+        LOG.debug("Setting size for " + keyBaseAlias + " to " + sjHint.getNumEntries() +
" based on the hint");
+        bloomFilterEval.setHintEntries(sjHint.getNumEntries());
+      }
       bloomFilterEval.setMaxEntries(parseContext.getConf().getLongVar(ConfVars.TEZ_MAX_BLOOM_FILTER_ENTRIES));
       bloomFilterEval.setMinEntries(parseContext.getConf().getLongVar(ConfVars.TEZ_MIN_BLOOM_FILTER_ENTRIES));
       bloomFilterEval.setFactor(parseContext.getConf().getFloatVar(ConfVars.TEZ_BLOOM_FILTER_FACTOR));
@@ -607,6 +629,9 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor
{
               bloomFilterFinalParams, false, Mode.FINAL);
       GenericUDAFBloomFilterEvaluator bloomFilterEval = (GenericUDAFBloomFilterEvaluator)
bloomFilter.getGenericUDAFEvaluator();
       bloomFilterEval.setSourceOperator(selectOp);
+      if (sjHint != null && sjHint.getNumEntries() > 0) {
+        bloomFilterEval.setHintEntries(sjHint.getNumEntries());
+      }
       bloomFilterEval.setMaxEntries(parseContext.getConf().getLongVar(ConfVars.TEZ_MAX_BLOOM_FILTER_ENTRIES));
       bloomFilterEval.setMinEntries(parseContext.getConf().getLongVar(ConfVars.TEZ_MIN_BLOOM_FILTER_ENTRIES));
       bloomFilterEval.setFactor(parseContext.getConf().getFloatVar(ConfVars.TEZ_BLOOM_FILTER_FACTOR));
@@ -635,23 +660,56 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor
{
       rsOp.getConf().setOutputOperators(outputOperators);
     }
 
+    createFinalRsForSemiJoinOp(parseContext, ts, groupByOpFinal, key,
+            keyBaseAlias, ctx.parent.getChildren().get(0), sjHint != null);
+
+    return true;
+  }
+
+  private void createFinalRsForSemiJoinOp(
+          ParseContext parseContext, TableScanOperator ts, GroupByOperator gb,
+          ExprNodeDesc key, String keyBaseAlias, ExprNodeDesc colExpr,
+          boolean isHint) throws SemanticException {
+    ArrayList<String> gbOutputNames = new ArrayList<>();
+    // One each for min, max and bloom filter
+    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(0));
+    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(1));
+    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(2));
+
+    int colPos = 0;
+    ArrayList<ExprNodeDesc> rsValueCols = new ArrayList<ExprNodeDesc>();
+    for (int i = 0; i < gbOutputNames.size() - 1; i++) {
+      ExprNodeColumnDesc expr = new ExprNodeColumnDesc(key.getTypeInfo(),
+              gbOutputNames.get(colPos++), "", false);
+      rsValueCols.add(expr);
+    }
+
+    // Bloom Filter uses binary
+    ExprNodeColumnDesc colBFExpr = new ExprNodeColumnDesc(TypeInfoFactory.binaryTypeInfo,
+            gbOutputNames.get(colPos++), "", false);
+    rsValueCols.add(colBFExpr);
+
     // Create the final Reduce Sink Operator
     ReduceSinkDesc rsDescFinal = PlanUtils.getReduceSinkDesc(
             new ArrayList<ExprNodeDesc>(), rsValueCols, gbOutputNames, false,
             -1, 0, 1, Operation.NOT_ACID);
     ReduceSinkOperator rsOpFinal = (ReduceSinkOperator)OperatorFactory.getAndMakeChild(
-            rsDescFinal, new RowSchema(groupByOpFinal.getSchema()), groupByOpFinal);
+            rsDescFinal, new RowSchema(gb.getSchema()), gb);
+    Map<String, ExprNodeDesc> columnExprMap = new HashMap<>();
     rsOpFinal.setColumnExprMap(columnExprMap);
 
-    LOG.debug("DynamicMinMaxPushdown: Saving RS to TS mapping: " + rsOpFinal + ": " + ts);
-    parseContext.getRsOpToTsOpMap().put(rsOpFinal, ts);
+    LOG.debug("DynamicSemiJoinPushdown: Saving RS to TS mapping: " + rsOpFinal + ": " + ts);
+    SemiJoinBranchInfo sjInfo = new SemiJoinBranchInfo(ts, isHint);
+    parseContext.getRsToSemiJoinBranchInfo().put(rsOpFinal, sjInfo);
 
     // for explain purpose
-    if (parseContext.getContext().getExplainConfig() != null
-        && parseContext.getContext().getExplainConfig().isFormatted()) {
-      List<String> outputOperators = new ArrayList<>();
+    if (parseContext.getContext().getExplainConfig() != null &&
+            parseContext.getContext().getExplainConfig().isFormatted()) {
+      List<String> outputOperators = rsOpFinal.getConf().getOutputOperators();
+      if (outputOperators == null) {
+        outputOperators = new ArrayList<>();
+      }
       outputOperators.add(ts.getOperatorId());
-      rsOpFinal.getConf().setOutputOperators(outputOperators);
     }
 
     // Save the info that is required at query time to resolve dynamic/runtime values.
@@ -666,10 +724,8 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor
{
     runtimeValuesInfo.setTableDesc(rsFinalTableDesc);
     runtimeValuesInfo.setDynamicValueIDs(dynamicValueIDs);
     runtimeValuesInfo.setColExprs(rsValueCols);
-    runtimeValuesInfo.setTsColExpr(ctx.parent.getChildren().get(0));
+    runtimeValuesInfo.setTsColExpr(colExpr);
     parseContext.getRsToRuntimeValuesInfoMap().put(rsOpFinal, runtimeValuesInfo);
-
-    return true;
   }
 
   private Map<Node, Object> collectDynamicPruningConditions(ExprNodeDesc pred, NodeProcessorCtx
ctx)

http://git-wip-us.apache.org/repos/asf/hive/blob/9d5d737d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
index 73a9b0f..d375d1b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
@@ -19,6 +19,8 @@
 package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
 
 
+import org.apache.hadoop.hive.ql.parse.*;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -72,19 +74,8 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
-import org.apache.hadoop.hive.ql.parse.JoinCond;
-import org.apache.hadoop.hive.ql.parse.JoinType;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec;
 import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
 import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression;
-import org.apache.hadoop.hive.ql.parse.PTFTranslator;
-import org.apache.hadoop.hive.ql.parse.ParseUtils;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
-import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.parse.UnparseTranslator;
-import org.apache.hadoop.hive.ql.parse.WindowingComponentizer;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec;
 import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -348,6 +339,9 @@ public class HiveOpConverter {
     // through Hive
     String[] baseSrc = new String[joinRel.getInputs().size()];
     String tabAlias = getHiveDerivedTableAlias();
+    Map<String, SemiJoinHint> semiJoinHints = semanticAnalyzer.parseSemiJoinHint(
+        semanticAnalyzer.getQB().getParseInfo().getHints());
+
     // 1. Convert inputs
     OpAttr[] inputs = new OpAttr[joinRel.getInputs().size()];
     List<Operator<?>> children = new ArrayList<Operator<?>>(joinRel.getInputs().size());
@@ -413,7 +407,7 @@ public class HiveOpConverter {
 
     // 6. Generate Join operator
     JoinOperator joinOp = genJoin(joinRel, joinExpressions, filterExpressions, children,
-            baseSrc, tabAlias);
+            baseSrc, tabAlias, semiJoinHints);
 
     // 7. Return result
     return new OpAttr(tabAlias, newVcolsInCalcite, joinOp);
@@ -726,7 +720,7 @@ public class HiveOpConverter {
       List<String> keepColNames) throws SemanticException {
     // 1. Generate RS operator
     // 1.1 Prune the tableNames, only count the tableNames that are not empty strings
-	// as empty string in table aliases is only allowed for virtual columns.
+  // as empty string in table aliases is only allowed for virtual columns.
     String tableAlias = null;
     Set<String> tableNames = input.getSchema().getTableNames();
     for (String tableName : tableNames) {
@@ -885,7 +879,8 @@ public class HiveOpConverter {
 
   private static JoinOperator genJoin(RelNode join, ExprNodeDesc[][] joinExpressions,
       List<List<ExprNodeDesc>> filterExpressions, List<Operator<?>>
children,
-      String[] baseSrc, String tabAlias) throws SemanticException {
+      String[] baseSrc, String tabAlias, Map<String, SemiJoinHint> semiJoinHints)
+          throws SemanticException {
 
     // 1. Extract join type
     JoinCondDesc[] joinCondns;
@@ -1011,6 +1006,7 @@ public class HiveOpConverter {
     // 4. We create the join operator with its descriptor
     JoinDesc desc = new JoinDesc(exprMap, outputColumnNames, noOuterJoin, joinCondns,
             filters, joinExpressions);
+    desc.setSemiJoinHints(semiJoinHints);
     desc.setReversedExprs(reversedExprs);
     desc.setFilterMap(filterMap);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/9d5d737d/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index c97b3e7..d10b6bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.ql.parse;
 
+import org.antlr.runtime.tree.Tree;
+
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
@@ -332,6 +334,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       skipCalcitePlan = true;
     } else {
       PreCboCtx cboCtx = (PreCboCtx) plannerCtx;
+      ASTNode oldHints = getQB().getParseInfo().getHints();
 
       // Note: for now, we don't actually pass the queryForCbo to CBO, because
       // it accepts qb, not AST, and can also access all the private stuff in
@@ -399,6 +402,15 @@ public class CalcitePlanner extends SemanticAnalyzer {
                 newAST = reAnalyzeCTASAfterCbo(newAST);
               }
             }
+            if (oldHints != null) {
+              if (getQB().getParseInfo().getHints() != null) {
+                LOG.warn("Hints are not null in the optimized tree; before CBO " + oldHints.dump()
+                    + "; after CBO " + getQB().getParseInfo().getHints().dump());
+              } else {
+                LOG.debug("Propagating hints to QB: " + oldHints);
+                getQB().getParseInfo().setHints(oldHints);
+              }
+            }
             Phase1Ctx ctx_1 = initPhase1Ctx();
             if (!doPhase1(newAST, getQB(), ctx_1, null)) {
               throw new RuntimeException("Couldn't do phase1 on CBO optimized query plan");
@@ -3462,6 +3474,24 @@ public class CalcitePlanner extends SemanticAnalyzer {
       return selRel;
     }
 
+    private void setQueryHints(QB qb) throws SemanticException {
+      QBParseInfo qbp = getQBParseInfo(qb);
+      String selClauseName = qbp.getClauseNames().iterator().next();
+      Tree selExpr0 = qbp.getSelForClause(selClauseName).getChild(0);
+
+      if (selExpr0.getType() != HiveParser.QUERY_HINT) return;
+      String hint = ctx.getTokenRewriteStream().toString(
+          selExpr0.getTokenStartIndex(), selExpr0.getTokenStopIndex());
+      LOG.debug("Handling query hints: " + hint);
+      ParseDriver pd = new ParseDriver();
+      try {
+        ASTNode hintNode = pd.parseHint(hint);
+        qbp.setHints((ASTNode) hintNode);
+      } catch (ParseException e) {
+        throw new SemanticException("failed to parse query hint: "+e.getMessage(), e);
+      }
+    }
+
     /**
      * NOTE: there can only be one select caluse since we don't handle multi
      * destination insert.
@@ -3960,7 +3990,12 @@ public class CalcitePlanner extends SemanticAnalyzer {
         throw new CalciteSemanticException("Unsupported", UnsupportedFeature.Others);
 
       }
+
       // 1.3 process join
+      // 1.3.1 process hints
+      setQueryHints(qb);
+
+      // 1.3.2 process the actual join
       if (qb.getParseInfo().getJoinExpr() != null) {
         srcRel = genJoinLogicalPlan(qb.getParseInfo().getJoinExpr(), aliasToRel);
       } else {


Mime
View raw message