hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pxi...@apache.org
Subject [9/9] hive git commit: HIVE-11927: Implement/Enable constant related optimization rules in Calcite: enable HiveReduceExpressionsRule to fold constants (Pengcheng Xiong, reviewed by Laljo John Pullokkaran)
Date Wed, 23 Dec 2015 05:06:52 GMT
HIVE-11927: Implement/Enable constant related optimization rules in Calcite: enable HiveReduceExpressionsRule to fold constants (Pengcheng Xiong, reviewed by Laljo John Pullokkaran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b340ecb5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b340ecb5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b340ecb5

Branch: refs/heads/master
Commit: b340ecb5e163277e86def59b454b8c041ece39d5
Parents: af96223
Author: Pengcheng Xiong <pxiong@apache.org>
Authored: Tue Dec 22 21:06:14 2015 -0800
Committer: Pengcheng Xiong <pxiong@apache.org>
Committed: Tue Dec 22 21:06:14 2015 -0800

----------------------------------------------------------------------
 .../calcite/CalciteSemanticException.java       |   2 +-
 .../optimizer/calcite/HiveRexExecutorImpl.java  |  80 ++
 .../rules/HiveReduceExpressionsRule.java        | 905 +++++++++++++++++++
 .../calcite/translator/ASTBuilder.java          |  30 +-
 .../calcite/translator/RexNodeConverter.java    |  48 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |  84 +-
 ql/src/test/queries/clientpositive/cbo_const.q  |  52 ++
 .../queries/clientpositive/constantfolding.q    |  88 ++
 .../clientpositive/annotate_stats_select.q.out  |  58 +-
 .../bucketizedhiveinputformat.q.out             |   2 +
 ql/src/test/results/clientpositive/cast1.q.out  |  10 +-
 .../test/results/clientpositive/cbo_const.q.out | 334 +++++++
 .../cbo_rp_cross_product_check_2.q.out          |   4 +-
 .../clientpositive/cbo_rp_lineage2.q.out        |   2 +-
 .../clientpositive/constantfolding.q.out        | 305 +++++++
 .../clientpositive/cross_product_check_1.q.out  |   4 +-
 .../clientpositive/cross_product_check_2.q.out  |   4 +-
 .../clientpositive/dynamic_rdd_cache.q.out      |  46 +-
 .../dynpart_sort_optimization2.q.out            |  45 +-
 .../results/clientpositive/groupby_ppd.q.out    |  18 +-
 .../clientpositive/groupby_sort_1_23.q.out      |  12 +-
 .../clientpositive/groupby_sort_skew_1_23.q.out |  12 +-
 .../results/clientpositive/input_part1.q.out    |   4 +-
 .../results/clientpositive/input_part5.q.out    |   2 +-
 .../results/clientpositive/input_part6.q.out    |   2 +-
 .../test/results/clientpositive/lineage2.q.out  |   2 +-
 .../test/results/clientpositive/lineage3.q.out  |  10 +-
 .../list_bucket_query_oneskew_2.q.out           |  50 +-
 .../llap/bucket_map_join_tez1.q.out             |   4 +-
 .../llap/vector_join_part_col_char.q.out        |   2 +-
 .../clientpositive/orc_predicate_pushdown.q.out |  22 +-
 .../parquet_predicate_pushdown.q.out            |  22 +-
 .../clientpositive/partition_multilevels.q.out  |  44 +-
 .../results/clientpositive/perf/query31.q.out   | 338 +++----
 .../results/clientpositive/perf/query39.q.out   |  58 +-
 .../results/clientpositive/perf/query42.q.out   | 171 ++--
 .../results/clientpositive/perf/query52.q.out   | 163 ++--
 .../results/clientpositive/perf/query64.q.out   | 362 ++++----
 .../results/clientpositive/perf/query66.q.out   | 627 ++++++-------
 .../results/clientpositive/perf/query75.q.out   | 643 ++++++-------
 .../results/clientpositive/pointlookup2.q.out   |   6 +-
 .../results/clientpositive/quotedid_basic.q.out |  62 +-
 .../clientpositive/quotedid_partition.q.out     |  22 +-
 .../spark/bucket_map_join_tez1.q.out            |   4 +-
 .../spark/bucketizedhiveinputformat.q.out       |   2 +
 .../spark/cross_product_check_1.q.out           |   4 +-
 .../spark/cross_product_check_2.q.out           |   4 +-
 .../spark/dynamic_rdd_cache.q.out               |  46 +-
 .../spark/groupby_sort_1_23.q.out               |  12 +-
 .../spark/groupby_sort_skew_1_23.q.out          |  12 +-
 .../clientpositive/spark/union_remove_25.q.out  |  16 +-
 .../clientpositive/spark/union_view.q.out       |  60 +-
 .../results/clientpositive/subquery_notin.q.out |  18 +-
 .../subquery_notin_having.q.java1.7.out         |  10 +-
 .../subquery_unqualcolumnrefs.q.out             |   2 +-
 .../tez/bucket_map_join_tez1.q.out              |   4 +-
 .../tez/cross_product_check_1.q.out             |   4 +-
 .../tez/cross_product_check_2.q.out             |   4 +-
 .../tez/dynpart_sort_optimization2.q.out        |  34 +-
 .../clientpositive/tez/explainuser_1.q.out      | 260 +++---
 .../tez/vector_decimal_round.q.out              |   6 +-
 ql/src/test/results/clientpositive/udf1.q.out   |  40 +-
 .../results/clientpositive/udf_10_trims.q.out   |   2 +-
 .../clientpositive/udf_folder_constants.q.out   |   8 +-
 .../clientpositive/union_remove_25.q.out        |  20 +-
 .../results/clientpositive/union_view.q.out     |  60 +-
 .../clientpositive/vector_decimal_round.q.out   |   6 +-
 67 files changed, 3660 insertions(+), 1739 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
index 336745b..f6b4124 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
@@ -31,7 +31,7 @@ public class CalciteSemanticException extends SemanticException {
 
   public enum UnsupportedFeature {
     Distinct_without_an_aggreggation, Duplicates_in_RR, Filter_expression_with_non_boolean_return_type, 
-    Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal, 
+    Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal, Invalid_interval,
     Less_than_equal_greater_than, Multi_insert, Others, Same_name_in_multiple_expressions, 
     Schema_less_table, Select_alias_in_having_clause, Select_transform, Subquery, 
     Table_sample_clauses, UDTF, Union_type, Unique_join

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
new file mode 100644
index 0000000..f7958c6
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.calcite;
+
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+
+public class HiveRexExecutorImpl implements RelOptPlanner.Executor {
+
+  private final RelOptCluster cluster;
+
+  protected final Logger LOG;
+
+  public HiveRexExecutorImpl(RelOptCluster cluster) {
+    this.cluster = cluster;
+    LOG = LoggerFactory.getLogger(this.getClass().getName());
+  }
+
+  @Override
+  public void reduce(RexBuilder rexBuilder, List<RexNode> constExps, List<RexNode> reducedValues) {
+    RexNodeConverter rexNodeConverter = new RexNodeConverter(cluster);
+    for (RexNode rexNode : constExps) {
+      // initialize the converter
+      ExprNodeConverter converter = new ExprNodeConverter("", null, null, null,
+          new HashSet<Integer>(), cluster.getTypeFactory());
+      // convert RexNode to ExprNodeGenericFuncDesc
+      ExprNodeDesc expr = rexNode.accept(converter);
+      if (expr instanceof ExprNodeGenericFuncDesc) {
+        // folding the constant
+        ExprNodeDesc constant = ConstantPropagateProcFactory
+            .foldExpr((ExprNodeGenericFuncDesc) expr);
+        if (constant != null) {
+          try {
+            // convert constant back to RexNode
+            reducedValues.add(rexNodeConverter.convert((ExprNodeConstantDesc) constant));
+          } catch (Exception e) {
+            LOG.warn(e.getMessage());
+            reducedValues.add(rexNode);
+          }
+        } else {
+          reducedValues.add(rexNode);
+        }
+      } else {
+        reducedValues.add(rexNode);
+      }
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
new file mode 100644
index 0000000..50e139b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptPredicateList;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexDynamicParam;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.rex.RexRangeRef;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlRowOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Stacks;
+import org.apache.calcite.util.Util;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * Collection of planner rules that apply various simplifying transformations on
+ * RexNode trees. Currently, there are two transformations:
+ *
+ * <ul>
+ * <li>Constant reduction, which evaluates constant subtrees, replacing them
+ * with a corresponding RexLiteral
+ * <li>Removal of redundant casts, which occurs when the argument into the cast
+ * is the same as the type of the resulting cast expression
+ * </ul>
+ */
+public abstract class HiveReduceExpressionsRule extends RelOptRule {
+  //~ Static fields/initializers ---------------------------------------------
+
+  /**
+   * Regular expression that matches the description of all instances of this
+   * rule and {@link ValuesReduceRule} also. Use
+   * it to prevent the planner from invoking these rules.
+   */
+  public static final Pattern EXCLUSION_PATTERN =
+      Pattern.compile("Reduce(Expressions|Values)Rule.*");
+
+  /**
+   * Singleton rule that reduces constants inside a
+   * {@link org.apache.calcite.rel.logical.HiveFilter}.
+   */
+  public static final HiveReduceExpressionsRule FILTER_INSTANCE =
+      new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER);
+
+  /**
+   * Singleton rule that reduces constants inside a
+   * {@link org.apache.calcite.rel.logical.HiveProject}.
+   */
+  public static final HiveReduceExpressionsRule PROJECT_INSTANCE =
+      new ProjectReduceExpressionsRule(HiveProject.class, HiveRelFactories.HIVE_BUILDER);
+
+  /**
+   * Singleton rule that reduces constants inside a
+   * {@link org.apache.calcite.rel.core.HiveJoin}.
+   */
+  public static final HiveReduceExpressionsRule JOIN_INSTANCE =
+      new JoinReduceExpressionsRule(HiveJoin.class, HiveRelFactories.HIVE_BUILDER);
+
+  /**
+   * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Filter}.
+   * If the condition is a constant, the filter is removed (if TRUE) or replaced with
+   * an empty {@link org.apache.calcite.rel.core.Values} (if FALSE or NULL).
+   */
+  public static class FilterReduceExpressionsRule extends HiveReduceExpressionsRule {
+
+    public FilterReduceExpressionsRule(Class<? extends Filter> filterClass,
+        RelBuilderFactory relBuilderFactory) {
+      super(filterClass, relBuilderFactory, "HiveReduceExpressionsRule(Filter)");
+    }
+
+    @Override public void onMatch(RelOptRuleCall call) {
+      final Filter filter = call.rel(0);
+      final List<RexNode> expList =
+          Lists.newArrayList(filter.getCondition());
+      RexNode newConditionExp;
+      boolean reduced;
+      final RelOptPredicateList predicates =
+          RelMetadataQuery.getPulledUpPredicates(filter.getInput());
+      if (reduceExpressions(filter, expList, predicates)) {
+        assert expList.size() == 1;
+        newConditionExp = expList.get(0);
+        reduced = true;
+      } else {
+        // No reduction, but let's still test the original
+        // predicate to see if it was already a constant,
+        // in which case we don't need any runtime decision
+        // about filtering.
+        newConditionExp = filter.getCondition();
+        reduced = false;
+      }
+      if (newConditionExp.isAlwaysTrue()) {
+        call.transformTo(
+            filter.getInput());
+      }
+      // TODO: support LogicalValues
+      else if (newConditionExp instanceof RexLiteral
+          || RexUtil.isNullLiteral(newConditionExp, true)) {
+        // call.transformTo(call.builder().values(filter.getRowType()).build());
+        return;
+      }
+      else if (reduced) {
+        call.transformTo(call.builder().
+            push(filter.getInput()).filter(expList.get(0)).build());
+      } else {
+        if (newConditionExp instanceof RexCall) {
+          RexCall rexCall = (RexCall) newConditionExp;
+          boolean reverse =
+              rexCall.getOperator()
+                  == SqlStdOperatorTable.NOT;
+          if (reverse) {
+            rexCall = (RexCall) rexCall.getOperands().get(0);
+          }
+          reduceNotNullableFilter(call, filter, rexCall, reverse);
+        }
+        return;
+      }
+
+      // New plan is absolutely better than old plan.
+      call.getPlanner().setImportance(filter, 0.0);
+    }
+
+    private void reduceNotNullableFilter(
+        RelOptRuleCall call,
+        Filter filter,
+        RexCall rexCall,
+        boolean reverse) {
+      // If the expression is a IS [NOT] NULL on a non-nullable
+      // column, then we can either remove the filter or replace
+      // it with an Empty.
+      boolean alwaysTrue;
+      switch (rexCall.getKind()) {
+      case IS_NULL:
+      case IS_UNKNOWN:
+        alwaysTrue = false;
+        break;
+      case IS_NOT_NULL:
+        alwaysTrue = true;
+        break;
+      default:
+        return;
+      }
+      if (reverse) {
+        alwaysTrue = !alwaysTrue;
+      }
+      RexNode operand = rexCall.getOperands().get(0);
+      if (operand instanceof RexInputRef) {
+        RexInputRef inputRef = (RexInputRef) operand;
+        if (!inputRef.getType().isNullable()) {
+          if (alwaysTrue) {
+            call.transformTo(filter.getInput());
+          } else {
+            // TODO: support LogicalValues
+            // call.transformTo(call.builder().values(filter.getRowType()).build());
+            return;
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Project}.
+   */
+  public static class ProjectReduceExpressionsRule extends HiveReduceExpressionsRule {
+
+    public ProjectReduceExpressionsRule(Class<? extends Project> projectClass,
+        RelBuilderFactory relBuilderFactory) {
+      super(projectClass, relBuilderFactory, "HiveReduceExpressionsRule(Project)");
+    }
+
+    public boolean matches(RelOptRuleCall call) {
+      Project project = call.rel(0);
+      HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class);
+
+      // If this operator has been visited already by the rule,
+      // we do not need to apply the optimization
+      if (registry != null && registry.getVisited(this).contains(project)) {
+        return false;
+      }
+
+      return true;
+    }
+
+    @Override public void onMatch(RelOptRuleCall call) {
+      Project project = call.rel(0);
+      // Register that we have visited this operator in this rule
+      HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class);
+      if (registry != null) {
+        registry.registerVisited(this, project);
+      }
+      final RelOptPredicateList predicates =
+          RelMetadataQuery.getPulledUpPredicates(project.getInput());
+      final List<RexNode> expList =
+          Lists.newArrayList(project.getProjects());
+      if (reduceExpressions(project, expList, predicates)) {
+        RelNode newProject = call.builder().push(project.getInput())
+            .project(expList, project.getRowType().getFieldNames()).build();
+        if (registry != null) {
+          registry.registerVisited(this, newProject);
+        }
+        call.transformTo(newProject);
+
+        // New plan is absolutely better than old plan.
+        call.getPlanner().setImportance(project, 0.0);
+      }
+    }
+  }
+
+  /**
+   * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.HiveJoin}.
+   */
+  public static class JoinReduceExpressionsRule extends HiveReduceExpressionsRule {
+
+    public JoinReduceExpressionsRule(Class<? extends HiveJoin> joinClass,
+        RelBuilderFactory relBuilderFactory) {
+      super(joinClass, relBuilderFactory, "HiveReduceExpressionsRule(HiveJoin)");
+    }
+
+    @Override public void onMatch(RelOptRuleCall call) {
+      final HiveJoin join = call.rel(0);
+      final List<RexNode> expList = Lists.newArrayList(join.getCondition());
+      final int fieldCount = join.getLeft().getRowType().getFieldCount();
+      final RelOptPredicateList leftPredicates =
+          RelMetadataQuery.getPulledUpPredicates(join.getLeft());
+      final RelOptPredicateList rightPredicates =
+          RelMetadataQuery.getPulledUpPredicates(join.getRight());
+      final RelOptPredicateList predicates =
+          leftPredicates.union(rightPredicates.shift(fieldCount));
+      if (!reduceExpressions(join, expList, predicates)) {
+        return;
+      }
+      final JoinInfo joinInfo = JoinInfo.of(join.getLeft(), join.getRight(), expList.get(0));
+      if (!joinInfo.isEqui()) {
+        // This kind of join must be an equi-join, and the condition is
+        // no longer an equi-join. SemiJoin is an example of this.
+        return;
+      }
+      call.transformTo(
+          join.copy(
+              join.getTraitSet(),
+              expList.get(0),
+              join.getLeft(),
+              join.getRight(),
+              join.getJoinType(),
+              join.isSemiJoinDone()));
+
+      // New plan is absolutely better than old plan.
+      call.getPlanner().setImportance(join, 0.0);
+    }
+  }
+
+  //~ Constructors -----------------------------------------------------------
+
+  /**
+   * Creates a HiveReduceExpressionsRule.
+   *
+   * @param clazz class of rels to which this rule should apply
+   */
+  protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz,
+      RelBuilderFactory relBuilderFactory, String desc) {
+    super(operand(clazz, any()), relBuilderFactory, desc);
+  }
+
+  //~ Methods ----------------------------------------------------------------
+
+  /**
+   * Reduces a list of expressions.
+   *
+   * @param rel     Relational expression
+   * @param expList List of expressions, modified in place
+   * @param predicates Constraints known to hold on input expressions
+   * @return whether reduction found something to change, and succeeded
+   */
+  protected static boolean reduceExpressions(RelNode rel, List<RexNode> expList,
+      RelOptPredicateList predicates) {
+    RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
+
+    // Replace predicates on CASE to CASE on predicates.
+    new CaseShuttle().mutate(expList);
+
+    // Find reducible expressions.
+    final List<RexNode> constExps = Lists.newArrayList();
+    List<Boolean> addCasts = Lists.newArrayList();
+    final List<RexNode> removableCasts = Lists.newArrayList();
+    final ImmutableMap<RexNode, RexLiteral> constants =
+        predicateConstants(predicates);
+    findReducibleExps(rel.getCluster().getTypeFactory(), expList, constants,
+        constExps, addCasts, removableCasts);
+    if (constExps.isEmpty() && removableCasts.isEmpty()) {
+      return false;
+    }
+
+    // Remove redundant casts before reducing constant expressions.
+    // If the argument to the redundant cast is a reducible constant,
+    // reducing that argument to a constant first will result in not being
+    // able to locate the original cast expression.
+    if (!removableCasts.isEmpty()) {
+      final List<RexNode> reducedExprs = Lists.newArrayList();
+      for (RexNode exp : removableCasts) {
+        RexCall call = (RexCall) exp;
+        reducedExprs.add(call.getOperands().get(0));
+      }
+      RexReplacer replacer =
+          new RexReplacer(
+              rexBuilder,
+              removableCasts,
+              reducedExprs,
+              Collections.nCopies(removableCasts.size(), false));
+      replacer.mutate(expList);
+    }
+
+    if (constExps.isEmpty()) {
+      return true;
+    }
+
+    final List<RexNode> constExps2 = Lists.newArrayList(constExps);
+    if (!constants.isEmpty()) {
+      //noinspection unchecked
+      final List<Map.Entry<RexNode, RexNode>> pairs =
+          (List<Map.Entry<RexNode, RexNode>>) (List)
+              Lists.newArrayList(constants.entrySet());
+      RexReplacer replacer =
+          new RexReplacer(
+              rexBuilder,
+              Pair.left(pairs),
+              Pair.right(pairs),
+              Collections.nCopies(pairs.size(), false));
+      replacer.mutate(constExps2);
+    }
+
+    // Compute the values they reduce to.
+    RelOptPlanner.Executor executor =
+        rel.getCluster().getPlanner().getExecutor();
+    if (executor == null) {
+      // Cannot reduce expressions: caller has not set an executor in their
+      // environment. Caller should execute something like the following before
+      // invoking the planner:
+      //
+      // final RexExecutorImpl executor =
+      //   new RexExecutorImpl(Schemas.createDataContext(null));
+      // rootRel.getCluster().getPlanner().setExecutor(executor);
+      return false;
+    }
+
+    final List<RexNode> reducedValues = Lists.newArrayList();
+    executor.reduce(rexBuilder, constExps2, reducedValues);
+
+    // For Project, we have to be sure to preserve the result
+    // types, so always cast regardless of the expression type.
+    // For other RelNodes like Filter, in general, this isn't necessary,
+    // and the presence of casts could hinder other rules such as sarg
+    // analysis, which require bare literals.  But there are special cases,
+    // like when the expression is a UDR argument, that need to be
+    // handled as special cases.
+    if (rel instanceof Project) {
+      addCasts = Collections.nCopies(reducedValues.size(), true);
+    }
+
+    RexReplacer replacer =
+        new RexReplacer(
+            rexBuilder,
+            constExps,
+            reducedValues,
+            addCasts);
+    replacer.mutate(expList);
+    return true;
+  }
+
+  /**
+   * Locates expressions that can be reduced to literals or converted to
+   * expressions with redundant casts removed.
+   *
+   * @param typeFactory    Type factory
+   * @param exps           list of candidate expressions to be examined for
+   *                       reduction
+   * @param constants      List of expressions known to be constant
+   * @param constExps      returns the list of expressions that can be constant
+   *                       reduced
+   * @param addCasts       indicator for each expression that can be constant
+   *                       reduced, whether a cast of the resulting reduced
+   *                       expression is potentially necessary
+   * @param removableCasts returns the list of cast expressions where the cast
+   */
+  protected static void findReducibleExps(RelDataTypeFactory typeFactory,
+      List<RexNode> exps, ImmutableMap<RexNode, RexLiteral> constants,
+      List<RexNode> constExps, List<Boolean> addCasts,
+      List<RexNode> removableCasts) {
+    ReducibleExprLocator gardener =
+        new ReducibleExprLocator(typeFactory, constants, constExps,
+            addCasts, removableCasts);
+    for (RexNode exp : exps) {
+      gardener.analyze(exp);
+    }
+    assert constExps.size() == addCasts.size();
+  }
+
+  protected static ImmutableMap<RexNode, RexLiteral> predicateConstants(
+      RelOptPredicateList predicates) {
+    // We cannot use an ImmutableMap.Builder here. If there are multiple entries
+    // with the same key (e.g. "WHERE deptno = 1 AND deptno = 2"), it doesn't
+    // matter which we take, so the latter will replace the former.
+    // The basic idea is to find all the pairs of RexNode = RexLiteral
+    // (1) If 'predicates' contain a non-EQUALS, we bail out.
+    // (2) It is OK if a RexNode is equal to the same RexLiteral several times,
+    // (e.g. "WHERE deptno = 1 AND deptno = 1")
+    // (3) It will return false if there are inconsistent constraints (e.g.
+    // "WHERE deptno = 1 AND deptno = 2")
+    final Map<RexNode, RexLiteral> map = new HashMap<>();
+    final Set<RexNode> excludeSet = new HashSet<>();
+    for (RexNode predicate : predicates.pulledUpPredicates) {
+      gatherConstraints(map, predicate, excludeSet);
+    }
+    final ImmutableMap.Builder<RexNode, RexLiteral> builder =
+        ImmutableMap.builder();
+    for (Map.Entry<RexNode, RexLiteral> entry : map.entrySet()) {
+      RexNode rexNode = entry.getKey();
+      if (!overlap(rexNode, excludeSet)) {
+        builder.put(rexNode, entry.getValue());
+      }
+    }
+    return builder.build();
+  }
+
+  private static boolean overlap(RexNode rexNode, Set<RexNode> set) {
+    if (rexNode instanceof RexCall) {
+      for (RexNode r : ((RexCall) rexNode).getOperands()) {
+        if (overlap(r, set)) {
+          return true;
+        }
+      }
+      return false;
+    } else {
+      return set.contains(rexNode);
+    }
+  }
+
+  /** Tries to decompose the RexNode which is a RexCall into non-literal
+   * RexNodes. */
+  private static void decompose(Set<RexNode> set, RexNode rexNode) {
+    if (rexNode instanceof RexCall) {
+      for (RexNode r : ((RexCall) rexNode).getOperands()) {
+        decompose(set, r);
+      }
+    } else if (!(rexNode instanceof RexLiteral)) {
+      set.add(rexNode);
+    }
+  }
+
+  private static void gatherConstraints(Map<RexNode, RexLiteral> map,
+      RexNode predicate, Set<RexNode> excludeSet) {
+    if (predicate.getKind() != SqlKind.EQUALS) {
+      decompose(excludeSet, predicate);
+      return;
+    }
+    final List<RexNode> operands = ((RexCall) predicate).getOperands();
+    if (operands.size() != 2) {
+      decompose(excludeSet, predicate);
+      return;
+    }
+    // if it reaches here, we have rexNode equals rexNode
+    final RexNode left = operands.get(0);
+    final RexNode right = operands.get(1);
+    // note that literals are immutable too and they can only be compared through
+    // values.
+    if (right instanceof RexLiteral && !excludeSet.contains(left)) {
+      RexLiteral existedValue = map.get(left);
+      if (existedValue == null) {
+        map.put(left, (RexLiteral) right);
+      } else {
+        if (!existedValue.getValue().equals(((RexLiteral) right).getValue())) {
+          // we found conflict values.
+          map.remove(left);
+          excludeSet.add(left);
+        }
+      }
+    } else if (left instanceof RexLiteral && !excludeSet.contains(right)) {
+      RexLiteral existedValue = map.get(right);
+      if (existedValue == null) {
+        map.put(right, (RexLiteral) left);
+      } else {
+        if (!existedValue.getValue().equals(((RexLiteral) left).getValue())) {
+          map.remove(right);
+          excludeSet.add(right);
+        }
+      }
+    }
+  }
+
+  /** Pushes predicates into a CASE.
+   *
+   * <p>We have a loose definition of 'predicate': any boolean expression will
+   * do, except CASE. For example '(CASE ...) = 5' or '(CASE ...) IS NULL'.
+   */
+  protected static RexCall pushPredicateIntoCase(RexCall call) {
+    if (call.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) {
+      return call;
+    }
+    switch (call.getKind()) {
+    case CASE:
+    case AND:
+    case OR:
+      return call; // don't push CASE into CASE!
+    }
+    int caseOrdinal = -1;
+    final List<RexNode> operands = call.getOperands();
+    for (int i = 0; i < operands.size(); i++) {
+      RexNode operand = operands.get(i);
+      switch (operand.getKind()) {
+      case CASE:
+        caseOrdinal = i;
+      }
+    }
+    if (caseOrdinal < 0) {
+      return call;
+    }
+    // Convert
+    //   f(CASE WHEN p1 THEN v1 ... END, arg)
+    // to
+    //   CASE WHEN p1 THEN f(v1, arg) ... END
+    final RexCall case_ = (RexCall) operands.get(caseOrdinal);
+    final List<RexNode> nodes = new ArrayList<>();
+    for (int i = 0; i < case_.getOperands().size(); i++) {
+      RexNode node = case_.getOperands().get(i);
+      if (!RexUtil.isCasePredicate(case_, i)) {
+        node = substitute(call, caseOrdinal, node);
+      }
+      nodes.add(node);
+    }
+    return case_.clone(call.getType(), nodes);
+  }
+
+  /** Converts op(arg0, ..., argOrdinal, ..., argN) to op(arg0,..., node, ..., argN). */
+  protected static RexNode substitute(RexCall call, int ordinal, RexNode node) {
+    final List<RexNode> newOperands = Lists.newArrayList(call.getOperands());
+    newOperands.set(ordinal, node);
+    return call.clone(call.getType(), newOperands);
+  }
+
+  //~ Inner Classes ----------------------------------------------------------
+
+  /**
+   * Replaces expressions with their reductions. Note that we only have to
+   * look for RexCall, since nothing else is reducible in the first place.
+   */
+  protected static class RexReplacer extends RexShuttle {
+    private final RexBuilder rexBuilder;
+    private final List<RexNode> reducibleExps;
+    private final List<RexNode> reducedValues;
+    private final List<Boolean> addCasts;
+
+    RexReplacer(
+        RexBuilder rexBuilder,
+        List<RexNode> reducibleExps,
+        List<RexNode> reducedValues,
+        List<Boolean> addCasts) {
+      this.rexBuilder = rexBuilder;
+      this.reducibleExps = reducibleExps;
+      this.reducedValues = reducedValues;
+      this.addCasts = addCasts;
+    }
+
+    @Override public RexNode visitInputRef(RexInputRef inputRef) {
+      RexNode node = visit(inputRef);
+      if (node == null) {
+        return super.visitInputRef(inputRef);
+      }
+      return node;
+    }
+
+    @Override public RexNode visitCall(RexCall call) {
+      RexNode node = visit(call);
+      if (node != null) {
+        return node;
+      }
+      node = super.visitCall(call);
+      if (node != call) {
+        node = RexUtil.simplify(rexBuilder, node);
+      }
+      return node;
+    }
+
+    private RexNode visit(final RexNode call) {
+      int i = reducibleExps.indexOf(call);
+      if (i == -1) {
+        return null;
+      }
+      RexNode replacement = reducedValues.get(i);
+      if (addCasts.get(i)
+          && (replacement.getType() != call.getType())) {
+        // Handle change from nullable to NOT NULL by claiming
+        // that the result is still nullable, even though
+        // we know it isn't.
+        //
+        // Also, we cannot reduce CAST('abc' AS VARCHAR(4)) to 'abc'.
+        // If we make 'abc' of type VARCHAR(4), we may later encounter
+        // the same expression in a Project's digest where it has
+        // type VARCHAR(3), and that's wrong.
+        replacement = rexBuilder.makeAbstractCast(call.getType(), replacement);
+      }
+      return replacement;
+    }
+  }
+
+  /**
+   * Helper class used to locate expressions that either can be reduced to
+   * literals or contain redundant casts.
+   */
+  protected static class ReducibleExprLocator extends RexVisitorImpl<Void> {
+    /** Whether an expression is constant, and if so, whether it can be
+     * reduced to a simpler constant. */
+    enum Constancy {
+      NON_CONSTANT, REDUCIBLE_CONSTANT, IRREDUCIBLE_CONSTANT
+    }
+
+    private final RelDataTypeFactory typeFactory;
+
+    private final List<Constancy> stack;
+
+    private final ImmutableMap<RexNode, RexLiteral> constants;
+
+    private final List<RexNode> constExprs;
+
+    private final List<Boolean> addCasts;
+
+    private final List<RexNode> removableCasts;
+
+    private final List<SqlOperator> parentCallTypeStack;
+
+    ReducibleExprLocator(RelDataTypeFactory typeFactory,
+        ImmutableMap<RexNode, RexLiteral> constants, List<RexNode> constExprs,
+        List<Boolean> addCasts, List<RexNode> removableCasts) {
+      // go deep
+      super(true);
+      this.typeFactory = typeFactory;
+      this.constants = constants;
+      this.constExprs = constExprs;
+      this.addCasts = addCasts;
+      this.removableCasts = removableCasts;
+      this.stack = Lists.newArrayList();
+      this.parentCallTypeStack = Lists.newArrayList();
+    }
+
+    public void analyze(RexNode exp) {
+      assert stack.isEmpty();
+
+      exp.accept(this);
+
+      // Deal with top of stack
+      assert stack.size() == 1;
+      assert parentCallTypeStack.isEmpty();
+      Constancy rootConstancy = stack.get(0);
+      if (rootConstancy == Constancy.REDUCIBLE_CONSTANT) {
+        // The entire subtree was constant, so add it to the result.
+        addResult(exp);
+      }
+      stack.clear();
+    }
+
+    private Void pushVariable() {
+      stack.add(Constancy.NON_CONSTANT);
+      return null;
+    }
+
+    private void addResult(RexNode exp) {
+      // Cast of literal can't be reduced, so skip those (otherwise we'd
+      // go into an infinite loop as we add them back).
+      if (exp.getKind() == SqlKind.CAST) {
+        RexCall cast = (RexCall) exp;
+        RexNode operand = cast.getOperands().get(0);
+        if (operand instanceof RexLiteral) {
+          return;
+        }
+      }
+      constExprs.add(exp);
+
+      // In the case where the expression corresponds to a UDR argument,
+      // we need to preserve casts.  Note that this only applies to
+      // the topmost argument, not expressions nested within the UDR
+      // call.
+      //
+      // REVIEW zfong 6/13/08 - Are there other expressions where we
+      // also need to preserve casts?
+      if (parentCallTypeStack.isEmpty()) {
+        addCasts.add(false);
+      } else {
+        addCasts.add(isUdf(Stacks.peek(parentCallTypeStack)));
+      }
+    }
+
+    private Boolean isUdf(SqlOperator operator) {
+      // return operator instanceof UserDefinedRoutine
+      return false;
+    }
+
+    public Void visitInputRef(RexInputRef inputRef) {
+      if (constants.containsKey(inputRef)) {
+        stack.add(Constancy.REDUCIBLE_CONSTANT);
+        return null;
+      }
+      return pushVariable();
+    }
+
+    public Void visitLiteral(RexLiteral literal) {
+      stack.add(Constancy.IRREDUCIBLE_CONSTANT);
+      return null;
+    }
+
+    public Void visitOver(RexOver over) {
+      // assume non-constant (running SUM(1) looks constant but isn't)
+      analyzeCall(over, Constancy.NON_CONSTANT);
+      return null;
+    }
+
+    public Void visitCorrelVariable(RexCorrelVariable correlVariable) {
+      return pushVariable();
+    }
+
+    public Void visitCall(RexCall call) {
+      // assume REDUCIBLE_CONSTANT until proven otherwise
+      analyzeCall(call, Constancy.REDUCIBLE_CONSTANT);
+      return null;
+    }
+
+    private void analyzeCall(RexCall call, Constancy callConstancy) {
+      Stacks.push(parentCallTypeStack, call.getOperator());
+
+      // visit operands, pushing their states onto stack
+      super.visitCall(call);
+
+      // look for NON_CONSTANT operands
+      int operandCount = call.getOperands().size();
+      List<Constancy> operandStack = Util.last(stack, operandCount);
+      for (Constancy operandConstancy : operandStack) {
+        if (operandConstancy == Constancy.NON_CONSTANT) {
+          callConstancy = Constancy.NON_CONSTANT;
+        }
+      }
+
+      // Even if all operands are constant, the call itself may
+      // be non-deterministic.
+      if (!call.getOperator().isDeterministic()) {
+        callConstancy = Constancy.NON_CONSTANT;
+      } else if (call.getOperator().isDynamicFunction()) {
+        // We can reduce the call to a constant, but we can't
+        // cache the plan if the function is dynamic.
+        // For now, treat it same as non-deterministic.
+        callConstancy = Constancy.NON_CONSTANT;
+      }
+
+      // Row operator itself can't be reduced to a literal, but if
+      // the operands are constants, we still want to reduce those
+      if ((callConstancy == Constancy.REDUCIBLE_CONSTANT)
+          && (call.getOperator() instanceof SqlRowOperator)) {
+        callConstancy = Constancy.NON_CONSTANT;
+      }
+
+      if (callConstancy == Constancy.NON_CONSTANT) {
+        // any REDUCIBLE_CONSTANT children are now known to be maximal
+        // reducible subtrees, so they can be added to the result
+        // list
+        for (int iOperand = 0; iOperand < operandCount; ++iOperand) {
+          Constancy constancy = operandStack.get(iOperand);
+          if (constancy == Constancy.REDUCIBLE_CONSTANT) {
+            addResult(call.getOperands().get(iOperand));
+          }
+        }
+
+        // if this cast expression can't be reduced to a literal,
+        // then see if we can remove the cast
+        if (call.getOperator() == SqlStdOperatorTable.CAST) {
+          reduceCasts(call);
+        }
+      }
+
+      // pop operands off of the stack
+      operandStack.clear();
+
+      // pop this parent call operator off the stack
+      Stacks.pop(parentCallTypeStack, call.getOperator());
+
+      // push constancy result for this call onto stack
+      stack.add(callConstancy);
+    }
+
+    private void reduceCasts(RexCall outerCast) {
+      List<RexNode> operands = outerCast.getOperands();
+      if (operands.size() != 1) {
+        return;
+      }
+      RelDataType outerCastType = outerCast.getType();
+      RelDataType operandType = operands.get(0).getType();
+      if (operandType.equals(outerCastType)) {
+        removableCasts.add(outerCast);
+        return;
+      }
+
+      // See if the reduction
+      // CAST((CAST x AS type) AS type NOT NULL)
+      // -> CAST(x AS type NOT NULL)
+      // applies.  TODO jvs 15-Dec-2008:  consider
+      // similar cases for precision changes.
+      if (!(operands.get(0) instanceof RexCall)) {
+        return;
+      }
+      RexCall innerCast = (RexCall) operands.get(0);
+      if (innerCast.getOperator() != SqlStdOperatorTable.CAST) {
+        return;
+      }
+      if (innerCast.getOperands().size() != 1) {
+        return;
+      }
+      RelDataType outerTypeNullable =
+          typeFactory.createTypeWithNullability(outerCastType, true);
+      RelDataType innerTypeNullable =
+          typeFactory.createTypeWithNullability(operandType, true);
+      if (outerTypeNullable != innerTypeNullable) {
+        return;
+      }
+      if (operandType.isNullable()) {
+        removableCasts.add(innerCast);
+      }
+    }
+
+    public Void visitDynamicParam(RexDynamicParam dynamicParam) {
+      return pushVariable();
+    }
+
+    public Void visitRangeRef(RexRangeRef rangeRef) {
+      return pushVariable();
+    }
+
+    public Void visitFieldAccess(RexFieldAccess fieldAccess) {
+      return pushVariable();
+    }
+  }
+
+  /** Shuttle that pushes predicates into a CASE. */
+  protected static class CaseShuttle extends RexShuttle {
+    @Override public RexNode visitCall(RexCall call) {
+      for (;;) {
+        call = (RexCall) super.visitCall(call);
+        final RexCall old = call;
+        call = pushPredicateIntoCase(call);
+        if (call == old) {
+          return call;
+        }
+      }
+    }
+  }
+}
+
+// End HiveReduceExpressionsRule.java

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
index 425514d..d39744b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
@@ -150,11 +150,33 @@ class ASTBuilder {
 
     switch (sqlType) {
     case BINARY:
-      ByteString bs = (ByteString) literal.getValue();
-      val = bs.byteAt(0);
-      type = HiveParser.BigintLiteral;
+    case DATE:
+    case TIME:
+    case TIMESTAMP:
+    case INTERVAL_YEAR_MONTH:
+    case INTERVAL_DAY_TIME:
+      if (literal.getValue() == null) {
+        return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node();
+      }
       break;
     case TINYINT:
+    case SMALLINT:
+    case INTEGER:
+    case BIGINT:
+    case DOUBLE:
+    case DECIMAL:
+    case FLOAT:
+    case REAL:
+    case VARCHAR:
+    case CHAR:
+    case BOOLEAN:
+      if (literal.getValue3() == null) {
+        return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node();
+      }
+    }
+
+    switch (sqlType) {
+    case TINYINT:
       if (useTypeQualInLiteral) {
         val = literal.getValue3() + "Y";
       } else {
@@ -244,6 +266,8 @@ class ASTBuilder {
       type = HiveParser.TOK_NULL;
       break;
 
+    //binary type should not be seen.
+    case BINARY:
     default:
       throw new RuntimeException("Unsupported Type: " + sqlType);
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
index 631a4ca..122546f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
@@ -46,12 +46,9 @@ import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.util.ConversionUtil;
 import org.apache.calcite.util.NlsString;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.common.type.Decimal128;
 import org.apache.hadoop.hive.common.type.HiveChar;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
 import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
 import org.apache.hadoop.hive.common.type.HiveVarchar;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -69,8 +66,8 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate;
@@ -91,8 +88,6 @@ import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.ImmutableMap;
 
 public class RexNodeConverter {
-  private static final Logger LOG = LoggerFactory.getLogger(RexNodeConverter.class);
-
   private static class InputCtx {
     private final RelDataType                   calciteInpDataType;
     private final ImmutableMap<String, Integer> hiveNameToPosMap;
@@ -112,6 +107,11 @@ public class RexNodeConverter {
   private final ImmutableList<InputCtx> inputCtxs;
   private final boolean                 flattenExpr;
 
+  //Constructor used by HiveRexExecutorImpl
+  public RexNodeConverter(RelOptCluster cluster) {
+    this(cluster, new ArrayList<InputCtx>(), false);
+  }
+
   public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType,
       ImmutableMap<String, Integer> nameToPosMap, int offset, boolean flattenExpr) {
     this.cluster = cluster;
@@ -259,6 +259,9 @@ public class RexNodeConverter {
       GenericUDF udf = func.getGenericUDF();
       if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar)
           || (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate)
+          // Calcite can not specify the scale for timestamp. As a result, all
+          // the millisecond part will be lost
+          || (udf instanceof GenericUDFTimestamp)
           || (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) {
         castExpr = cluster.getRexBuilder().makeAbstractCast(
             TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()),
@@ -321,6 +324,10 @@ public class RexNodeConverter {
         coi);
 
     RexNode calciteLiteral = null;
+    // If value is null, the type should also be VOID.
+    if (value == null) {
+      hiveTypeCategory = PrimitiveCategory.VOID;
+    }
     // TODO: Verify if we need to use ConstantObjectInspector to unwrap data
     switch (hiveTypeCategory) {
     case BOOLEAN:
@@ -378,6 +385,10 @@ public class RexNodeConverter {
       calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Float) value), calciteDataType);
       break;
     case DOUBLE:
+      // TODO: The best solution is to support NaN in expression reduction.
+      if (Double.isNaN((Double) value)) {
+        throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+      }
       calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Double) value), calciteDataType);
       break;
     case CHAR:
@@ -417,14 +428,22 @@ public class RexNodeConverter {
           new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1,1)));
       break;
     case INTERVAL_DAY_TIME:
+      // Calcite RexBuilder L525 divides value by the multiplier.
+      // Need to get CAlCITE-1020 in.
+      throw new CalciteSemanticException("INTERVAL_DAY_TIME is not well supported",
+          UnsupportedFeature.Invalid_interval);
       // Calcite day-time interval is millis value as BigDecimal
       // Seconds converted to millis
-      BigDecimal secsValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000);
-      // Nanos converted to millis
-      BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getNanos(), 6);
-      calciteLiteral = rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
-          new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.SECOND, new SqlParserPos(1,1)));
-      break;
+      // BigDecimal secsValueBd = BigDecimal
+      // .valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000);
+      // // Nanos converted to millis
+      // BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime)
+      // value).getNanos(), 6);
+      // calciteLiteral =
+      // rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+      // new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+      // SqlParserPos(1, 1)));
+      // break;
     case VOID:
       calciteLiteral = cluster.getRexBuilder().makeLiteral(null,
           cluster.getTypeFactory().createSqlType(SqlTypeName.NULL), true);
@@ -438,11 +457,6 @@ public class RexNodeConverter {
     return calciteLiteral;
   }
 
-  private RexNode createNullLiteral(ExprNodeDesc expr) throws CalciteSemanticException {
-    return cluster.getRexBuilder().makeNullLiteral(
-        TypeConverter.convert(expr.getTypeInfo(), cluster.getTypeFactory()).getSqlTypeName());
-  }
-
   public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprNode,
       List<RelNode> inputRels, LinkedHashMap<RelNode, RowResolver> relToHiveRR,
       Map<RelNode, ImmutableMap<String, Integer>> relToHiveColNameCalcitePosMap, boolean flattenExpr)

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 87b18b7..21423c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -40,6 +40,7 @@ import org.antlr.runtime.tree.TreeVisitor;
 import org.antlr.runtime.tree.TreeVisitorAction;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptPlanner.Executor;
 import org.apache.calcite.plan.RelOptQuery;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.plan.RelOptSchema;
@@ -119,6 +120,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveHepPlannerContext;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveVolcanoPlannerContext;
 import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
@@ -154,6 +156,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePartitionPruneRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePreFilteringRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectSortTransposeRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveReduceExpressionsRule;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry;
 import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortJoinReduceRule;
@@ -881,9 +884,12 @@ public class CalcitePlanner extends SemanticAnalyzer {
       // Create MD provider
       HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf);
 
+      // Create executor
+      Executor executorProvider = new HiveRexExecutorImpl(cluster);
+
       // 2. Apply pre-join order optimizations
       calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan,
-              mdProvider.getMetadataProvider());
+              mdProvider.getMetadataProvider(), executorProvider);
 
       // 3. Apply join order optimizations: reordering MST algorithm
       //    If join optimizations failed because of missing stats, we continue with
@@ -935,7 +941,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
 
       // 4. Run other optimizations that do not need stats
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+      calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
               HepMatchOrder.BOTTOM_UP,
               ProjectRemoveRule.INSTANCE, UnionMergeRule.INSTANCE,
               new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY),
@@ -979,7 +985,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       // aggregation columns (HIVE-10627)
       if (profilesCBO.contains(ExtendedCBOProfile.WINDOWING_POSTPROCESSING)) {
         perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
                 HepMatchOrder.BOTTOM_UP, HiveWindowingFixRule.INSTANCE);
         perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Window fixing rule");
       }
@@ -988,7 +994,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
         perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
         // 8.1. Merge join into multijoin operators (if possible)
-        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(),
+        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null,
                 HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT_INCLUDE_OUTER,
                 HiveJoinProjectTransposeRule.LEFT_PROJECT_INCLUDE_OUTER,
                 HiveJoinProjectTransposeRule.RIGHT_PROJECT_INCLUDE_OUTER,
@@ -998,15 +1004,15 @@ public class CalcitePlanner extends SemanticAnalyzer {
         HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null,
             HiveRelFactories.HIVE_BUILDER.create(cluster, null));
         calciteOptimizedPlan = fieldTrimmer.trim(calciteOptimizedPlan);
-        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
                 HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE,
                 new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY));
-        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(),
+        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null,
             new HiveFilterProjectTSTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
                     HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class));
 
         // 8.2.  Introduce exchange operators below join/multijoin operators
-        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+        calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
                 HepMatchOrder.BOTTOM_UP, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_JOIN,
                 HiveInsertExchange4JoinRule.EXCHANGE_BELOW_MULTIJOIN);
         perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Translation from Calcite tree to Hive tree");
@@ -1031,9 +1037,11 @@ public class CalcitePlanner extends SemanticAnalyzer {
      *          original plan
      * @param mdProvider
      *          meta data provider
+     * @param executorProvider
+     *          executor
      * @return
      */
-    private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProvider mdProvider) {
+    private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProvider mdProvider, Executor executorProvider) {
       // TODO: Decorelation of subquery should be done before attempting
       // Partition Pruning; otherwise Expression evaluation may try to execute
       // corelated sub query.
@@ -1048,7 +1056,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
         // Its not clear, if this rewrite is always performant on MR, since extra map phase
         // introduced for 2nd MR job may offset gains of this multi-stage aggregation.
         // We need a cost model for MR to enable this on MR.
-        basePlan = hepPlan(basePlan, true, mdProvider, HiveExpandDistinctAggregatesRule.INSTANCE);
+        basePlan = hepPlan(basePlan, true, mdProvider, null, HiveExpandDistinctAggregatesRule.INSTANCE);
         perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
          "Calcite: Prejoin ordering transformation, Distinct aggregate rewrite");
       }
@@ -1059,7 +1067,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       // Ex: select * from R1 join R2 where ((R1.x=R2.x) and R1.y<10) or
       // ((R1.x=R2.x) and R1.z=10)) and rand(1) < 0.1
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, false, mdProvider, HepMatchOrder.ARBITRARY,
+      basePlan = hepPlan(basePlan, false, mdProvider, null, HepMatchOrder.ARBITRARY,
           HivePreFilteringRule.INSTANCE);
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, factor out common filter elements and separating deterministic vs non-deterministic UDF");
@@ -1070,7 +1078,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       // TODO: Add in ReduceExpressionrules (Constant folding) to below once
       // HIVE-11927 is fixed.
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, true, mdProvider, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
+      basePlan = hepPlan(basePlan, true, mdProvider, null, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
           HiveFilterSetOpTransposeRule.INSTANCE, HiveFilterSortTransposeRule.INSTANCE, HiveFilterJoinRule.JOIN,
           HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class,
               HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule(
@@ -1078,6 +1086,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, PPD for old join syntax");
 
+
       // TODO: Transitive inference, constant prop & Predicate push down has to
       // do multiple passes till no more inference is left
       // Currently doing so would result in a spin. Just checking for if inferred
@@ -1085,7 +1094,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       // could have been mutated by constant folding/prop
       // 4. Transitive inference for join on clauses
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, true, mdProvider, new HiveJoinPushTransitivePredicatesRule(
+      basePlan = hepPlan(basePlan, true, mdProvider, null, new HiveJoinPushTransitivePredicatesRule(
           Join.class, HiveRelFactories.HIVE_FILTER_FACTORY));
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, Transitive inference for join on clauses");
@@ -1102,10 +1111,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
             HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_TRANSPOSE_REDUCTION_PERCENTAGE);
         final long reductionTuples = HiveConf.getLongVar(conf,
             HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_TRANSPOSE_REDUCTION_TUPLES);
-        basePlan = hepPlan(basePlan, true, mdProvider, HiveSortMergeRule.INSTANCE,
+        basePlan = hepPlan(basePlan, true, mdProvider, null, HiveSortMergeRule.INSTANCE,
             HiveSortProjectTransposeRule.INSTANCE, HiveSortJoinReduceRule.INSTANCE,
             HiveSortUnionReduceRule.INSTANCE);
-        basePlan = hepPlan(basePlan, true, mdProvider, HepMatchOrder.BOTTOM_UP,
+        basePlan = hepPlan(basePlan, true, mdProvider, null, HepMatchOrder.BOTTOM_UP,
             new HiveSortRemoveRule(reductionProportion, reductionTuples),
             HiveProjectSortTransposeRule.INSTANCE);
         perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
@@ -1114,7 +1123,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
 
       // 6. Add not null filters
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE);
+      basePlan = hepPlan(basePlan, true, mdProvider, null, HiveJoinAddNotNullRule.INSTANCE);
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, Add not null filters");
 
@@ -1122,7 +1131,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       // TODO: Add in ReduceExpressionrules (Constant folding) to below once
       // HIVE-11927 is fixed.
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, true, mdProvider, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
+      basePlan = hepPlan(basePlan, true, mdProvider, null, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
           HiveFilterSetOpTransposeRule.INSTANCE, HiveFilterSortTransposeRule.INSTANCE, HiveFilterJoinRule.JOIN,
           HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class,
               HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule(
@@ -1132,18 +1141,26 @@ public class CalcitePlanner extends SemanticAnalyzer {
 
       // 8. Push Down Semi Joins
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, true, mdProvider, SemiJoinJoinTransposeRule.INSTANCE,
+      basePlan = hepPlan(basePlan, true, mdProvider, null, SemiJoinJoinTransposeRule.INSTANCE,
           SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE);
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, Push Down Semi Joins");
 
-      // 9. Apply Partition Pruning
+      // 9. Constant folding
+      perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
+      basePlan = hepPlan(basePlan, true, mdProvider, executorProvider,
+          HiveReduceExpressionsRule.PROJECT_INSTANCE, HiveReduceExpressionsRule.FILTER_INSTANCE,
+          HiveReduceExpressionsRule.JOIN_INSTANCE);
+      perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
+          "Calcite: Prejoin ordering transformation, Constant folding");
+
+      // 10. Apply Partition Pruning
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, false, mdProvider, new HivePartitionPruneRule(conf));
+      basePlan = hepPlan(basePlan, false, mdProvider, null, new HivePartitionPruneRule(conf));
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, Partition Pruning");
 
-      // 10. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP)
+      // 11. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP)
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
       HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null,
           HiveRelFactories.HIVE_BUILDER.create(cluster, null));
@@ -1151,19 +1168,19 @@ public class CalcitePlanner extends SemanticAnalyzer {
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, Projection Pruning");
 
-      // 11. Merge Project-Project if possible
+      // 12. Merge Project-Project if possible
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, false, mdProvider, new ProjectMergeRule(true,
+      basePlan = hepPlan(basePlan, false, mdProvider, null, new ProjectMergeRule(true,
           HiveRelFactories.HIVE_PROJECT_FACTORY));
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
         "Calcite: Prejoin ordering transformation, Merge Project-Project");
 
-      // 12. Rerun PPD through Project as column pruning would have introduced
+      // 13. Rerun PPD through Project as column pruning would have introduced
       // DT above scans; By pushing filter just above TS, Hive can push it into
       // storage (incase there are filters on non partition cols). This only
       // matches FIL-PROJ-TS
       perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
-      basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTSTransposeRule(
+      basePlan = hepPlan(basePlan, true, mdProvider, null, new HiveFilterProjectTSTransposeRule(
           Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class,
           HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class));
       perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
@@ -1178,12 +1195,13 @@ public class CalcitePlanner extends SemanticAnalyzer {
      * @param basePlan
      * @param followPlanChanges
      * @param mdProvider
+     * @param executorProvider
      * @param rules
      * @return optimized RelNode
      */
     private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges,
-        RelMetadataProvider mdProvider, RelOptRule... rules) {
-      return hepPlan(basePlan, followPlanChanges, mdProvider,
+        RelMetadataProvider mdProvider, Executor executorProvider, RelOptRule... rules) {
+      return hepPlan(basePlan, followPlanChanges, mdProvider, executorProvider,
               HepMatchOrder.TOP_DOWN, rules);
     }
 
@@ -1193,12 +1211,14 @@ public class CalcitePlanner extends SemanticAnalyzer {
      * @param basePlan
      * @param followPlanChanges
      * @param mdProvider
+     * @param executorProvider
      * @param order
      * @param rules
      * @return optimized RelNode
      */
-    private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadataProvider mdProvider,
-            HepMatchOrder order, RelOptRule... rules) {
+    private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges,
+        RelMetadataProvider mdProvider, Executor executorProvider, HepMatchOrder order,
+        RelOptRule... rules) {
 
       RelNode optimizedRelNode = basePlan;
       HepProgramBuilder programBuilder = new HepProgramBuilder();
@@ -1214,6 +1234,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       HiveRulesRegistry registry = new HiveRulesRegistry();
       HiveHepPlannerContext context = new HiveHepPlannerContext(registry);
       HepPlanner planner = new HepPlanner(programBuilder.build(), context);
+
       List<RelMetadataProvider> list = Lists.newArrayList();
       list.add(mdProvider);
       planner.registerMetadataProviders(list);
@@ -1221,10 +1242,9 @@ public class CalcitePlanner extends SemanticAnalyzer {
       basePlan.getCluster().setMetadataProvider(
           new CachingRelMetadataProvider(chainedProvider, planner));
 
-      // Executor is required for constant-reduction rules; see [CALCITE-566]
-      final RexExecutorImpl executor =
-          new RexExecutorImpl(Schemas.createDataContext(null));
-      basePlan.getCluster().getPlanner().setExecutor(executor);
+      if (executorProvider != null) {
+        basePlan.getCluster().getPlanner().setExecutor(executorProvider);
+      }
 
       planner.setRoot(basePlan);
       optimizedRelNode = planner.findBestExp();

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/queries/clientpositive/cbo_const.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cbo_const.q b/ql/src/test/queries/clientpositive/cbo_const.q
new file mode 100644
index 0000000..e38e2da
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cbo_const.q
@@ -0,0 +1,52 @@
+set hive.mapred.mode=nonstrict;
+set hive.cbo.enable=true;
+
+select
+  interval_day_time('2 1:2:3'),
+  interval_day_time(cast('2 1:2:3' as string)),
+  interval_day_time(cast('2 1:2:3' as varchar(10))),
+  interval_day_time(cast('2 1:2:3' as char(10))),
+  interval_day_time('2 1:2:3') = interval '2 1:2:3' day to second
+from src limit 1;
+
+select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08';
+
+drop view t1;
+
+create table t1_new (key string, value string) partitioned by (ds string);
+
+insert overwrite table t1_new partition (ds = '2011-10-15')
+select 'key1', 'value1' from src tablesample (1 rows);
+
+insert overwrite table t1_new partition (ds = '2011-10-16')
+select 'key2', 'value2' from src tablesample (1 rows);
+
+create view t1 partitioned on (ds) as
+select * from
+(
+select key, value, ds from t1_new
+union all
+select key, value, ds from t1_new
+)subq;
+
+select * from t1 where ds = '2011-10-15';
+
+
+explain select array(1,2,3) from src;
+
+EXPLAIN
+select key from (SELECT key from src where key = 1+3)s;
+
+select * from (select key from src where key = '1')subq;
+
+select '1';
+
+select * from (select '1')subq;
+
+select * from (select key from src where false)subq;
+
+EXPLAIN
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3);
+

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/queries/clientpositive/constantfolding.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/constantfolding.q b/ql/src/test/queries/clientpositive/constantfolding.q
new file mode 100644
index 0000000..4ddb710
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/constantfolding.q
@@ -0,0 +1,88 @@
+set hive.mapred.mode=nonstrict;
+set hive.optimize.ppd=true;
+
+-- SORT_QUERY_RESULTS
+
+select * from (select 'k2' as key, '1 ' as value from src limit 2)b
+union all 
+select * from (select 'k3' as key, '' as value from src limit 2)b
+union all 
+select * from (select 'k4' as key, ' ' as value from src limit 2)c;
+  
+
+drop table if exists union_all_bug_test_1;
+drop table if exists union_all_bug_test_2;
+create table if not exists union_all_bug_test_1
+(
+f1 int,
+f2 int
+);
+
+create table if not exists union_all_bug_test_2
+(
+f1 int
+);
+
+insert into table union_all_bug_test_1 values (1,1);
+insert into table union_all_bug_test_2 values (1);
+insert into table union_all_bug_test_1 values (0,0);
+insert into table union_all_bug_test_2 values (0);
+
+
+
+SELECT f1
+FROM (
+
+SELECT
+f1
+, if('helloworld' like '%hello%' ,f1,f2) as filter
+FROM union_all_bug_test_1
+
+union all
+
+select
+f1
+, 0 as filter
+from union_all_bug_test_2
+) A
+WHERE (filter = 1 and f1 = 1);
+
+
+select percentile(cast(key as bigint), array()) from src where false;
+
+select unbase64("0xe23") from src limit 1;
+
+SELECT key,randum123, h4
+FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a
+WHERE a.h4 <= 3 limit 1;
+
+select null from src limit 1;
+
+-- numRows: 2 rawDataSize: 80
+explain select cast("1970-12-31 15:59:58.174" as TIMESTAMP) from src;
+
+-- numRows: 2 rawDataSize: 112
+explain select cast("1970-12-31 15:59:58.174" as DATE) from src;
+
+CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE;
+
+FROM src INSERT OVERWRITE TABLE dest1 SELECT '  abc  ' WHERE src.key = 86;
+
+EXPLAIN
+SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0),
+       LOG(-1), ROUND(LOG2(3.0),12), LOG2(0.0), LOG2(-1),
+       ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12),
+       LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12),
+       POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5),
+       POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)),
+       POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), 
+       POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1;
+
+SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0),
+       LOG(-1), ROUND(LOG2(3.0),12), LOG2(0.0), LOG2(-1),
+       ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12),
+       LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12),
+       POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5),
+       POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)),
+       POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), 
+       POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1;

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/annotate_stats_select.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_select.q.out b/ql/src/test/results/clientpositive/annotate_stats_select.q.out
index c4d59c8..b158d85 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_select.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_select.q.out
@@ -925,24 +925,46 @@ POSTHOOK: query: -- inner select - numRows: 2 rawDataSize: 24
 explain select x from (select i1,11.0 as x from alltypes_orc limit 10) temp
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-0 is a root stage
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-0
-    Fetch Operator
-      limit: 10
-      Processor Tree:
-        TableScan
-          alias: alltypes_orc
-          Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: alltypes_orc
+            Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+            Select Operator
+              Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+              Limit
+                Number of rows: 10
+                Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+                Reduce Output Operator
+                  sort order: 
+                  Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+                  TopN Hash Memory Usage: 0.1
+      Reduce Operator Tree:
+        Limit
+          Number of rows: 10
+          Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
           Select Operator
             expressions: 11.0 (type: double)
             outputColumnNames: _col0
             Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
-            Limit
-              Number of rows: 10
+            File Output Operator
+              compressed: false
               Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
-              ListSink
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
 
 PREHOOK: query: -- inner select - numRows: 2 rawDataSize: 104
 -- outer select - numRows: 2 rawDataSize: 186
@@ -1024,21 +1046,21 @@ STAGE PLANS:
             alias: alltypes_orc
             Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
             Select Operator
-              Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+              Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
               Limit
                 Number of rows: 10
-                Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   sort order: 
-                  Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
                   TopN Hash Memory Usage: 0.1
       Reduce Operator Tree:
         Limit
           Number of rows: 10
-          Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+          Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
           Limit
             Number of rows: 10
-            Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+            Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
             File Output Operator
               compressed: false
               table:
@@ -1052,12 +1074,12 @@ STAGE PLANS:
           TableScan
             Reduce Output Operator
               sort order: 
-              Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+              Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
               TopN Hash Memory Usage: 0.1
       Reduce Operator Tree:
         Limit
           Number of rows: 10
-          Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+          Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
           Select Operator
             expressions: 'hello' (type: string), 11.0 (type: double)
             outputColumnNames: _col0, _col1

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out b/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
index 277b0f7..cfb95be 100644
--- a/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
+++ b/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
@@ -22,6 +22,8 @@ POSTHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@T2
+Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
 PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM (
 SELECT tmp1.name as name FROM (
   SELECT name, 'MMM' AS n FROM T1) tmp1 

http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/cast1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cast1.q.out b/ql/src/test/results/clientpositive/cast1.q.out
index 0bdecba..48a0c14 100644
--- a/ql/src/test/results/clientpositive/cast1.q.out
+++ b/ql/src/test/results/clientpositive/cast1.q.out
@@ -105,11 +105,11 @@ POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT 3 + 2, 3.0 + 2, 3
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 POSTHOOK: Output: default@dest1
-POSTHOOK: Lineage: dest1.c1 EXPRESSION []
-POSTHOOK: Lineage: dest1.c2 EXPRESSION []
-POSTHOOK: Lineage: dest1.c3 EXPRESSION []
-POSTHOOK: Lineage: dest1.c4 EXPRESSION []
-POSTHOOK: Lineage: dest1.c5 EXPRESSION []
+POSTHOOK: Lineage: dest1.c1 SIMPLE []
+POSTHOOK: Lineage: dest1.c2 SIMPLE []
+POSTHOOK: Lineage: dest1.c3 SIMPLE []
+POSTHOOK: Lineage: dest1.c4 SIMPLE []
+POSTHOOK: Lineage: dest1.c5 SIMPLE []
 POSTHOOK: Lineage: dest1.c6 EXPRESSION []
 POSTHOOK: Lineage: dest1.c7 EXPRESSION []
 PREHOOK: query: select dest1.* FROM dest1


Mime
View raw message