hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpull...@apache.org
Subject svn commit: r1644224 [3/5] - in /hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/ ql/src/java/org...
Date Tue, 09 Dec 2014 23:02:24 GMT
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java?rev=1644224&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java Tue Dec  9 23:02:22 2014
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+
+import org.apache.calcite.avatica.ByteString;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.ParseDriver;
+
+class ASTBuilder {
+
+  static ASTBuilder construct(int tokenType, String text) {
+    ASTBuilder b = new ASTBuilder();
+    b.curr = createAST(tokenType, text);
+    return b;
+  }
+
+  static ASTNode createAST(int tokenType, String text) {
+    return (ASTNode) ParseDriver.adaptor.create(tokenType, text);
+  }
+
+  static ASTNode destNode() {
+    return ASTBuilder
+        .construct(HiveParser.TOK_DESTINATION, "TOK_DESTINATION")
+        .add(
+            ASTBuilder.construct(HiveParser.TOK_DIR, "TOK_DIR").add(HiveParser.TOK_TMP_FILE,
+                "TOK_TMP_FILE")).node();
+  }
+
+  static ASTNode table(TableScan scan) {
+    RelOptHiveTable hTbl = (RelOptHiveTable) scan.getTable();
+    ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABREF, "TOK_TABREF").add(
+        ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME")
+            .add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName())
+            .add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName()));
+
+    // NOTE: Calcite considers tbls to be equal if their names are the same. Hence
+    // we need to provide Calcite the fully qualified table name (dbname.tblname)
+    // and not the user provided aliases.
+    // However in HIVE DB name can not appear in select list; in case of join
+    // where table names differ only in DB name, Hive would require user
+    // introducing explicit aliases for tbl.
+    b.add(HiveParser.Identifier, hTbl.getTableAlias());
+    return b.node();
+  }
+
+  static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType, ASTNode cond,
+      boolean semiJoin) {
+    ASTBuilder b = null;
+
+    switch (joinType) {
+    case INNER:
+      if (semiJoin) {
+        b = ASTBuilder.construct(HiveParser.TOK_LEFTSEMIJOIN, "TOK_LEFTSEMIJOIN");
+      } else {
+        b = ASTBuilder.construct(HiveParser.TOK_JOIN, "TOK_JOIN");
+      }
+      break;
+    case LEFT:
+      b = ASTBuilder.construct(HiveParser.TOK_LEFTOUTERJOIN, "TOK_LEFTOUTERJOIN");
+      break;
+    case RIGHT:
+      b = ASTBuilder.construct(HiveParser.TOK_RIGHTOUTERJOIN, "TOK_RIGHTOUTERJOIN");
+      break;
+    case FULL:
+      b = ASTBuilder.construct(HiveParser.TOK_FULLOUTERJOIN, "TOK_FULLOUTERJOIN");
+      break;
+    }
+
+    b.add(left).add(right).add(cond);
+    return b.node();
+  }
+
+  static ASTNode subQuery(ASTNode qry, String alias) {
+    return ASTBuilder.construct(HiveParser.TOK_SUBQUERY, "TOK_SUBQUERY").add(qry)
+        .add(HiveParser.Identifier, alias).node();
+  }
+
+  static ASTNode qualifiedName(String tableName, String colName) {
+    ASTBuilder b = ASTBuilder
+        .construct(HiveParser.DOT, ".")
+        .add(
+            ASTBuilder.construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add(
+                HiveParser.Identifier, tableName)).add(HiveParser.Identifier, colName);
+    return b.node();
+  }
+
+  static ASTNode unqualifiedName(String colName) {
+    ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL").add(
+        HiveParser.Identifier, colName);
+    return b.node();
+  }
+
+  static ASTNode where(ASTNode cond) {
+    return ASTBuilder.construct(HiveParser.TOK_WHERE, "TOK_WHERE").add(cond).node();
+  }
+
+  static ASTNode having(ASTNode cond) {
+    return ASTBuilder.construct(HiveParser.TOK_HAVING, "TOK_HAVING").add(cond).node();
+  }
+
+  static ASTNode limit(Object value) {
+    return ASTBuilder.construct(HiveParser.TOK_LIMIT, "TOK_LIMIT")
+        .add(HiveParser.Number, value.toString()).node();
+  }
+
+  static ASTNode selectExpr(ASTNode expr, String alias) {
+    return ASTBuilder.construct(HiveParser.TOK_SELEXPR, "TOK_SELEXPR").add(expr)
+        .add(HiveParser.Identifier, alias).node();
+  }
+
+  static ASTNode literal(RexLiteral literal) {
+    return literal(literal, false);
+  }
+
+  static ASTNode literal(RexLiteral literal, boolean useTypeQualInLiteral) {
+    Object val = null;
+    int type = 0;
+    SqlTypeName sqlType = literal.getType().getSqlTypeName();
+
+    switch (sqlType) {
+    case BINARY:
+      ByteString bs = (ByteString) literal.getValue();
+      val = bs.byteAt(0);
+      type = HiveParser.BigintLiteral;
+      break;
+    case TINYINT:
+      if (useTypeQualInLiteral) {
+        val = literal.getValue3() + "Y";
+      } else {
+        val = literal.getValue3();
+      }
+      type = HiveParser.TinyintLiteral;
+      break;
+    case SMALLINT:
+      if (useTypeQualInLiteral) {
+        val = literal.getValue3() + "S";
+      } else {
+        val = literal.getValue3();
+      }
+      type = HiveParser.SmallintLiteral;
+      break;
+    case INTEGER:
+      val = literal.getValue3();
+      type = HiveParser.BigintLiteral;
+      break;
+    case BIGINT:
+      if (useTypeQualInLiteral) {
+        val = literal.getValue3() + "L";
+      } else {
+        val = literal.getValue3();
+      }
+      type = HiveParser.BigintLiteral;
+      break;
+    case DOUBLE:
+      val = literal.getValue3() + "D";
+      type = HiveParser.Number;
+      break;
+    case DECIMAL:
+      val = literal.getValue3() + "BD";
+      type = HiveParser.DecimalLiteral;
+      break;
+    case FLOAT:
+    case REAL:
+      val = literal.getValue3();
+      type = HiveParser.Number;
+      break;
+    case VARCHAR:
+    case CHAR:
+      val = literal.getValue3();
+      String escapedVal = BaseSemanticAnalyzer.escapeSQLString(String.valueOf(val));
+      type = HiveParser.StringLiteral;
+      val = "'" + escapedVal + "'";
+      break;
+    case BOOLEAN:
+      val = literal.getValue3();
+      type = ((Boolean) val).booleanValue() ? HiveParser.KW_TRUE : HiveParser.KW_FALSE;
+      break;
+    case DATE: {
+      val = literal.getValue();
+      type = HiveParser.TOK_DATELITERAL;
+      DateFormat df = new SimpleDateFormat("yyyy-MM-dd");
+      val = df.format(((Calendar) val).getTime());
+      val = "'" + val + "'";
+    }
+      break;
+    case TIME:
+    case TIMESTAMP: {
+      val = literal.getValue();
+      type = HiveParser.TOK_TIMESTAMPLITERAL;
+      DateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
+      val = df.format(((Calendar) val).getTime());
+      val = "'" + val + "'";
+    }
+      break;
+    case NULL:
+      type = HiveParser.TOK_NULL;
+      break;
+
+    default:
+      throw new RuntimeException("Unsupported Type: " + sqlType);
+    }
+
+    return (ASTNode) ParseDriver.adaptor.create(type, String.valueOf(val));
+  }
+
+  ASTNode curr;
+
+  ASTNode node() {
+    return curr;
+  }
+
+  ASTBuilder add(int tokenType, String text) {
+    ParseDriver.adaptor.addChild(curr, createAST(tokenType, text));
+    return this;
+  }
+
+  ASTBuilder add(ASTBuilder b) {
+    ParseDriver.adaptor.addChild(curr, b.curr);
+    return this;
+  }
+
+  ASTBuilder add(ASTNode n) {
+    if (n != null) {
+      ParseDriver.adaptor.addChild(curr, n);
+    }
+    return this;
+  }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java?rev=1644224&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java Tue Dec  9 23:02:22 2014
@@ -0,0 +1,668 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelVisitor;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.SemiJoin;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.core.Union;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexFieldCollation;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.rex.RexWindow;
+import org.apache.calcite.rex.RexWindowBound;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.BitSets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter.HiveToken;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.ParseDriver;
+
+import com.google.common.collect.Iterables;
+
+public class ASTConverter {
+  private static final Log LOG = LogFactory.getLog(ASTConverter.class);
+
+  private RelNode          root;
+  private HiveAST          hiveAST;
+  private RelNode          from;
+  private Filter           where;
+  private Aggregate        groupBy;
+  private Filter           having;
+  private Project          select;
+  private Sort             order;
+  private Sort             limit;
+
+  private Schema           schema;
+
+  private long             derivedTableCount;
+
+  ASTConverter(RelNode root, long dtCounterInitVal) {
+    this.root = root;
+    hiveAST = new HiveAST();
+    this.derivedTableCount = dtCounterInitVal;
+  }
+
+  public static ASTNode convert(final RelNode relNode, List<FieldSchema> resultSchema)
+      throws CalciteSemanticException {
+    RelNode root = PlanModifierForASTConv.convertOpTree(relNode, resultSchema);
+    ASTConverter c = new ASTConverter(root, 0);
+    return c.convert();
+  }
+
+  private ASTNode convert() {
+    /*
+     * 1. Walk RelNode Graph; note from, where, gBy.. nodes.
+     */
+    new QBVisitor().go(root);
+
+    /*
+     * 2. convert from node.
+     */
+    QueryBlockInfo qb = convertSource(from);
+    schema = qb.schema;
+    hiveAST.from = ASTBuilder.construct(HiveParser.TOK_FROM, "TOK_FROM").add(qb.ast).node();
+
+    /*
+     * 3. convert filterNode
+     */
+    if (where != null) {
+      ASTNode cond = where.getCondition().accept(new RexVisitor(schema));
+      hiveAST.where = ASTBuilder.where(cond);
+    }
+
+    /*
+     * 4. GBy
+     */
+    if (groupBy != null) {
+      ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_GROUPBY, "TOK_GROUPBY");
+      for (int i : BitSets.toIter(groupBy.getGroupSet())) {
+        RexInputRef iRef = new RexInputRef(i, groupBy.getCluster().getTypeFactory()
+            .createSqlType(SqlTypeName.ANY));
+        b.add(iRef.accept(new RexVisitor(schema)));
+      }
+
+      if (!groupBy.getGroupSet().isEmpty())
+        hiveAST.groupBy = b.node();
+      schema = new Schema(schema, groupBy);
+    }
+
+    /*
+     * 5. Having
+     */
+    if (having != null) {
+      ASTNode cond = having.getCondition().accept(new RexVisitor(schema));
+      hiveAST.having = ASTBuilder.having(cond);
+    }
+
+    /*
+     * 6. Project
+     */
+    ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT");
+
+    if (select.getChildExps().isEmpty()) {
+      RexLiteral r = select.getCluster().getRexBuilder().makeExactLiteral(new BigDecimal(1));
+      ASTNode selectExpr = ASTBuilder.selectExpr(ASTBuilder.literal(r), "1");
+      b.add(selectExpr);
+    } else {
+      int i = 0;
+
+      for (RexNode r : select.getChildExps()) {
+        ASTNode selectExpr = ASTBuilder.selectExpr(r.accept(
+             new RexVisitor(schema, r instanceof RexLiteral)),
+                  select.getRowType().getFieldNames().get(i++));
+        b.add(selectExpr);
+      }
+    }
+    hiveAST.select = b.node();
+
+    /*
+     * 7. Order Use in Order By from the block above. RelNode has no pointer to
+     * parent hence we need to go top down; but OB at each block really belong
+     * to its src/from. Hence the need to pass in sort for each block from
+     * its parent.
+     */
+    convertOBToASTNode((HiveSort) order);
+
+    // 8. Limit
+    convertLimitToASTNode((HiveSort) limit);
+
+    return hiveAST.getAST();
+  }
+
+  private void convertLimitToASTNode(HiveSort limit) {
+    if (limit != null) {
+      HiveSort hiveLimit = (HiveSort) limit;
+      RexNode limitExpr = hiveLimit.getFetchExpr();
+      if (limitExpr != null) {
+        Object val = ((RexLiteral) limitExpr).getValue2();
+        hiveAST.limit = ASTBuilder.limit(val);
+      }
+    }
+  }
+
+  private void convertOBToASTNode(HiveSort order) {
+    if (order != null) {
+      HiveSort hiveSort = (HiveSort) order;
+      if (!hiveSort.getCollation().getFieldCollations().isEmpty()) {
+        // 1 Add order by token
+        ASTNode orderAst = ASTBuilder.createAST(HiveParser.TOK_ORDERBY, "TOK_ORDERBY");
+
+        schema = new Schema((HiveSort) hiveSort);
+        Map<Integer, RexNode> obRefToCallMap = hiveSort.getInputRefToCallMap();
+        RexNode obExpr;
+        ASTNode astCol;
+        for (RelFieldCollation c : hiveSort.getCollation().getFieldCollations()) {
+
+          // 2 Add Direction token
+          ASTNode directionAST = c.getDirection() == RelFieldCollation.Direction.ASCENDING ? ASTBuilder
+              .createAST(HiveParser.TOK_TABSORTCOLNAMEASC, "TOK_TABSORTCOLNAMEASC") : ASTBuilder
+              .createAST(HiveParser.TOK_TABSORTCOLNAMEDESC, "TOK_TABSORTCOLNAMEDESC");
+
+          // 3 Convert OB expr (OB Expr is usually an input ref except for top
+          // level OB; top level OB will have RexCall kept in a map.)
+          obExpr = null;
+          if (obRefToCallMap != null)
+            obExpr = obRefToCallMap.get(c.getFieldIndex());
+
+          if (obExpr != null) {
+            astCol = obExpr.accept(new RexVisitor(schema));
+          } else {
+            ColumnInfo cI = schema.get(c.getFieldIndex());
+            /*
+             * The RowResolver setup for Select drops Table associations. So
+             * setup ASTNode on unqualified name.
+             */
+            astCol = ASTBuilder.unqualifiedName(cI.column);
+          }
+
+          // 4 buildup the ob expr AST
+          directionAST.addChild(astCol);
+          orderAst.addChild(directionAST);
+        }
+        hiveAST.order = orderAst;
+      }
+    }
+  }
+
+  private Schema getRowSchema(String tblAlias) {
+    return new Schema(select, tblAlias);
+  }
+
+  private QueryBlockInfo convertSource(RelNode r) {
+    Schema s;
+    ASTNode ast;
+
+    if (r instanceof TableScan) {
+      TableScan f = (TableScan) r;
+      s = new Schema(f);
+      ast = ASTBuilder.table(f);
+    } else if (r instanceof Join) {
+      Join join = (Join) r;
+      QueryBlockInfo left = convertSource(join.getLeft());
+      QueryBlockInfo right = convertSource(join.getRight());
+      s = new Schema(left.schema, right.schema);
+      ASTNode cond = join.getCondition().accept(new RexVisitor(s));
+      boolean semiJoin = join instanceof SemiJoin;
+      ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond, semiJoin);
+      if (semiJoin)
+        s = left.schema;
+    } else if (r instanceof Union) {
+      RelNode leftInput = ((Union) r).getInput(0);
+      RelNode rightInput = ((Union) r).getInput(1);
+
+      ASTConverter leftConv = new ASTConverter(leftInput, this.derivedTableCount);
+      ASTConverter rightConv = new ASTConverter(rightInput, this.derivedTableCount);
+      ASTNode leftAST = leftConv.convert();
+      ASTNode rightAST = rightConv.convert();
+
+      ASTNode unionAST = getUnionAllAST(leftAST, rightAST);
+
+      String sqAlias = nextAlias();
+      ast = ASTBuilder.subQuery(unionAST, sqAlias);
+      s = new Schema((Union) r, sqAlias);
+    } else {
+      ASTConverter src = new ASTConverter(r, this.derivedTableCount);
+      ASTNode srcAST = src.convert();
+      String sqAlias = nextAlias();
+      s = src.getRowSchema(sqAlias);
+      ast = ASTBuilder.subQuery(srcAST, sqAlias);
+    }
+    return new QueryBlockInfo(s, ast);
+  }
+
+  class QBVisitor extends RelVisitor {
+
+    public void handle(Filter filter) {
+      RelNode child = filter.getInput();
+      if (child instanceof Aggregate && !((Aggregate) child).getGroupSet().isEmpty()) {
+        ASTConverter.this.having = filter;
+      } else {
+        ASTConverter.this.where = filter;
+      }
+    }
+
+    public void handle(Project project) {
+      if (ASTConverter.this.select == null) {
+        ASTConverter.this.select = project;
+      } else {
+        ASTConverter.this.from = project;
+      }
+    }
+
+    @Override
+    public void visit(RelNode node, int ordinal, RelNode parent) {
+
+      if (node instanceof TableScan) {
+        ASTConverter.this.from = node;
+      } else if (node instanceof Filter) {
+        handle((Filter) node);
+      } else if (node instanceof Project) {
+        handle((Project) node);
+      } else if (node instanceof Join) {
+        ASTConverter.this.from = node;
+      } else if (node instanceof Union) {
+        ASTConverter.this.from = node;
+      } else if (node instanceof Aggregate) {
+        ASTConverter.this.groupBy = (Aggregate) node;
+      } else if (node instanceof Sort) {
+        if (ASTConverter.this.select != null) {
+          ASTConverter.this.from = node;
+        } else {
+          Sort hiveSortRel = (Sort) node;
+          if (hiveSortRel.getCollation().getFieldCollations().isEmpty())
+            ASTConverter.this.limit = hiveSortRel;
+          else
+            ASTConverter.this.order = hiveSortRel;
+        }
+      }
+      /*
+       * once the source node is reached; stop traversal for this QB
+       */
+      if (ASTConverter.this.from == null) {
+        node.childrenAccept(this);
+      }
+    }
+
+  }
+
+  static class RexVisitor extends RexVisitorImpl<ASTNode> {
+
+    private final Schema schema;
+    private boolean useTypeQualInLiteral;
+
+    protected RexVisitor(Schema schema) {
+      this(schema, false);
+    }
+
+    protected RexVisitor(Schema schema, boolean useTypeQualInLiteral) {
+      super(true);
+      this.schema = schema;
+      this.useTypeQualInLiteral = useTypeQualInLiteral;
+    }
+
+    @Override
+    public ASTNode visitFieldAccess(RexFieldAccess fieldAccess) {
+      return ASTBuilder.construct(HiveParser.DOT, ".").add(super.visitFieldAccess(fieldAccess))
+          .add(HiveParser.Identifier, fieldAccess.getField().getName()).node();
+    }
+
+    @Override
+    public ASTNode visitInputRef(RexInputRef inputRef) {
+      ColumnInfo cI = schema.get(inputRef.getIndex());
+      if (cI.agg != null) {
+        return (ASTNode) ParseDriver.adaptor.dupTree(cI.agg);
+      }
+
+      if (cI.table == null || cI.table.isEmpty())
+        return ASTBuilder.unqualifiedName(cI.column);
+      else
+        return ASTBuilder.qualifiedName(cI.table, cI.column);
+
+    }
+
+    @Override
+    public ASTNode visitLiteral(RexLiteral literal) {
+      return ASTBuilder.literal(literal, useTypeQualInLiteral);
+    }
+
+    private ASTNode getPSpecAST(RexWindow window) {
+      ASTNode pSpecAst = null;
+
+      ASTNode dByAst = null;
+      if (window.partitionKeys != null && !window.partitionKeys.isEmpty()) {
+        dByAst = ASTBuilder.createAST(HiveParser.TOK_DISTRIBUTEBY, "TOK_DISTRIBUTEBY");
+        for (RexNode pk : window.partitionKeys) {
+          ASTNode astCol = pk.accept(this);
+          dByAst.addChild(astCol);
+        }
+      }
+
+      ASTNode oByAst = null;
+      if (window.orderKeys != null && !window.orderKeys.isEmpty()) {
+        oByAst = ASTBuilder.createAST(HiveParser.TOK_ORDERBY, "TOK_ORDERBY");
+        for (RexFieldCollation ok : window.orderKeys) {
+          ASTNode astNode = ok.getDirection() == RelFieldCollation.Direction.ASCENDING ? ASTBuilder
+              .createAST(HiveParser.TOK_TABSORTCOLNAMEASC, "TOK_TABSORTCOLNAMEASC") : ASTBuilder
+              .createAST(HiveParser.TOK_TABSORTCOLNAMEDESC, "TOK_TABSORTCOLNAMEDESC");
+          ASTNode astCol = ok.left.accept(this);
+          astNode.addChild(astCol);
+          oByAst.addChild(astNode);
+        }
+      }
+
+      if (dByAst != null || oByAst != null) {
+        pSpecAst = ASTBuilder.createAST(HiveParser.TOK_PARTITIONINGSPEC, "TOK_PARTITIONINGSPEC");
+        if (dByAst != null)
+          pSpecAst.addChild(dByAst);
+        if (oByAst != null)
+          pSpecAst.addChild(oByAst);
+      }
+
+      return pSpecAst;
+    }
+
+    private ASTNode getWindowBound(RexWindowBound wb) {
+      ASTNode wbAST = null;
+
+      if (wb.isCurrentRow()) {
+        wbAST = ASTBuilder.createAST(HiveParser.KW_CURRENT, "CURRENT");
+      } else {
+        if (wb.isPreceding())
+          wbAST = ASTBuilder.createAST(HiveParser.KW_PRECEDING, "PRECEDING");
+        else
+          wbAST = ASTBuilder.createAST(HiveParser.KW_FOLLOWING, "FOLLOWING");
+        if (wb.isUnbounded()) {
+          wbAST.addChild(ASTBuilder.createAST(HiveParser.KW_UNBOUNDED, "UNBOUNDED"));
+        } else {
+          ASTNode offset = wb.getOffset().accept(this);
+          wbAST.addChild(offset);
+        }
+      }
+
+      return wbAST;
+    }
+
+    private ASTNode getWindowRangeAST(RexWindow window) {
+      ASTNode wRangeAst = null;
+
+      ASTNode startAST = null;
+      RexWindowBound ub = window.getUpperBound();
+      if (ub != null) {
+        startAST = getWindowBound(ub);
+      }
+
+      ASTNode endAST = null;
+      RexWindowBound lb = window.getLowerBound();
+      if (lb != null) {
+        endAST = getWindowBound(lb);
+      }
+
+      if (startAST != null || endAST != null) {
+        // NOTE: in Hive AST Rows->Range(Physical) & Range -> Values (logical)
+        if (window.isRows())
+          wRangeAst = ASTBuilder.createAST(HiveParser.TOK_WINDOWRANGE, "TOK_WINDOWRANGE");
+        else
+          wRangeAst = ASTBuilder.createAST(HiveParser.TOK_WINDOWVALUES, "TOK_WINDOWVALUES");
+        if (startAST != null)
+          wRangeAst.addChild(startAST);
+        if (endAST != null)
+          wRangeAst.addChild(endAST);
+      }
+
+      return wRangeAst;
+    }
+
+    @Override
+    public ASTNode visitOver(RexOver over) {
+      if (!deep) {
+        return null;
+      }
+
+      // 1. Translate the UDAF
+      final ASTNode wUDAFAst = visitCall(over);
+
+      // 2. Add TOK_WINDOW as child of UDAF
+      ASTNode wSpec = ASTBuilder.createAST(HiveParser.TOK_WINDOWSPEC, "TOK_WINDOWSPEC");
+      wUDAFAst.addChild(wSpec);
+
+      // 3. Add Part Spec & Range Spec as child of TOK_WINDOW
+      final RexWindow window = over.getWindow();
+      final ASTNode wPSpecAst = getPSpecAST(window);
+      final ASTNode wRangeAst = getWindowRangeAST(window);
+      if (wPSpecAst != null)
+        wSpec.addChild(wPSpecAst);
+      if (wRangeAst != null)
+        wSpec.addChild(wRangeAst);
+
+      return wUDAFAst;
+    }
+
+    @Override
+    public ASTNode visitCall(RexCall call) {
+      if (!deep) {
+        return null;
+      }
+
+      SqlOperator op = call.getOperator();
+      List<ASTNode> astNodeLst = new LinkedList<ASTNode>();
+      if (op.kind == SqlKind.CAST) {
+        HiveToken ht = TypeConverter.hiveToken(call.getType());
+        ASTBuilder astBldr = ASTBuilder.construct(ht.type, ht.text);
+        if (ht.args != null) {
+          for (String castArg : ht.args)
+            astBldr.add(HiveParser.Identifier, castArg);
+        }
+        astNodeLst.add(astBldr.node());
+      }
+
+      for (RexNode operand : call.operands) {
+        astNodeLst.add(operand.accept(this));
+      }
+
+      if (isFlat(call))
+        return SqlFunctionConverter.buildAST(op, astNodeLst, 0);
+      else
+        return SqlFunctionConverter.buildAST(op, astNodeLst);
+    }
+  }
+
+  static class QueryBlockInfo {
+    Schema  schema;
+    ASTNode ast;
+
+    public QueryBlockInfo(Schema schema, ASTNode ast) {
+      super();
+      this.schema = schema;
+      this.ast = ast;
+    }
+  }
+
+  /*
+   * represents the schema exposed by a QueryBlock.
+   */
+  static class Schema extends ArrayList<ColumnInfo> {
+
+    private static final long serialVersionUID = 1L;
+
+    Schema(TableScan scan) {
+      String tabName = ((RelOptHiveTable) scan.getTable()).getTableAlias();
+      for (RelDataTypeField field : scan.getRowType().getFieldList()) {
+        add(new ColumnInfo(tabName, field.getName()));
+      }
+    }
+
+    Schema(Project select, String alias) {
+      for (RelDataTypeField field : select.getRowType().getFieldList()) {
+        add(new ColumnInfo(alias, field.getName()));
+      }
+    }
+
+    Schema(Union unionRel, String alias) {
+      for (RelDataTypeField field : unionRel.getRowType().getFieldList()) {
+        add(new ColumnInfo(alias, field.getName()));
+      }
+    }
+
+    Schema(Schema left, Schema right) {
+      for (ColumnInfo cI : Iterables.concat(left, right)) {
+        add(cI);
+      }
+    }
+
+    Schema(Schema src, Aggregate gBy) {
+      for (int i : BitSets.toIter(gBy.getGroupSet())) {
+        ColumnInfo cI = src.get(i);
+        add(cI);
+      }
+      List<AggregateCall> aggs = gBy.getAggCallList();
+      for (AggregateCall agg : aggs) {
+        int argCount = agg.getArgList().size();
+        ASTBuilder b = agg.isDistinct() ? ASTBuilder.construct(HiveParser.TOK_FUNCTIONDI,
+            "TOK_FUNCTIONDI") : argCount == 0 ? ASTBuilder.construct(HiveParser.TOK_FUNCTIONSTAR,
+            "TOK_FUNCTIONSTAR") : ASTBuilder.construct(HiveParser.TOK_FUNCTION, "TOK_FUNCTION");
+        b.add(HiveParser.Identifier, agg.getAggregation().getName());
+        for (int i : agg.getArgList()) {
+          RexInputRef iRef = new RexInputRef(i, gBy.getCluster().getTypeFactory()
+              .createSqlType(SqlTypeName.ANY));
+          b.add(iRef.accept(new RexVisitor(src)));
+        }
+        add(new ColumnInfo(null, b.node()));
+      }
+    }
+
+    /**
+     * Assumption:<br>
+     * 1. Project will always be child of Sort.<br>
+     * 2. In Calcite every projection in Project is uniquely named
+     * (unambigous) without using table qualifier (table name).<br>
+     * 
+     * @param order
+     *          Hive Sort Node
+     * @return Schema
+     */
+    public Schema(HiveSort order) {
+      Project select = (Project) order.getInput();
+      for (String projName : select.getRowType().getFieldNames()) {
+        add(new ColumnInfo(null, projName));
+      }
+    }
+  }
+
+  /*
+   * represents Column information exposed by a QueryBlock.
+   */
+  static class ColumnInfo {
+    String  table;
+    String  column;
+    ASTNode agg;
+
+    ColumnInfo(String table, String column) {
+      super();
+      this.table = table;
+      this.column = column;
+    }
+
+    ColumnInfo(String table, ASTNode agg) {
+      super();
+      this.table = table;
+      this.agg = agg;
+    }
+
+    ColumnInfo(String alias, ColumnInfo srcCol) {
+      this.table = alias;
+      this.column = srcCol.column;
+      this.agg = srcCol.agg;
+    }
+  }
+
+  private String nextAlias() {
+    String tabAlias = String.format("$hdt$_%d", derivedTableCount);
+    derivedTableCount++;
+    return tabAlias;
+  }
+
+  static class HiveAST {
+
+    ASTNode from;
+    ASTNode where;
+    ASTNode groupBy;
+    ASTNode having;
+    ASTNode select;
+    ASTNode order;
+    ASTNode limit;
+
+    public ASTNode getAST() {
+      ASTBuilder b = ASTBuilder
+          .construct(HiveParser.TOK_QUERY, "TOK_QUERY")
+          .add(from)
+          .add(
+              ASTBuilder.construct(HiveParser.TOK_INSERT, "TOK_INSERT").add(ASTBuilder.destNode())
+                  .add(select).add(where).add(groupBy).add(having).add(order).add(limit));
+      return b.node();
+    }
+  }
+
+  public ASTNode getUnionAllAST(ASTNode leftAST, ASTNode rightAST) {
+
+    ASTNode unionTokAST = ASTBuilder.construct(HiveParser.TOK_UNION, "TOK_UNION").add(leftAST)
+        .add(rightAST).node();
+
+    return unionTokAST;
+  }
+
+  public static boolean isFlat(RexCall call) {
+    boolean flat = false;
+    if (call.operands != null && call.operands.size() > 2) {
+      SqlOperator op = call.getOperator();
+      if (op.getKind() == SqlKind.AND || op.getKind() == SqlKind.OR) {
+        flat = true;
+      }
+    }
+
+    return flat;
+  }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java?rev=1644224&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java Tue Dec  9 23:02:22 2014
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexVisitorImpl;
+
+/*
+ * convert a RexNode to an ExprNodeDesc
+ */
+public class ExprNodeConverter extends RexVisitorImpl<ExprNodeDesc> {
+
+  RelDataType rType;
+  String      tabAlias;
+  boolean     partitioningExpr;
+
+  public ExprNodeConverter(String tabAlias, RelDataType rType, boolean partitioningExpr) {
+    super(true);
+    /*
+     * hb: 6/25/14 for now we only support expressions that only contain
+     * partition cols. there is no use case for supporting generic expressions.
+     * for supporting generic exprs., we need to give the converter information
+     * on whether a column is a partition column or not, whether a column is a
+     * virtual column or not.
+     */
+    assert partitioningExpr == true;
+    this.tabAlias = tabAlias;
+    this.rType = rType;
+    this.partitioningExpr = partitioningExpr;
+  }
+
+  @Override
+  public ExprNodeDesc visitInputRef(RexInputRef inputRef) {
+    RelDataTypeField f = rType.getFieldList().get(inputRef.getIndex());
+    return new ExprNodeColumnDesc(TypeConverter.convert(f.getType()), f.getName(), tabAlias,
+        partitioningExpr);
+  }
+
+  @Override
+  public ExprNodeDesc visitCall(RexCall call) {
+    ExprNodeGenericFuncDesc gfDesc = null;
+
+    if (!deep) {
+      return null;
+    }
+
+    List<ExprNodeDesc> args = new LinkedList<ExprNodeDesc>();
+
+    for (RexNode operand : call.operands) {
+      args.add(operand.accept(this));
+    }
+
+    // If Expr is flat (and[p,q,r,s] or[p,q,r,s]) then recursively build the
+    // exprnode
+    if (ASTConverter.isFlat(call)) {
+      ArrayList<ExprNodeDesc> tmpExprArgs = new ArrayList<ExprNodeDesc>();
+      tmpExprArgs.addAll(args.subList(0, 2));
+      gfDesc = new ExprNodeGenericFuncDesc(TypeConverter.convert(call.getType()),
+          SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType(), 2), tmpExprArgs);
+      for (int i = 2; i < call.operands.size(); i++) {
+        tmpExprArgs = new ArrayList<ExprNodeDesc>();
+        tmpExprArgs.add(gfDesc);
+        tmpExprArgs.add(args.get(i));
+        gfDesc = new ExprNodeGenericFuncDesc(TypeConverter.convert(call.getType()),
+            SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType(), 2), tmpExprArgs);
+      }
+    } else {
+      GenericUDF hiveUdf = SqlFunctionConverter.getHiveUDF(
+          call.getOperator(), call.getType(), args.size());
+      if (hiveUdf == null) {
+        throw new RuntimeException("Cannot find UDF for " + call.getType() + " " + call.getOperator()
+            + "[" + call.getOperator().getKind() + "]/" + args.size());
+      }
+      gfDesc = new ExprNodeGenericFuncDesc(TypeConverter.convert(call.getType()), hiveUdf, args);
+    }
+
+    return gfDesc;
+  }
+
+  @Override
+  public ExprNodeDesc visitLiteral(RexLiteral literal) {
+    RelDataType lType = literal.getType();
+
+    switch (literal.getType().getSqlTypeName()) {
+    case BOOLEAN:
+      return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.valueOf(RexLiteral
+          .booleanValue(literal)));
+    case TINYINT:
+      return new ExprNodeConstantDesc(TypeInfoFactory.byteTypeInfo, Byte.valueOf(((Number) literal
+          .getValue3()).byteValue()));
+    case SMALLINT:
+      return new ExprNodeConstantDesc(TypeInfoFactory.shortTypeInfo,
+          Short.valueOf(((Number) literal.getValue3()).shortValue()));
+    case INTEGER:
+      return new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo,
+          Integer.valueOf(((Number) literal.getValue3()).intValue()));
+    case BIGINT:
+      return new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo, Long.valueOf(((Number) literal
+          .getValue3()).longValue()));
+    case FLOAT:
+      return new ExprNodeConstantDesc(TypeInfoFactory.floatTypeInfo,
+          Float.valueOf(((Number) literal.getValue3()).floatValue()));
+    case DOUBLE:
+      return new ExprNodeConstantDesc(TypeInfoFactory.doubleTypeInfo,
+          Double.valueOf(((Number) literal.getValue3()).doubleValue()));
+    case DATE:
+      return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo,
+        new Date(((Calendar)literal.getValue()).getTimeInMillis()));
+    case TIMESTAMP: {
+      Object value = literal.getValue3();
+      if (value instanceof Long) {
+        value = new Timestamp((Long)value);
+      }
+      return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, value);
+    }
+    case BINARY:
+      return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, literal.getValue3());
+    case DECIMAL:
+      return new ExprNodeConstantDesc(TypeInfoFactory.getDecimalTypeInfo(lType.getPrecision(),
+          lType.getScale()), literal.getValue3());
+    case VARCHAR:
+      return new ExprNodeConstantDesc(TypeInfoFactory.getVarcharTypeInfo(lType.getPrecision()),
+          new HiveVarchar((String) literal.getValue3(), lType.getPrecision()));
+    case CHAR:
+      return new ExprNodeConstantDesc(TypeInfoFactory.getCharTypeInfo(lType.getPrecision()),
+          new HiveChar((String) literal.getValue3(), lType.getPrecision()));
+    case OTHER:
+    default:
+      return new ExprNodeConstantDesc(TypeInfoFactory.voidTypeInfo, literal.getValue3());
+    }
+  }
+
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinCondTypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinCondTypeCheckProcFactory.java?rev=1644224&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinCondTypeCheckProcFactory.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinCondTypeCheckProcFactory.java Tue Dec  9 23:02:22 2014
@@ -0,0 +1,316 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.TypeCheckCtx;
+import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
+
+/**
+ * JoinCondTypeCheckProcFactory is used by Calcite planner(CBO) to generate Join Conditions from Join Condition AST.
+ * Reasons for sub class:
+ * 1. Additional restrictions on what is supported in Join Conditions
+ * 2. Column handling is different
+ * 3. Join Condn expr has two input RR as opposed to one.
+ */
+
+/**
+ * TODO:<br>
+ * 1. Could we use combined RR instead of list of RR ?<br>
+ * 2. Use Column Processing from TypeCheckProcFactory<br>
+ * 3. Why not use GB expr ?
+ */
+public class JoinCondTypeCheckProcFactory extends TypeCheckProcFactory {
+
+  public static Map<ASTNode, ExprNodeDesc> genExprNode(ASTNode expr, TypeCheckCtx tcCtx)
+      throws SemanticException {
+    return TypeCheckProcFactory.genExprNode(expr, tcCtx, new JoinCondTypeCheckProcFactory());
+  }
+
+  /**
+   * Processor for table columns.
+   */
+  public static class JoinCondColumnExprProcessor extends ColumnExprProcessor {
+
+    @Override
+    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+        Object... nodeOutputs) throws SemanticException {
+
+      JoinTypeCheckCtx ctx = (JoinTypeCheckCtx) procCtx;
+      if (ctx.getError() != null) {
+        return null;
+      }
+
+      ASTNode expr = (ASTNode) nd;
+      ASTNode parent = stack.size() > 1 ? (ASTNode) stack.get(stack.size() - 2) : null;
+
+      if (expr.getType() != HiveParser.TOK_TABLE_OR_COL) {
+        ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr), expr);
+        return null;
+      }
+
+      assert (expr.getChildCount() == 1);
+      String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText());
+
+      boolean qualifiedAccess = (parent != null && parent.getType() == HiveParser.DOT);
+
+      ColumnInfo colInfo = null;
+      if (!qualifiedAccess) {
+        colInfo = getColInfo(ctx, null, tableOrCol, expr);
+        // It's a column.
+        return new ExprNodeColumnDesc(colInfo);
+      } else if (hasTableAlias(ctx, tableOrCol, expr)) {
+        return null;
+      } else {
+        // Qualified column access for which table was not found
+        throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr));
+      }
+    }
+
+    private static boolean hasTableAlias(JoinTypeCheckCtx ctx, String tabName, ASTNode expr)
+        throws SemanticException {
+      int tblAliasCnt = 0;
+      for (RowResolver rr : ctx.getInputRRList()) {
+        if (rr.hasTableAlias(tabName))
+          tblAliasCnt++;
+      }
+
+      if (tblAliasCnt > 1) {
+        throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr));
+      }
+
+      return (tblAliasCnt == 1) ? true : false;
+    }
+
+    private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias,
+        ASTNode expr) throws SemanticException {
+      ColumnInfo tmp;
+      ColumnInfo cInfoToRet = null;
+
+      for (RowResolver rr : ctx.getInputRRList()) {
+        tmp = rr.get(tabName, colAlias);
+        if (tmp != null) {
+          if (cInfoToRet != null) {
+            throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr));
+          }
+          cInfoToRet = tmp;
+        }
+      }
+
+      return cInfoToRet;
+    }
+  }
+
+  /**
+   * Factory method to get ColumnExprProcessor.
+   *
+   * @return ColumnExprProcessor.
+   */
+  @Override
+  public ColumnExprProcessor getColumnExprProcessor() {
+    return new JoinCondColumnExprProcessor();
+  }
+
+  /**
+   * The default processor for typechecking.
+   */
+  public static class JoinCondDefaultExprProcessor extends DefaultExprProcessor {
+    @Override
+    protected List<String> getReferenceableColumnAliases(TypeCheckCtx ctx) {
+      JoinTypeCheckCtx jCtx = (JoinTypeCheckCtx) ctx;
+      List<String> possibleColumnNames = new ArrayList<String>();
+      for (RowResolver rr : jCtx.getInputRRList()) {
+        possibleColumnNames.addAll(rr.getReferenceableColumnAliases(null, -1));
+      }
+
+      return possibleColumnNames;
+    }
+
+    @Override
+    protected ExprNodeColumnDesc processQualifiedColRef(TypeCheckCtx ctx, ASTNode expr,
+        Object... nodeOutputs) throws SemanticException {
+      String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0)
+          .getText());
+      // NOTE: tableAlias must be a valid non-ambiguous table alias,
+      // because we've checked that in TOK_TABLE_OR_COL's process method.
+      ColumnInfo colInfo = getColInfo((JoinTypeCheckCtx) ctx, tableAlias,
+          ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString(), expr);
+
+      if (colInfo == null) {
+        ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);
+        return null;
+      }
+      return new ExprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName(), tableAlias,
+          colInfo.getIsVirtualCol());
+    }
+
+    private static ColumnInfo getColInfo(JoinTypeCheckCtx ctx, String tabName, String colAlias,
+        ASTNode expr) throws SemanticException {
+      ColumnInfo tmp;
+      ColumnInfo cInfoToRet = null;
+
+      for (RowResolver rr : ctx.getInputRRList()) {
+        tmp = rr.get(tabName, colAlias);
+        if (tmp != null) {
+          if (cInfoToRet != null) {
+            throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr));
+          }
+          cInfoToRet = tmp;
+        }
+      }
+
+      return cInfoToRet;
+    }
+
+    @Override
+    protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, FunctionInfo fi,
+        List<ExprNodeDesc> children, GenericUDF genericUDF) throws SemanticException {
+      super.validateUDF(expr, isFunction, ctx, fi, children, genericUDF);
+
+      JoinTypeCheckCtx jCtx = (JoinTypeCheckCtx) ctx;
+
+      // Join Condition can not contain disjunctions
+      if (genericUDF instanceof GenericUDFOPOr) {
+        throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_3.getMsg(expr));
+      }
+
+      // Non Conjunctive elements have further limitations in Join conditions
+      if (!(genericUDF instanceof GenericUDFOPAnd)) {
+        // Non Comparison UDF other than 'and' can not use inputs from both side
+        if (!(genericUDF instanceof GenericUDFBaseCompare)) {
+          if (genericUDFargsRefersToBothInput(genericUDF, children, jCtx.getInputRRList())) {
+            throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr));
+          }
+        } else if (genericUDF instanceof GenericUDFBaseCompare) {
+          // Comparisons of non literals LHS/RHS can not refer to inputs from
+          // both sides
+          if (children.size() == 2 && !(children.get(0) instanceof ExprNodeConstantDesc)
+              && !(children.get(1) instanceof ExprNodeConstantDesc)) {
+            if (comparisonUDFargsRefersToBothInput((GenericUDFBaseCompare) genericUDF, children,
+                jCtx.getInputRRList())) {
+              throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(expr));
+            }
+          }
+        }
+      }
+    }
+
+    private static boolean genericUDFargsRefersToBothInput(GenericUDF udf,
+        List<ExprNodeDesc> children, List<RowResolver> inputRRList) {
+      boolean argsRefersToBothInput = false;
+
+      Map<Integer, ExprNodeDesc> hasCodeToColDescMap = new HashMap<Integer, ExprNodeDesc>();
+      for (ExprNodeDesc child : children) {
+        ExprNodeDescUtils.getExprNodeColumnDesc(child, hasCodeToColDescMap);
+      }
+      Set<Integer> inputRef = getInputRef(hasCodeToColDescMap.values(), inputRRList);
+
+      if (inputRef.size() > 1)
+        argsRefersToBothInput = true;
+
+      return argsRefersToBothInput;
+    }
+
+    private static boolean comparisonUDFargsRefersToBothInput(GenericUDFBaseCompare comparisonUDF,
+        List<ExprNodeDesc> children, List<RowResolver> inputRRList) {
+      boolean argsRefersToBothInput = false;
+
+      Map<Integer, ExprNodeDesc> lhsHashCodeToColDescMap = new HashMap<Integer, ExprNodeDesc>();
+      Map<Integer, ExprNodeDesc> rhsHashCodeToColDescMap = new HashMap<Integer, ExprNodeDesc>();
+      ExprNodeDescUtils.getExprNodeColumnDesc(children.get(0), lhsHashCodeToColDescMap);
+      ExprNodeDescUtils.getExprNodeColumnDesc(children.get(1), rhsHashCodeToColDescMap);
+      Set<Integer> lhsInputRef = getInputRef(lhsHashCodeToColDescMap.values(), inputRRList);
+      Set<Integer> rhsInputRef = getInputRef(rhsHashCodeToColDescMap.values(), inputRRList);
+
+      if (lhsInputRef.size() > 1 || rhsInputRef.size() > 1)
+        argsRefersToBothInput = true;
+
+      return argsRefersToBothInput;
+    }
+
+    private static Set<Integer> getInputRef(Collection<ExprNodeDesc> colDescSet,
+        List<RowResolver> inputRRList) {
+      String tableAlias;
+      RowResolver inputRR;
+      Set<Integer> inputLineage = new HashSet<Integer>();
+
+      for (ExprNodeDesc col : colDescSet) {
+        ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) col;
+        tableAlias = colDesc.getTabAlias();
+
+        for (int i = 0; i < inputRRList.size(); i++) {
+          inputRR = inputRRList.get(i);
+
+          // If table Alias is present check if InputRR has that table and then
+          // check for internal name
+          // else if table alias is null then check with internal name in all
+          // inputRR.
+          if (tableAlias != null) {
+            if (inputRR.hasTableAlias(tableAlias)) {
+              if (inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) {
+                inputLineage.add(i);
+              }
+            }
+          } else {
+            if (inputRR.getInvRslvMap().containsKey(colDesc.getColumn())) {
+              inputLineage.add(i);
+            }
+          }
+        }
+      }
+
+      return inputLineage;
+    }
+  }
+
+  /**
+   * Factory method to get DefaultExprProcessor.
+   *
+   * @return DefaultExprProcessor.
+   */
+  @Override
+  public DefaultExprProcessor getDefaultExprProcessor() {
+    return new JoinCondDefaultExprProcessor();
+  }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java?rev=1644224&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/JoinTypeCheckCtx.java Tue Dec  9 23:02:22 2014
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.parse.JoinType;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.TypeCheckCtx;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * JoinTypeCheckCtx is used by Calcite planner(CBO) to generate Join Conditions from Join Condition AST.
+ * Reasons for sub class:
+ * 1. Join Conditions can not handle:
+ *    a. Stateful Functions
+ *    b. Distinct
+ *    c. '*' expr
+ *    d. '.*' expr
+ *    e. Windowing expr
+ *    f. Complex type member access
+ *    g. Array Index Access
+ *    h. Sub query
+ *    i. GB expr elimination
+ * 2. Join Condn expr has two input RR as opposed to one.
+ */
+
+/**
+ * TODO:<br>
+ * 1. Could we use combined RR instead of list of RR ?<br>
+ * 2. Why not use GB expr ?
+ */
+public class JoinTypeCheckCtx extends TypeCheckCtx {
+  private final ImmutableList<RowResolver> inputRRLst;
+  private final boolean outerJoin;
+
+  public JoinTypeCheckCtx(RowResolver leftRR, RowResolver rightRR, JoinType hiveJoinType)
+      throws SemanticException {
+    super(RowResolver.getCombinedRR(leftRR, rightRR), false, false, false, false, false, false,
+        false, false);
+    this.inputRRLst = ImmutableList.of(leftRR, rightRR);
+    this.outerJoin = (hiveJoinType == JoinType.LEFTOUTER) || (hiveJoinType == JoinType.RIGHTOUTER)
+        || (hiveJoinType == JoinType.FULLOUTER);
+  }
+
+  /**
+   * @return the inputRR List
+   */
+  public List<RowResolver> getInputRRList() {
+    return inputRRLst;
+  }
+
+  public boolean isOuterJoin() {
+    return outerJoin;
+  }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java?rev=1644224&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java Tue Dec  9 23:02:22 2014
@@ -0,0 +1,392 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.SingleRel;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.SetOp;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.rules.MultiJoin;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.util.Pair;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+public class PlanModifierForASTConv {
+  private static final Log LOG = LogFactory.getLog(PlanModifierForASTConv.class);
+
+  public static RelNode convertOpTree(RelNode rel, List<FieldSchema> resultSchema)
+      throws CalciteSemanticException {
+    RelNode newTopNode = rel;
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Original plan for PlanModifier\n " + RelOptUtil.toString(newTopNode));
+    }
+
+    if (!(newTopNode instanceof Project) && !(newTopNode instanceof Sort)) {
+      newTopNode = introduceDerivedTable(newTopNode);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Plan after top-level introduceDerivedTable\n "
+            + RelOptUtil.toString(newTopNode));
+      }
+    }
+
+    convertOpTree(newTopNode, (RelNode) null);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Plan after nested convertOpTree\n " + RelOptUtil.toString(newTopNode));
+    }
+
+    Pair<RelNode, RelNode> topSelparentPair = HiveCalciteUtil.getTopLevelSelect(newTopNode);
+    fixTopOBSchema(newTopNode, topSelparentPair, resultSchema);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Plan after fixTopOBSchema\n " + RelOptUtil.toString(newTopNode));
+    }
+
+    topSelparentPair = HiveCalciteUtil.getTopLevelSelect(newTopNode);
+    newTopNode = renameTopLevelSelectInResultSchema(newTopNode, topSelparentPair, resultSchema);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Final plan after modifier\n " + RelOptUtil.toString(newTopNode));
+    }
+    return newTopNode;
+  }
+
+  private static void convertOpTree(RelNode rel, RelNode parent) {
+
+    if (rel instanceof HepRelVertex) {
+      throw new RuntimeException("Found HepRelVertex");
+    } else if (rel instanceof Join) {
+      if (!validJoinParent(rel, parent)) {
+        introduceDerivedTable(rel, parent);
+      }
+    } else if (rel instanceof MultiJoin) {
+      throw new RuntimeException("Found MultiJoin");
+    } else if (rel instanceof RelSubset) {
+      throw new RuntimeException("Found RelSubset");
+    } else if (rel instanceof SetOp) {
+      // TODO: Handle more than 2 inputs for setop
+      if (!validSetopParent(rel, parent))
+        introduceDerivedTable(rel, parent);
+
+      SetOp setop = (SetOp) rel;
+      for (RelNode inputRel : setop.getInputs()) {
+        if (!validSetopChild(inputRel)) {
+          introduceDerivedTable(inputRel, setop);
+        }
+      }
+    } else if (rel instanceof SingleRel) {
+      if (rel instanceof Filter) {
+        if (!validFilterParent(rel, parent)) {
+          introduceDerivedTable(rel, parent);
+        }
+      } else if (rel instanceof HiveSort) {
+        if (!validSortParent(rel, parent)) {
+          introduceDerivedTable(rel, parent);
+        }
+        if (!validSortChild((HiveSort) rel)) {
+          introduceDerivedTable(((HiveSort) rel).getInput(), rel);
+        }
+      } else if (rel instanceof HiveAggregate) {
+        RelNode newParent = parent;
+        if (!validGBParent(rel, parent)) {
+          newParent = introduceDerivedTable(rel, parent);
+        }
+        // check if groupby is empty and there is no other cols in aggr
+        // this should only happen when newParent is constant.
+        if (isEmptyGrpAggr(rel)) {
+          replaceEmptyGroupAggr(rel, newParent);
+        }
+      }
+    }
+
+    List<RelNode> childNodes = rel.getInputs();
+    if (childNodes != null) {
+      for (RelNode r : childNodes) {
+        convertOpTree(r, rel);
+      }
+    }
+  }
+
+  private static void fixTopOBSchema(final RelNode rootRel,
+      Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema)
+      throws CalciteSemanticException {
+    if (!(topSelparentPair.getKey() instanceof Sort)
+        || !HiveCalciteUtil.orderRelNode(topSelparentPair.getKey())) {
+      return;
+    }
+    HiveSort obRel = (HiveSort) topSelparentPair.getKey();
+    Project obChild = (Project) topSelparentPair.getValue();
+    if (obChild.getRowType().getFieldCount() <= resultSchema.size()) {
+      return;
+    }
+
+    RelDataType rt = obChild.getRowType();
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    Set<Integer> collationInputRefs = new HashSet(
+        RelCollationImpl.ordinals(obRel.getCollation()));
+    ImmutableMap.Builder<Integer, RexNode> inputRefToCallMapBldr = ImmutableMap.builder();
+    for (int i = resultSchema.size(); i < rt.getFieldCount(); i++) {
+      if (collationInputRefs.contains(i)) {
+        inputRefToCallMapBldr.put(i, obChild.getChildExps().get(i));
+      }
+    }
+    ImmutableMap<Integer, RexNode> inputRefToCallMap = inputRefToCallMapBldr.build();
+
+    if ((obChild.getRowType().getFieldCount() - inputRefToCallMap.size()) != resultSchema.size()) {
+      LOG.error(generateInvalidSchemaMessage(obChild, resultSchema, inputRefToCallMap.size()));
+      throw new CalciteSemanticException("Result Schema didn't match Optimized Op Tree Schema");
+    }
+    // This removes order-by only expressions from the projections.
+    HiveProject replacementProjectRel = HiveProject.create(obChild.getInput(), obChild
+        .getChildExps().subList(0, resultSchema.size()), obChild.getRowType().getFieldNames()
+        .subList(0, resultSchema.size()));
+    obRel.replaceInput(0, replacementProjectRel);
+    obRel.setInputRefToCallMap(inputRefToCallMap);
+  }
+
+  private static String generateInvalidSchemaMessage(Project topLevelProj,
+      List<FieldSchema> resultSchema, int fieldsForOB) {
+    String errorDesc = "Result Schema didn't match Calcite Optimized Op Tree; schema: ";
+    for (FieldSchema fs : resultSchema) {
+      errorDesc += "[" + fs.getName() + ":" + fs.getType() + "], ";
+    }
+    errorDesc += " projection fields: ";
+    for (RexNode exp : topLevelProj.getChildExps()) {
+      errorDesc += "[" + exp.toString() + ":" + exp.getType() + "], ";
+    }
+    if (fieldsForOB != 0) {
+      errorDesc += fieldsForOB + " fields removed due to ORDER BY  ";
+    }
+    return errorDesc.substring(0, errorDesc.length() - 2);
+  }
+
+  private static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel,
+      Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema)
+      throws CalciteSemanticException {
+    RelNode parentOforiginalProjRel = topSelparentPair.getKey();
+    HiveProject originalProjRel = (HiveProject) topSelparentPair.getValue();
+
+    // Assumption: top portion of tree could only be
+    // (limit)?(OB)?(Project)....
+    List<RexNode> rootChildExps = originalProjRel.getChildExps();
+    if (resultSchema.size() != rootChildExps.size()) {
+      // Safeguard against potential issues in CBO RowResolver construction. Disable CBO for now.
+      LOG.error(generateInvalidSchemaMessage(originalProjRel, resultSchema, 0));
+      throw new CalciteSemanticException("Result Schema didn't match Optimized Op Tree Schema");
+    }
+
+    List<String> newSelAliases = new ArrayList<String>();
+    String colAlias;
+    for (int i = 0; i < rootChildExps.size(); i++) {
+      colAlias = resultSchema.get(i).getName();
+      if (colAlias.startsWith("_")) {
+        colAlias = colAlias.substring(1);
+      }
+      newSelAliases.add(colAlias);
+    }
+
+    HiveProject replacementProjectRel = HiveProject.create(originalProjRel.getInput(),
+        originalProjRel.getChildExps(), newSelAliases);
+
+    if (rootRel == originalProjRel) {
+      return replacementProjectRel;
+    } else {
+      parentOforiginalProjRel.replaceInput(0, replacementProjectRel);
+      return rootRel;
+    }
+  }
+
+  private static RelNode introduceDerivedTable(final RelNode rel) {
+    List<RexNode> projectList = HiveCalciteUtil.getProjsFromBelowAsInputRef(rel);
+
+    HiveProject select = HiveProject.create(rel.getCluster(), rel, projectList,
+        rel.getRowType(), rel.getCollationList());
+
+    return select;
+  }
+
+  private static RelNode introduceDerivedTable(final RelNode rel, RelNode parent) {
+    int i = 0;
+    int pos = -1;
+    List<RelNode> childList = parent.getInputs();
+
+    for (RelNode child : childList) {
+      if (child == rel) {
+        pos = i;
+        break;
+      }
+      i++;
+    }
+
+    if (pos == -1) {
+      throw new RuntimeException("Couldn't find child node in parent's inputs");
+    }
+
+    RelNode select = introduceDerivedTable(rel);
+
+    parent.replaceInput(pos, select);
+    
+    return select;
+  }
+
+  private static boolean validJoinParent(RelNode joinNode, RelNode parent) {
+    boolean validParent = true;
+
+    if (parent instanceof Join) {
+      if (((Join) parent).getRight() == joinNode) {
+        validParent = false;
+      }
+    } else if (parent instanceof SetOp) {
+      validParent = false;
+    }
+
+    return validParent;
+  }
+
+  private static boolean validFilterParent(RelNode filterNode, RelNode parent) {
+    boolean validParent = true;
+
+    // TOODO: Verify GB having is not a seperate filter (if so we shouldn't
+    // introduce derived table)
+    if (parent instanceof Filter || parent instanceof Join
+        || parent instanceof SetOp) {
+      validParent = false;
+    }
+
+    return validParent;
+  }
+
+  private static boolean validGBParent(RelNode gbNode, RelNode parent) {
+    boolean validParent = true;
+
+    // TOODO: Verify GB having is not a seperate filter (if so we shouldn't
+    // introduce derived table)
+    if (parent instanceof Join || parent instanceof SetOp
+        || parent instanceof Aggregate
+        || (parent instanceof Filter && ((Aggregate) gbNode).getGroupSet().isEmpty())) {
+      validParent = false;
+    }
+
+    return validParent;
+  }
+
+  private static boolean validSortParent(RelNode sortNode, RelNode parent) {
+    boolean validParent = true;
+
+    if (parent != null && !(parent instanceof Project)
+        && !((parent instanceof Sort) || HiveCalciteUtil.orderRelNode(parent)))
+      validParent = false;
+
+    return validParent;
+  }
+
+  private static boolean validSortChild(HiveSort sortNode) {
+    boolean validChild = true;
+    RelNode child = sortNode.getInput();
+
+    if (!(HiveCalciteUtil.limitRelNode(sortNode) && HiveCalciteUtil.orderRelNode(child))
+        && !(child instanceof Project)) {
+      validChild = false;
+    }
+
+    return validChild;
+  }
+
+  private static boolean validSetopParent(RelNode setop, RelNode parent) {
+    boolean validChild = true;
+
+    if (parent != null && !(parent instanceof Project)) {
+      validChild = false;
+    }
+
+    return validChild;
+  }
+
+  private static boolean validSetopChild(RelNode setopChild) {
+    boolean validChild = true;
+
+    if (!(setopChild instanceof Project)) {
+      validChild = false;
+    }
+
+    return validChild;
+  }
+  
+  private static boolean isEmptyGrpAggr(RelNode gbNode) {
+    // Verify if both groupset and aggrfunction are empty)
+    Aggregate aggrnode = (Aggregate) gbNode;
+    if (aggrnode.getGroupSet().isEmpty() && aggrnode.getAggCallList().isEmpty()) {
+      return true;
+    }
+    return false;
+  }
+  
+  private static void replaceEmptyGroupAggr(final RelNode rel, RelNode parent) {
+    // If this function is called, the parent should only include constant
+    List<RexNode> exps = parent.getChildExps();
+    for (RexNode rexNode : exps) {
+      if (rexNode.getKind() != SqlKind.LITERAL) {
+        throw new RuntimeException("We expect " + parent.toString()
+            + " to contain only constants. However, " + rexNode.toString() + " is "
+            + rexNode.getKind());
+      }
+    }
+    HiveAggregate oldAggRel = (HiveAggregate) rel;
+    RelDataTypeFactory typeFactory = oldAggRel.getCluster().getTypeFactory();
+    RelDataType longType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, typeFactory);
+    RelDataType intType = TypeConverter.convert(TypeInfoFactory.intTypeInfo, typeFactory);
+    // Create the dummy aggregation.
+    SqlAggFunction countFn = (SqlAggFunction) SqlFunctionConverter.getCalciteAggFn("count",
+        ImmutableList.of(intType), longType);
+    // TODO: Using 0 might be wrong; might need to walk down to find the
+    // proper index of a dummy.
+    List<Integer> argList = ImmutableList.of(0);
+    AggregateCall dummyCall = new AggregateCall(countFn, false, argList, longType, null);
+    Aggregate newAggRel = oldAggRel.copy(oldAggRel.getTraitSet(), oldAggRel.getInput(),
+        oldAggRel.indicator, oldAggRel.getGroupSet(), oldAggRel.getGroupSets(),
+        ImmutableList.of(dummyCall));
+    RelNode select = introduceDerivedTable(newAggRel);
+    parent.replaceInput(0, select);
+  }
+}



Mime
View raw message