hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1634728 - in /hive/branches/branch-0.14/ql/src: java/org/apache/hadoop/hive/ql/optimizer/index/ java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/ test/results/clientpositive/
Date Mon, 27 Oct 2014 23:02:37 GMT
Author: sershe
Date: Mon Oct 27 23:02:37 2014
New Revision: 1634728

URL: http://svn.apache.org/r1634728
Log:
HIVE-8021 : CBO: support CTAS and insert ... select (Sergey Shelukhin, reviewed by Laljo John Pullokkaran)

Added:
    hive/branches/branch-0.14/ql/src/test/queries/clientpositive/insert0.q
    hive/branches/branch-0.14/ql/src/test/results/clientpositive/insert0.q.out
Modified:
    hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java
    hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
    hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/branch-0.14/ql/src/test/queries/clientpositive/cbo_correctness.q
    hive/branches/branch-0.14/ql/src/test/queries/clientpositive/ctas_colname.q
    hive/branches/branch-0.14/ql/src/test/queries/clientpositive/decimal_serde.q
    hive/branches/branch-0.14/ql/src/test/results/clientpositive/ctas_colname.q.out
    hive/branches/branch-0.14/ql/src/test/results/clientpositive/decimal_serde.q.out

Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java?rev=1634728&r1=1634727&r2=1634728&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java Mon Oct 27 23:02:37 2014
@@ -112,7 +112,7 @@ public final class RewriteParseContextGe
     ((SemanticAnalyzer) sem).initParseCtx(subPCtx);
 
     LOG.info("Starting Sub-query Semantic Analysis");
-    sem.doPhase1(child, qb, sem.initPhase1Ctx());
+    sem.doPhase1(child, qb, sem.initPhase1Ctx(), null);
     LOG.info("Completed phase 1 of Sub-query Semantic Analysis");
 
     sem.getMetaData(qb);

Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1634728&r1=1634727&r2=1634728&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Mon Oct 27 23:02:37 2014
@@ -79,7 +79,7 @@ import com.google.common.annotations.Vis
  *
  */
 public abstract class BaseSemanticAnalyzer {
-  private static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName());
+  protected static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName());
   protected final Hive db;
   protected final HiveConf conf;
   protected List<Task<? extends Serializable>> rootTasks;

Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1634728&r1=1634727&r2=1634728&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Mon Oct 27 23:02:37 2014
@@ -33,6 +33,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -368,7 +369,7 @@ public class SemanticAnalyzer extends Ba
   //flag for partial scan during analyze ... compute statistics
   protected boolean partialscan;
 
-  private volatile boolean runCBO = true;
+  private volatile boolean runCBO = true; // TODO: why is this volatile?
   private volatile boolean disableJoinMerge = false;
 
   /*
@@ -380,6 +381,9 @@ public class SemanticAnalyzer extends Ba
    */
   private ArrayList<String> ctesExpanded;
 
+  /** Not thread-safe. */
+  private ASTSearcher astSearcher = new ASTSearcher();
+
   private static class Phase1Ctx {
     String dest;
     int nextNum;
@@ -512,7 +516,7 @@ public class SemanticAnalyzer extends Ba
     case HiveParser.TOK_QUERY: {
       QB qb = new QB(id, alias, true);
       Phase1Ctx ctx_1 = initPhase1Ctx();
-      doPhase1(ast, qb, ctx_1);
+      doPhase1(ast, qb, ctx_1, null);
 
       qbexpr.setOpcode(QBExpr.Opcode.NULLOP);
       qbexpr.setQB(qb);
@@ -1177,6 +1181,30 @@ public class SemanticAnalyzer extends Ba
     return alias;
   }
 
+  /** The context that doPhase1 uses to populate information pertaining
+   *  to CBO (currently, this is used for CTAS and insert-as-select). */
+  private static class PreCboCtx {
+    enum Type {
+      NONE,
+      INSERT,
+      CTAS,
+
+      UNEXPECTED
+    }
+    public ASTNode nodeOfInterest;
+    public Type type = Type.NONE;
+    public void set(Type type, ASTNode ast) {
+      if (this.type != Type.NONE) {
+        STATIC_LOG.warn("Setting " + type + " when already " + this.type
+            + "; node " + ast.dump() + " vs old node " + nodeOfInterest.dump());
+        this.type = Type.UNEXPECTED;
+        return;
+      }
+      this.type = type;
+      this.nodeOfInterest = ast;
+    }
+  }
+
   /**
    * Phase 1: (including, but not limited to):
    *
@@ -1194,7 +1222,7 @@ public class SemanticAnalyzer extends Ba
    * @throws SemanticException
    */
   @SuppressWarnings({"fallthrough", "nls"})
-  public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1)
+  public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PreCboCtx cboCtx)
       throws SemanticException {
 
     boolean phase1Result = true;
@@ -1238,20 +1266,26 @@ public class SemanticAnalyzer extends Ba
         String currentDatabase = SessionState.get().getCurrentDatabase();
         String tab_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), currentDatabase);
         qbp.addInsertIntoTable(tab_name);
-        // TODO: is this supposed to fall thru?
 
       case HiveParser.TOK_DESTINATION:
         ctx_1.dest = "insclause-" + ctx_1.nextNum;
         ctx_1.nextNum++;
+        boolean isTmpFileDest = false;
+        if (ast.getChildCount() > 0 && ast.getChild(0) instanceof ASTNode) {
+          ASTNode ch = (ASTNode)ast.getChild(0);
+          if (ch.getToken().getType() == HiveParser.TOK_DIR
+              && ch.getChildCount() > 0 && ch.getChild(0) instanceof ASTNode) {
+            ch = (ASTNode)ch.getChild(0);
+            isTmpFileDest = ch.getToken().getType() == HiveParser.TOK_TMP_FILE;
+          }
+        }
 
         // is there a insert in the subquery
-        if (qbp.getIsSubQ()) {
-          ASTNode ch = (ASTNode) ast.getChild(0);
-          if ((ch.getToken().getType() != HiveParser.TOK_DIR)
-              || (((ASTNode) ch.getChild(0)).getToken().getType() != HiveParser.TOK_TMP_FILE)) {
-            throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY
-                .getMsg(ast));
-          }
+        if (qbp.getIsSubQ() && !isTmpFileDest) {
+          throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY.getMsg(ast));
+        }
+        if (cboCtx != null && !isTmpFileDest) {
+          cboCtx.set(PreCboCtx.Type.INSERT, ast);
         }
         qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0));
 
@@ -1480,12 +1514,18 @@ public class SemanticAnalyzer extends Ba
       int child_count = ast.getChildCount();
       for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) {
         // Recurse
-        phase1Result = phase1Result && doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1);
+        phase1Result = phase1Result && doPhase1(
+            (ASTNode)ast.getChild(child_pos), qb, ctx_1, cboCtx);
       }
     }
     return phase1Result;
   }
 
+  private void traceLogAst(ASTNode ast, String what) {
+    if (!LOG.isTraceEnabled()) return;
+    LOG.trace(what + ast.dump());
+  }
+
   private void getMetaData(QBExpr qbexpr, ReadEntity parentInput)
       throws SemanticException {
     if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
@@ -1760,6 +1800,7 @@ public class SemanticAnalyzer extends Ba
                 qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts);
               }
             } else {
+              // This is the only place where isQuery is set to true; it defaults to false.
               qb.setIsQuery(true);
               fname = ctx.getMRTmpPath().toString();
               ctx.setResDir(new Path(fname));
@@ -2464,7 +2505,7 @@ public class SemanticAnalyzer extends Ba
       ISubQueryJoinInfo subQueryPredicate) throws SemanticException {
     qbSQ.setSubQueryDef(subQueryPredicate.getSubQuery());
     Phase1Ctx ctx_1 = initPhase1Ctx();
-    doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1);
+    doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1, null);
     getMetaData(qbSQ);
     Operator op = genPlan(qbSQ);
     return op;
@@ -6197,8 +6238,7 @@ public class SemanticAnalyzer extends Ba
       ArrayList<ColumnInfo> colInfos = inputRR.getColumnInfos();
 
       // CTAS case: the file output format and serde are defined by the create
-      // table command
-      // rather than taking the default value
+      // table command rather than taking the default value
       List<FieldSchema> field_schemas = null;
       CreateTableDesc tblDesc = qb.getTableDesc();
       if (tblDesc != null) {
@@ -6219,7 +6259,8 @@ public class SemanticAnalyzer extends Ba
           if (!("".equals(nm[0])) && nm[1] != null) {
             colName = unescapeIdentifier(colInfo.getAlias()).toLowerCase(); // remove ``
           }
-          col.setName(colName);;
+          String ctasColName = fixCtasColumnName(colName, colInfo, inputRR);
+          col.setName(ctasColName);
           col.setType(colInfo.getType().getTypeName());
           field_schemas.add(col);
         }
@@ -6397,6 +6438,14 @@ public class SemanticAnalyzer extends Ba
     return output;
   }
 
+  private static String fixCtasColumnName(String colName, ColumnInfo colInfo, RowResolver rr) {
+    int lastDot = colName.lastIndexOf('.');
+    if (lastDot < 0) return colName; // alias is not fully qualified
+    String nqColumnName = colName.substring(lastDot + 1);
+    STATIC_LOG.debug("Replacing " + colName + " (produced by CBO) by " + nqColumnName);
+    return nqColumnName;
+  }
+
   // Check constraints on acid tables.  This includes
   // * no insert overwrites
   // * no use of vectorization
@@ -9875,11 +9924,14 @@ public class SemanticAnalyzer extends Ba
 
     // analyze and process the position alias
     processPositionAlias(ast);
+    // Check configuration for CBO first.
+    runCBO = runCBO && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED);
 
     // analyze create table command
+    PreCboCtx cboCtx = runCBO ? new PreCboCtx() : null;
     if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) {
       // if it is not CTAS, we don't need to go further and just return
-      if ((child = analyzeCreateTable(ast, qb)) == null) {
+      if ((child = analyzeCreateTable(ast, qb, cboCtx)) == null) {
         return;
       }
     } else {
@@ -9888,7 +9940,8 @@ public class SemanticAnalyzer extends Ba
 
     // analyze create view command
     if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW ||
-        (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) {
+        (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW
+          && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) {
       child = analyzeCreateView(ast, qb);
       SessionState.get().setCommandType(HiveOperation.CREATEVIEW);
       if (child == null) {
@@ -9901,7 +9954,7 @@ public class SemanticAnalyzer extends Ba
 
     // continue analyzing from the child ASTNode.
     Phase1Ctx ctx_1 = initPhase1Ctx();
-    if (!doPhase1(child, qb, ctx_1)) {
+    if (!doPhase1(child, qb, ctx_1, cboCtx)) {
       // if phase1Result false return
       return;
     }
@@ -9911,19 +9964,16 @@ public class SemanticAnalyzer extends Ba
     getMetaData(qb);
     LOG.info("Completed getting MetaData in Semantic Analysis");
 
-
+    // Note: for now, we don't actually pass the queryForCbo to CBO, because it accepts qb, not
+    //    AST, and can also access all the private stuff in SA. We rely on the fact that CBO
+    //    ignores the unknown tokens (create table, destination), so if the query is otherwise ok,
+    //    it is as if we did remove those and gave CBO the proper AST. That is kinda hacky.
     if (runCBO) {
-      boolean tokenTypeIsQuery = ast.getToken().getType() == HiveParser.TOK_QUERY
-          || ast.getToken().getType() == HiveParser.TOK_EXPLAIN;
-      if (!tokenTypeIsQuery || createVwDesc != null
-          || !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)
-          || !canHandleQuery(qb, true) || !HiveOptiqUtil.validateASTForCBO(ast)) {
-        runCBO = false;
-      }
-
-      if (runCBO) {
-        disableJoinMerge = true;
+      ASTNode queryForCbo = ast;
+      if (cboCtx.type == PreCboCtx.Type.CTAS) {
+        queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query
       }
+      runCBO = canHandleAstForCbo(queryForCbo, qb, cboCtx);
     }
 
     // Save the result schema derived from the sink operator produced
@@ -9933,6 +9983,7 @@ public class SemanticAnalyzer extends Ba
     Operator sinkOp = null;
 
     if (runCBO) {
+      disableJoinMerge = true;
       OptiqBasedPlanner optiqPlanner = new OptiqBasedPlanner();
       boolean reAnalyzeAST = false;
 
@@ -9940,10 +9991,17 @@ public class SemanticAnalyzer extends Ba
         // 1. Gen Optimized AST
         ASTNode newAST = optiqPlanner.getOptimizedAST(prunedPartitions);
 
+        // 1.1. Fix up the query for insert/ctas
+        newAST = fixUpCtasAndInsertAfterCbo(ast, newAST, cboCtx);
+
         // 2. Regen OP plan from optimized AST
         init(false);
+        if (cboCtx.type == PreCboCtx.Type.CTAS) {
+          // Redo create-table analysis, because it's not part of doPhase1.
+          newAST = reAnalyzeCtasAfterCbo(newAST);
+        }
         ctx_1 = initPhase1Ctx();
-        if (!doPhase1(newAST, qb, ctx_1)) {
+        if (!doPhase1(newAST, qb, ctx_1, null)) {
           throw new RuntimeException(
               "Couldn't do phase1 on CBO optimized query plan");
         }
@@ -10089,6 +10147,118 @@ public class SemanticAnalyzer extends Ba
     return;
   }
 
+  private ASTNode fixUpCtasAndInsertAfterCbo(
+      ASTNode originalAst, ASTNode newAst, PreCboCtx cboCtx) throws SemanticException {
+    switch (cboCtx.type) {
+    case NONE: return newAst; // nothing to do
+    case CTAS: {
+      // Patch the optimized query back into original CTAS AST, replacing the original query.
+      replaceASTChild(cboCtx.nodeOfInterest, newAst);
+      return originalAst;
+    }
+    case INSERT: {
+      // We need to patch the dest back to original into new query.
+      // This makes assumptions about the structure of the AST.
+      ASTNode newDest = astSearcher.simpleBreadthFirstSearch(
+          newAst, HiveParser.TOK_QUERY, HiveParser.TOK_INSERT, HiveParser.TOK_DESTINATION);
+      if (newDest == null) {
+        LOG.error("Cannot find destination after CBO; new ast is "+ newAst.dump());
+        throw new SemanticException("Cannot find destination after CBO");
+      }
+      replaceASTChild(newDest, cboCtx.nodeOfInterest);
+      return newAst;
+    }
+    default: throw new AssertionError("Unexpected type " + cboCtx.type);
+    }
+  }
+
+  private ASTNode reAnalyzeCtasAfterCbo(ASTNode newAst) throws SemanticException {
+    // analyzeCreateTable uses this.ast, but doPhase1 doesn't, so only reset it here.
+    this.ast = newAst;
+    newAst = analyzeCreateTable(newAst, qb, null);
+    if (newAst == null) {
+      LOG.error("analyzeCreateTable failed to initialize CTAS after CBO;"
+          + " new ast is " + this.ast.dump());
+      throw new SemanticException("analyzeCreateTable failed to initialize CTAS after CBO");
+    }
+    return newAst;
+  }
+
+  private boolean canHandleAstForCbo(ASTNode ast, QB qb, PreCboCtx cboCtx) {
+     int root = ast.getToken().getType();
+     boolean needToLogMessage = LOG.isInfoEnabled();
+     boolean isSupportedRoot =
+         root == HiveParser.TOK_QUERY || root == HiveParser.TOK_EXPLAIN || qb.isCTAS();
+     // Check AST.
+     // Assumption: If top level QB is query then everything below it must also be Query
+     // Can there be an insert or CTAS that wouldn't
+     //        be supported and would require additional checks similar to IsQuery?
+     boolean isSupportedType =
+         qb.getIsQuery() || qb.isCTAS() || cboCtx.type == PreCboCtx.Type.INSERT;
+     boolean result = isSupportedRoot && isSupportedType && createVwDesc == null;
+     if (!result) {
+       if (needToLogMessage) {
+         String msg = "";
+         if (!isSupportedRoot) msg += "doesn't have QUERY or EXPLAIN as root and not a CTAS; ";
+         if (!isSupportedType) msg += "is not a query, CTAS, or insert; ";
+         if (createVwDesc != null) msg += "has create view; ";
+
+         if (msg.isEmpty()) msg += "has some unspecified limitations; ";
+         LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2));
+       }
+       return false;
+     }
+     // Now check QB in more detail. canHandleQbForCbo returns null if query can be handled.
+     String msg = canHandleQbForCbo(qb, true, needToLogMessage);
+     if (msg == null) {
+       return true;
+     }
+     if (needToLogMessage) {
+       LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2));
+     }
+     return false;
+  }
+
+  private class ASTSearcher {
+    private final LinkedList<ASTNode> searchQueue = new LinkedList<ASTNode>();
+    /**
+     * Performs breadth-first search of the AST for a nested set of tokens. Tokens don't have to be
+     * each others' direct children, they can be separated by layers of other tokens. For each token
+     * in the list, the first one found is matched and there's no backtracking; thus, if AST has
+     * multiple instances of some token, of which only one matches, it is not guaranteed to be found.
+     * We use this for simple things.
+     * Not thread-safe - reuses searchQueue.
+     */
+    public ASTNode simpleBreadthFirstSearch(ASTNode ast, int... tokens) {
+      searchQueue.clear();
+      searchQueue.add(ast);
+      for (int i = 0; i < tokens.length; ++i) {
+        boolean found = false;
+        int token = tokens[i];
+        while (!searchQueue.isEmpty() && !found) {
+          ASTNode next = searchQueue.poll();
+          found = next.getType() == token;
+          if (found) {
+            if (i == tokens.length - 1) return next;
+            searchQueue.clear();
+          }
+          for (int j = 0; j < next.getChildCount(); ++j) {
+            searchQueue.add((ASTNode)next.getChild(j));
+          }
+        }
+        if (!found) return null;
+      }
+      return null;
+    }
+  }
+
+  private void replaceASTChild(ASTNode child, ASTNode newChild) {
+    ASTNode parent = (ASTNode)child.parent;
+    int childIndex = child.childIndex;
+    parent.deleteChild(childIndex);
+    parent.insertChild(childIndex, newChild);
+  }
+
   private void putAccessedColumnsToReadEntity(HashSet<ReadEntity> inputs, ColumnAccessInfo columnAccessInfo) {
     Map<String, List<String>> tableToColumnAccessMap = columnAccessInfo.getTableToColumnAccessMap();
     if (tableToColumnAccessMap != null && !tableToColumnAccessMap.isEmpty()) {
@@ -10585,8 +10755,8 @@ public class SemanticAnalyzer extends Ba
    * the semantic analyzer need to deal with the select statement with respect
    * to the SerDe and Storage Format.
    */
-  private ASTNode analyzeCreateTable(ASTNode ast, QB qb)
-      throws SemanticException {
+  private ASTNode analyzeCreateTable(
+      ASTNode ast, QB qb, PreCboCtx cboCtx) throws SemanticException {
     String[] qualifiedTabName = getQualifiedTableName((ASTNode) ast.getChild(0));
     String dbDotTab = getDotName(qualifiedTabName);
 
@@ -10676,6 +10846,9 @@ public class SemanticAnalyzer extends Ba
           throw new SemanticException(ErrorMsg.CTAS_EXTTBL_COEXISTENCE.getMsg());
         }
         command_type = CTAS;
+        if (cboCtx != null) {
+          cboCtx.set(PreCboCtx.Type.CTAS, child);
+        }
         selectStmt = child;
         break;
       case HiveParser.TOK_TABCOLLIST:
@@ -12247,27 +12420,43 @@ public class SemanticAnalyzer extends Ba
 
   /**** Temporary Place Holder For Optiq plan Gen, Optimizer ****/
 
-  /*
-   * Entry point to Optimizations using Optiq.
+  /**
+   * Entry point to Optimizations using Optiq. Checks whether Optiq can handle the query.
+   * @param qbToChk Query block to check.
+   * @param verbose Whether return value should be verbose in case of failure.
+   * @return null if the query can be handled; non-null reason string if it cannot be.
    */
-  private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) {
-    boolean runOptiqPlanner = false;
+  private String canHandleQbForCbo(QB qbToChk, boolean topLevelQB, boolean verbose) {
     // Assumption:
     // 1. If top level QB is query then everything below it must also be Query
     // 2. Nested Subquery will return false for qbToChk.getIsQuery()
-    if ((!topLevelQB || qbToChk.getIsQuery())
-        && (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict"))
-        && (!topLevelQB || (queryProperties.getJoinCount() > 1) || conf.getBoolVar(ConfVars.HIVE_IN_TEST))
-        && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy()
-        && !queryProperties.hasSortBy() && !queryProperties.hasPTF()
-        && !queryProperties.usesScript() && !queryProperties.hasMultiDestQuery()
-        && !queryProperties.hasLateralViews()) {
-      runOptiqPlanner = true;
-    } else {
-      LOG.info("Can not invoke CBO; query contains operators not supported for CBO.");
-    }
+    boolean isInTest = conf.getBoolVar(ConfVars.HIVE_IN_TEST);
+    boolean isStrictTest = isInTest
+        && !conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict");
+    boolean hasEnoughJoins = !topLevelQB || (queryProperties.getJoinCount() > 1) || isInTest;
+    if (!isStrictTest && hasEnoughJoins && !queryProperties.hasClusterBy()
+        && !queryProperties.hasDistributeBy() && !queryProperties.hasSortBy()
+        && !queryProperties.hasPTF() && !queryProperties.usesScript()
+        && !queryProperties.hasMultiDestQuery() && !queryProperties.hasLateralViews()) {
+      return null; // Ok to run CBO.
+    }
+
+    // Not ok to run CBO, build error message.
+    String msg = "";
+    if (verbose) {
+      if (isStrictTest) msg += "is in test running in mode other than nonstrict; ";
+      if (!hasEnoughJoins) msg += "has too few joins; ";
+      if (queryProperties.hasClusterBy()) msg += "has cluster by; ";
+      if (queryProperties.hasDistributeBy()) msg += "has distribute by; ";
+      if (queryProperties.hasSortBy()) msg += "has sort by; ";
+      if (queryProperties.hasPTF()) msg += "has PTF; ";
+      if (queryProperties.usesScript()) msg += "uses scripts; ";
+      if (queryProperties.hasMultiDestQuery()) msg += "is a multi-destination query; ";
+      if (queryProperties.hasLateralViews()) msg += "has lateral views; ";
 
-    return runOptiqPlanner;
+      if (msg.isEmpty()) msg += "has some unspecified limitations; ";
+    }
+    return msg;
   }
 
   private class OptiqBasedPlanner implements Frameworks.PlannerAction<RelNode> {
@@ -12984,7 +13173,7 @@ public class SemanticAnalyzer extends Ba
           QB qbSQ = new QB(subQuery.getOuterQueryId(), subQuery.getAlias(), true);
           qbSQ.setSubQueryDef(subQuery.getSubQuery());
           Phase1Ctx ctx_1 = initPhase1Ctx();
-          doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1);
+          doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1, null);
           getMetaData(qbSQ);
           RelNode subQueryRelNode = genLogicalPlan(qbSQ, false);
           aliasToRel.put(subQuery.getAlias(), subQueryRelNode);
@@ -13012,7 +13201,7 @@ public class SemanticAnalyzer extends Ba
             QB qbSQ_nic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true);
             qbSQ_nic.setSubQueryDef(notInCheck.getSubQuery());
             ctx_1 = initPhase1Ctx();
-            doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1);
+            doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1, null);
             getMetaData(qbSQ_nic);
             RelNode subQueryNICRelNode = genLogicalPlan(qbSQ_nic, false);
             aliasToRel.put(notInCheck.getAlias(), subQueryNICRelNode);
@@ -14069,11 +14258,14 @@ public class SemanticAnalyzer extends Ba
       // First generate all the opInfos for the elements in the from clause
       Map<String, RelNode> aliasToRel = new HashMap<String, RelNode>();
 
-      // 0. Check if we can handle the query
-      // This check is needed here because of SubQuery
-      if (!canHandleQuery(qb, false)) {
-        String msg = String.format("CBO Can not handle Sub Query");
-        LOG.debug(msg);
+      // 0. Check if we can handle the SubQuery;
+      // canHandleQbForCbo returns null if the query can be handled.
+      String reason = canHandleQbForCbo(qb, false, LOG.isDebugEnabled());
+      if (reason != null) {
+        String msg = "CBO can not handle Sub Query";
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(msg + " because it: " + reason);
+        }
         throw new OptiqSemanticException(msg);
       }
 

Modified: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/cbo_correctness.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/cbo_correctness.q?rev=1634728&r1=1634727&r2=1634728&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/cbo_correctness.q (original)
+++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/cbo_correctness.q Mon Oct 27 23:02:37 2014
@@ -485,4 +485,4 @@ select unionsrc.key, count(1) FROM (sele
 -- Windowing
 select *, rank() over(partition by key order by value) as rr from src1;
 
-select *, rank() over(partition by key order by value) from src1;
\ No newline at end of file
+select *, rank() over(partition by key order by value) from src1;

Modified: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/ctas_colname.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/ctas_colname.q?rev=1634728&r1=1634727&r2=1634728&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/ctas_colname.q (original)
+++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/ctas_colname.q Mon Oct 27 23:02:37 2014
@@ -3,9 +3,11 @@
 -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table
 
 -- group by
+
+
 explain
-create table summary as select *, sum(key), count(value) from src;
-create table summary as select *, sum(key), count(value) from src;
+create table summary as select *, key + 1, concat(value, value) from src limit 20;
+create table summary as select *, key + 1, concat(value, value) from src limit 20;
 describe formatted summary;
 select * from summary;
 

Modified: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/decimal_serde.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/decimal_serde.q?rev=1634728&r1=1634727&r2=1634728&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/decimal_serde.q (original)
+++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/decimal_serde.q Mon Oct 27 23:02:37 2014
@@ -15,12 +15,15 @@ SELECT * FROM DECIMAL_TEXT ORDER BY key,
 CREATE TABLE DECIMAL_RC
 STORED AS RCFile AS
 SELECT * FROM DECIMAL_TEXT;
+describe formatted DECIMAL_RC;
 
 CREATE TABLE DECIMAL_LAZY_COL
 ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
 STORED AS RCFile AS
 SELECT * FROM DECIMAL_RC;
 
+describe formatted DECIMAL_LAZY_COL;
+
 CREATE TABLE DECIMAL_SEQUENCE
 ROW FORMAT DELIMITED
 FIELDS TERMINATED BY '\001'

Added: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/insert0.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/insert0.q?rev=1634728&view=auto
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/insert0.q (added)
+++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/insert0.q Mon Oct 27 23:02:37 2014
@@ -0,0 +1,38 @@
+set hive.cbo.enable=true;
+
+DROP TABLE insert_into1;
+DROP TABLE ctas_table;
+DROP TABLE ctas_part;
+
+CREATE TABLE insert_into1 (key int, value string);
+
+INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10;
+
+select * from insert_into1 order by key;
+
+INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10;
+
+select * from insert_into1 order by key;
+
+create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10;
+
+describe extended ctas_table;
+
+select * from ctas_table order by key;
+
+
+set hive.exec.dynamic.partition=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+create table ctas_part (key int, value string) partitioned by (modkey bigint);
+
+insert overwrite table ctas_part partition (modkey) 
+select key, value, ceil(key / 100) from src where key is not null limit 10;
+
+select * from ctas_part order by key;
+
+
+
+DROP TABLE insert_into1;
+DROP TABLE ctas_table;
+DROP TABLE ctas_part;
\ No newline at end of file

Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/ctas_colname.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/ctas_colname.q.out?rev=1634728&r1=1634727&r2=1634728&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/ctas_colname.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/ctas_colname.q.out Mon Oct 27 23:02:37 2014
@@ -3,16 +3,20 @@ PREHOOK: query: -- SORT_QUERY_RESULTS
 -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table
 
 -- group by
+
+
 explain
-create table summary as select *, sum(key), count(value) from src
+create table summary as select *, key + 1, concat(value, value) from src limit 20
 PREHOOK: type: CREATETABLE_AS_SELECT
 POSTHOOK: query: -- SORT_QUERY_RESULTS
 
 -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table
 
 -- group by
+
+
 explain
-create table summary as select *, sum(key), count(value) from src
+create table summary as select *, key + 1, concat(value, value) from src limit 20
 POSTHOOK: type: CREATETABLE_AS_SELECT
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
@@ -28,31 +32,27 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Select Operator
-              expressions: key (type: string), value (type: string)
-              outputColumnNames: key, value
+              expressions: key (type: string), value (type: string), (key + 1) (type: double), concat(value, value) (type: string)
+              outputColumnNames: _col0, _col1, _col2, _col3
               Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
-              Group By Operator
-                aggregations: sum(key), count(value)
-                mode: hash
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+              Limit
+                Number of rows: 20
+                Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   sort order: 
-                  Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
-                  value expressions: _col0 (type: double), _col1 (type: bigint)
+                  Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+                  value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _col3 (type: string)
       Reduce Operator Tree:
-        Group By Operator
-          aggregations: sum(VALUE._col0), count(VALUE._col1)
-          mode: mergepartial
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
-          Select Operator
-            expressions: _col0 (type: double), _col1 (type: bigint), _col0 (type: double), _col1 (type: bigint)
-            outputColumnNames: _col0, _col1, _col2, _col3
-            Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+        Select Operator
+          expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: string)
+          outputColumnNames: _col0, _col1, _col2, _col3
+          Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+          Limit
+            Number of rows: 20
+            Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
             File Output Operator
               compressed: false
-              Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
               table:
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -68,7 +68,7 @@ STAGE PLANS:
   Stage: Stage-3
       Create Table Operator:
         Create Table
-          columns: _col0 double, _col1 bigint, _c1 double, _c2 bigint
+          columns: key string, value string, _c1 double, _c2 string
           input format: org.apache.hadoop.mapred.TextInputFormat
           output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
           serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
@@ -77,12 +77,12 @@ STAGE PLANS:
   Stage: Stage-2
     Stats-Aggr Operator
 
-PREHOOK: query: create table summary as select *, sum(key), count(value) from src
+PREHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20
 PREHOOK: type: CREATETABLE_AS_SELECT
 PREHOOK: Input: default@src
 PREHOOK: Output: database:default
 PREHOOK: Output: default@summary
-POSTHOOK: query: create table summary as select *, sum(key), count(value) from src
+POSTHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20
 POSTHOOK: type: CREATETABLE_AS_SELECT
 POSTHOOK: Input: default@src
 POSTHOOK: Output: database:default
@@ -95,10 +95,10 @@ POSTHOOK: type: DESCTABLE
 POSTHOOK: Input: default@summary
 # col_name            	data_type           	comment             
 	 	 
-_col0               	double              	                    
-_col1               	bigint              	                    
+key                 	string              	                    
+value               	string              	                    
 _c1                 	double              	                    
-_c2                 	bigint              	                    
+_c2                 	string              	                    
 	 	 
 # Detailed Table Information	 	 
 Database:           	default             	 
@@ -110,9 +110,9 @@ Table Type:         	MANAGED_TABLE      
 Table Parameters:	 	 
 	COLUMN_STATS_ACCURATE	true                
 	numFiles            	1                   
-	numRows             	1                   
-	rawDataSize         	25                  
-	totalSize           	26                  
+	numRows             	20                  
+	rawDataSize         	620                 
+	totalSize           	640                 
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -133,7 +133,26 @@ POSTHOOK: query: select * from summary
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@summary
 #### A masked pattern was here ####
-130091.0	500	130091.0	500
+128	val_128	129.0	val_128val_128
+150	val_150	151.0	val_150val_150
+165	val_165	166.0	val_165val_165
+193	val_193	194.0	val_193val_193
+213	val_213	214.0	val_213val_213
+224	val_224	225.0	val_224val_224
+238	val_238	239.0	val_238val_238
+255	val_255	256.0	val_255val_255
+265	val_265	266.0	val_265val_265
+27	val_27	28.0	val_27val_27
+273	val_273	274.0	val_273val_273
+278	val_278	279.0	val_278val_278
+311	val_311	312.0	val_311val_311
+369	val_369	370.0	val_369val_369
+401	val_401	402.0	val_401val_401
+409	val_409	410.0	val_409val_409
+484	val_484	485.0	val_484val_484
+66	val_66	67.0	val_66val_66
+86	val_86	87.0	val_86val_86
+98	val_98	99.0	val_98val_98
 PREHOOK: query: -- window functions
 explain
 create table x4 as select *, rank() over(partition by key order by value) as rr from src1

Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/decimal_serde.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/decimal_serde.q.out?rev=1634728&r1=1634727&r2=1634728&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/decimal_serde.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/decimal_serde.q.out Mon Oct 27 23:02:37 2014
@@ -96,6 +96,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
 POSTHOOK: Input: default@decimal_text
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@DECIMAL_RC
+PREHOOK: query: describe formatted DECIMAL_RC
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@decimal_rc
+POSTHOOK: query: describe formatted DECIMAL_RC
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@decimal_rc
+# col_name            	data_type           	comment             
+	 	 
+key                 	decimal(10,0)       	                    
+value               	int                 	                    
+	 	 
+# Detailed Table Information	 	 
+Database:           	default             	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+#### A masked pattern was here ####
+Table Type:         	MANAGED_TABLE       	 
+Table Parameters:	 	 
+	COLUMN_STATS_ACCURATE	true                
+	numFiles            	1                   
+	numRows             	38                  
+	rawDataSize         	157                 
+	totalSize           	278                 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe	 
+InputFormat:        	org.apache.hadoop.hive.ql.io.RCFileInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.RCFileOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
 PREHOOK: query: CREATE TABLE DECIMAL_LAZY_COL
 ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
 STORED AS RCFile AS
@@ -112,6 +148,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
 POSTHOOK: Input: default@decimal_rc
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@DECIMAL_LAZY_COL
+PREHOOK: query: describe formatted DECIMAL_LAZY_COL
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@decimal_lazy_col
+POSTHOOK: query: describe formatted DECIMAL_LAZY_COL
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@decimal_lazy_col
+# col_name            	data_type           	comment             
+	 	 
+key                 	decimal(10,0)       	                    
+value               	int                 	                    
+	 	 
+# Detailed Table Information	 	 
+Database:           	default             	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+#### A masked pattern was here ####
+Table Type:         	MANAGED_TABLE       	 
+Table Parameters:	 	 
+	COLUMN_STATS_ACCURATE	true                
+	numFiles            	1                   
+	numRows             	38                  
+	rawDataSize         	157                 
+	totalSize           	278                 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe	 
+InputFormat:        	org.apache.hadoop.hive.ql.io.RCFileInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.RCFileOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
 PREHOOK: query: CREATE TABLE DECIMAL_SEQUENCE
 ROW FORMAT DELIMITED
 FIELDS TERMINATED BY '\001'

Added: hive/branches/branch-0.14/ql/src/test/results/clientpositive/insert0.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/insert0.q.out?rev=1634728&view=auto
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/insert0.q.out (added)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/insert0.q.out Mon Oct 27 23:02:37 2014
@@ -0,0 +1,208 @@
+PREHOOK: query: DROP TABLE insert_into1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE insert_into1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE ctas_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE ctas_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE ctas_part
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE ctas_part
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE insert_into1 (key int, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: CREATE TABLE insert_into1 (key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@insert_into1
+PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@insert_into1
+POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from insert_into1 order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_into1 order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+0	val_0
+0	val_0
+0	val_0
+10	val_10
+100	val_100
+100	val_100
+103	val_103
+103	val_103
+104	val_104
+104	val_104
+PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@insert_into1
+POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from insert_into1 order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_into1 order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+0	val_0
+0	val_0
+0	val_0
+10	val_10
+90	val_90
+90	val_90
+92	val_92
+95	val_95
+95	val_95
+96	val_96
+97	val_97
+97	val_97
+98	val_98
+98	val_98
+100	val_100
+100	val_100
+103	val_103
+103	val_103
+104	val_104
+104	val_104
+PREHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas_table
+POSTHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas_table
+PREHOOK: query: describe extended ctas_table
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ctas_table
+POSTHOOK: query: describe extended ctas_table
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ctas_table
+key                 	string              	                    
+foo                 	bigint              	                    
+	 	 
+#### A masked pattern was here ####
+PREHOOK: query: select * from ctas_table order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas_table order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas_table
+#### A masked pattern was here ####
+0	3
+10	1
+100	2
+103	2
+104	2
+105	1
+11	1
+111	1
+113	2
+114	1
+PREHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas_part
+PREHOOK: query: insert overwrite table ctas_part partition (modkey) 
+select key, value, ceil(key / 100) from src where key is not null limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: insert overwrite table ctas_part partition (modkey) 
+select key, value, ceil(key / 100) from src where key is not null limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ctas_part@modkey=1
+POSTHOOK: Output: default@ctas_part@modkey=2
+POSTHOOK: Output: default@ctas_part@modkey=3
+POSTHOOK: Output: default@ctas_part@modkey=4
+POSTHOOK: Output: default@ctas_part@modkey=5
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from ctas_part order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas_part
+PREHOOK: Input: default@ctas_part@modkey=1
+PREHOOK: Input: default@ctas_part@modkey=2
+PREHOOK: Input: default@ctas_part@modkey=3
+PREHOOK: Input: default@ctas_part@modkey=4
+PREHOOK: Input: default@ctas_part@modkey=5
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas_part order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas_part
+POSTHOOK: Input: default@ctas_part@modkey=1
+POSTHOOK: Input: default@ctas_part@modkey=2
+POSTHOOK: Input: default@ctas_part@modkey=3
+POSTHOOK: Input: default@ctas_part@modkey=4
+POSTHOOK: Input: default@ctas_part@modkey=5
+#### A masked pattern was here ####
+27	val_27	1
+86	val_86	1
+98	val_98	1
+165	val_165	2
+238	val_238	3
+255	val_255	3
+278	val_278	3
+311	val_311	4
+409	val_409	5
+484	val_484	5
+PREHOOK: query: DROP TABLE insert_into1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@insert_into1
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: DROP TABLE insert_into1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@insert_into1
+POSTHOOK: Output: default@insert_into1
+PREHOOK: query: DROP TABLE ctas_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas_table
+PREHOOK: Output: default@ctas_table
+POSTHOOK: query: DROP TABLE ctas_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas_table
+POSTHOOK: Output: default@ctas_table
+PREHOOK: query: DROP TABLE ctas_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas_part
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: DROP TABLE ctas_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas_part
+POSTHOOK: Output: default@ctas_part



Mime
View raw message