hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject [1/2] hive git commit: HIVE-10698 : query on view results fails with table not found error if view is created with subquery alias (CTE). (Pengcheng Xiong via Ashutosh Chauhan, John Pullokkaran)
Date Sun, 31 May 2015 07:50:20 GMT
Repository: hive
Updated Branches:
  refs/heads/master 35e91f267 -> c6a09ce30


HIVE-10698 : query on view results fails with table not found error if view is created with
subquery alias (CTE). (Pengcheng Xiong via Ashutosh Chauhan, John Pullokkaran)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c6a09ce3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c6a09ce3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c6a09ce3

Branch: refs/heads/master
Commit: c6a09ce30c0e9a1a6cc9eb3be37c1363997823dd
Parents: 66acd26
Author: Pengcheng Xiong <pxiong@hortonworks.com>
Authored: Thu May 14 20:42:00 2015 -0700
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Sun May 31 00:49:46 2015 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/parse/HiveParser.g    |   4 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  66 ++---
 ql/src/test/queries/clientpositive/cteViews.q   |  41 ++++
 .../test/results/clientpositive/cteViews.q.out  | 242 +++++++++++++++++++
 4 files changed, 309 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c6a09ce3/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 4ec1925..b267bd2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -2127,7 +2127,7 @@ queryStatementExpression[boolean topLevel]
     (w=withClause {topLevel}?)?
     queryStatementExpressionBody[topLevel] {
       if ($w.tree != null) {
-      adaptor.addChild($queryStatementExpressionBody.tree, $w.tree);
+      $queryStatementExpressionBody.tree.insertChild(0, $w.tree);
       }
     }
     ->  queryStatementExpressionBody
@@ -2302,7 +2302,7 @@ selectStatementWithCTE
     (w=withClause)?
     selectStatement[true] {
       if ($w.tree != null) {
-      adaptor.addChild($selectStatement.tree, $w.tree);
+      $selectStatement.tree.insertChild(0, $w.tree);
       }
     }
     ->  selectStatement

http://git-wip-us.apache.org/repos/asf/hive/blob/c6a09ce3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 351c267..d609732 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -709,10 +709,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
 
     qb.getParseInfo().setSrcForAlias(alias, tableTree);
 
-    unparseTranslator.addTableNameTranslation(tableTree, SessionState.get().getCurrentDatabase());
-    if (aliasIndex != 0) {
-      unparseTranslator.addIdentifierTranslation((ASTNode) tabref
-          .getChild(aliasIndex));
+    // if alias to CTE contains the alias, we do not do the translation because
+    // cte is actually a subquery.
+    if (!this.aliasToCTEs.containsKey(alias)) {
+      unparseTranslator.addTableNameTranslation(tableTree, SessionState.get().getCurrentDatabase());
+      if (aliasIndex != 0) {
+        unparseTranslator.addIdentifierTranslation((ASTNode) tabref.getChild(aliasIndex));
+      }
     }
 
     return alias;
@@ -952,19 +955,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
    *
    */
   private ASTNode findCTEFromName(QB qb, String cteName) {
-
-    /*
-     * When saving a view definition all table references in the AST are qualified; including
CTE references.
-     * Where as CTE definitions have no DB qualifier; so we strip out the DB qualifier before
searching in
-     * <code>aliasToCTEs</code> map.
-     */
-    String currDB = SessionState.get().getCurrentDatabase();
-    if ( currDB != null && cteName.startsWith(currDB) &&
-        cteName.length() > currDB.length() &&
-        cteName.charAt(currDB.length()) == '.'   ) {
-      cteName = cteName.substring(currDB.length() + 1);
-    }
-
     StringBuffer qId = new StringBuffer();
     if (qb.getId() != null) {
       qId.append(qb.getId());
@@ -997,14 +987,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     cteAlias = cteAlias == null ? cteName : cteAlias;
     ASTNode cteQryNode = findCTEFromName(qb, cteName);
     QBExpr cteQBExpr = new QBExpr(cteAlias);
-
-    String cteText = ctx.getTokenRewriteStream().toString(
-        cteQryNode.getTokenStartIndex(), cteQryNode.getTokenStopIndex());
-    final ASTNodeOrigin cteOrigin = new ASTNodeOrigin("CTE", cteName,
-        cteText, cteAlias, cteQryNode);
-    cteQryNode = (ASTNode) ParseDriver.adaptor.dupTree(cteQryNode);
-    SubQueryUtils.setOriginDeep(cteQryNode, cteOrigin);
-
     doPhase1QBExpr(cteQryNode, cteQBExpr, qb.getId(), cteAlias);
     qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr);
   }
@@ -1571,31 +1553,31 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
 
       for (String alias : tabAliases) {
         String tab_name = qb.getTabNameForAlias(alias);
+        
+        // we first look for this alias from CTE, and then from catalog.
+        /*
+         * if this s a CTE reference: Add its AST as a SubQuery to this QB.
+         */
+        ASTNode cteNode = findCTEFromName(qb, tab_name.toLowerCase());
+        if (cteNode != null) {
+          String cte_name = tab_name.toLowerCase();
+          if (ctesExpanded.contains(cte_name)) {
+            throw new SemanticException("Recursive cte " + tab_name + " detected (cycle:
"
+                + StringUtils.join(ctesExpanded, " -> ") + " -> " + tab_name + ").");
+          }
+          addCTEAsSubQuery(qb, cte_name, alias);
+          sqAliasToCTEName.put(alias, cte_name);
+          continue;
+        }
+
         Table tab = db.getTable(tab_name, false);
         if (tab == null) {
-          /*
-           * if this s a CTE reference:
-           * Add its AST as a SubQuery to this QB.
-           */
-          ASTNode cteNode = findCTEFromName(qb, tab_name.toLowerCase());
-          if ( cteNode != null ) {
-            String cte_name = tab_name.toLowerCase();
-            if (ctesExpanded.contains(cte_name)) {
-              throw new SemanticException("Recursive cte " + tab_name +
-                  " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") +
-                  " -> " + tab_name + ").");
-            }
-            addCTEAsSubQuery(qb, cte_name, alias);
-            sqAliasToCTEName.put(alias, cte_name);
-            continue;
-          }
           ASTNode src = qb.getParseInfo().getSrcForAlias(alias);
           if (null != src) {
             throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(src));
           } else {
             throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(alias));
           }
-
         }
 
         // Disallow INSERT INTO on bucketized tables

http://git-wip-us.apache.org/repos/asf/hive/blob/c6a09ce3/ql/src/test/queries/clientpositive/cteViews.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cteViews.q b/ql/src/test/queries/clientpositive/cteViews.q
new file mode 100644
index 0000000..c076841
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cteViews.q
@@ -0,0 +1,41 @@
+with src1 as (select key from src order by key limit 5)
+select * from src1;
+
+use default;
+drop view v;
+create view v as with cte as (select key, value from src order by key limit 5)
+select key from cte;
+
+describe extended v;
+
+create database bug;
+use bug;
+select * from default.v;
+drop database bug;
+
+use default;
+drop view v;
+create view v as with cte as (select * from src  order by key limit 5)
+select * from cte;
+
+describe extended v;
+
+create database bug;
+use bug;
+select * from default.v;
+drop database bug;
+
+
+use default;
+drop view v;
+create view v as with src1 as (select key from src order by key limit 5)
+select * from src1;
+
+describe extended v;
+
+create database bug;
+use bug;
+select * from default.v;
+use default;
+drop view v;
+drop database bug;

http://git-wip-us.apache.org/repos/asf/hive/blob/c6a09ce3/ql/src/test/results/clientpositive/cteViews.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cteViews.q.out b/ql/src/test/results/clientpositive/cteViews.q.out
new file mode 100644
index 0000000..6291784
--- /dev/null
+++ b/ql/src/test/results/clientpositive/cteViews.q.out
@@ -0,0 +1,242 @@
+PREHOOK: query: with src1 as (select key from src order by key limit 5)
+select * from src1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: with src1 as (select key from src order by key limit 5)
+select * from src1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0
+0
+0
+10
+100
+PREHOOK: query: use default
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:default
+POSTHOOK: query: use default
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:default
+PREHOOK: query: drop view v
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: drop view v
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: create view v as with cte as (select key, value from src order by key limit
5)
+select key from cte
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as with cte as (select key, value from src order by key limit
5)
+select key from cte
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key                 	string              	                    
+	 	 
+#### A masked pattern was here ####
+select key from cte, viewExpandedText:with cte as (select `src`.`key`, `src`.`value` from
`default`.`src` order by key limit 5)	 	 
+select `cte`.`key` from cte, tableType:VIRTUAL_VIEW)		 
+PREHOOK: query: create database bug
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:bug
+POSTHOOK: query: create database bug
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:bug
+PREHOOK: query: use bug
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:bug
+POSTHOOK: query: use bug
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:bug
+PREHOOK: query: select * from default.v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from default.v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0
+0
+0
+10
+100
+PREHOOK: query: drop database bug
+PREHOOK: type: DROPDATABASE
+PREHOOK: Input: database:bug
+PREHOOK: Output: database:bug
+POSTHOOK: query: drop database bug
+POSTHOOK: type: DROPDATABASE
+POSTHOOK: Input: database:bug
+POSTHOOK: Output: database:bug
+PREHOOK: query: use default
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:default
+POSTHOOK: query: use default
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:default
+PREHOOK: query: drop view v
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@v
+PREHOOK: Output: default@v
+POSTHOOK: query: drop view v
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@v
+POSTHOOK: Output: default@v
+PREHOOK: query: create view v as with cte as (select * from src  order by key limit 5)
+select * from cte
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as with cte as (select * from src  order by key limit 5)
+select * from cte
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key                 	string              	                    
+value               	string              	                    
+	 	 
+#### A masked pattern was here ####
+select * from cte, viewExpandedText:with cte as (select `src`.`key`, `src`.`value` from `default`.`src`
 order by `src`.`key` limit 5)	 	 
+select `cte`.`key`, `cte`.`value` from cte, tableType:VIRTUAL_VIEW)		 
+PREHOOK: query: create database bug
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:bug
+POSTHOOK: query: create database bug
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:bug
+PREHOOK: query: use bug
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:bug
+POSTHOOK: query: use bug
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:bug
+PREHOOK: query: select * from default.v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from default.v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0	val_0
+0	val_0
+0	val_0
+10	val_10
+100	val_100
+PREHOOK: query: drop database bug
+PREHOOK: type: DROPDATABASE
+PREHOOK: Input: database:bug
+PREHOOK: Output: database:bug
+POSTHOOK: query: drop database bug
+POSTHOOK: type: DROPDATABASE
+POSTHOOK: Input: database:bug
+POSTHOOK: Output: database:bug
+PREHOOK: query: use default
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:default
+POSTHOOK: query: use default
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:default
+PREHOOK: query: drop view v
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@v
+PREHOOK: Output: default@v
+POSTHOOK: query: drop view v
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@v
+POSTHOOK: Output: default@v
+PREHOOK: query: create view v as with src1 as (select key from src order by key limit 5)
+select * from src1
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as with src1 as (select key from src order by key limit 5)
+select * from src1
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key                 	string              	                    
+	 	 
+#### A masked pattern was here ####
+select * from src1, viewExpandedText:with src1 as (select `src`.`key` from `default`.`src`
order by key limit 5)	 	 
+select `src1`.`key` from src1, tableType:VIRTUAL_VIEW)		 
+PREHOOK: query: create database bug
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:bug
+POSTHOOK: query: create database bug
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:bug
+PREHOOK: query: use bug
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:bug
+POSTHOOK: query: use bug
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:bug
+PREHOOK: query: select * from default.v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from default.v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0
+0
+0
+10
+100
+PREHOOK: query: use default
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:default
+POSTHOOK: query: use default
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:default
+PREHOOK: query: drop view v
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@v
+PREHOOK: Output: default@v
+POSTHOOK: query: drop view v
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@v
+POSTHOOK: Output: default@v
+PREHOOK: query: drop database bug
+PREHOOK: type: DROPDATABASE
+PREHOOK: Input: database:bug
+PREHOOK: Output: database:bug
+POSTHOOK: query: drop database bug
+POSTHOOK: type: DROPDATABASE
+POSTHOOK: Input: database:bug
+POSTHOOK: Output: database:bug


Mime
View raw message