Mailing-List: contact commits-help@eagle.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@eagle.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: mw@apache.org
To: commits@eagle.incubator.apache.org
Date: Mon, 25 Jul 2016 09:37:06 -0000
Message-Id: <aa69dfdbbfa6473b88f98230e3ba9634@git.apache.org>
In-Reply-To: <b2a7c386090548888790cf3d8b534b77@git.apache.org>
References: <b2a7c386090548888790cf3d8b534b77@git.apache.org>
Subject: [29/47] incubator-eagle git commit: EAGLE-330 Fix Hive ql.Parser Hive
 ql.Parser cant parser a hive query sql with keywords
 https://issues.apache.org/jira/browse/EAGLE-330
archived-at: Mon, 25 Jul 2016 09:36:48 -0000

EAGLE-330 Fix Hive ql.Parser
Hive ql.Parser cant parser a hive query sql with keywords
https://issues.apache.org/jira/browse/EAGLE-330

Author: @wgyumg <wgyumg@gmail.com>
Reviewer: @pkuwm <hlu@ebay.com>

Closes: #228


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/cbf3c76c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/cbf3c76c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/cbf3c76c

Branch: refs/heads/master
Commit: cbf3c76ce148b73379698d9626df6e1baa5162ab
Parents: f51474d
Author: yonzhang <yonzhang2012@gmail.com>
Authored: Tue Jun 7 16:01:41 2016 -0700
Committer: yonzhang <yonzhang2012@gmail.com>
Committed: Tue Jun 7 16:01:41 2016 -0700

----------------------------------------------------------------------
 .../apache/eagle/security/hive/ql/Parser.java   | 46 ++++++++++++++++++--
 .../eagle/security/hive/ql/TestParser.java      | 30 ++++++++++++-
 2 files changed, 70 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/cbf3c76c/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java
index 5ab0f4b..19ce003 100644
--- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java
+++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java
@@ -16,11 +16,19 @@
  */
 package org.apache.eagle.security.hive.ql;
 
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenRewriteStream;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.tree.CommonTree;
+import org.antlr.runtime.tree.CommonTreeAdaptor;
+import org.antlr.runtime.tree.TreeAdaptor;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.parse.ASTErrorNode;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.HiveParser;
 import org.apache.hadoop.hive.ql.parse.ParseDriver;
-import org.apache.hadoop.hive.ql.parse.ParseException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,6 +41,21 @@ import java.util.Set;
 public class Parser {
   private static final Logger LOG = LoggerFactory.getLogger(Parser.class);
 
+  static final TreeAdaptor adaptor = new CommonTreeAdaptor() {
+    @Override
+    public Object create(Token payload) {
+      return new ASTNode(payload);
+    }
+    @Override
+    public Object dupNode(Object t) {
+      return create(((CommonTree)t).token);
+    }
+    @Override
+    public Object errorNode(TokenStream input, Token start, Token stop, RecognitionException e) {
+      return new ASTErrorNode(input, start, stop, e);
+    }
+  };
+
   private Set<String> tableSet;
   private Set<String> columnSet;
   private Map<String, String> columnAliasMap;
@@ -66,11 +89,26 @@ public class Parser {
    * Parse an Hive QL into an Abstract Syntax Tree(AST).
    * @param query
    * @return
-   * @throws ParseException
+   * @throws RecognitionException
    */
-  public ASTNode generateAST(String query) throws ParseException {
+  public ASTNode generateAST(String query) throws RecognitionException {
+    // https://issues.apache.org/jira/browse/HIVE-10731
+    // https://issues.apache.org/jira/browse/HIVE-6617
+    HiveConf hiveConf = new HiveConf();
+    hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS, false);
+
     ParseDriver pd = new ParseDriver();
-    return pd.parse(query);
+    ParseDriver.ANTLRNoCaseStringStream antlrStream = pd.new ANTLRNoCaseStringStream(query);
+    ParseDriver.HiveLexerX lexer = pd.new HiveLexerX(antlrStream);
+    lexer.setHiveConf(hiveConf);
+    TokenRewriteStream tokens = new TokenRewriteStream(lexer);
+
+    HiveParser parser = new HiveParser(tokens);
+    parser.setHiveConf(hiveConf);
+    parser.setTreeAdaptor(adaptor);
+    HiveParser.statement_return r = parser.statement();
+
+    return (ASTNode)r.getTree();
   }
 
   private void parseQL(ASTNode ast) {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/cbf3c76c/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java b/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java
index a5f5f82..0bd994f 100644
--- a/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java
+++ b/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java
@@ -22,7 +22,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.ParseException;
+import org.antlr.runtime.RecognitionException;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -67,7 +67,7 @@ public class TestParser {
     }
   }
 
-  public void printQueryAST(String query) throws ParseException {
+  public void printQueryAST(String query) throws RecognitionException {
     ASTNode root = parser.generateAST(query);
     printTree(root, 0);
   }
@@ -209,6 +209,32 @@ public class TestParser {
     }
 
     @Test
+    public void testQueryWithKeyWords() throws Exception {
+        String query =  "SELECT id, user FROM db.table1";
+        String expectedOperation = "SELECT";
+        String expectedInsertTable = null;
+        Map<String, Set<String>> expectedTableColumn = new HashMap<String, Set<String>>();
+        Set<String> set = new HashSet<String>();
+        set.add("id");
+        set.add("user");
+        expectedTableColumn.put("db.table1", set);
+        _testParsingQuery(query, expectedOperation, expectedInsertTable, expectedTableColumn);
+    }
+
+    @Test
+    public void testInsertTableWithKeyWords() throws Exception {
+        String query =  "INSERT OVERWRITE TABLE table2 SELECT id, user FROM db.table1";
+        String expectedOperation = "SELECT";
+        String expectedInsertTable = "table2";
+        Map<String, Set<String>> expectedTableColumn = new HashMap<String, Set<String>>();
+        Set<String> set = new HashSet<String>();
+        set.add("id");
+        set.add("user");
+        expectedTableColumn.put("db.table1", set);
+        _testParsingQuery(query, expectedOperation, expectedInsertTable, expectedTableColumn);
+    }
+
+    @Test
     public void testCreateTable() throws Exception {
         String query = "CREATE TABLE page_view(viewTime INT, userid BIGINT,\n" +
                 "                page_url STRING, referrer_url STRING,\n" +