hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pxi...@apache.org
Subject hive git commit: HIVE-15578: Simplify IdentifiersParser (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Date Wed, 25 Jan 2017 19:07:06 GMT
Repository: hive
Updated Branches:
  refs/heads/master 608194fc5 -> 92f31d07a


HIVE-15578: Simplify IdentifiersParser (Pengcheng Xiong, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92f31d07
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92f31d07
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92f31d07

Branch: refs/heads/master
Commit: 92f31d07aa988d4a460aac56e369bfa386361776
Parents: 608194f
Author: Pengcheng Xiong <pxiong@apache.org>
Authored: Wed Jan 25 11:06:52 2017 -0800
Committer: Pengcheng Xiong <pxiong@apache.org>
Committed: Wed Jan 25 11:06:52 2017 -0800

----------------------------------------------------------------------
 .../hadoop/hive/ql/parse/IdentifiersParser.g    | 86 ++++++++------------
 .../clientnegative/char_pad_convert_fail2.q.out |  2 +-
 .../invalid_select_expression.q.out             |  2 +-
 .../ptf_negative_DistributeByOrderBy.q.out      |  3 +-
 .../ptf_negative_PartitionBySortBy.q.out        |  3 +-
 .../clientnegative/ptf_window_boundaries.q.out  |  2 +-
 .../clientnegative/ptf_window_boundaries2.q.out |  2 +-
 .../llap/offset_limit_ppd_optimizer.q.out       |  2 +-
 8 files changed, 42 insertions(+), 60 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index b90210c..43fd69e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -76,15 +76,14 @@ rollupOldSyntax
 @init { gParent.pushMsg("rollup old syntax", state); }
 @after { gParent.popMsg(state); }
     :
-    expression
-    ( COMMA expression)*
+    expressionsNotInParenthese
     ((rollup=KW_WITH KW_ROLLUP) | (cube=KW_WITH KW_CUBE)) ?
     (sets=KW_GROUPING KW_SETS
     LPAREN groupingSetExpression ( COMMA groupingSetExpression)*  RPAREN ) ?
-    -> {rollup != null}? ^(TOK_ROLLUP_GROUPBY expression+)
-    -> {cube != null}? ^(TOK_CUBE_GROUPBY expression+)
-    -> {sets != null}? ^(TOK_GROUPING_SETS expression+ groupingSetExpression+)
-    -> ^(TOK_GROUPBY expression+)
+    -> {rollup != null}? ^(TOK_ROLLUP_GROUPBY expressionsNotInParenthese)
+    -> {cube != null}? ^(TOK_CUBE_GROUPBY expressionsNotInParenthese)
+    -> {sets != null}? ^(TOK_GROUPING_SETS expressionsNotInParenthese groupingSetExpression+)
+    -> ^(TOK_GROUPBY expressionsNotInParenthese)
     ;
 
 
@@ -92,7 +91,7 @@ groupingSetExpression
 @init {gParent.pushMsg("grouping set expression", state); }
 @after {gParent.popMsg(state); }
    :
-   (LPAREN) => groupingSetExpressionMultiple 
+   (groupingSetExpressionMultiple) => groupingSetExpressionMultiple 
    |
    groupingExpressionSingle
    ;
@@ -130,7 +129,7 @@ havingCondition
 
 expressionsInParenthese
     :
-    LPAREN expression (COMMA expression)* RPAREN -> expression+
+    LPAREN! expressionsNotInParenthese RPAREN!
     ;
 
 expressionsNotInParenthese
@@ -138,6 +137,13 @@ expressionsNotInParenthese
     expression (COMMA expression)* -> expression+
     ;
 
+expressions
+    :
+    (expressionsInParenthese) => expressionsInParenthese
+    |
+    expressionsNotInParenthese
+    ;
+
 columnRefOrderInParenthese
     :
     LPAREN columnRefOrder (COMMA columnRefOrder)* RPAREN -> columnRefOrder+
@@ -160,36 +166,21 @@ clusterByClause
 @init { gParent.pushMsg("cluster by clause", state); }
 @after { gParent.popMsg(state); }
     :
-    KW_CLUSTER KW_BY
-    (
-    (LPAREN) => expressionsInParenthese -> ^(TOK_CLUSTERBY expressionsInParenthese)
-    |
-    expressionsNotInParenthese -> ^(TOK_CLUSTERBY expressionsNotInParenthese)
-    )
+    KW_CLUSTER KW_BY expressions -> ^(TOK_CLUSTERBY expressions)
     ;
 
 partitionByClause
 @init  { gParent.pushMsg("partition by clause", state); }
 @after { gParent.popMsg(state); }
     :
-    KW_PARTITION KW_BY
-    (
-    (LPAREN) => expressionsInParenthese -> ^(TOK_DISTRIBUTEBY expressionsInParenthese)
-    |
-    expressionsNotInParenthese -> ^(TOK_DISTRIBUTEBY expressionsNotInParenthese)
-    )
+    KW_PARTITION KW_BY expressions -> ^(TOK_DISTRIBUTEBY expressions) 
     ;
 
 distributeByClause
 @init { gParent.pushMsg("distribute by clause", state); }
 @after { gParent.popMsg(state); }
     :
-    KW_DISTRIBUTE KW_BY
-    (
-    (LPAREN) => expressionsInParenthese -> ^(TOK_DISTRIBUTEBY expressionsInParenthese)
-    |
-    expressionsNotInParenthese -> ^(TOK_DISTRIBUTEBY expressionsNotInParenthese)
-    )
+    KW_DISTRIBUTE KW_BY expressions -> ^(TOK_DISTRIBUTEBY expressions) 
     ;
 
 sortByClause
@@ -225,9 +216,7 @@ functionName
 @init { gParent.pushMsg("function name", state); }
 @after { gParent.popMsg(state); }
     : // Keyword IF is also a function name
-    (KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE) => (KW_IF | KW_ARRAY | KW_MAP
| KW_STRUCT | KW_UNIONTYPE)
-    | 
-    (functionIdentifier) => functionIdentifier
+    functionIdentifier
     |
     sql11ReservedKeywordsUsedAsFunctionName -> Identifier[$sql11ReservedKeywordsUsedAsFunctionName.start]
     ;
@@ -323,6 +312,7 @@ constant
     | NumberLiteral
     | charSetStringLiteral
     | booleanValue
+    | KW_NULL -> TOK_NULL
     ;
 
 stringLiteralSequence
@@ -389,17 +379,16 @@ expression
 
 atomExpression
     :
-    (KW_NULL) => KW_NULL -> TOK_NULL
-    | (intervalExpression)=>intervalExpression
+    (intervalExpression)=>intervalExpression
     | (constant) => constant
     | castExpression
     | extractExpression
     | floorExpression
     | caseExpression
     | whenExpression
-    | (LPAREN KW_SELECT)=> (subQueryExpression)
+    | (subQueryExpression)=> (subQueryExpression)
         -> ^(TOK_SUBQUERY_EXPR TOK_SUBQUERY_OP subQueryExpression)
-    | (functionName LPAREN) => function
+    | (function) => function
     | tableOrColumn
     | LPAREN! expression RPAREN!
     ;
@@ -519,40 +508,35 @@ subQueryExpression
 
 precedenceEqualExpression
     :
-    (LPAREN precedenceBitwiseOrExpression COMMA) => precedenceEqualExpressionMutiple
+    (precedenceEqualExpressionSingle) => precedenceEqualExpressionSingle
     |
-    precedenceEqualExpressionSingle
+    precedenceEqualExpressionMutiple
     ;
-
+ 
 precedenceEqualExpressionSingle
     :
     (left=precedenceBitwiseOrExpression -> $left)
     (
-       (KW_NOT precedenceEqualNegatableOperator notExpr=precedenceBitwiseOrExpression)
+      (KW_NOT precedenceEqualNegatableOperator notExpr=precedenceBitwiseOrExpression)
        -> ^(KW_NOT ^(precedenceEqualNegatableOperator $precedenceEqualExpressionSingle
$notExpr))
     | (precedenceEqualOperator equalExpr=precedenceBitwiseOrExpression)
        -> ^(precedenceEqualOperator $precedenceEqualExpressionSingle $equalExpr)
     | (KW_NOT KW_IN LPAREN KW_SELECT)=>  (KW_NOT KW_IN subQueryExpression)
        -> ^(KW_NOT ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpressionSingle))
-    | (KW_NOT KW_IN expressions)
-       -> ^(KW_NOT ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressions))
-    | (KW_IN LPAREN KW_SELECT)=>  (KW_IN subQueryExpression) 
+    | (KW_NOT KW_IN expressionsInParenthese)
+       -> ^(KW_NOT ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressionsInParenthese))
+    | (KW_IN LPAREN KW_SELECT)=>  (KW_IN subQueryExpression)
        -> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpressionSingle)
-    | (KW_IN expressions)
-       -> ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressions)
+    | (KW_IN expressionsInParenthese)
+       -> ^(TOK_FUNCTION KW_IN $precedenceEqualExpressionSingle expressionsInParenthese)
     | ( KW_NOT KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression)
)
-       -> ^(TOK_FUNCTION Identifier["between"] KW_TRUE $left $min $max)
+       -> ^(TOK_FUNCTION Identifier["between"] KW_TRUE $precedenceEqualExpressionSingle
$min $max)
     | ( KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression)
)
-       -> ^(TOK_FUNCTION Identifier["between"] KW_FALSE $left $min $max)
+       -> ^(TOK_FUNCTION Identifier["between"] KW_FALSE $precedenceEqualExpressionSingle
$min $max)
     )*
-    | (KW_EXISTS LPAREN KW_SELECT)=> (KW_EXISTS subQueryExpression) 
-	-> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_EXISTS) subQueryExpression)
+    | KW_EXISTS subQueryExpression -> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_EXISTS)
subQueryExpression)    
     ;
 
-expressions
-    :
-    LPAREN expression (COMMA expression)* RPAREN -> expression+
-    ;
 
 //we transform the (col0, col1) in ((v00,v01),(v10,v11)) into struct(col0, col1) in (struct(v00,v01),struct(v10,v11))
 precedenceEqualExpressionMutiple
@@ -770,5 +754,5 @@ nonReserved
 //The following SQL2011 reserved keywords are used as function name only, but not as identifiers.
 sql11ReservedKeywordsUsedAsFunctionName
     :
-    KW_BIGINT | KW_BINARY | KW_BOOLEAN | KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP | KW_DATE
| KW_DOUBLE | KW_FLOAT | KW_GROUPING | KW_INT | KW_SMALLINT | KW_TIMESTAMP
+    KW_IF | KW_ARRAY | KW_MAP | KW_BIGINT | KW_BINARY | KW_BOOLEAN | KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP
| KW_DATE | KW_DOUBLE | KW_FLOAT | KW_GROUPING | KW_INT | KW_SMALLINT | KW_TIMESTAMP
     ;

http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out b/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out
index 90f9356..2d08fb3 100644
--- a/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out
+++ b/ql/src/test/results/clientnegative/char_pad_convert_fail2.q.out
@@ -40,4 +40,4 @@ POSTHOOK: query: load data local inpath '../../data/files/over1k' into table
ove
 POSTHOOK: type: LOAD
 #### A masked pattern was here ####
 POSTHOOK: Output: default@over1k
-FAILED: ParseException line 7:12 cannot recognize input near '{' '"key1"' ':' in function
specification
+FAILED: ParseException line 7:7 cannot recognize input near 'lpad' '(' '{' in expression
specification

http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/invalid_select_expression.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/invalid_select_expression.q.out b/ql/src/test/results/clientnegative/invalid_select_expression.q.out
index 63a682a..3650ca7 100644
--- a/ql/src/test/results/clientnegative/invalid_select_expression.q.out
+++ b/ql/src/test/results/clientnegative/invalid_select_expression.q.out
@@ -1 +1 @@
-FAILED: ParseException line 1:32 cannot recognize input near '.' 'foo' '<EOF>' in expression
specification
+FAILED: ParseException line 1:26 cannot recognize input near 'foo' '>' '.' in expression
specification

http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out b/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
index cb7f154..a0f3fe2 100644
--- a/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
+++ b/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
@@ -1,2 +1 @@
-FAILED: ParseException line 3:46 missing ) at 'order' near 'p_mfgr'
-line 3:61 missing EOF at ')' near 'p_mfgr'
+FAILED: ParseException line 3:0 cannot recognize input near 'sum' '(' 'p_retailprice' in
expression specification

http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out b/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
index 5daf86c..a0f3fe2 100644
--- a/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
+++ b/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
@@ -1,2 +1 @@
-FAILED: ParseException line 3:45 missing ) at 'sort' near 'p_mfgr'
-line 3:59 missing EOF at ')' near 'p_mfgr'
+FAILED: ParseException line 3:0 cannot recognize input near 'sum' '(' 'p_retailprice' in
expression specification

http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out b/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
index c76feee..3808f2c 100644
--- a/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
+++ b/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
@@ -1 +1 @@
-FAILED: ParseException line 2:44 mismatched input 'following' expecting KW_PRECEDING near
'unbounded' in windowframestartboundary
+FAILED: ParseException line 2:4 cannot recognize input near 'sum' '(' 'p_retailprice' in
expression specification

http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out b/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
index 9ed8be5..3808f2c 100644
--- a/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
+++ b/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
@@ -1 +1 @@
-FAILED: ParseException line 2:45 mismatched input 'following' expecting KW_PRECEDING near
'unbounded' in windowframestartboundary
+FAILED: ParseException line 2:4 cannot recognize input near 'sum' '(' 'p_retailprice' in
expression specification

http://git-wip-us.apache.org/repos/asf/hive/blob/92f31d07/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
index 26ee6f4..fb8e3ad 100644
--- a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
+++ b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
@@ -1301,7 +1301,7 @@ STAGE PLANS:
         Reducer 2 <- Map 1 (SIMPLE_EDGE)
         Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
         Reducer 5 <- Map 4 (SIMPLE_EDGE)
-        Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+        Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
 #### A masked pattern was here ####
       Vertices:
         Map 1 


Mime
View raw message