hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1455421 - in /hive/branches/ptf-windowing/ql/src: java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/ test/results/clientpositive/
Date Tue, 12 Mar 2013 06:48:45 GMT
Author: hashutosh
Date: Tue Mar 12 06:48:45 2013
New Revision: 1455421

URL: http://svn.apache.org/r1455421
Log:
HIVE-4108 [jira] Allow over() clause to contain an order by with no partition by
(Harish Butani via Ashutosh Chauhan)

Summary:
allow over clause with only an order spec

HIVE-4073 allows over() to be called with no partition by and no order by. We should allow
only an order by.

>From the review of HIVE-4073:

Ashutosh

Can you also add following test. This should also work.

select p_name, p_retailprice,
avg(p_retailprice) over(order by p_name)
from part
partition by p_name;

Harish

This test will not work (:
The grammar needs to be changed so:

partitioningSpec
@init { msgs.push("partitioningSpec clause"); }
@after { msgs.pop(); }
:
partitionByClause orderByClause? -> ^(TOK_PARTITIONINGSPEC partitionByClause orderByClause?)
|
orderByClause -> ^(TOK_PARTITIONINGSPEC orderByClause) |
distributeByClause sortByClause? -> ^(TOK_PARTITIONINGSPEC distributeByClause sortByClause?)
|
sortByClause? -> ^(TOK_PARTITIONINGSPEC sortByClause) |
clusterByClause -> ^(TOK_PARTITIONINGSPEC clusterByClause)
;

And the SemanticAnalyzer::processPTFPartitionSpec has to handle this shape of the AST Tree.
The PTFTranslator also needs changes. Do this as another Jira

Test Plan: added new tests with over clause containing only order spec.

Reviewers: JIRA, ashutoshc

Reviewed By: ashutoshc

Differential Revision: https://reviews.facebook.net/D9309

Modified:
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_npath.q
    hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_windowspec.q
    hive/branches/ptf-windowing/ql/src/test/results/clientpositive/ptf_npath.q.out
    hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_windowspec.q.out

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1455421&r1=1455420&r2=1455421&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
(original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
Tue Mar 12 06:48:45 2013
@@ -208,7 +208,9 @@ partitioningSpec
 @after { gParent.msgs.pop(); } 
    :
    partitionByClause orderByClause? -> ^(TOK_PARTITIONINGSPEC partitionByClause orderByClause?)
|
+   orderByClause -> ^(TOK_PARTITIONINGSPEC orderByClause) |
    distributeByClause sortByClause? -> ^(TOK_PARTITIONINGSPEC distributeByClause sortByClause?)
|
+   sortByClause -> ^(TOK_PARTITIONINGSPEC sortByClause) |
    clusterByClause -> ^(TOK_PARTITIONINGSPEC clusterByClause)
    ;
 

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1455421&r1=1455420&r2=1455421&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
(original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
Tue Mar 12 06:48:45 2013
@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Stack;
 
+import org.antlr.runtime.CommonToken;
 import org.antlr.runtime.tree.TreeWizard;
 import org.antlr.runtime.tree.TreeWizard.ContextVisitor;
 import org.apache.commons.logging.Log;
@@ -471,6 +472,7 @@ public class PTFTranslator {
       PartitionedTableFunctionSpec spec)
       throws SemanticException {
 
+    applyConstantPartition(spec);
     if ( spec.getPartition() == null ) {
       return;
     }
@@ -480,6 +482,24 @@ public class PTFTranslator {
     def.setOrder(orderDef);
   }
 
+  /*
+   * If this the first PPTF in the chain and there is no partition specified
+   * then assume the user wants to include the entire input in 1 partition.
+   */
+  private static void applyConstantPartition( PartitionedTableFunctionSpec spec) {
+    if ( spec.getPartition() != null ) {
+      return;
+    }
+    PTFInputSpec iSpec = spec.getInput();
+    if ( iSpec instanceof PTFInputSpec ) {
+        PartitionSpec partSpec = new PartitionSpec();
+        PartitionExpression partExpr = new PartitionExpression();
+        partExpr.setExpression(new ASTNode(new CommonToken(HiveParser.Number, "0")));
+        partSpec.addExpression(partExpr);
+        spec.setPartition(partSpec);
+    }
+  }
+
   private PartitionDef translate(ShapeDetails inpShape, PartitionSpec spec)
       throws SemanticException
   {
@@ -1310,6 +1330,7 @@ public class PTFTranslator {
     }
 
     PartitionedTableFunctionSpec prevFn = (PartitionedTableFunctionSpec) ptfChain.pop();
+    applyConstantPartition(prevFn);
     PartitionSpec partSpec = prevFn.getPartition();
     OrderSpec orderSpec = prevFn.getOrder();
 

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1455421&r1=1455420&r2=1455421&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Tue Mar 12 06:48:45 2013
@@ -9831,28 +9831,9 @@ public class SemanticAnalyzer extends Ba
 
     PTFInvocationSpec spec = new PTFInvocationSpec();
     spec.setFunction(ptfSpec);
-    ensurePTFChainHasPartitioning(spec, ptf);
     qb.addPTFNodeToSpec(ptf, spec);
   }
 
-  /*
-   * ensure that the PTF chain has a partitioning specification associated. This method
-   * should be called when a PTF chain is encountered as a fromSource.
-   * (from the processPTF method)
-   */
-  private void ensurePTFChainHasPartitioning(PTFInvocationSpec qSpec, ASTNode node)
-      throws SemanticException {
-    if(qSpec == null){
-      return;
-    }
-    PartitionedTableFunctionSpec ptfSpec = qSpec.getStartOfChain();
-    PartitionSpec pSpec = ptfSpec.getPartition();
-    if ( pSpec == null ) {
-      throw new SemanticException(generateErrorMessage(node,
-                  "No partition specification associated with start of PTF chain "));
-    }
-   }
-
 //--------------------------- Windowing handling -----------------------------------
 
   /*
@@ -10113,11 +10094,11 @@ public class SemanticAnalyzer extends Ba
     int type = firstChild.getType();
     int exprCnt;
 
-    PartitionSpec pSpec = processPartitionSpec(firstChild);
-    partitioning.setPartSpec(pSpec);
 
-    if ( type == HiveParser.TOK_DISTRIBUTEBY )
+    if ( type == HiveParser.TOK_DISTRIBUTEBY || type == HiveParser.TOK_CLUSTERBY )
     {
+      PartitionSpec pSpec = processPartitionSpec(firstChild);
+      partitioning.setPartSpec(pSpec);
       ASTNode sortNode = pSpecNode.getChildCount() > 1 ? (ASTNode) pSpecNode.getChild(1)
: null;
       if ( sortNode != null )
       {
@@ -10125,6 +10106,11 @@ public class SemanticAnalyzer extends Ba
         partitioning.setOrderSpec(oSpec);
       }
     }
+    else if ( type == HiveParser.TOK_SORTBY || type == HiveParser.TOK_ORDERBY ) {
+      ASTNode sortNode = firstChild;
+      OrderSpec oSpec = processOrderSpec(sortNode);
+      partitioning.setOrderSpec(oSpec);
+    }
     return partitioning;
   }
 

Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_npath.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_npath.q?rev=1455421&r1=1455420&r2=1455421&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_npath.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/ptf_npath.q Tue Mar 12
06:48:45 2013
@@ -22,3 +22,15 @@ from npath(on 
       arg2('LATE'), arg3(arr_delay > 15), 
     arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath as
tpath') 
    );       
+
+-- 2. Npath on 1 partition
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath 
+from npath(on 
+        flights_tiny 
+        sort by year, month, day_of_month  
+      arg1('LATE.LATE+'), 
+      arg2('LATE'), arg3(arr_delay > 15), 
+    arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath as
tpath') 
+   )
+where fl_num = 1142;       
+   
\ No newline at end of file

Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_windowspec.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_windowspec.q?rev=1455421&r1=1455420&r2=1455421&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_windowspec.q
(original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_windowspec.q
Tue Mar 12 06:48:45 2013
@@ -31,3 +31,6 @@ select s, avg(f) over (partition by bo o
 
 select s, sum(i) over() from over10k limit 100;
 
+select f, sum(f) over (order by f range between unbounded preceding and current row) from
over10k limit 100;
+
+

Modified: hive/branches/ptf-windowing/ql/src/test/results/clientpositive/ptf_npath.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientpositive/ptf_npath.q.out?rev=1455421&r1=1455420&r2=1455421&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientpositive/ptf_npath.q.out (original)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientpositive/ptf_npath.q.out Tue Mar
12 06:48:45 2013
@@ -71,3 +71,32 @@ Washington	7291	2010	10	27	2	[{"origin_c
 Chicago	897	2010	10	20	4	[{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":20,"arr_delay":24.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":21,"arr_delay":77.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":22,"arr_delay":24.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":24,"arr_delay":113.0,"fl_num":"897"}]
 Chicago	897	2010	10	21	3	[{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":77.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":22,"arr_delay":24.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":24,"arr_delay":113.0,"fl_num":"897"}]
 Chicago	897	2010	10	22	2	[{"origin_city_name":"Chicago","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":24.0,"fl_num":"897"},{"origin_city_name":"Chicago","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":24,"arr_delay":113.0,"fl_num":"897"}]
+PREHOOK: query: -- 2. Npath on 1 partition
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath 
+from npath(on 
+        flights_tiny 
+        sort by year, month, day_of_month  
+      arg1('LATE.LATE+'), 
+      arg2('LATE'), arg3(arr_delay > 15), 
+    arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath as
tpath') 
+   )
+where fl_num = 1142
+PREHOOK: type: QUERY
+PREHOOK: Input: default@flights_tiny
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2. Npath on 1 partition
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath 
+from npath(on 
+        flights_tiny 
+        sort by year, month, day_of_month  
+      arg1('LATE.LATE+'), 
+      arg2('LATE'), arg3(arr_delay > 15), 
+    arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath as
tpath') 
+   )
+where fl_num = 1142
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@flights_tiny
+#### A masked pattern was here ####
+Baltimore	1142	2010	10	21	2	[{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":21,"arr_delay":105.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":21,"arr_delay":28.0,"fl_num":"1599"}]
+Baltimore	1142	2010	10	22	2	[{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":22,"arr_delay":54.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":22,"arr_delay":18.0,"fl_num":"1599"}]
+Baltimore	1142	2010	10	25	3	[{"origin_city_name":"Baltimore","dest_city_name":"New York","year":2010,"month":10,"day_of_month":25,"arr_delay":92.0,"fl_num":"1142"},{"origin_city_name":"Baltimore","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":25,"arr_delay":106.0,"fl_num":"1599"},{"origin_city_name":"Chicago","dest_city_name":"New
York","year":2010,"month":10,"day_of_month":25,"arr_delay":31.0,"fl_num":"361"}]

Modified: hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_windowspec.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_windowspec.q.out?rev=1455421&r1=1455420&r2=1455421&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_windowspec.q.out
(original)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_windowspec.q.out
Tue Mar 12 06:48:45 2013
@@ -795,3 +795,111 @@ oscar king	656584379
 calvin hernandez	656584379
 nick ichabod	656584379
 bob ovid	656584379
+PREHOOK: query: select f, sum(f) over (order by f range between unbounded preceding and current
row) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select f, sum(f) over (order by f range between unbounded preceding and
current row) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+0.01	0.009999999776482582
+0.01	0.019999999552965164
+0.02	0.03999999910593033
+0.02	0.05999999865889549
+0.02	0.07999999821186066
+0.03	0.1099999975413084
+0.03	0.13999999687075615
+0.04	0.17999999597668648
+0.05	0.22999999672174454
+0.07	0.29999999701976776
+0.07	0.369999997317791
+0.08	0.44999999552965164
+0.08	0.5299999937415123
+0.08	0.609999991953373
+0.09	0.6999999955296516
+0.09	0.7899999991059303
+0.1	0.8900000005960464
+0.11	1.0
+0.12	1.119999997317791
+0.12	1.239999994635582
+0.13	1.3699999898672104
+0.14	1.5099999904632568
+0.14	1.6499999910593033
+0.15	1.7999999970197678
+0.15	1.9500000029802322
+0.16	2.1099999994039536
+0.16	2.269999995827675
+0.17	2.439999997615814
+0.18	2.6200000047683716
+0.19	2.810000002384186
+0.2	3.010000005364418
+0.2	3.2100000083446503
+0.23	3.4400000125169754
+0.23	3.6700000166893005
+0.23	3.9000000208616257
+0.27	4.170000031590462
+0.27	4.440000042319298
+0.27	4.710000053048134
+0.28	4.990000054240227
+0.28	5.27000005543232
+0.29	5.560000047087669
+0.31	5.870000049471855
+0.32	6.190000042319298
+0.35	6.540000036358833
+0.36	6.900000050663948
+0.36	7.260000064969063
+0.37	7.630000069737434
+0.39	8.02000005543232
+0.39	8.410000041127205
+0.42	8.830000028014183
+0.43	9.26000003516674
+0.43	9.690000042319298
+0.43	10.120000049471855
+0.44	10.56000004708767
+0.44	11.000000044703484
+0.48	11.480000033974648
+0.5	11.980000033974648
+0.5	12.480000033974648
+0.51	12.990000024437904
+0.52	13.510000005364418
+0.52	14.029999986290932
+0.52	14.549999967217445
+0.53	15.079999938607216
+0.54	15.619999960064888
+0.55	16.169999971985817
+0.56	16.729999974370003
+0.57	17.299999967217445
+0.58	17.879999950528145
+0.59	18.4699999243021
+0.6	19.06999994814396
+0.61	19.679999962449074
+0.62	20.299999967217445
+0.62	20.919999971985817
+0.62	21.53999997675419
+0.67	22.20999999344349
+0.68	22.890000000596046
+0.68	23.570000007748604
+0.69	24.260000005364418
+0.69	24.950000002980232
+0.71	25.65999998152256
+0.75	26.40999998152256
+0.76	27.169999971985817
+0.77	27.93999995291233
+0.78	28.7199999243021
+0.79	29.509999945759773
+0.79	30.299999967217445
+0.79	31.089999988675117
+0.8	31.890000000596046
+0.82	32.70999999344349
+0.83	33.53999997675419
+0.84	34.379999950528145
+0.84	35.2199999243021
+0.85	36.06999994814396
+0.87	36.93999995291233
+0.88	37.81999994814396
+0.89	38.709999933838844
+0.9	39.609999909996986
+0.91	40.51999993622303
+0.92	41.43999995291233
+0.93	42.36999996006489



Mime
View raw message