hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1476829 - in /hive/branches/branch-0.11/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/plan/ test/queries/clientpositive/ test/results/clientpositive/
Date Sun, 28 Apr 2013 20:34:39 GMT
Author: hashutosh
Date: Sun Apr 28 20:34:38 2013
New Revision: 1476829

URL: http://svn.apache.org/r1476829
Log:
HIVE-4358 : Check for Map side processing in PTFOp is no longer valid (Harish Butani via Ashutosh
Chauhan)

Modified:
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
    hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q
    hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
(original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
Sun Apr 28 20:34:38 2013
@@ -72,14 +72,7 @@ public class PTFOperator extends Operato
 		hiveConf = new HiveConf(jobConf, PTFOperator.class);
 		// if the parent is ExtractOperator, this invocation is from reduce-side
 		Operator<? extends OperatorDesc> parentOp = getParentOperators().get(0);
-		if (parentOp instanceof ExtractOperator)
-		{
-			isMapOperator = false;
-		}
-		else
-		{
-			isMapOperator = true;
-		}
+		isMapOperator = conf.isMapSide();
 
 		reconstructQueryDef(hiveConf);
     inputPart = createFirstPartitionForChain(

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Sun Apr 28 20:34:38 2013
@@ -10631,6 +10631,7 @@ public class SemanticAnalyzer extends Ba
       {
         RowResolver ptfMapRR = tabDef.getRawInputShape().getRr();
 
+        ptfDesc.setMapSide(true);
         input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc,
             new RowSchema(ptfMapRR.getColumnInfos()),
             input), ptfMapRR);

Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java Sun
Apr 28 20:34:38 2013
@@ -46,6 +46,10 @@ public class PTFDesc extends AbstractOpe
 
   PartitionedTableFunctionDef funcDef;
   LeadLagInfo llInfo;
+  /*
+   * is this PTFDesc for a Map-Side PTF Operation?
+   */
+  boolean isMapSide = false;
 
   static{
     PTFUtils.makeTransient(PTFDesc.class, "llInfo");
@@ -75,6 +79,14 @@ public class PTFDesc extends AbstractOpe
     return funcDef != null && (funcDef instanceof WindowTableFunctionDef);
   }
 
+  public boolean isMapSide() {
+    return isMapSide;
+  }
+
+  public void setMapSide(boolean isMapSide) {
+    this.isMapSide = isMapSide;
+  }
+
   public abstract static class PTFInputDef {
     String expressionTreeString;
     ShapeDetails outputShape;

Modified: hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q (original)
+++ hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q Sun Apr 28 20:34:38
2013
@@ -224,6 +224,11 @@ window w1 as (distribute by p_mfgr sort 
 ) sq
 order by p_mfgr, p_brand;
 
+select p_mfgr, p_brand, s, 
+round(sum(s),2) over w1  as s1
+from mfgr_price_view 
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row);
+
 -- 23. testCreateViewWithWindowingQuery
 create view IF NOT EXISTS mfgr_brand_price_view as 
 select p_mfgr, p_brand, 

Modified: hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out (original)
+++ hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out Sun Apr 28
20:34:38 2013
@@ -1066,6 +1066,38 @@ Manufacturer#4	Brand#42	2581.68	7337.62
 Manufacturer#5	Brand#51	1611.66	7672.66
 Manufacturer#5	Brand#52	3254.17	7672.66
 Manufacturer#5	Brand#53	2806.83	7672.66
+PREHOOK: query: select p_mfgr, p_brand, s, 
+round(sum(s),2) over w1  as s1
+from mfgr_price_view 
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mfgr_price_view
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_brand, s, 
+round(sum(s),2) over w1  as s1
+from mfgr_price_view 
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mfgr_price_view
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1	Brand#12	4800.84	4800.84
+Manufacturer#1	Brand#14	2346.3	7147.14
+Manufacturer#1	Brand#15	1602.59	8749.73
+Manufacturer#2	Brand#22	3491.38	3491.38
+Manufacturer#2	Brand#23	2031.98	5523.36
+Manufacturer#2	Brand#24	1698.66	7222.02
+Manufacturer#2	Brand#25	1701.6	5432.24
+Manufacturer#3	Brand#31	1671.68	1671.68
+Manufacturer#3	Brand#32	3333.37	5005.05
+Manufacturer#3	Brand#34	1337.29	6342.34
+Manufacturer#3	Brand#35	1190.27	5860.93
+Manufacturer#4	Brand#41	4755.94	4755.94
+Manufacturer#4	Brand#42	2581.68	7337.62
+Manufacturer#5	Brand#51	1611.66	1611.66
+Manufacturer#5	Brand#52	3254.17	4865.83
+Manufacturer#5	Brand#53	2806.83	7672.66
 PREHOOK: query: -- 23. testCreateViewWithWindowingQuery
 create view IF NOT EXISTS mfgr_brand_price_view as 
 select p_mfgr, p_brand, 



Mime
View raw message