hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1641640 - in /hive/branches/spark/ql/src: java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java test/results/clientpositive/spark/limit_partition_metadataonly.q.out
Date Tue, 25 Nov 2014 15:44:24 GMT
Author: xuefu
Date: Tue Nov 25 15:44:24 2014
New Revision: 1641640

URL: http://svn.apache.org/r1641640
Log:
HIVE-8788: UT: fix partition test case [Spark Branch] (Chinna via Xuefu)

Added:
    hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
Modified:
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java?rev=1641640&r1=1641639&r2=1641640&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java Tue
Nov 25 15:44:24 2014
@@ -64,6 +64,8 @@ import org.apache.hadoop.hive.ql.plan.Ba
 import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
 import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
 import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.plan.SparkWork;
 import org.apache.hadoop.hive.ql.plan.StatsWork;
 import org.apache.hadoop.hive.ql.plan.UnionWork;
@@ -214,6 +216,20 @@ public class SparkTask extends Task<Spar
     return result;
   }
 
+  @Override
+  public Operator<? extends OperatorDesc> getReducer(MapWork mapWork) {
+    List<BaseWork> children = getWork().getChildren(mapWork);
+    if (children.size() != 1) {
+      return null;
+    }
+
+    if (!(children.get(0) instanceof ReduceWork)) {
+      return null;
+    }
+
+    return ((ReduceWork) children.get(0)).getReducer();
+  }
+
   public SparkCounters getSparkCounters() {
     return sparkCounters;
   }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java?rev=1641640&r1=1641639&r2=1641640&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
Tue Nov 25 15:44:24 2014
@@ -58,6 +58,7 @@ import org.apache.hadoop.hive.ql.lib.Rul
 import org.apache.hadoop.hive.ql.lib.TypeRule;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck;
+import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer;
 import org.apache.hadoop.hive.ql.optimizer.physical.NullScanOptimizer;
 import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
 import org.apache.hadoop.hive.ql.optimizer.physical.SparkMapJoinResolver;
@@ -278,6 +279,12 @@ public class SparkCompiler extends TaskC
       LOG.debug("Skipping null scan query optimization");
     }
 
+    if (conf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) {
+      physicalCtx = new MetadataOnlyOptimizer().resolve(physicalCtx);
+    } else {
+      LOG.debug("Skipping metadata only query optimization");
+    }
+
     if (conf.getBoolVar(HiveConf.ConfVars.HIVE_CHECK_CROSS_PRODUCT)) {
       physicalCtx = new CrossProductCheck().resolve(physicalCtx);
     } else {

Added: hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out?rev=1641640&view=auto
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
(added)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
Tue Nov 25 15:44:24 2014
@@ -0,0 +1,608 @@
+PREHOOK: query: explain select ds from srcpart where hr=11 and ds='2008-04-08'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select ds from srcpart where hr=11 and ds='2008-04-08'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: srcpart
+          Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+          Select Operator
+            expressions: '2008-04-08' (type: string)
+            outputColumnNames: _col0
+            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+            ListSink
+
+PREHOOK: query: select ds from srcpart where hr=11 and ds='2008-04-08'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select ds from srcpart where hr=11 and ds='2008-04-08'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+PREHOOK: query: explain select distinct hr from srcpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select distinct hr from srcpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: srcpart
+                  Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    expressions: hr (type: string)
+                    outputColumnNames: hr
+                    Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column
stats: NONE
+                    Group By Operator
+                      keys: hr (type: string)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column
stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE
Column stats: NONE
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                Select Operator
+                  expressions: _col0 (type: string)
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select distinct hr from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct hr from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+11
+12



Mime
View raw message