hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject svn commit: r1642026 - in /hive/branches/spark: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/test/results/clientpositive/spark/
Date Thu, 27 Nov 2014 06:05:19 GMT
Author: szehon
Date: Thu Nov 27 06:05:19 2014
New Revision: 1642026

URL: http://svn.apache.org/r1642026
Log:
HIVE-8924 : Investigate test failure for join_empty.q [Spark Branch] (Szehon, reviewed by
Xuefu Zhang)

Modified:
    hive/branches/spark/itests/src/test/resources/testconfiguration.properties
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java
    hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
    hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out
    hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out
    hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out
    hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out

Modified: hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties Thu Nov 27
06:05:19 2014
@@ -657,6 +657,7 @@ spark.query.files=add_part_multiple.q, \
   join_cond_pushdown_unqual2.q, \
   join_cond_pushdown_unqual3.q, \
   join_cond_pushdown_unqual4.q, \
+  join_empty.q \
   join_filters.q, \
   join_filters_overlap.q, \
   join_hive_626.q, \

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java
Thu Nov 27 06:05:19 2014
@@ -21,7 +21,9 @@ package org.apache.hadoop.hive.ql.optimi
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -42,6 +44,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
 import org.apache.hadoop.hive.ql.lib.TaskGraphWalker;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.BaseWork;
+import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.SparkWork;
@@ -63,7 +66,24 @@ public class SparkMapJoinResolver implem
   // Check whether the specified BaseWork's operator tree contains a operator
   // of the specified operator class
   private boolean containsOp(BaseWork work, Class<?> clazz) {
-    for (Operator<? extends OperatorDesc> op : work.getAllOperators()) {
+    Set<Operator<? extends OperatorDesc>> ops = new HashSet<Operator<?
extends OperatorDesc>>();
+    if (work instanceof MapWork) {
+      Collection<Operator<?>> opSet = ((MapWork) work).getAliasToWork().values();
+      Stack<Operator<?>> opStack = new Stack<Operator<?>>();
+      opStack.addAll(opSet);
+
+      while (!opStack.empty()) {
+        Operator<?> op = opStack.pop();
+        ops.add(op);
+        if (op.getChildOperators() != null) {
+          opStack.addAll(op.getChildOperators());
+        }
+      }
+    } else {
+      ops.addAll(work.getAllOperators());
+    }
+
+    for (Operator<? extends OperatorDesc> op : ops) {
       if (clazz.isInstance(op)) {
         return true;
       }

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
Files hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out (original)
and hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out Thu
Nov 27 06:05:19 2014 differ

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
Files hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out (original)
and hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out Thu
Nov 27 06:05:19 2014 differ

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out Thu Nov 27
06:05:19 2014
@@ -43,18 +43,26 @@ PREHOOK: type: QUERY
 POSTHOOK: query: explain select * from v where ds='2011-09-01'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-2 is a root stage
+  Stage-1 depends on stages: Stage-2
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-2
     Spark
-      Edges:
-        Map 2 <- Map 1 (NONE, 0)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+#### A masked pattern was here ####
+      Vertices:
         Map 2 
+            Local Work:
+              Map Reduce Local Work
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
Files hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out
(original) and hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out
Thu Nov 27 06:05:19 2014 differ

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out Thu Nov
27 06:05:19 2014
@@ -106,18 +106,26 @@ TOK_QUERY
 
 
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-2 is a root stage
+  Stage-1 depends on stages: Stage-2
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-2
     Spark
-      Edges:
-        Map 2 <- Map 1 (NONE, 0)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+#### A masked pattern was here ####
+      Vertices:
         Map 2 
+            Local Work:
+              Map Reduce Local Work
 
   Stage: Stage-0
     Fetch Operator
@@ -231,18 +239,26 @@ TOK_QUERY
 
 
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-2 is a root stage
+  Stage-1 depends on stages: Stage-2
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-2
     Spark
-      Edges:
-        Map 2 <- Map 1 (NONE, 0)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+#### A masked pattern was here ####
+      Vertices:
         Map 2 
+            Local Work:
+              Map Reduce Local Work
 
   Stage: Stage-0
     Fetch Operator



Mime
View raw message