hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1650565 [3/3] - in /hive/branches/spark: ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/ ql/src/test/results/clientpositive/...
Date Fri, 09 Jan 2015 15:30:43 GMT
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out Fri Jan
 9 15:30:41 2015
@@ -199,8 +199,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
-        Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT,
3)
+        Reducer 2 <- Map 1 (GROUP, 2)
+        Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT,
2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out Fri
Jan  9 15:30:41 2015
@@ -153,7 +153,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 1)
         Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:
@@ -274,7 +274,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 1)
         Reducer 3 <- Reducer 2 (GROUP, 1)
 #### A masked pattern was here ####
       Vertices:

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_join1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_join1.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_join1.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_join1.q.out Fri
Jan  9 15:30:41 2015
@@ -40,7 +40,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -136,7 +136,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -234,7 +234,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out Fri Jan  9
15:30:41 2015
@@ -66,8 +66,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 4 <- Map 3 (GROUP, 3), Map 5 (GROUP, 3)
-        Reducer 2 <- Map 1 (GROUP, 3), Reducer 4 (GROUP, 3)
+        Reducer 4 <- Map 3 (GROUP, 2), Map 5 (GROUP, 2)
+        Reducer 2 <- Map 1 (GROUP, 2), Reducer 4 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out Fri Jan  9
15:30:41 2015
@@ -187,8 +187,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out Fri
Jan  9 15:30:41 2015
@@ -261,8 +261,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
-        Reducer 4 <- Map 3 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
+        Reducer 4 <- Map 3 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -430,8 +430,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
-        Reducer 4 <- Map 3 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
+        Reducer 4 <- Map 3 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
(original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
Fri Jan  9 15:30:41 2015
@@ -119,7 +119,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
         Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
(original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
Fri Jan  9 15:30:41 2015
@@ -44,7 +44,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -149,7 +149,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out Fri
Jan  9 15:30:41 2015
@@ -119,7 +119,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out Fri
Jan  9 15:30:41 2015
@@ -119,7 +119,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
(original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
Fri Jan  9 15:30:41 2015
@@ -231,7 +231,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out Fri
Jan  9 15:30:41 2015
@@ -115,7 +115,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
         Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out
(original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out
Fri Jan  9 15:30:41 2015
@@ -272,12 +272,14 @@ PREHOOK: query: EXPLAIN
 SELECT CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt)
AS STRING)) AS `field`
     FROM vectortab2korc 
     GROUP BY CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS
STRING)),'-'),CAST(YEAR(dt) AS STRING))
+    ORDER BY `field`
     LIMIT 50
 PREHOOK: type: QUERY
 POSTHOOK: query: EXPLAIN
 SELECT CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt)
AS STRING)) AS `field`
     FROM vectortab2korc 
     GROUP BY CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS
STRING)),'-'),CAST(YEAR(dt) AS STRING))
+    ORDER BY `field`
     LIMIT 50
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
@@ -288,7 +290,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
+        Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -318,20 +321,27 @@ STAGE PLANS:
                 mode: mergepartial
                 outputColumnNames: _col0
                 Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column
stats: NONE
-                Select Operator
-                  expressions: _col0 (type: string)
-                  outputColumnNames: _col0
+                Reduce Output Operator
+                  key expressions: _col0 (type: string)
+                  sort order: +
                   Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column
stats: NONE
-                  Limit
-                    Number of rows: 50
+            Execution mode: vectorized
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0
+                Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column
stats: NONE
+                Limit
+                  Number of rows: 50
+                  Statistics: Num rows: 50 Data size: 22950 Basic stats: COMPLETE Column
stats: NONE
+                  File Output Operator
+                    compressed: false
                     Statistics: Num rows: 50 Data size: 22950 Basic stats: COMPLETE Column
stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 50 Data size: 22950 Basic stats: COMPLETE Column
stats: NONE
-                      table:
-                          input format: org.apache.hadoop.mapred.TextInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
             Execution mode: vectorized
 
   Stage: Stage-0
@@ -343,6 +353,7 @@ STAGE PLANS:
 PREHOOK: query: SELECT CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1
AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) AS `field`
     FROM vectortab2korc 
     GROUP BY CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS
STRING)),'-'),CAST(YEAR(dt) AS STRING))
+    ORDER BY `field`
     LIMIT 50
 PREHOOK: type: QUERY
 PREHOOK: Input: default@vectortab2korc
@@ -350,57 +361,58 @@ PREHOOK: Input: default@vectortab2korc
 POSTHOOK: query: SELECT CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1
AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) AS `field`
     FROM vectortab2korc 
     GROUP BY CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS
STRING)),'-'),CAST(YEAR(dt) AS STRING))
+    ORDER BY `field`
     LIMIT 50
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@vectortab2korc
 #### A masked pattern was here ####
-Quarter 3-2051
-Quarter 3-2045
-Quarter 4-2095
-Quarter 1-2020
-Quarter 3-2042
-Quarter 2-1971
-Quarter 1-2044
-Quarter 4-1987
-Quarter 1-2047
-Quarter 1-2101
-Quarter 4-2080
-Quarter 3-2087
-Quarter 4-2047
-Quarter 2-2061
-Quarter 2-2103
+NULL
+Quarter 1-1970
+Quarter 1-1971
+Quarter 1-1972
+Quarter 1-1973
+Quarter 1-1974
+Quarter 1-1975
+Quarter 1-1976
+Quarter 1-1977
 Quarter 1-1978
-Quarter 4-1984
-Quarter 3-1982
-Quarter 2-2001
-Quarter 3-2024
-Quarter 2-2076
-Quarter 1-2074
-Quarter 3-2105
-Quarter 4-1999
-Quarter 3-2072
-Quarter 3-2000
-Quarter 2-2040
-Quarter 4-1972
-Quarter 4-2041
-Quarter 1-2026
-Quarter 3-2015
-Quarter 1-2041
-Quarter 3-1979
-Quarter 3-1970
-Quarter 3-2066
-Quarter 1-2077
-Quarter 4-2059
-Quarter 1-2086
-Quarter 3-2030
-Quarter 2-2067
-Quarter 4-2065
-Quarter 3-2021
-Quarter 3-2012
-Quarter 2-2037
+Quarter 1-1979
+Quarter 1-1980
+Quarter 1-1981
+Quarter 1-1982
+Quarter 1-1983
+Quarter 1-1984
+Quarter 1-1985
+Quarter 1-1986
 Quarter 1-1987
-Quarter 4-2014
-Quarter 4-2038
-Quarter 4-1975
-Quarter 1-2053
-Quarter 4-2068
+Quarter 1-1988
+Quarter 1-1989
+Quarter 1-1990
+Quarter 1-1991
+Quarter 1-1992
+Quarter 1-1993
+Quarter 1-1994
+Quarter 1-1995
+Quarter 1-1996
+Quarter 1-1997
+Quarter 1-1998
+Quarter 1-1999
+Quarter 1-2000
+Quarter 1-2001
+Quarter 1-2002
+Quarter 1-2003
+Quarter 1-2004
+Quarter 1-2005
+Quarter 1-2006
+Quarter 1-2007
+Quarter 1-2008
+Quarter 1-2009
+Quarter 1-2010
+Quarter 1-2011
+Quarter 1-2012
+Quarter 1-2013
+Quarter 1-2014
+Quarter 1-2015
+Quarter 1-2016
+Quarter 1-2017
+Quarter 1-2018

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out Fri
Jan  9 15:30:41 2015
@@ -326,7 +326,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
         Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
(original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
Fri Jan  9 15:30:41 2015
@@ -1866,7 +1866,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
         Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:
@@ -2074,7 +2074,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
         Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:
@@ -2329,7 +2329,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
         Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:
@@ -2662,7 +2662,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
         Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out Fri
Jan  9 15:30:41 2015
@@ -228,8 +228,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -513,9 +513,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -831,7 +831,7 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -1087,8 +1087,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -1375,8 +1375,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -1672,9 +1672,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (GROUP, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (GROUP, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -1937,8 +1937,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT,
3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT,
2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -2250,8 +2250,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 3)
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Reducer 4 (PARTITION-LEVEL SORT,
3)
+        Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT,
2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -2572,8 +2572,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -2849,8 +2849,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -3130,8 +3130,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -3418,9 +3418,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -3745,8 +3745,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -4130,9 +4130,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT,
3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT,
2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -4457,8 +4457,8 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (GROUP, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (GROUP, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -4728,9 +4728,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (GROUP, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (GROUP, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -5197,11 +5197,11 @@ STAGE PLANS:
   Stage: Stage-2
     Spark
       Edges:
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 3)
-        Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -5711,9 +5711,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -6067,10 +6067,10 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
-        Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
+        Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -6421,9 +6421,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -6770,10 +6770,10 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
-        Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
+        Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -7169,9 +7169,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -7518,9 +7518,9 @@ STAGE PLANS:
   Stage: Stage-1
     Spark
       Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
-        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 

Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java
(original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java
Fri Jan  9 15:30:41 2015
@@ -72,4 +72,9 @@ public interface SparkClient extends Ser
    * Get the count of executors.
    */
   Future<Integer> getExecutorCount();
+
+  /**
+   * Get default parallelism. For standalone mode, this can be used to get total number of
cores.
+   */
+  Future<Integer> getDefaultParallelism();
 }

Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1650565&r1=1650564&r2=1650565&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
(original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Fri Jan  9 15:30:41 2015
@@ -155,6 +155,11 @@ class SparkClientImpl implements SparkCl
     return submit(new GetExecutorCountJob());
   }
 
+  @Override
+  public Future<Integer> getDefaultParallelism() {
+    return submit(new GetDefaultParallelismJob());
+  }
+
   void cancel(String jobId) {
     protocol.cancel(jobId);
   }
@@ -489,10 +494,20 @@ class SparkClientImpl implements SparkCl
 
       @Override
       public Integer call(JobContext jc) throws Exception {
-        int count = jc.sc().sc().getExecutorMemoryStatus().size();
+        // minus 1 here otherwise driver is also counted as an executor
+        int count = jc.sc().sc().getExecutorMemoryStatus().size() - 1;
         return Integer.valueOf(count);
       }
 
   }
 
+  private static class GetDefaultParallelismJob implements Job<Integer> {
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    public Integer call(JobContext jc) throws Exception {
+      return jc.sc().sc().defaultParallelism();
+    }
+  }
+
 }



Mime
View raw message