hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pxi...@apache.org
Subject [2/2] hive git commit: HIVE-11387: CBO: Calcite Operator To Hive Operator (Calcite Return Path) : fix reduce_deduplicate optimization (reviewed by Jesus Camacho Rodriguez, Hari Subramaniyan)
Date Tue, 11 Aug 2015 18:15:36 GMT
HIVE-11387: CBO: Calcite Operator To Hive Operator (Calcite Return Path) : fix reduce_deduplicate optimization (reviewed by Jesus Camacho Rodriguez, Hari Subramaniyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a851d030
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a851d030
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a851d030

Branch: refs/heads/branch-1
Commit: a851d0308c93bc76cf67abf898fb9f26a1c59a70
Parents: be21c94
Author: Pengcheng Xiong <pxiong@apache.org>
Authored: Tue Aug 11 11:15:09 2015 -0700
Committer: Pengcheng Xiong <pxiong@apache.org>
Committed: Tue Aug 11 11:15:09 2015 -0700

----------------------------------------------------------------------
 .../correlation/AbstractCorrelationProcCtx.java |   7 +
 .../correlation/CorrelationUtilities.java       |  11 +-
 .../correlation/ReduceSinkDeDuplication.java    |   6 +-
 ...i_insert_move_tasks_share_dependencies.q.out | 336 +++---------
 ql/src/test/results/clientpositive/ptf.q.out    |  27 +-
 ...i_insert_move_tasks_share_dependencies.q.out | 512 +++++++------------
 .../test/results/clientpositive/spark/ptf.q.out |  17 +-
 .../spark/union_remove_6_subq.q.out             |  22 +-
 .../clientpositive/spark/vectorized_ptf.q.out   |  21 +-
 .../clientpositive/tez/explainuser_1.q.out      |  67 +--
 .../test/results/clientpositive/tez/ptf.q.out   |  15 +-
 .../clientpositive/tez/vectorized_ptf.q.out     |  19 +-
 .../clientpositive/union_remove_6_subq.q.out    |  34 +-
 .../results/clientpositive/vectorized_ptf.q.out |  67 +--
 14 files changed, 326 insertions(+), 835 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a851d030/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/AbstractCorrelationProcCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/AbstractCorrelationProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/AbstractCorrelationProcCtx.java
index 174685b..5b673df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/AbstractCorrelationProcCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/AbstractCorrelationProcCtx.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.optimizer.correlation;
 
 import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATIONMINREDUCER;
 import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESCRIPTOPERATORTRUST;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVEMAPSIDEAGGREGATE;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -39,11 +40,13 @@ abstract class AbstractCorrelationProcCtx implements NodeProcessorCtx {
   // only one reducer if this configuration does not prevents
   private final int minReducer;
   private final Set<Operator<?>> removedOps;
+  private final boolean isMapAggr;
 
   public AbstractCorrelationProcCtx(ParseContext pctx) {
     removedOps = new HashSet<Operator<?>>();
     trustScript = pctx.getConf().getBoolVar(HIVESCRIPTOPERATORTRUST);
     minReducer = pctx.getConf().getIntVar(HIVEOPTREDUCEDEDUPLICATIONMINREDUCER);
+    isMapAggr = pctx.getConf().getBoolVar(HIVEMAPSIDEAGGREGATE);
     this.pctx = pctx;
   }
 
@@ -70,4 +73,8 @@ abstract class AbstractCorrelationProcCtx implements NodeProcessorCtx {
   public boolean addRemovedOperator(Operator<?> rsOp) {
     return removedOps.add(rsOp);
   }
+
+  public boolean isMapAggr() {
+    return isMapAggr;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a851d030/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java
index 64bef21..7bb49be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java
@@ -29,6 +29,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.ForwardOperator;
@@ -44,6 +45,7 @@ import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.Utilities.ReduceField;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication.ReduceSinkDeduplicateProcCtx;
 import org.apache.hadoop.hive.ql.parse.ParseContext;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.AggregationDesc;
@@ -163,10 +165,10 @@ public final class CorrelationUtilities {
     return type.isInstance(parent) ? (T)parent : null;
   }
 
-  protected static Operator<?> getStartForGroupBy(ReduceSinkOperator cRS)
+  protected static Operator<?> getStartForGroupBy(ReduceSinkOperator cRS, ReduceSinkDeduplicateProcCtx dedupCtx)
       throws SemanticException {
     Operator<? extends Serializable> parent = getSingleParent(cRS);
-    return parent instanceof GroupByOperator ? parent : cRS;  // skip map-aggr GBY
+    return parent instanceof GroupByOperator && dedupCtx.isMapAggr() ? parent : cRS;  // skip map-aggr GBY
   }
 
 
@@ -240,6 +242,7 @@ public final class CorrelationUtilities {
           || cursor instanceof FilterOperator
           || cursor instanceof ForwardOperator
           || cursor instanceof ScriptOperator
+          || cursor instanceof GroupByOperator
           || cursor instanceof ReduceSinkOperator)) {
         return null;
       }
@@ -395,7 +398,7 @@ public final class CorrelationUtilities {
 
     Operator<?> parent = getSingleParent(cRS);
 
-    if (parent instanceof GroupByOperator) {
+    if ((parent instanceof GroupByOperator) && procCtx.isMapAggr()) {
       // pRS-cGBYm-cRS-cGBYr (map aggregation) --> pRS-cGBYr(COMPLETE)
       // copies desc of cGBYm to cGBYr and remove cGBYm and cRS
       GroupByOperator cGBYm = (GroupByOperator) parent;
@@ -440,7 +443,7 @@ public final class CorrelationUtilities {
     removeOperator(cRS, cGBYr, parent, context);
     procCtx.addRemovedOperator(cRS);
 
-    if (parent instanceof GroupByOperator) {
+    if ((parent instanceof GroupByOperator) && procCtx.isMapAggr()) {
       removeOperator(parent, cGBYr, getSingleParent(parent), context);
       procCtx.addRemovedOperator(cGBYr);
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/a851d030/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
index 8ac9ca7..30ebd5b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
@@ -500,7 +500,7 @@ public class ReduceSinkDeDuplication implements Transform {
     public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY,
         ReduceSinkDeduplicateProcCtx dedupCtx)
         throws SemanticException {
-      Operator<?> start = CorrelationUtilities.getStartForGroupBy(cRS);
+      Operator<?> start = CorrelationUtilities.getStartForGroupBy(cRS, dedupCtx);
       GroupByOperator pGBY =
           CorrelationUtilities.findPossibleParent(
               start, GroupByOperator.class, dedupCtx.trustScript());
@@ -547,7 +547,7 @@ public class ReduceSinkDeDuplication implements Transform {
     public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY,
         ReduceSinkDeduplicateProcCtx dedupCtx)
         throws SemanticException {
-      Operator<?> start = CorrelationUtilities.getStartForGroupBy(cRS);
+      Operator<?> start = CorrelationUtilities.getStartForGroupBy(cRS, dedupCtx);
       JoinOperator pJoin =
           CorrelationUtilities.findPossibleParent(
               start, JoinOperator.class, dedupCtx.trustScript());
@@ -590,7 +590,7 @@ public class ReduceSinkDeDuplication implements Transform {
     public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY,
         ReduceSinkDeduplicateProcCtx dedupCtx)
         throws SemanticException {
-      Operator<?> start = CorrelationUtilities.getStartForGroupBy(cRS);
+      Operator<?> start = CorrelationUtilities.getStartForGroupBy(cRS, dedupCtx);
       ReduceSinkOperator pRS =
           CorrelationUtilities.findPossibleParent(
               start, ReduceSinkOperator.class, dedupCtx.trustScript());

http://git-wip-us.apache.org/repos/asf/hive/blob/a851d030/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out b/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out
index 935ae75..81d9808 100644
--- a/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out
+++ b/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out
@@ -2821,11 +2821,10 @@ from src
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-2 is a root stage
-  Stage-3 depends on stages: Stage-2
-  Stage-4 depends on stages: Stage-3, Stage-5
-  Stage-0 depends on stages: Stage-3
-  Stage-5 depends on stages: Stage-2
-  Stage-1 depends on stages: Stage-5
+  Stage-3 depends on stages: Stage-2, Stage-4
+  Stage-0 depends on stages: Stage-2
+  Stage-4 depends on stages: Stage-2
+  Stage-1 depends on stages: Stage-4
 
 STAGE PLANS:
   Stage: Stage-2
@@ -2840,7 +2839,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: key (type: string), value (type: string)
                 sort order: ++
-                Map-reduce partition columns: key (type: string), value (type: string)
+                Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Forward
@@ -2855,10 +2854,11 @@ STAGE PLANS:
               Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
+                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
                 table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Filter Operator
             predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
             Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -2875,29 +2875,6 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-3
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col1 (type: string)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-            table:
-                input format: org.apache.hadoop.mapred.TextInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-  Stage: Stage-4
     Dependency Collection
 
   Stage: Stage-0
@@ -2906,7 +2883,7 @@ STAGE PLANS:
           hdfs directory: false
 #### A masked pattern was here ####
 
-  Stage: Stage-5
+  Stage: Stage-4
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -2957,11 +2934,10 @@ from src
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-2 is a root stage
-  Stage-3 depends on stages: Stage-2
-  Stage-4 depends on stages: Stage-3, Stage-5
-  Stage-0 depends on stages: Stage-3
-  Stage-5 depends on stages: Stage-2
-  Stage-1 depends on stages: Stage-5
+  Stage-3 depends on stages: Stage-2, Stage-4
+  Stage-0 depends on stages: Stage-2
+  Stage-4 depends on stages: Stage-2
+  Stage-1 depends on stages: Stage-4
 
 STAGE PLANS:
   Stage: Stage-2
@@ -2976,7 +2952,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: key (type: string), value (type: string)
                 sort order: ++
-                Map-reduce partition columns: key (type: string), value (type: string)
+                Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Forward
@@ -2991,10 +2967,11 @@ STAGE PLANS:
               Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
+                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
                 table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Filter Operator
             predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
             Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -3011,29 +2988,6 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-3
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col1 (type: string)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-            table:
-                input format: org.apache.hadoop.mapred.TextInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-  Stage: Stage-4
     Dependency Collection
 
   Stage: Stage-0
@@ -3042,7 +2996,7 @@ STAGE PLANS:
           hdfs directory: false
 #### A masked pattern was here ####
 
-  Stage: Stage-5
+  Stage: Stage-4
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -3093,11 +3047,10 @@ from src
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-2 is a root stage
-  Stage-3 depends on stages: Stage-2
-  Stage-4 depends on stages: Stage-3, Stage-5
-  Stage-0 depends on stages: Stage-3
-  Stage-5 depends on stages: Stage-2
-  Stage-1 depends on stages: Stage-5
+  Stage-3 depends on stages: Stage-2, Stage-4
+  Stage-0 depends on stages: Stage-2
+  Stage-4 depends on stages: Stage-2
+  Stage-1 depends on stages: Stage-4
 
 STAGE PLANS:
   Stage: Stage-2
@@ -3112,7 +3065,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: key (type: string), value (type: string)
                 sort order: ++
-                Map-reduce partition columns: key (type: string), value (type: string)
+                Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Forward
@@ -3127,10 +3080,11 @@ STAGE PLANS:
               Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
+                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
                 table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Filter Operator
             predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
             Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -3147,29 +3101,6 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-3
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col1 (type: string)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-            table:
-                input format: org.apache.hadoop.mapred.TextInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-  Stage: Stage-4
     Dependency Collection
 
   Stage: Stage-0
@@ -3178,7 +3109,7 @@ STAGE PLANS:
           hdfs directory: false
 #### A masked pattern was here ####
 
-  Stage: Stage-5
+  Stage: Stage-4
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -3229,11 +3160,10 @@ from src
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-2 is a root stage
-  Stage-3 depends on stages: Stage-2
-  Stage-4 depends on stages: Stage-3, Stage-5
-  Stage-0 depends on stages: Stage-3
-  Stage-5 depends on stages: Stage-2
-  Stage-1 depends on stages: Stage-5
+  Stage-3 depends on stages: Stage-2, Stage-4
+  Stage-0 depends on stages: Stage-2
+  Stage-4 depends on stages: Stage-2
+  Stage-1 depends on stages: Stage-4
 
 STAGE PLANS:
   Stage: Stage-2
@@ -3248,7 +3178,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: key (type: string), value (type: string)
                 sort order: ++
-                Map-reduce partition columns: key (type: string), value (type: string)
+                Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Forward
@@ -3263,10 +3193,11 @@ STAGE PLANS:
               Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
+                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
                 table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Filter Operator
             predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
             Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -3283,29 +3214,6 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-3
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col1 (type: string)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-            table:
-                input format: org.apache.hadoop.mapred.TextInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-  Stage: Stage-4
     Dependency Collection
 
   Stage: Stage-0
@@ -3314,7 +3222,7 @@ STAGE PLANS:
           hdfs directory: false
 #### A masked pattern was here ####
 
-  Stage: Stage-5
+  Stage: Stage-4
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -3369,15 +3277,14 @@ insert overwrite table src_multi2 select * where key > 10 and key < 20
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-4 is a root stage
-  Stage-6 depends on stages: Stage-4, Stage-8, Stage-9
+  Stage-6 depends on stages: Stage-4, Stage-8
   Stage-0 depends on stages: Stage-6
   Stage-5 depends on stages: Stage-0
   Stage-1 depends on stages: Stage-6
   Stage-7 depends on stages: Stage-1
+  Stage-2 depends on stages: Stage-4
   Stage-8 depends on stages: Stage-4
-  Stage-2 depends on stages: Stage-8
-  Stage-9 depends on stages: Stage-4
-  Stage-3 depends on stages: Stage-9
+  Stage-3 depends on stages: Stage-8
 
 STAGE PLANS:
   Stage: Stage-4
@@ -3422,7 +3329,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: key (type: string), value (type: string)
                 sort order: ++
-                Map-reduce partition columns: key (type: string), value (type: string)
+                Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Forward
@@ -3437,10 +3344,11 @@ STAGE PLANS:
               Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
+                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
                 table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Filter Operator
             predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
             Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -3485,36 +3393,13 @@ STAGE PLANS:
   Stage: Stage-7
     Stats-Aggr Operator
 
-  Stage: Stage-8
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col1 (type: string)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-            table:
-                input format: org.apache.hadoop.mapred.TextInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
   Stage: Stage-2
     Move Operator
       files:
           hdfs directory: false
 #### A masked pattern was here ####
 
-  Stage: Stage-9
+  Stage: Stage-8
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -3616,15 +3501,14 @@ insert overwrite table src_multi2 select * where key > 10 and key < 20
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-4 is a root stage
-  Stage-6 depends on stages: Stage-4, Stage-8, Stage-9
+  Stage-6 depends on stages: Stage-4, Stage-8
   Stage-0 depends on stages: Stage-6
   Stage-5 depends on stages: Stage-0
   Stage-1 depends on stages: Stage-6
   Stage-7 depends on stages: Stage-1
+  Stage-2 depends on stages: Stage-4
   Stage-8 depends on stages: Stage-4
-  Stage-2 depends on stages: Stage-8
-  Stage-9 depends on stages: Stage-4
-  Stage-3 depends on stages: Stage-9
+  Stage-3 depends on stages: Stage-8
 
 STAGE PLANS:
   Stage: Stage-4
@@ -3669,7 +3553,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: key (type: string), value (type: string)
                 sort order: ++
-                Map-reduce partition columns: key (type: string), value (type: string)
+                Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Forward
@@ -3684,10 +3568,11 @@ STAGE PLANS:
               Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
+                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
                 table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Filter Operator
             predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
             Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -3732,36 +3617,13 @@ STAGE PLANS:
   Stage: Stage-7
     Stats-Aggr Operator
 
-  Stage: Stage-8
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col1 (type: string)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-            table:
-                input format: org.apache.hadoop.mapred.TextInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
   Stage: Stage-2
     Move Operator
       files:
           hdfs directory: false
 #### A masked pattern was here ####
 
-  Stage: Stage-9
+  Stage: Stage-8
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -3865,7 +3727,7 @@ STAGE DEPENDENCIES:
   Stage-4 is a root stage
   Stage-11 depends on stages: Stage-4 , consists of Stage-8, Stage-7, Stage-9
   Stage-8
-  Stage-6 depends on stages: Stage-8, Stage-7, Stage-10, Stage-14, Stage-13, Stage-16, Stage-18, Stage-19
+  Stage-6 depends on stages: Stage-8, Stage-7, Stage-10, Stage-14, Stage-13, Stage-16, Stage-4, Stage-18
   Stage-0 depends on stages: Stage-6
   Stage-5 depends on stages: Stage-0
   Stage-1 depends on stages: Stage-6
@@ -3878,10 +3740,9 @@ STAGE DEPENDENCIES:
   Stage-13
   Stage-15
   Stage-16 depends on stages: Stage-15
+  Stage-2 depends on stages: Stage-4
   Stage-18 depends on stages: Stage-4
-  Stage-2 depends on stages: Stage-18
-  Stage-19 depends on stages: Stage-4
-  Stage-3 depends on stages: Stage-19
+  Stage-3 depends on stages: Stage-18
 
 STAGE PLANS:
   Stage: Stage-4
@@ -3926,7 +3787,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: key (type: string), value (type: string)
                 sort order: ++
-                Map-reduce partition columns: key (type: string), value (type: string)
+                Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Forward
@@ -3941,10 +3802,11 @@ STAGE PLANS:
               Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
+                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
                 table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Filter Operator
             predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
             Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -4067,36 +3929,13 @@ STAGE PLANS:
           hdfs directory: true
 #### A masked pattern was here ####
 
-  Stage: Stage-18
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col1 (type: string)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-            table:
-                input format: org.apache.hadoop.mapred.TextInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
   Stage: Stage-2
     Move Operator
       files:
           hdfs directory: false
 #### A masked pattern was here ####
 
-  Stage: Stage-19
+  Stage: Stage-18
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -4200,7 +4039,7 @@ STAGE DEPENDENCIES:
   Stage-4 is a root stage
   Stage-11 depends on stages: Stage-4 , consists of Stage-8, Stage-7, Stage-9
   Stage-8
-  Stage-6 depends on stages: Stage-8, Stage-7, Stage-10, Stage-14, Stage-13, Stage-16, Stage-18, Stage-19
+  Stage-6 depends on stages: Stage-8, Stage-7, Stage-10, Stage-14, Stage-13, Stage-16, Stage-4, Stage-18
   Stage-0 depends on stages: Stage-6
   Stage-5 depends on stages: Stage-0
   Stage-1 depends on stages: Stage-6
@@ -4213,10 +4052,9 @@ STAGE DEPENDENCIES:
   Stage-13
   Stage-15
   Stage-16 depends on stages: Stage-15
+  Stage-2 depends on stages: Stage-4
   Stage-18 depends on stages: Stage-4
-  Stage-2 depends on stages: Stage-18
-  Stage-19 depends on stages: Stage-4
-  Stage-3 depends on stages: Stage-19
+  Stage-3 depends on stages: Stage-18
 
 STAGE PLANS:
   Stage: Stage-4
@@ -4261,7 +4099,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: key (type: string), value (type: string)
                 sort order: ++
-                Map-reduce partition columns: key (type: string), value (type: string)
+                Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Forward
@@ -4276,10 +4114,11 @@ STAGE PLANS:
               Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
+                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
                 table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Filter Operator
             predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
             Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -4402,36 +4241,13 @@ STAGE PLANS:
           hdfs directory: true
 #### A masked pattern was here ####
 
-  Stage: Stage-18
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col1 (type: string)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-          outputColumnNames: _col0, _col1
-          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-            table:
-                input format: org.apache.hadoop.mapred.TextInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
   Stage: Stage-2
     Move Operator
       files:
           hdfs directory: false
 #### A masked pattern was here ####
 
-  Stage: Stage-19
+  Stage: Stage-18
     Map Reduce
       Map Operator Tree:
           TableScan

http://git-wip-us.apache.org/repos/asf/hive/blob/a851d030/ql/src/test/results/clientpositive/ptf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ptf.q.out b/ql/src/test/results/clientpositive/ptf.q.out
index e61703c..9d34e4e 100644
--- a/ql/src/test/results/clientpositive/ptf.q.out
+++ b/ql/src/test/results/clientpositive/ptf.q.out
@@ -880,8 +880,7 @@ POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-2 depends on stages: Stage-1
-  Stage-3 depends on stages: Stage-2
-  Stage-0 depends on stages: Stage-3
+  Stage-0 depends on stages: Stage-2
 
 STAGE PLANS:
   Stage: Stage-1
@@ -938,7 +937,7 @@ STAGE PLANS:
             Reduce Output Operator
               key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
               sort order: +++
-              Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+              Map-reduce partition columns: _col0 (type: string)
               Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Group By Operator
@@ -946,28 +945,6 @@ STAGE PLANS:
           mode: mergepartial
           outputColumnNames: _col0, _col1, _col2
           Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            table:
-                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
-  Stage: Stage-3
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Reduce Output Operator
-              key expressions: _col0 (type: string), _col1 (type: string)
-              sort order: ++
-              Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col2 (type: int)
-      Reduce Operator Tree:
-        Select Operator
-          expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int)
-          outputColumnNames: _col0, _col1, _col2
-          Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
           PTF Operator
             Function definitions:
                 Input definition

http://git-wip-us.apache.org/repos/asf/hive/blob/a851d030/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
index 2bcf1bf..9bc6345 100644
--- a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
@@ -2353,10 +2353,8 @@ STAGE PLANS:
   Stage: Stage-2
     Spark
       Edges:
-        Reducer 5 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 5 (PARTITION-LEVEL SORT, 2)
-        Reducer 4 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -2370,35 +2368,9 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: key (type: string), value (type: string)
                       sort order: ++
-                      Map-reduce partition columns: key (type: string), value (type: string)
+                      Map-reduce partition columns: key (type: string)
                       Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
-        Reducer 3 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 5 
+        Reducer 2 
             Reduce Operator Tree:
               Forward
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -2410,16 +2382,13 @@ STAGE PLANS:
                     mode: complete
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
+                    File Output Operator
+                      compressed: false
                       Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string)
-        Reducer 6 
-            Reduce Operator Tree:
-              Forward
-                Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Filter Operator
                   predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
                   Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -2434,6 +2403,19 @@ STAGE PLANS:
                       Map-reduce partition columns: _col1 (type: string)
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col0 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-3
     Dependency Collection
@@ -2480,10 +2462,8 @@ STAGE PLANS:
   Stage: Stage-2
     Spark
       Edges:
-        Reducer 5 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 5 (PARTITION-LEVEL SORT, 2)
-        Reducer 4 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -2497,35 +2477,9 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: key (type: string), value (type: string)
                       sort order: ++
-                      Map-reduce partition columns: key (type: string), value (type: string)
+                      Map-reduce partition columns: key (type: string)
                       Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
-        Reducer 3 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 5 
+        Reducer 2 
             Reduce Operator Tree:
               Forward
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -2537,16 +2491,13 @@ STAGE PLANS:
                     mode: complete
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
+                    File Output Operator
+                      compressed: false
                       Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string)
-        Reducer 6 
-            Reduce Operator Tree:
-              Forward
-                Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Filter Operator
                   predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
                   Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -2561,6 +2512,19 @@ STAGE PLANS:
                       Map-reduce partition columns: _col1 (type: string)
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col0 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-3
     Dependency Collection
@@ -2607,10 +2571,8 @@ STAGE PLANS:
   Stage: Stage-2
     Spark
       Edges:
-        Reducer 5 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 5 (PARTITION-LEVEL SORT, 2)
-        Reducer 4 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -2624,35 +2586,9 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: key (type: string), value (type: string)
                       sort order: ++
-                      Map-reduce partition columns: key (type: string), value (type: string)
+                      Map-reduce partition columns: key (type: string)
                       Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
-        Reducer 3 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 5 
+        Reducer 2 
             Reduce Operator Tree:
               Forward
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -2664,16 +2600,13 @@ STAGE PLANS:
                     mode: complete
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
+                    File Output Operator
+                      compressed: false
                       Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string)
-        Reducer 6 
-            Reduce Operator Tree:
-              Forward
-                Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Filter Operator
                   predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
                   Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -2688,6 +2621,19 @@ STAGE PLANS:
                       Map-reduce partition columns: _col1 (type: string)
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col0 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-3
     Dependency Collection
@@ -2734,10 +2680,8 @@ STAGE PLANS:
   Stage: Stage-2
     Spark
       Edges:
-        Reducer 5 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 5 (PARTITION-LEVEL SORT, 2)
-        Reducer 4 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -2751,35 +2695,9 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: key (type: string), value (type: string)
                       sort order: ++
-                      Map-reduce partition columns: key (type: string), value (type: string)
+                      Map-reduce partition columns: key (type: string)
                       Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
-        Reducer 3 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 5 
+        Reducer 2 
             Reduce Operator Tree:
               Forward
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -2791,16 +2709,13 @@ STAGE PLANS:
                     mode: complete
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
+                    File Output Operator
+                      compressed: false
                       Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string)
-        Reducer 6 
-            Reduce Operator Tree:
-              Forward
-                Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Filter Operator
                   predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
                   Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -2815,6 +2730,19 @@ STAGE PLANS:
                       Map-reduce partition columns: _col1 (type: string)
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col0 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-3
     Dependency Collection
@@ -2869,10 +2797,8 @@ STAGE PLANS:
   Stage: Stage-4
     Spark
       Edges:
-        Reducer 5 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 5 (PARTITION-LEVEL SORT, 2)
-        Reducer 4 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -2916,35 +2842,9 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: key (type: string), value (type: string)
                       sort order: ++
-                      Map-reduce partition columns: key (type: string), value (type: string)
+                      Map-reduce partition columns: key (type: string)
                       Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
-        Reducer 3 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 5 
+        Reducer 2 
             Reduce Operator Tree:
               Forward
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -2956,16 +2856,13 @@ STAGE PLANS:
                     mode: complete
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
+                    File Output Operator
+                      compressed: false
                       Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string)
-        Reducer 6 
-            Reduce Operator Tree:
-              Forward
-                Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Filter Operator
                   predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
                   Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -2980,6 +2877,19 @@ STAGE PLANS:
                       Map-reduce partition columns: _col1 (type: string)
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col0 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-5
     Dependency Collection
@@ -3107,10 +3017,8 @@ STAGE PLANS:
   Stage: Stage-4
     Spark
       Edges:
-        Reducer 5 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 5 (PARTITION-LEVEL SORT, 2)
-        Reducer 4 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -3154,35 +3062,9 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: key (type: string), value (type: string)
                       sort order: ++
-                      Map-reduce partition columns: key (type: string), value (type: string)
+                      Map-reduce partition columns: key (type: string)
                       Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
-        Reducer 3 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 5 
+        Reducer 2 
             Reduce Operator Tree:
               Forward
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3194,16 +3076,13 @@ STAGE PLANS:
                     mode: complete
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
+                    File Output Operator
+                      compressed: false
                       Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string)
-        Reducer 6 
-            Reduce Operator Tree:
-              Forward
-                Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Filter Operator
                   predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
                   Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -3218,6 +3097,19 @@ STAGE PLANS:
                       Map-reduce partition columns: _col1 (type: string)
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col0 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-5
     Dependency Collection
@@ -3345,10 +3237,8 @@ STAGE PLANS:
   Stage: Stage-4
     Spark
       Edges:
-        Reducer 5 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 5 (PARTITION-LEVEL SORT, 2)
-        Reducer 4 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -3392,35 +3282,9 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: key (type: string), value (type: string)
                       sort order: ++
-                      Map-reduce partition columns: key (type: string), value (type: string)
+                      Map-reduce partition columns: key (type: string)
                       Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
-        Reducer 3 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 5 
+        Reducer 2 
             Reduce Operator Tree:
               Forward
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3432,16 +3296,13 @@ STAGE PLANS:
                     mode: complete
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
+                    File Output Operator
+                      compressed: false
                       Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string)
-        Reducer 6 
-            Reduce Operator Tree:
-              Forward
-                Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Filter Operator
                   predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
                   Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -3456,6 +3317,19 @@ STAGE PLANS:
                       Map-reduce partition columns: _col1 (type: string)
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col0 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-5
     Dependency Collection
@@ -3583,10 +3457,8 @@ STAGE PLANS:
   Stage: Stage-4
     Spark
       Edges:
-        Reducer 5 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 5 (PARTITION-LEVEL SORT, 2)
-        Reducer 4 <- Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -3630,35 +3502,9 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: key (type: string), value (type: string)
                       sort order: ++
-                      Map-reduce partition columns: key (type: string), value (type: string)
+                      Map-reduce partition columns: key (type: string)
                       Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
-        Reducer 3 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 5 
+        Reducer 2 
             Reduce Operator Tree:
               Forward
                 Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3670,16 +3516,13 @@ STAGE PLANS:
                     mode: complete
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
+                    File Output Operator
+                      compressed: false
                       Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string)
-        Reducer 6 
-            Reduce Operator Tree:
-              Forward
-                Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Filter Operator
                   predicate: ((KEY._col0 > 10) and (KEY._col0 < 20)) (type: boolean)
                   Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
@@ -3694,6 +3537,19 @@ STAGE PLANS:
                       Map-reduce partition columns: _col1 (type: string)
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col0 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-5
     Dependency Collection

http://git-wip-us.apache.org/repos/asf/hive/blob/a851d030/ql/src/test/results/clientpositive/spark/ptf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ptf.q.out b/ql/src/test/results/clientpositive/spark/ptf.q.out
index 647b83e..6beeaf4 100644
--- a/ql/src/test/results/clientpositive/spark/ptf.q.out
+++ b/ql/src/test/results/clientpositive/spark/ptf.q.out
@@ -868,8 +868,7 @@ STAGE PLANS:
     Spark
       Edges:
         Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
-        Reducer 3 <- Reducer 2 (GROUP, 2)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
+        Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -915,7 +914,7 @@ STAGE PLANS:
                       Reduce Output Operator
                         key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
                         sort order: +++
-                        Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+                        Map-reduce partition columns: _col0 (type: string)
                         Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
         Reducer 3 
             Reduce Operator Tree:
@@ -924,18 +923,6 @@ STAGE PLANS:
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2
                 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
-                Reduce Output Operator
-                  key expressions: _col0 (type: string), _col1 (type: string)
-                  sort order: ++
-                  Map-reduce partition columns: _col0 (type: string)
-                  Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
-                  value expressions: _col2 (type: int)
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int)
-                outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
                 PTF Operator
                   Function definitions:
                       Input definition

http://git-wip-us.apache.org/repos/asf/hive/blob/a851d030/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
index 45d1528..9e0f2b5 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
@@ -383,9 +383,8 @@ STAGE PLANS:
     Spark
       Edges:
         Reducer 2 <- Map 1 (GROUP, 2)
-        Reducer 6 <- Map 1 (GROUP, 2)
-        Reducer 3 <- Reducer 2 (GROUP, 2), Reducer 6 (GROUP, 2)
-        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2)
+        Reducer 5 <- Map 1 (GROUP, 2)
+        Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 2), Reducer 5 (GROUP PARTITION-LEVEL SORT, 2)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -429,7 +428,7 @@ STAGE PLANS:
                     Reduce Output Operator
                       key expressions: _col0 (type: string), _col1 (type: bigint)
                       sort order: ++
-                      Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint)
+                      Map-reduce partition columns: _col0 (type: string)
                       Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
         Reducer 3 
             Reduce Operator Tree:
@@ -438,17 +437,6 @@ STAGE PLANS:
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
-                Reduce Output Operator
-                  key expressions: _col0 (type: string), _col1 (type: bigint)
-                  sort order: ++
-                  Map-reduce partition columns: _col0 (type: string)
-                  Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
-        Reducer 4 
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 PTF Operator
                   Function definitions:
                       Input definition
@@ -480,7 +468,7 @@ STAGE PLANS:
                           input format: org.apache.hadoop.mapred.TextInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-        Reducer 6 
+        Reducer 5 
             Reduce Operator Tree:
               Group By Operator
                 aggregations: count(VALUE._col0)
@@ -496,7 +484,7 @@ STAGE PLANS:
                   Reduce Output Operator
                     key expressions: _col0 (type: string), _col1 (type: bigint)
                     sort order: ++
-                    Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint)
+                    Map-reduce partition columns: _col0 (type: string)
                     Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
 
   Stage: Stage-0


Mime
View raw message