hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1505870 [1/2] - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/lib/ java/org/apache/hadoop/hive/ql/optimizer/ java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/plan/ test/queri...
Date Tue, 23 Jul 2013 00:50:40 GMT
Author: hashutosh
Date: Tue Jul 23 00:50:39 2013
New Revision: 1505870

URL: http://svn.apache.org/r1505870
Log:
HIVE-4502 : NPE - subquery smb joins fails (Navis via Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_25.q
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java
    hive/trunk/ql/src/test/queries/clientpositive/auto_sortmerge_join_6.q
    hive/trunk/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin2.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin3.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin4.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketsortoptimize_insert_7.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer3.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer7.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapjoin_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin9.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_14.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_6.q.out
    hive/trunk/ql/src/test/results/clientpositive/stats11.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Tue Jul 23 00:50:39 2013
@@ -86,6 +86,7 @@ public abstract class Task<T extends Ser
 
   protected String id;
   protected T work;
+
   public static enum FeedType {
     DYNAMIC_PARTITIONS, // list of dynamic partitions
   };

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java Tue Jul 23 00:50:39 2013
@@ -75,6 +75,13 @@ public class DefaultGraphWalker implemen
    * @throws SemanticException
    */
   public void dispatch(Node nd, Stack<Node> ndStack) throws SemanticException {
+    dispatchAndReturn(nd, ndStack);
+  }
+
+  /**
+   * Returns dispatch result
+   */
+  public <T> T dispatchAndReturn(Node nd, Stack<Node> ndStack) throws SemanticException {
     Object[] nodeOutputs = null;
     if (nd.getChildren() != null) {
       nodeOutputs = new Object[nd.getChildren().size()];
@@ -86,6 +93,7 @@ public class DefaultGraphWalker implemen
 
     Object retVal = dispatcher.dispatch(nd, ndStack, nodeOutputs);
     retMap.put(nd, retVal);
+    return (T) retVal;
   }
 
   /**

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Tue Jul 23 00:50:39 2013
@@ -91,6 +91,8 @@ public class GenMRFileSink1 implements N
     ParseContext parseCtx = ctx.getParseCtx();
     boolean chDir = false;
     Task<? extends Serializable> currTask = ctx.getCurrTask();
+    ctx.addRootIfPossible(currTask);
+
     FileSinkOperator fsOp = (FileSinkOperator) nd;
     boolean isInsertTable = // is INSERT OVERWRITE TABLE
     fsOp.getConf().getTableInfo().getTableName() != null &&
@@ -106,7 +108,7 @@ public class GenMRFileSink1 implements N
     if (fileSinkDescs != null) {
       Task<? extends Serializable> childTask = fileSinkDescs.get(fsOp.getConf());
       processLinkedFileDesc(ctx, childTask);
-      return null;
+      return true;
     }
 
     // Has the user enabled merging of files for map-only jobs or for all jobs
@@ -181,7 +183,7 @@ public class GenMRFileSink1 implements N
       }
     }
 
-    return null;
+    return true;
   }
 
   /*
@@ -189,26 +191,12 @@ public class GenMRFileSink1 implements N
    * Use the task created by the first linked file descriptor
    */
   private void processLinkedFileDesc(GenMRProcContext ctx,
-    Task<? extends Serializable> childTask)
-    throws SemanticException {
-    Operator<? extends OperatorDesc> currTopOp = ctx.getCurrTopOp();
-    String currAliasId = ctx.getCurrAliasId();
-    List<Operator<? extends OperatorDesc>> seenOps = ctx.getSeenOps();
-    List<Task<? extends Serializable>> rootTasks = ctx.getRootTasks();
+      Task<? extends Serializable> childTask) throws SemanticException {
     Task<? extends Serializable> currTask = ctx.getCurrTask();
-
-    if (currTopOp != null) {
-      if (!seenOps.contains(currTopOp)) {
-        seenOps.add(currTopOp);
-        GenMapRedUtils.setTaskPlan(currAliasId, currTopOp,
-          (MapredWork) currTask.getWork(), false, ctx);
-      }
-
-      if (!rootTasks.contains(currTask)
-          && (currTask.getParentTasks() == null
-              || currTask.getParentTasks().isEmpty())) {
-        rootTasks.add(currTask);
-      }
+    Operator<? extends OperatorDesc> currTopOp = ctx.getCurrTopOp();
+    if (currTopOp != null && !ctx.isSeenOp(currTask, currTopOp)) {
+      String currAliasId = ctx.getCurrAliasId();
+      GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, false, ctx);
     }
 
     if (childTask != null) {
@@ -702,8 +690,6 @@ public class GenMRFileSink1 implements N
     String currAliasId = ctx.getCurrAliasId();
     HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
         ctx.getOpTaskMap();
-    List<Operator<? extends OperatorDesc>> seenOps = ctx.getSeenOps();
-    List<Task<? extends Serializable>> rootTasks = ctx.getRootTasks();
 
     // Set the move task to be dependent on the current task
     if (mvTask != null) {
@@ -717,22 +703,13 @@ public class GenMRFileSink1 implements N
     if (currTopOp != null) {
       Task<? extends Serializable> mapTask = opTaskMap.get(null);
       if (mapTask == null) {
-        if (!seenOps.contains(currTopOp)) {
-          seenOps.add(currTopOp);
-          GenMapRedUtils.setTaskPlan(currAliasId, currTopOp,
-              (MapredWork) currTask.getWork(), false, ctx);
+        if (!ctx.isSeenOp(currTask, currTopOp)) {
+          GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, false, ctx);
         }
         opTaskMap.put(null, currTask);
-        if (!rootTasks.contains(currTask)
-            && (currTask.getParentTasks() == null
-                || currTask.getParentTasks().isEmpty())) {
-          rootTasks.add(currTask);
-        }
       } else {
-        if (!seenOps.contains(currTopOp)) {
-          seenOps.add(currTopOp);
-          GenMapRedUtils.setTaskPlan(currAliasId, currTopOp,
-              (MapredWork) mapTask.getWork(), false, ctx);
+        if (!ctx.isSeenOp(currTask, currTopOp)) {
+          GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, mapTask, false, ctx);
         } else {
           UnionOperator currUnionOp = ctx.getCurrUnionOp();
           if (currUnionOp != null) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java Tue Jul 23 00:50:39 2013
@@ -53,8 +53,7 @@ public class GenMROperator implements No
         .getMapCurrCtx();
     GenMapRedCtx mapredCtx = mapCurrCtx.get(stack.get(stack.size() - 2));
     mapCurrCtx.put((Operator<? extends OperatorDesc>) nd, new GenMapRedCtx(
-        mapredCtx.getCurrTask(), mapredCtx.getCurrTopOp(), mapredCtx
-        .getCurrAliasId()));
-    return null;
+        mapredCtx.getCurrTask(), mapredCtx.getCurrAliasId()));
+    return true;
   }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java Tue Jul 23 00:50:39 2013
@@ -55,7 +55,6 @@ public class GenMRProcContext implements
    */
   public static class GenMapRedCtx {
     Task<? extends Serializable> currTask;
-    Operator<? extends OperatorDesc> currTopOp;
     String currAliasId;
 
     public GenMapRedCtx() {
@@ -64,15 +63,10 @@ public class GenMRProcContext implements
     /**
      * @param currTask
      *          the current task
-     * @param currTopOp
-     *          the current top operator being traversed
      * @param currAliasId
-     *          the current alias for the to operator
      */
-    public GenMapRedCtx(Task<? extends Serializable> currTask,
-        Operator<? extends OperatorDesc> currTopOp, String currAliasId) {
+    public GenMapRedCtx(Task<? extends Serializable> currTask, String currAliasId) {
       this.currTask = currTask;
-      this.currTopOp = currTopOp;
       this.currAliasId = currAliasId;
     }
 
@@ -84,13 +78,6 @@ public class GenMRProcContext implements
     }
 
     /**
-     * @return current top operator
-     */
-    public Operator<? extends OperatorDesc> getCurrTopOp() {
-      return currTopOp;
-    }
-
-    /**
      * @return current alias
      */
     public String getCurrAliasId() {
@@ -103,13 +90,13 @@ public class GenMRProcContext implements
    *
    */
   public static class GenMRUnionCtx {
-    Task<? extends Serializable> uTask;
+    final Task<? extends Serializable> uTask;
     List<String> taskTmpDir;
     List<TableDesc> tt_desc;
     List<Operator<? extends OperatorDesc>> listTopOperators;
 
-    public GenMRUnionCtx() {
-      uTask = null;
+    public GenMRUnionCtx(Task<? extends Serializable> uTask) {
+      this.uTask = uTask;
       taskTmpDir = new ArrayList<String>();
       tt_desc = new ArrayList<TableDesc>();
       listTopOperators = new ArrayList<Operator<? extends OperatorDesc>>();
@@ -119,10 +106,6 @@ public class GenMRProcContext implements
       return uTask;
     }
 
-    public void setUTask(Task<? extends Serializable> uTask) {
-      this.uTask = uTask;
-    }
-
     public void addTaskTmpDir(String taskTmpDir) {
       this.taskTmpDir.add(taskTmpDir);
     }
@@ -156,8 +139,10 @@ public class GenMRProcContext implements
   private HiveConf conf;
   private
     HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap;
+  private
+    HashMap<Task<? extends Serializable>, List<Operator<? extends OperatorDesc>>> taskToSeenOps;
+
   private HashMap<UnionOperator, GenMRUnionCtx> unionTaskMap;
-  private List<Operator<? extends OperatorDesc>> seenOps;
   private List<FileSinkOperator> seenFileSinkOps;
 
   private ParseContext parseCtx;
@@ -169,7 +154,6 @@ public class GenMRProcContext implements
   private Operator<? extends OperatorDesc> currTopOp;
   private UnionOperator currUnionOp;
   private String currAliasId;
-  private List<Operator<? extends OperatorDesc>> rootOps;
   private DependencyCollectionTask dependencyTaskForMultiInsert;
 
   // If many fileSinkDescs are linked to each other, it is a good idea to keep track of
@@ -213,14 +197,13 @@ public class GenMRProcContext implements
   public GenMRProcContext(
       HiveConf conf,
       HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap,
-      List<Operator<? extends OperatorDesc>> seenOps, ParseContext parseCtx,
+      ParseContext parseCtx,
       List<Task<MoveWork>> mvTask,
       List<Task<? extends Serializable>> rootTasks,
       LinkedHashMap<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx,
       Set<ReadEntity> inputs, Set<WriteEntity> outputs) {
     this.conf = conf;
     this.opTaskMap = opTaskMap;
-    this.seenOps = seenOps;
     this.mvTask = mvTask;
     this.parseCtx = parseCtx;
     this.rootTasks = rootTasks;
@@ -231,9 +214,9 @@ public class GenMRProcContext implements
     currTopOp = null;
     currUnionOp = null;
     currAliasId = null;
-    rootOps = new ArrayList<Operator<? extends OperatorDesc>>();
-    rootOps.addAll(parseCtx.getTopOps().values());
     unionTaskMap = new HashMap<UnionOperator, GenMRUnionCtx>();
+    taskToSeenOps = new HashMap<Task<? extends Serializable>,
+        List<Operator<? extends OperatorDesc>>>();
     dependencyTaskForMultiInsert = null;
     linkedFileDescTasks = null;
   }
@@ -255,11 +238,17 @@ public class GenMRProcContext implements
     this.opTaskMap = opTaskMap;
   }
 
-  /**
-   * @return operators already visited
-   */
-  public List<Operator<? extends OperatorDesc>> getSeenOps() {
-    return seenOps;
+  public boolean isSeenOp(Task task, Operator operator) {
+    List<Operator<?extends OperatorDesc>> seenOps = taskToSeenOps.get(task);
+    return seenOps != null && seenOps.contains(operator);
+  }
+
+  public void addSeenOp(Task task, Operator operator) {
+    List<Operator<?extends OperatorDesc>> seenOps = taskToSeenOps.get(task);
+    if (seenOps == null) {
+      taskToSeenOps.put(task, seenOps = new ArrayList<Operator<? extends OperatorDesc>>());
+    }
+    seenOps.add(operator);
   }
 
   /**
@@ -270,14 +259,6 @@ public class GenMRProcContext implements
   }
 
   /**
-   * @param seenOps
-   *          operators already visited
-   */
-  public void setSeenOps(List<Operator<? extends OperatorDesc>> seenOps) {
-    this.seenOps = seenOps;
-  }
-
-  /**
    * @param seenFileSinkOps
    *          file sink operators already visited
    */
@@ -286,21 +267,6 @@ public class GenMRProcContext implements
   }
 
   /**
-   * @return top operators for tasks
-   */
-  public List<Operator<? extends OperatorDesc>> getRootOps() {
-    return rootOps;
-  }
-
-  /**
-   * @param rootOps
-   *          top operators for tasks
-   */
-  public void setRootOps(List<Operator<? extends OperatorDesc>> rootOps) {
-    this.rootOps = rootOps;
-  }
-
-  /**
    * @return current parse context
    */
   public ParseContext getParseCtx() {
@@ -345,6 +311,15 @@ public class GenMRProcContext implements
     this.rootTasks = rootTasks;
   }
 
+  public boolean addRootIfPossible(Task<? extends Serializable> task) {
+    if (task.getParentTasks() == null || task.getParentTasks().isEmpty()) {
+      if (!rootTasks.contains(task)) {
+        return rootTasks.add(task);
+      }
+    }
+    return false;
+  }
+
   /**
    * @return operator to task mappings
    */

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java Tue Jul 23 00:50:39 2013
@@ -19,7 +19,6 @@
 package org.apache.hadoop.hive.ql.optimizer;
 
 import java.io.Serializable;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Stack;
 
@@ -64,19 +63,20 @@ public class GenMRRedSink1 implements No
     GenMapRedCtx mapredCtx = mapCurrCtx.get(stack.get(stack.size() - 2));
     Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
     MapredWork currPlan = (MapredWork) currTask.getWork();
-    Operator<? extends OperatorDesc> currTopOp = mapredCtx.getCurrTopOp();
     String currAliasId = mapredCtx.getCurrAliasId();
+
+    if (op.getNumChild() != 1) {
+      throw new IllegalStateException("Expecting operator " + op + " to have one child. " +
+          "But found multiple children : " + op.getChildOperators());
+    }
     Operator<? extends OperatorDesc> reducer = op.getChildOperators().get(0);
-    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap = ctx
-        .getOpTaskMap();
-    Task<? extends Serializable> opMapTask = opTaskMap.get(reducer);
+    Task<? extends Serializable> oldTask = ctx.getOpTaskMap().get(reducer);
 
-    ctx.setCurrTopOp(currTopOp);
     ctx.setCurrAliasId(currAliasId);
     ctx.setCurrTask(currTask);
 
     // If the plan for this reducer does not exist, initialize the plan
-    if (opMapTask == null) {
+    if (oldTask == null) {
       if (currPlan.getReducer() == null) {
         GenMapRedUtils.initPlan(op, ctx);
       } else {
@@ -85,14 +85,18 @@ public class GenMRRedSink1 implements No
     } else {
       // This will happen in case of joins. The current plan can be thrown away
       // after being merged with the original plan
-      GenMapRedUtils.joinPlan(op, null, opMapTask, ctx, -1, false);
-      currTask = opMapTask;
+      GenMapRedUtils.joinPlan(currTask, oldTask, ctx);
+      currTask = oldTask;
       ctx.setCurrTask(currTask);
     }
 
-    mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(),
-        ctx.getCurrAliasId()));
-    return null;
-  }
+    mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrAliasId()));
 
+    if (GenMapRedUtils.hasBranchFinished(nodeOutputs)) {
+      ctx.addRootIfPossible(currTask);
+      return false;
+    }
+
+    return true;
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java Tue Jul 23 00:50:39 2013
@@ -57,28 +57,32 @@ public class GenMRRedSink2 implements No
         .getMapCurrCtx();
     GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(0));
     Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
-    Operator<? extends OperatorDesc> currTopOp = mapredCtx.getCurrTopOp();
     String currAliasId = mapredCtx.getCurrAliasId();
     Operator<? extends OperatorDesc> reducer = op.getChildOperators().get(0);
     Map<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap = ctx
         .getOpTaskMap();
-    Task<? extends Serializable> opMapTask = opTaskMap.get(reducer);
+    Task<? extends Serializable> oldTask = opTaskMap.get(reducer);
 
-    ctx.setCurrTopOp(currTopOp);
     ctx.setCurrAliasId(currAliasId);
     ctx.setCurrTask(currTask);
 
-    if (opMapTask == null) {
+    if (oldTask == null) {
       GenMapRedUtils.splitPlan(op, ctx);
     } else {
-      GenMapRedUtils.joinPlan(op, currTask, opMapTask, ctx, -1, true);
-      currTask = opMapTask;
+      GenMapRedUtils.splitPlan(op, currTask, oldTask, ctx);
+      currTask = oldTask;
       ctx.setCurrTask(currTask);
     }
 
-    mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(),
+    mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(),
         ctx.getCurrAliasId()));
-    return null;
+
+    if (GenMapRedUtils.hasBranchFinished(nodeOutputs)) {
+      ctx.addRootIfPossible(currTask);
+      return false;
+    }
+
+    return true;
   }
 
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java Tue Jul 23 00:50:39 2013
@@ -101,11 +101,11 @@ public class GenMRRedSink3 implements No
       ctx.setCurrTask(reducerTask);
     }
 
-    mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(),
+    mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(),
         ctx.getCurrAliasId()));
 
     // the union operator has been processed
     ctx.setCurrUnionOp(null);
-    return null;
+    return true;
   }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java Tue Jul 23 00:50:39 2013
@@ -84,7 +84,7 @@ public class GenMRTableScan1 implements 
       if (currOp == op) {
         String currAliasId = alias;
         ctx.setCurrAliasId(currAliasId);
-        mapCurrCtx.put(op, new GenMapRedCtx(currTask, currTopOp, currAliasId));
+        mapCurrCtx.put(op, new GenMapRedCtx(currTask, currAliasId));
 
         QBParseInfo parseInfo = parseCtx.getQB().getParseInfo();
         if (parseInfo.isAnalyzeCommand()) {
@@ -139,12 +139,12 @@ public class GenMRTableScan1 implements 
             Table source = parseCtx.getQB().getMetaData().getTableForAlias(alias);
             PrunedPartitionList partList = new PrunedPartitionList(source, confirmedPartns,
                 new HashSet<Partition>(), null);
-            GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currWork, false, ctx, partList);
+            GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, false, ctx, partList);
           } else { // non-partitioned table
-            GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currWork, false, ctx);
+            GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, false, ctx);
           }
         }
-        return null;
+        return true;
       }
     }
     assert false;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java Tue Jul 23 00:50:39 2013
@@ -82,14 +82,13 @@ public class GenMRUnion1 implements Node
     UnionParseContext uPrsCtx = uCtx.getUnionParseContext(union);
     ctx.getMapCurrCtx().put(
         (Operator<? extends OperatorDesc>) union,
-        new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(),
+        new GenMapRedCtx(ctx.getCurrTask(),
             ctx.getCurrAliasId()));
 
     // if the union is the first time seen, set current task to GenMRUnionCtx
     uCtxTask = ctx.getUnionTask(union);
     if (uCtxTask == null) {
-      uCtxTask = new GenMRUnionCtx();
-      uCtxTask.setUTask(ctx.getCurrTask());
+      uCtxTask = new GenMRUnionCtx(ctx.getCurrTask());
       ctx.setUnionTask(union, uCtxTask);
     }
 
@@ -101,7 +100,7 @@ public class GenMRUnion1 implements Node
       }
     }
 
-    return null;
+    return true;
   }
 
   /**
@@ -192,14 +191,11 @@ public class GenMRUnion1 implements Node
     // The current plan can be thrown away after being merged with the union
     // plan
     Task<? extends Serializable> uTask = uCtxTask.getUTask();
-    MapredWork plan = (MapredWork) uTask.getWork();
     ctx.setCurrTask(uTask);
-    List<Operator<? extends OperatorDesc>> seenOps = ctx.getSeenOps();
     Operator<? extends OperatorDesc> topOp = ctx.getCurrTopOp();
-    if (!seenOps.contains(topOp) && topOp != null) {
-      seenOps.add(topOp);
+    if (topOp != null && !ctx.isSeenOp(uTask, topOp)) {
       GenMapRedUtils.setTaskPlan(ctx.getCurrAliasId(), ctx
-          .getCurrTopOp(), plan, false, ctx);
+          .getCurrTopOp(), uTask, false, ctx);
     }
   }
 
@@ -230,8 +226,7 @@ public class GenMRUnion1 implements Node
       // All inputs of this UnionOperator are in the same Reducer.
       // We do not need to break the operator tree.
       mapCurrCtx.put((Operator<? extends OperatorDesc>) nd,
-          new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(),
-              ctx.getCurrAliasId()));
+        new GenMapRedCtx(ctx.getCurrTask(),ctx.getCurrAliasId()));
       return null;
     }
 
@@ -255,10 +250,9 @@ public class GenMRUnion1 implements Node
     // union is encountered for the first time
     GenMRUnionCtx uCtxTask = ctx.getUnionTask(union);
     if (uCtxTask == null) {
-      uCtxTask = new GenMRUnionCtx();
       uPlan = GenMapRedUtils.getMapRedWork(parseCtx);
       uTask = TaskFactory.get(uPlan, parseCtx.getConf());
-      uCtxTask.setUTask(uTask);
+      uCtxTask = new GenMRUnionCtx(uTask);
       ctx.setUnionTask(union, uCtxTask);
     }
     else {
@@ -293,9 +287,9 @@ public class GenMRUnion1 implements Node
     ctx.setCurrTask(uTask);
 
     mapCurrCtx.put((Operator<? extends OperatorDesc>) nd,
-        new GenMapRedCtx(ctx.getCurrTask(), null, null));
+        new GenMapRedCtx(ctx.getCurrTask(), null));
 
-    return null;
+    return true;
   }
 
   private boolean shouldBeRootTask(

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Tue Jul 23 00:50:39 2013
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.exec.Ta
 import org.apache.hadoop.hive.ql.exec.UnionOperator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.mr.ExecDriver;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
@@ -60,7 +61,6 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.FetchWork;
 import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
 import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc;
-import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
 import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
 import org.apache.hadoop.hive.ql.plan.MapredWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -81,6 +81,10 @@ public final class GenMapRedUtils {
     LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils");
   }
 
+  private static boolean needsTagging(Operator<? extends OperatorDesc> reducer) {
+    return (reducer.getClass() == JoinOperator.class ||
+        reducer.getClass() == DemuxOperator.class);
+  }
   /**
    * Initialize the current plan by adding it to root tasks.
    *
@@ -107,25 +111,15 @@ public final class GenMapRedUtils {
 
     plan.setNumReduceTasks(desc.getNumReducers());
 
-    List<Task<? extends Serializable>> rootTasks = opProcCtx.getRootTasks();
-
-    if (!rootTasks.contains(currTask)
-        && (currTask.getParentTasks() == null
-            || currTask.getParentTasks().isEmpty())) {
-      rootTasks.add(currTask);
-    }
-    if (reducer.getClass() == JoinOperator.class ||
-        reducer.getClass() == DemuxOperator.class) {
+    if (needsTagging(reducer)) {
       plan.setNeedsTagging(true);
     }
 
     assert currTopOp != null;
-    List<Operator<? extends OperatorDesc>> seenOps = opProcCtx.getSeenOps();
     String currAliasId = opProcCtx.getCurrAliasId();
 
-    if (!seenOps.contains(currTopOp)) {
-      seenOps.add(currTopOp);
-      setTaskPlan(currAliasId, currTopOp, plan, false, opProcCtx);
+    if (!opProcCtx.isSeenOp(currTask, currTopOp)) {
+      setTaskPlan(currAliasId, currTopOp, currTask, false, opProcCtx);
     }
 
     currTopOp = null;
@@ -160,8 +154,7 @@ public final class GenMapRedUtils {
 
     plan.setNumReduceTasks(desc.getNumReducers());
 
-    if (reducer.getClass() == JoinOperator.class ||
-        reducer.getClass() == DemuxOperator.class) {
+    if (needsTagging(reducer)) {
       plan.setNeedsTagging(true);
     }
 
@@ -169,16 +162,14 @@ public final class GenMapRedUtils {
   }
 
   private static void setUnionPlan(GenMRProcContext opProcCtx,
-      boolean local, MapredWork plan, GenMRUnionCtx uCtx,
+      boolean local, Task<? extends Serializable> currTask, GenMRUnionCtx uCtx,
       boolean mergeTask) throws SemanticException {
     Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();
 
     if (currTopOp != null) {
-      List<Operator<? extends OperatorDesc>> seenOps = opProcCtx.getSeenOps();
       String currAliasId = opProcCtx.getCurrAliasId();
-      if (!seenOps.contains(currTopOp) || mergeTask) {
-        seenOps.add(currTopOp);
-        setTaskPlan(currAliasId, currTopOp, plan, local, opProcCtx);
+      if (mergeTask || !opProcCtx.isSeenOp(currTask, currTopOp)) {
+        setTaskPlan(currAliasId, currTopOp, currTask, local, opProcCtx);
       }
       currTopOp = null;
       opProcCtx.setCurrTopOp(currTopOp);
@@ -194,6 +185,7 @@ public final class GenMapRedUtils {
         List<Operator<? extends OperatorDesc>> topOperators =
             uCtx.getListTopOperators();
 
+        MapredWork plan = (MapredWork) currTask.getWork();
         for (int pos = 0; pos < size; pos++) {
           String taskTmpDir = taskTmpDirLst.get(pos);
           TableDesc tt_desc = tt_descLst.get(pos);
@@ -217,14 +209,12 @@ public final class GenMapRedUtils {
   public static void initUnionPlan(GenMRProcContext opProcCtx, UnionOperator currUnionOp,
       Task<? extends Serializable> currTask, boolean local)
       throws SemanticException {
-    MapredWork plan = (MapredWork) currTask.getWork();
-
     // In case of lateral views followed by a join, the same tree
     // can be traversed more than one
     if (currUnionOp != null) {
       GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp);
       assert uCtx != null;
-      setUnionPlan(opProcCtx, local, plan, uCtx, false);
+      setUnionPlan(opProcCtx, local, currTask, uCtx, false);
     }
   }
 
@@ -236,12 +226,11 @@ public final class GenMapRedUtils {
       Task<? extends Serializable> currentUnionTask,
       Task<? extends Serializable> existingTask, boolean local)
       throws SemanticException {
-    MapredWork plan = (MapredWork) existingTask.getWork();
     assert currUnionOp != null;
     GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp);
     assert uCtx != null;
 
-    setUnionPlan(opProcCtx, local, plan, uCtx, true);
+    setUnionPlan(opProcCtx, local, existingTask, uCtx, true);
 
     List<Task<? extends Serializable>> parTasks = null;
     if (opProcCtx.getRootTasks().contains(currentUnionTask)) {
@@ -276,104 +265,105 @@ public final class GenMapRedUtils {
   }
 
   /**
-   * Merge the current task with the task for the current reducer.
+   * Merge the current task into the old task for the reducer
    *
-   * @param op
-   *          operator being processed
+   * @param currTask
+   *          the current task for the current reducer
    * @param oldTask
    *          the old task for the current reducer
-   * @param task
-   *          the current task for the current reducer
    * @param opProcCtx
    *          processing context
-   * @param pos
-   *          position of the parent in the stack
    */
-  public static void joinPlan(Operator<? extends OperatorDesc> op,
-      Task<? extends Serializable> oldTask, Task<? extends Serializable> task,
-      GenMRProcContext opProcCtx, int pos, boolean split)
+  public static void joinPlan(Task<? extends Serializable> currTask,
+      Task<? extends Serializable> oldTask, GenMRProcContext opProcCtx)
       throws SemanticException {
-    Task<? extends Serializable> currTask = task;
-    MapredWork plan = (MapredWork) currTask.getWork();
+    assert currTask != null && oldTask != null;
+
     Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();
     List<Task<? extends Serializable>> parTasks = null;
-
     // terminate the old task and make current task dependent on it
-    if (split) {
-      assert oldTask != null;
-      splitTasks(op, oldTask, currTask, opProcCtx, true, false, 0);
-    } else {
-      if ((oldTask != null) && (oldTask.getParentTasks() != null)
-          && !oldTask.getParentTasks().isEmpty()) {
-        parTasks = new ArrayList<Task<? extends Serializable>>();
-        parTasks.addAll(oldTask.getParentTasks());
-
-        Object[] parTaskArr = parTasks.toArray();
-        for (Object element : parTaskArr) {
-          ((Task<? extends Serializable>) element).removeDependentTask(oldTask);
-        }
+    if (currTask.getParentTasks() != null
+        && !currTask.getParentTasks().isEmpty()) {
+      parTasks = new ArrayList<Task<? extends Serializable>>();
+      parTasks.addAll(currTask.getParentTasks());
+
+      Object[] parTaskArr = parTasks.toArray();
+      for (Object element : parTaskArr) {
+        ((Task<? extends Serializable>) element).removeDependentTask(currTask);
       }
     }
 
     if (currTopOp != null) {
-      List<Operator<? extends OperatorDesc>> seenOps = opProcCtx.getSeenOps();
-      String currAliasId = opProcCtx.getCurrAliasId();
-
-      if (!seenOps.contains(currTopOp)) {
-        seenOps.add(currTopOp);
-        boolean local = false;
-        if (pos != -1) {
-          local = (pos == ((MapJoinDesc) op.getConf()).getPosBigTable()) ? false
-              : true;
-        }
-        setTaskPlan(currAliasId, currTopOp, plan, local, opProcCtx);
-      }
-      currTopOp = null;
-      opProcCtx.setCurrTopOp(currTopOp);
+      mergeInput(currTopOp, opProcCtx, oldTask, false);
     }
 
-    if ((oldTask != null) && (parTasks != null)) {
+    if (parTasks != null) {
       for (Task<? extends Serializable> parTask : parTasks) {
-        parTask.addDependentTask(currTask);
-        if (opProcCtx.getRootTasks().contains(currTask)) {
-          opProcCtx.getRootTasks().remove(currTask);
-        }
+        parTask.addDependentTask(oldTask);
       }
     }
 
-    opProcCtx.setCurrTask(currTask);
+    if (oldTask instanceof MapRedTask && currTask instanceof MapRedTask) {
+      ((MapRedTask)currTask).getWork().mergingInto(((MapRedTask) oldTask).getWork());
+    }
+
+    opProcCtx.setCurrTopOp(null);
+    opProcCtx.setCurrTask(oldTask);
   }
 
   /**
-   * Split the current plan by creating a temporary destination.
+   * If currTopOp is not set for input of the task, add input for to the task
+   */
+  static boolean mergeInput(Operator<? extends OperatorDesc> currTopOp,
+      GenMRProcContext opProcCtx, Task<? extends Serializable> task, boolean local)
+      throws SemanticException {
+    if (!opProcCtx.isSeenOp(task, currTopOp)) {
+      String currAliasId = opProcCtx.getCurrAliasId();
+      setTaskPlan(currAliasId, currTopOp, task, local, opProcCtx);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Met cRS in pRS(parentTask)-cRS-OP(childTask) case
+   * Split and link two tasks by temporary file : pRS-FS / TS-cRS-OP
+   */
+  static void splitPlan(ReduceSinkOperator cRS,
+      Task<? extends Serializable> parentTask, Task<? extends Serializable> childTask,
+      GenMRProcContext opProcCtx) throws SemanticException {
+    assert parentTask != null && childTask != null;
+    splitTasks(cRS, parentTask, childTask, opProcCtx);
+  }
+
+  /**
+   * Met cRS in pOP(parentTask with RS)-cRS-cOP(noTask) case
+   * Create new child task for cRS-cOP and link two tasks by temporary file : pOP-FS / TS-cRS-cOP
    *
-   * @param op
+   * @param cRS
    *          the reduce sink operator encountered
    * @param opProcCtx
    *          processing context
    */
-  public static void splitPlan(ReduceSinkOperator op, GenMRProcContext opProcCtx)
+  static void splitPlan(ReduceSinkOperator cRS, GenMRProcContext opProcCtx)
       throws SemanticException {
     // Generate a new task
     ParseContext parseCtx = opProcCtx.getParseCtx();
-    MapredWork cplan = getMapRedWork(parseCtx);
-    Task<? extends Serializable> redTask = TaskFactory.get(cplan, parseCtx
+    Task<? extends Serializable> parentTask = opProcCtx.getCurrTask();
+
+    MapredWork childPlan = getMapRedWork(parseCtx);
+    Task<? extends Serializable> childTask = TaskFactory.get(childPlan, parseCtx
         .getConf());
-    Operator<? extends OperatorDesc> reducer = op.getChildOperators().get(0);
+    Operator<? extends OperatorDesc> reducer = cRS.getChildOperators().get(0);
 
     // Add the reducer
-    cplan.setReducer(reducer);
-    ReduceSinkDesc desc = op.getConf();
-
-    cplan.setNumReduceTasks(new Integer(desc.getNumReducers()));
+    childPlan.setReducer(reducer);
+    ReduceSinkDesc desc = cRS.getConf();
+    childPlan.setNumReduceTasks(new Integer(desc.getNumReducers()));
 
-    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
-        opProcCtx.getOpTaskMap();
-    opTaskMap.put(reducer, redTask);
-    Task<? extends Serializable> currTask = opProcCtx.getCurrTask();
+    opProcCtx.getOpTaskMap().put(reducer, childTask);
 
-    splitTasks(op, currTask, redTask, opProcCtx, true, false, 0);
-    opProcCtx.getRootOps().add(op);
+    splitTasks(cRS, parentTask, childTask, opProcCtx);
   }
 
   /**
@@ -391,9 +381,9 @@ public final class GenMapRedUtils {
    *          processing context
    */
   public static void setTaskPlan(String alias_id,
-      Operator<? extends OperatorDesc> topOp, MapredWork plan, boolean local,
+      Operator<? extends OperatorDesc> topOp, Task<?> task, boolean local,
       GenMRProcContext opProcCtx) throws SemanticException {
-    setTaskPlan(alias_id, topOp, plan, local, opProcCtx, null);
+    setTaskPlan(alias_id, topOp, task, local, opProcCtx, null);
   }
 
   private static ReadEntity getParentViewInfo(String alias_id,
@@ -435,8 +425,9 @@ public final class GenMapRedUtils {
    *          pruned partition list. If it is null it will be computed on-the-fly.
    */
   public static void setTaskPlan(String alias_id,
-      Operator<? extends OperatorDesc> topOp, MapredWork plan, boolean local,
+      Operator<? extends OperatorDesc> topOp, Task<?> task, boolean local,
       GenMRProcContext opProcCtx, PrunedPartitionList pList) throws SemanticException {
+    MapredWork plan = (MapredWork) task.getWork();
     ParseContext parseCtx = opProcCtx.getParseCtx();
     Set<ReadEntity> inputs = opProcCtx.getInputs();
 
@@ -684,6 +675,7 @@ public final class GenMapRedUtils {
       }
       plan.setMapLocalWork(localPlan);
     }
+    opProcCtx.addSeenOp(task, topOp);
   }
 
   /**
@@ -854,20 +846,20 @@ public final class GenMapRedUtils {
 
   @SuppressWarnings("nls")
   /**
-   * Merge the tasks - by creating a temporary file between them.
+   * Split two tasks by creating a temporary file between them.
+   *
    * @param op reduce sink operator being processed
-   * @param oldTask the parent task
-   * @param task the child task
+   * @param parentTask the parent task
+   * @param childTask the child task
    * @param opProcCtx context
-   * @param setReducer does the reducer needs to be set
-   * @param pos position of the parent
    **/
-  public static void splitTasks(Operator<? extends OperatorDesc> op,
-      Task<? extends Serializable> parentTask,
-      Task<? extends Serializable> childTask, GenMRProcContext opProcCtx,
-      boolean setReducer, boolean local, int posn) throws SemanticException {
-    childTask.getWork();
-    Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();
+  private static void splitTasks(ReduceSinkOperator op,
+      Task<? extends Serializable> parentTask, Task<? extends Serializable> childTask,
+      GenMRProcContext opProcCtx) throws SemanticException {
+    if (op.getNumParent() != 1) {
+      throw new IllegalStateException("Expecting operator " + op + " to have one parent. " +
+          "But found multiple parents : " + op.getParentOperators());
+    }
 
     ParseContext parseCtx = opProcCtx.getParseCtx();
     parentTask.addDependentTask(childTask);
@@ -883,7 +875,7 @@ public final class GenMapRedUtils {
     Context baseCtx = parseCtx.getContext();
     String taskTmpDir = baseCtx.getMRTmpFileURI();
 
-    Operator<? extends OperatorDesc> parent = op.getParentOperators().get(posn);
+    Operator<? extends OperatorDesc> parent = op.getParentOperators().get(0);
     TableDesc tt_desc = PlanUtils.getIntermediateFileTableDesc(PlanUtils
         .getFieldSchemasFromRowSchema(parent.getSchema(), "temporarycol"));
 
@@ -925,41 +917,46 @@ public final class GenMapRedUtils {
     childOpList = new ArrayList<Operator<? extends OperatorDesc>>();
     childOpList.add(op);
     ts_op.setChildOperators(childOpList);
-    op.getParentOperators().set(posn, ts_op);
+    op.getParentOperators().set(0, ts_op);
 
     Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx =
         opProcCtx.getMapCurrCtx();
-    mapCurrCtx.put(ts_op, new GenMapRedCtx(childTask, null, null));
+    mapCurrCtx.put(ts_op, new GenMapRedCtx(childTask, null));
 
     String streamDesc = taskTmpDir;
     MapredWork cplan = (MapredWork) childTask.getWork();
 
-    if (setReducer) {
-      Operator<? extends OperatorDesc> reducer = op.getChildOperators().get(0);
+    Operator<? extends OperatorDesc> reducer = op.getChildOperators().get(0);
 
-      if (reducer.getClass() == JoinOperator.class) {
-        String origStreamDesc;
-        streamDesc = "$INTNAME";
-        origStreamDesc = streamDesc;
-        int pos = 0;
-        while (cplan.getAliasToWork().get(streamDesc) != null) {
-          streamDesc = origStreamDesc.concat(String.valueOf(++pos));
-        }
+    if (needsTagging(reducer)) {
+      String origStreamDesc;
+      streamDesc = "$INTNAME";
+      origStreamDesc = streamDesc;
+      int pos = 0;
+      while (cplan.getAliasToWork().get(streamDesc) != null) {
+        streamDesc = origStreamDesc.concat(String.valueOf(++pos));
       }
 
       // TODO: Allocate work to remove the temporary files and make that
       // dependent on the redTask
-      if (reducer.getClass() == JoinOperator.class ||
-          reducer.getClass() == DemuxOperator.class) {
-        cplan.setNeedsTagging(true);
-      }
+      cplan.setNeedsTagging(true);
     }
 
     // Add the path to alias mapping
-    setTaskPlan(taskTmpDir, streamDesc, ts_op, cplan, local, tt_desc);
+    setTaskPlan(taskTmpDir, streamDesc, ts_op, cplan, false, tt_desc);
     opProcCtx.setCurrTopOp(null);
     opProcCtx.setCurrAliasId(null);
     opProcCtx.setCurrTask(childTask);
+    opProcCtx.addRootIfPossible(parentTask);
+  }
+
+  static boolean hasBranchFinished(Object... children) {
+    for (Object child : children) {
+      if (child == null) {
+        return false;
+      }
+    }
+    return true;
   }
 
   private GenMapRedUtils() {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java Tue Jul 23 00:50:39 2013
@@ -144,38 +144,16 @@ public final class MapJoinFactory {
      *          position of the parent
      */
     private static void initMapJoinPlan(AbstractMapJoinOperator<? extends MapJoinDesc> op,
-        GenMRProcContext opProcCtx, int pos)
+        Task<? extends Serializable> currTask,
+        GenMRProcContext opProcCtx, boolean local)
         throws SemanticException {
-      Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx =
-          opProcCtx.getMapCurrCtx();
-      int parentPos = (pos == -1) ? 0 : pos;
-      GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(
-          parentPos));
-      Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
-      MapredWork plan = (MapredWork) currTask.getWork();
-      HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
-          opProcCtx.getOpTaskMap();
-      Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();
-
-      MapJoinDesc desc = (MapJoinDesc) op.getConf();
 
       // The map is overloaded to keep track of mapjoins also
-      opTaskMap.put(op, currTask);
-
-      List<Task<? extends Serializable>> rootTasks = opProcCtx.getRootTasks();
-      if(!rootTasks.contains(currTask)
-         && (currTask.getParentTasks() == null
-             || currTask.getParentTasks().isEmpty())) {
-        rootTasks.add(currTask);
-      }
-
-      assert currTopOp != null;
-      opProcCtx.getSeenOps().add(currTopOp);
+      opProcCtx.getOpTaskMap().put(op, currTask);
 
+      Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();
       String currAliasId = opProcCtx.getCurrAliasId();
-      boolean local = (pos == desc.getPosBigTable()) ? false : true;
-      GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, plan, local, opProcCtx);
-      setupBucketMapJoinInfo(plan, op);
+      GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, local, opProcCtx);
     }
 
     /**
@@ -191,29 +169,12 @@ public final class MapJoinFactory {
      * @param pos
      *          position of the parent in the stack
      */
-    public static void joinMapJoinPlan(AbstractMapJoinOperator<? extends OperatorDesc> op,
+    private static void joinMapJoinPlan(AbstractMapJoinOperator<? extends MapJoinDesc> op,
         Task<? extends Serializable> oldTask,
-        GenMRProcContext opProcCtx, int pos)
+        GenMRProcContext opProcCtx, boolean local)
         throws SemanticException {
-      MapredWork plan = (MapredWork) oldTask.getWork();
       Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();
-
-      List<Operator<? extends OperatorDesc>> seenOps = opProcCtx.getSeenOps();
-      String currAliasId = opProcCtx.getCurrAliasId();
-
-      if (!seenOps.contains(currTopOp)) {
-        seenOps.add(currTopOp);
-        boolean local = false;
-        if (pos != -1) {
-          local = (pos == ((MapJoinDesc) op.getConf()).getPosBigTable()) ? false
-              : true;
-        }
-        GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, plan, local, opProcCtx);
-        setupBucketMapJoinInfo(plan, op);
-      }
-      currTopOp = null;
-      opProcCtx.setCurrTopOp(currTopOp);
-      opProcCtx.setCurrTask(oldTask);
+      GenMapRedUtils.mergeInput(currTopOp, opProcCtx, oldTask, local);
     }
 
     /*
@@ -236,17 +197,14 @@ public final class MapJoinFactory {
 
       Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx = ctx
           .getMapCurrCtx();
-      GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(
-          pos));
+      GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(pos));
       Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
       MapredWork currPlan = (MapredWork) currTask.getWork();
-      Operator<? extends OperatorDesc> currTopOp = mapredCtx.getCurrTopOp();
       String currAliasId = mapredCtx.getCurrAliasId();
       HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
           ctx.getOpTaskMap();
-      Task<? extends Serializable> opMapTask = opTaskMap.get(mapJoin);
+      Task<? extends Serializable> oldTask = opTaskMap.get(mapJoin);
 
-      ctx.setCurrTopOp(currTopOp);
       ctx.setCurrAliasId(currAliasId);
       ctx.setCurrTask(currTask);
 
@@ -254,20 +212,23 @@ public final class MapJoinFactory {
       // If we are seeing this mapjoin for the second or later time then atleast one of the
       // branches for this mapjoin have been encounered. Join the plan with the plan created
       // the first time.
-      if (opMapTask == null) {
+      boolean local = pos != mapJoin.getConf().getPosBigTable();
+      if (oldTask == null) {
         assert currPlan.getReducer() == null;
-        initMapJoinPlan(mapJoin, ctx, pos);
+        initMapJoinPlan(mapJoin, currTask, ctx, local);
       } else {
         // The current plan can be thrown away after being merged with the
         // original plan
-        joinMapJoinPlan(mapJoin, opMapTask, ctx, pos);
-        currTask = opMapTask;
-        ctx.setCurrTask(currTask);
+        joinMapJoinPlan(mapJoin, oldTask, ctx, local);
+        ctx.setCurrTask(currTask = oldTask);
       }
+      MapredWork plan = (MapredWork) currTask.getWork();
+      setupBucketMapJoinInfo(plan, mapJoin);
+
+      mapCurrCtx.put(mapJoin, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrAliasId()));
 
-      mapCurrCtx.put(mapJoin, new GenMapRedCtx(ctx.getCurrTask(), ctx
-          .getCurrTopOp(), ctx.getCurrAliasId()));
-      return null;
+      // local aliases need not to hand over context further
+      return !local;
     }
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Tue Jul 23 00:50:39 2013
@@ -263,7 +263,8 @@ public class MapJoinProcessor implements
 
     } catch (Exception e) {
       e.printStackTrace();
-      throw new SemanticException("Generate New MapJoin Opertor Exeception " + e.getMessage());
+      throw new SemanticException("Failed to generate new mapJoin operator " +
+          "by exception : " + e.getMessage());
     }
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java Tue Jul 23 00:50:39 2013
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.parse;
 
 import java.util.List;
 
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
 import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
 import org.apache.hadoop.hive.ql.lib.Dispatcher;
 import org.apache.hadoop.hive.ql.lib.Node;
@@ -52,12 +51,11 @@ public class GenMapRedWalker extends Def
 
     // maintain the stack of operators encountered
     opStack.push(nd);
-    dispatch(nd, opStack);
+    Boolean result = dispatchAndReturn(nd, opStack);
 
-    // kids of reduce sink operator need not be traversed again
-    if ((children == null)
-        || ((nd instanceof ReduceSinkOperator) && (getDispatchedList()
-        .containsAll(children)))) {
+    // kids of reduce sink operator or mapjoin operators merged into root task
+    // need not be traversed again
+    if (children == null || result == Boolean.FALSE) {
       opStack.pop();
       return;
     }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java Tue Jul 23 00:50:39 2013
@@ -217,8 +217,7 @@ public class MapReduceCompiler {
     GenMRProcContext procCtx = new GenMRProcContext(
         conf,
         new HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>>(),
-        new ArrayList<Operator<? extends OperatorDesc>>(), tempParseContext,
-        mvTask, rootTasks,
+        tempParseContext, mvTask, rootTasks,
         new LinkedHashMap<Operator<? extends OperatorDesc>, GenMapRedCtx>(),
         inputs, outputs);
 
@@ -599,8 +598,7 @@ public class MapReduceCompiler {
     boolean hasNonLocalJob = false;
     for (ExecDriver mrtask : mrtasks) {
       try {
-        ContentSummary inputSummary = Utilities.getInputSummary
-            (ctx, (MapredWork) mrtask.getWork(), p);
+        ContentSummary inputSummary = Utilities.getInputSummary(ctx, mrtask.getWork(), p);
         int numReducers = getNumberOfReducers(mrtask.getWork(), conf);
 
         long estimatedInput;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java Tue Jul 23 00:50:39 2013
@@ -95,7 +95,7 @@ public class MapredWork extends Abstract
 
   private boolean mapperCannotSpanPartns;
 
-  // used to indicate the input is sorted, and so a BinarySearchRecordReader shoudl be used
+  // used to indicate the input is sorted, and so a BinarySearchRecordReader should be used
   private boolean inputFormatSorted = false;
 
   private transient boolean useBucketizedHiveInputFormat;
@@ -630,4 +630,9 @@ public class MapredWork extends Abstract
     return samplingType == 1 ? "SAMPLING_ON_PREV_MR" :
         samplingType == 2 ? "SAMPLING_ON_START" : null;
   }
+
+  public void mergingInto(MapredWork mapred) {
+    // currently, this is sole field affecting mergee task
+    mapred.useBucketizedHiveInputFormat |= useBucketizedHiveInputFormat;
+  }
 }

Modified: hive/trunk/ql/src/test/queries/clientpositive/auto_sortmerge_join_6.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/auto_sortmerge_join_6.q?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/auto_sortmerge_join_6.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/auto_sortmerge_join_6.q Tue Jul 23 00:50:39 2013
@@ -19,10 +19,29 @@ set hive.auto.convert.join=true;
 set hive.auto.convert.join.noconditionaltask=true;
 set hive.auto.convert.join.noconditionaltask.size=200;
 set hive.auto.convert.sortmerge.join.to.mapjoin=false;
+
 -- A SMB join is being followed by a regular join on a non-bucketed table on a different key
+
+-- Three tests below are all the same query with different alias, which changes dispatch order of GenMapRedWalker
+-- This is dependent to iteration order of HashMap, so can be meaningless in non-sun jdk
+-- b = TS[0]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]-SEL[9]-FS[10]
+-- c = TS[1]-RS[7]-JOIN[8]
+-- a = TS[2]-MAPJOIN[11]
 explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src c on c.value = a.value;
 select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src c on c.value = a.value;
 
+-- d = TS[0]-RS[7]-JOIN[8]-SEL[9]-FS[10]
+-- b = TS[1]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]
+-- a = TS[2]-MAPJOIN[11]
+explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src d on d.value = a.value;
+select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src d on d.value = a.value;
+
+-- b = TS[0]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]-SEL[9]-FS[10]
+-- a = TS[1]-MAPJOIN[11]
+-- h = TS[2]-RS[7]-JOIN[8]
+explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src h on h.value = a.value;
+select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src h on h.value = a.value;
+
 -- A SMB join is being followed by a regular join on a non-bucketed table on the same key
 explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src c on c.key = a.key;
 select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src c on c.key = a.key;

Added: hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_25.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_25.q?rev=1505870&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_25.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_25.q Tue Jul 23 00:50:39 2013
@@ -0,0 +1,41 @@
+set hive.enforce.bucketing=true;
+set hive.enforce.sorting=true;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.exec.max.dynamic.partitions.pernode=1000000;
+set hive.exec.max.dynamic.partitions=1000000;
+set hive.exec.max.created.files=1000000;
+set hive.map.aggr=true;
+
+create table smb_bucket_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; 
+create table smb_bucket_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE; 
+create table smb_bucket_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS STORED AS RCFILE;
+
+load data local inpath '../data/files/smbbucket_1.rc' overwrite into table smb_bucket_1;
+load data local inpath '../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2;
+load data local inpath '../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3;
+
+explain 
+select * from (select a.key from smb_bucket_1 a join smb_bucket_2 b on (a.key = b.key) where a.key = 5) t1 left outer join (select c.key from smb_bucket_2 c join smb_bucket_3 d on (c.key = d.key) where c.key=5) t2 on (t1.key=t2.key) where t2.key=5;
+
+set hive.optimize.bucketmapjoin=true;
+set hive.optimize.bucketmapjoin.sortedmerge=true;
+set hive.mapred.reduce.tasks.speculative.execution=false;
+set hive.auto.convert.join=true;
+set hive.auto.convert.sortmerge.join=true;
+set hive.auto.convert.sortmerge.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000000000;
+set hive.optimize.reducededuplication.min.reducer=1;
+set hive.optimize.mapjoin.mapreduce=true;
+set hive.auto.convert.sortmerge.join.bigtable.selection.policy=org.apache.hadoop.hive.ql.optimizer.LeftmostBigTableSelectorForAutoSMJ;
+
+-- explain
+-- select * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key left outer join src c on a.key=c.value
+
+-- select a.key from smb_bucket_1 a
+
+explain 
+select * from (select a.key from smb_bucket_1 a join smb_bucket_2 b on (a.key = b.key) where a.key = 5) t1 left outer join (select c.key from smb_bucket_2 c join smb_bucket_3 d on (c.key = d.key) where c.key=5) t2 on (t1.key=t2.key) where t2.key=5;
+
+select * from (select a.key from smb_bucket_1 a join smb_bucket_2 b on (a.key = b.key) where a.key = 5) t1 left outer join (select c.key from smb_bucket_2 c join smb_bucket_3 d on (c.key = d.key) where c.key=5) t2 on (t1.key=t2.key) where t2.key=5;
+

Modified: hive/trunk/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out Tue Jul 23 00:50:39 2013
@@ -162,7 +162,7 @@ ABSTRACT SYNTAX TREE:
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
+  Stage-3 depends on stages: Stage-1
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -234,7 +234,7 @@ STAGE PLANS:
                     input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -346,8 +346,8 @@ ABSTRACT SYNTAX TREE:
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-5
-  Stage-3 depends on stages: Stage-2
+  Stage-3 depends on stages: Stage-1, Stage-5
+  Stage-4 depends on stages: Stage-3
   Stage-5 is a root stage
   Stage-0 is a root stage
 
@@ -420,7 +420,7 @@ STAGE PLANS:
                   input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
         $INTNAME 
@@ -476,7 +476,7 @@ STAGE PLANS:
                   input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-3
+  Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####

Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out Tue Jul 23 00:50:39 2013
@@ -71,9 +71,21 @@ POSTHOOK: Lineage: tbl3.value SIMPLE [(s
 POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 PREHOOK: query: -- A SMB join is being followed by a regular join on a non-bucketed table on a different key
+
+-- Three tests below are all the same query with different alias, which changes dispatch order of GenMapRedWalker
+-- This is dependent to iteration order of HashMap, so can be meaningless in non-sun jdk
+-- b = TS[0]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]-SEL[9]-FS[10]
+-- c = TS[1]-RS[7]-JOIN[8]
+-- a = TS[2]-MAPJOIN[11]
 explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src c on c.value = a.value
 PREHOOK: type: QUERY
 POSTHOOK: query: -- A SMB join is being followed by a regular join on a non-bucketed table on a different key
+
+-- Three tests below are all the same query with different alias, which changes dispatch order of GenMapRedWalker
+-- This is dependent to iteration order of HashMap, so can be meaningless in non-sun jdk
+-- b = TS[0]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]-SEL[9]-FS[10]
+-- c = TS[1]-RS[7]-JOIN[8]
+-- a = TS[2]-MAPJOIN[11]
 explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src c on c.value = a.value
 POSTHOOK: type: QUERY
 POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -88,12 +100,12 @@ ABSTRACT SYNTAX TREE:
   (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME src) c) (= (. (TOK_TABLE_OR_COL c) value) (. (TOK_TABLE_OR_COL a) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
 
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
+  Stage-2 is a root stage
+  Stage-3 depends on stages: Stage-2
   Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
         a 
@@ -154,7 +166,7 @@ STAGE PLANS:
                     input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -209,6 +221,292 @@ POSTHOOK: Lineage: tbl3.value SIMPLE [(s
 POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 2654
+PREHOOK: query: -- d = TS[0]-RS[7]-JOIN[8]-SEL[9]-FS[10]
+-- b = TS[1]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]
+-- a = TS[2]-MAPJOIN[11]
+explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src d on d.value = a.value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- d = TS[0]-RS[7]-JOIN[8]-SEL[9]-FS[10]
+-- b = TS[1]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]
+-- a = TS[2]-MAPJOIN[11]
+explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src d on d.value = a.value
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl3.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME src) d) (= (. (TOK_TABLE_OR_COL d) value) (. (TOK_TABLE_OR_COL a) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        a 
+          TableScan
+            alias: a
+            Sorted Merge Bucket Map Join Operator
+              condition map:
+                   Inner Join 0 to 1
+              condition expressions:
+                0 {value}
+                1 
+              handleSkewJoin: false
+              keys:
+                0 [Column[key]]
+                1 [Column[key]]
+              outputColumnNames: _col1
+              Position of Big Table: 0
+              Reduce Output Operator
+                key expressions:
+                      expr: _col1
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: _col1
+                      type: string
+                tag: 0
+        d 
+          TableScan
+            alias: d
+            Reduce Output Operator
+              key expressions:
+                    expr: value
+                    type: string
+              sort order: +
+              Map-reduce partition columns:
+                    expr: value
+                    type: string
+              tag: 1
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 
+          handleSkewJoin: false
+          Select Operator
+            Group By Operator
+              aggregations:
+                    expr: count()
+              bucketGroup: false
+              mode: hash
+              outputColumnNames: _col0
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+            Reduce Output Operator
+              sort order: 
+              tag: -1
+              value expressions:
+                    expr: _col0
+                    type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          bucketGroup: false
+          mode: mergepartial
+          outputColumnNames: _col0
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: bigint
+            outputColumnNames: _col0
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+PREHOOK: query: select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src d on d.value = a.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src d on d.value = a.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl3.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+2654
+PREHOOK: query: -- b = TS[0]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]-SEL[9]-FS[10]
+-- a = TS[1]-MAPJOIN[11]
+-- h = TS[2]-RS[7]-JOIN[8]
+explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src h on h.value = a.value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- b = TS[0]-OP[13]-MAPJOIN[11]-RS[6]-JOIN[8]-SEL[9]-FS[10]
+-- a = TS[1]-MAPJOIN[11]
+-- h = TS[2]-RS[7]-JOIN[8]
+explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src h on h.value = a.value
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl3.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME src) h) (= (. (TOK_TABLE_OR_COL h) value) (. (TOK_TABLE_OR_COL a) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-3 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        a 
+          TableScan
+            alias: a
+            Sorted Merge Bucket Map Join Operator
+              condition map:
+                   Inner Join 0 to 1
+              condition expressions:
+                0 {value}
+                1 
+              handleSkewJoin: false
+              keys:
+                0 [Column[key]]
+                1 [Column[key]]
+              outputColumnNames: _col1
+              Position of Big Table: 0
+              Reduce Output Operator
+                key expressions:
+                      expr: _col1
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: _col1
+                      type: string
+                tag: 0
+        h 
+          TableScan
+            alias: h
+            Reduce Output Operator
+              key expressions:
+                    expr: value
+                    type: string
+              sort order: +
+              Map-reduce partition columns:
+                    expr: value
+                    type: string
+              tag: 1
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 
+          handleSkewJoin: false
+          Select Operator
+            Group By Operator
+              aggregations:
+                    expr: count()
+              bucketGroup: false
+              mode: hash
+              outputColumnNames: _col0
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-3
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+            Reduce Output Operator
+              sort order: 
+              tag: -1
+              value expressions:
+                    expr: _col0
+                    type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          bucketGroup: false
+          mode: mergepartial
+          outputColumnNames: _col0
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: bigint
+            outputColumnNames: _col0
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+PREHOOK: query: select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src h on h.value = a.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src h on h.value = a.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl3.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+2654
 PREHOOK: query: -- A SMB join is being followed by a regular join on a non-bucketed table on the same key
 explain select count(*) FROM tbl1 a JOIN tbl2 b ON a.key = b.key join src c on c.key = a.key
 PREHOOK: type: QUERY
@@ -703,12 +1001,12 @@ ABSTRACT SYNTAX TREE:
   (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME tbl4) c) (= (. (TOK_TABLE_OR_COL c) value) (. (TOK_TABLE_OR_COL a) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
 
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
+  Stage-2 is a root stage
+  Stage-3 depends on stages: Stage-2
   Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
         a 
@@ -769,7 +1067,7 @@ STAGE PLANS:
                     input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -842,12 +1140,12 @@ ABSTRACT SYNTAX TREE:
   (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME src) c) (= (. (TOK_TABLE_OR_COL c) value) (. (TOK_TABLE_OR_COL a) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
 
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
+  Stage-2 is a root stage
+  Stage-3 depends on stages: Stage-2
   Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
         a 
@@ -908,7 +1206,7 @@ STAGE PLANS:
                     input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -1746,12 +2044,12 @@ ABSTRACT SYNTAX TREE:
   (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME tbl4) c) (= (. (TOK_TABLE_OR_COL c) value) (. (TOK_TABLE_OR_COL a) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
 
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
+  Stage-2 is a root stage
+  Stage-3 depends on stages: Stage-2
   Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
         a 
@@ -1812,7 +2110,7 @@ STAGE PLANS:
                     input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####

Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out?rev=1505870&r1=1505869&r2=1505870&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out Tue Jul 23 00:50:39 2013
@@ -322,7 +322,7 @@ ABSTRACT SYNTAX TREE:
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
+  Stage-3 depends on stages: Stage-1
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -394,7 +394,7 @@ STAGE PLANS:
                     input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -506,13 +506,13 @@ ABSTRACT SYNTAX TREE:
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-9 depends on stages: Stage-1, Stage-5 , consists of Stage-10, Stage-11, Stage-2
-  Stage-10 has a backup stage: Stage-2
+  Stage-9 depends on stages: Stage-1, Stage-5 , consists of Stage-10, Stage-11, Stage-3
+  Stage-10 has a backup stage: Stage-3
   Stage-7 depends on stages: Stage-10
-  Stage-3 depends on stages: Stage-2, Stage-7, Stage-8
-  Stage-11 has a backup stage: Stage-2
+  Stage-4 depends on stages: Stage-3, Stage-7, Stage-8
+  Stage-11 has a backup stage: Stage-3
   Stage-8 depends on stages: Stage-11
-  Stage-2
+  Stage-3
   Stage-5 is a root stage
   Stage-0 is a root stage
 
@@ -640,7 +640,7 @@ STAGE PLANS:
       Local Work:
         Map Reduce Local Work
 
-  Stage: Stage-3
+  Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -722,7 +722,7 @@ STAGE PLANS:
       Local Work:
         Map Reduce Local Work
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
         $INTNAME 
@@ -2695,7 +2695,7 @@ STAGE DEPENDENCIES:
   Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
   Stage-7 has a backup stage: Stage-1
   Stage-4 depends on stages: Stage-7
-  Stage-2 depends on stages: Stage-1, Stage-4, Stage-5
+  Stage-3 depends on stages: Stage-1, Stage-4, Stage-5
   Stage-8 has a backup stage: Stage-1
   Stage-5 depends on stages: Stage-8
   Stage-1
@@ -2795,7 +2795,7 @@ STAGE PLANS:
                     input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -3067,13 +3067,13 @@ STAGE DEPENDENCIES:
   Stage-12 is a root stage , consists of Stage-18, Stage-19, Stage-1
   Stage-18 has a backup stage: Stage-1
   Stage-10 depends on stages: Stage-18
-  Stage-9 depends on stages: Stage-1, Stage-5, Stage-10, Stage-11, Stage-13, Stage-14 , consists of Stage-16, Stage-17, Stage-2
-  Stage-16 has a backup stage: Stage-2
+  Stage-9 depends on stages: Stage-1, Stage-5, Stage-10, Stage-11, Stage-13, Stage-14 , consists of Stage-16, Stage-17, Stage-3
+  Stage-16 has a backup stage: Stage-3
   Stage-7 depends on stages: Stage-16
-  Stage-3 depends on stages: Stage-2, Stage-7, Stage-8
-  Stage-17 has a backup stage: Stage-2
+  Stage-4 depends on stages: Stage-3, Stage-7, Stage-8
+  Stage-17 has a backup stage: Stage-3
   Stage-8 depends on stages: Stage-17
-  Stage-2
+  Stage-3
   Stage-19 has a backup stage: Stage-1
   Stage-11 depends on stages: Stage-19
   Stage-1
@@ -3234,7 +3234,7 @@ STAGE PLANS:
       Local Work:
         Map Reduce Local Work
 
-  Stage: Stage-3
+  Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -3316,7 +3316,7 @@ STAGE PLANS:
       Local Work:
         Map Reduce Local Work
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
         $INTNAME 



Mime
View raw message