hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r762665 - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/ ql/src/java/o...
Date Tue, 07 Apr 2009 09:53:47 GMT
Author: zshao
Date: Tue Apr  7 09:53:46 2009
New Revision: 762665

URL: http://svn.apache.org/viewvc?rev=762665&view=rev
Log:
HIVE-384. Fixing UNION ALL when the first job is a map-only job. (Namit Jain via zshao)

Added:
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/union14.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/union15.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union14.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union15.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/unionDesc.java
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map_skew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union10.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union11.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union12.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union3.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union5.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union7.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union8.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union9.q.out

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Apr  7 09:53:46 2009
@@ -119,12 +119,15 @@
     HIVE-375. LazySimpleSerDe directly creates a UTF8 buffer for primitive types.
     (Zheng Shao via namit)
 
-    HIVE-382 Fix for hash aggr, remove elements from hash table in the loop
+    HIVE-382. Fix for hash aggr, remove elements from hash table in the loop
     during close, rather than waiting till the end. (Namit Jain via rmurthy)
 
-    HIVE-391 Fix for UDAFCount which was not handling merging nulls
+    HIVE-391. Fix for UDAFCount which was not handling merging nulls
     (Namit Jain via rmurthy)
 
+    HIVE-384. Fixing UNION ALL when the first job is a map-only job.
+    (Namit Jain via zshao)
+
 Release 0.2.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Tue Apr  7 09:53:46 2009
@@ -63,14 +63,21 @@
   private BaseSemanticAnalyzer sem;
 
   public int countJobs(List<Task<? extends Serializable>> tasks) {
+    return countJobs(tasks, new ArrayList<Task<? extends Serializable>>());
+  }
+
+  public int countJobs(List<Task<? extends Serializable>> tasks, List<Task<? extends Serializable>> seenTasks) {
     if (tasks == null)
       return 0;
     int jobs = 0;
     for (Task<? extends Serializable> task : tasks) {
-      if (task.isMapRedTask()) {
-        jobs++;
+      if (!seenTasks.contains(task)) {
+        seenTasks.add(task);
+        if (task.isMapRedTask()) {
+          jobs++;
+        }
+        jobs += countJobs(task.getChildTasks(), seenTasks);
       }
-      jobs += countJobs(task.getChildTasks());
     }
     return jobs;
   }
@@ -160,7 +167,6 @@
     boolean noName = StringUtils.isEmpty(conf
         .getVar(HiveConf.ConfVars.HADOOPJOBNAME));
     int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
-    int jobs = 0;
 
     conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, command);
     
@@ -193,7 +199,7 @@
       sem.analyze(tree, ctx);
       LOG.info("Semantic Analysis Completed");
 
-      jobs = countJobs(sem.getRootTasks());
+      int jobs = countJobs(sem.getRootTasks());
       if (jobs > 0) {
         console.printInfo("Total MapReduce jobs = " + jobs);
       }
@@ -203,18 +209,8 @@
         SessionState.get().getHiveHistory().setIdToTableMap(sem.getIdToTableNameMap());
       }
       String jobname = Utilities.abbreviate(command, maxlen - 6);
-      int curJob = 0;
-      for (Task<? extends Serializable> rootTask : sem.getRootTasks()) {
-        // assumption that only top level tasks are map-reduce tasks
-        if (rootTask.isMapRedTask()) {
-          curJob++;
-          if (noName) {
-            conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname + "(" + curJob
-                + "/" + jobs + ")");
-          }
-        }
-        rootTask.initialize(conf);
-      }
+      
+      int curJobNo = 0;
 
       // A very simple runtime that keeps putting runnable takss
       // on a list and when a job completes, it puts the children at the back of
@@ -236,6 +232,16 @@
           SessionState.get().getHiveHistory().startTask(queryId, tsk,
               tsk.getClass().getName());
 
+        if (tsk.isMapRedTask()) {
+          curJobNo++;
+          if (noName) {
+            conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname + "(" + curJobNo
+                        + "/" + jobs + ")");
+          }
+        }
+
+        tsk.initialize(conf);
+
         int exitVal = tsk.execute();
         if (SessionState.get() != null) {
           SessionState.get().getHiveHistory().setTaskProperty(queryId,

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Tue Apr  7 09:53:46 2009
@@ -75,14 +75,6 @@
     }
 
     console = new LogHelper(LOG);
-
-    if(childTasks == null) {
-      return;
-    }
-
-    for(Task<? extends Serializable> t: childTasks) {
-      t.initialize(conf);
-    }
   }
 
   public abstract int execute();

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java?rev=762665&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java Tue Apr  7 09:53:46 2009
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Stack;
+import java.io.Serializable;
+
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.UnionOperator;
+import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.plan.mapredWork;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
+import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
+import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext.UnionParseContext;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.plan.reduceSinkDesc;
+
+/**
+ * Processor for the rule - table scan followed by reduce sink
+ */
+public class GenMRRedSink3 implements NodeProcessor {
+
+  public GenMRRedSink3() {
+  }
+
+  /**
+   * Reduce Scan encountered 
+   * @param nd the reduce sink operator encountered
+   * @param opProcCtx context
+   */
+  public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException {
+    ReduceSinkOperator op = (ReduceSinkOperator)nd;
+    GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
+
+    ParseContext parseCtx = ctx.getParseCtx();
+    UnionProcContext uCtx = parseCtx.getUCtx();
+
+    // union was map only - no special processing needed
+    if (uCtx.isMapOnlySubq())
+      return (new GenMRRedSink1()).process(nd, stack, opProcCtx, nodeOutputs);
+
+    // union consisted on a bunch of map-reduce jobs, and it has been split at the union
+    Operator<? extends Serializable> reducer = op.getChildOperators().get(0);
+    Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
+    GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(0));
+    Task<? extends Serializable> currTask    = mapredCtx.getCurrTask();
+    mapredWork plan = (mapredWork) currTask.getWork();
+    HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = ctx.getOpTaskMap();
+    
+    opTaskMap.put(reducer, currTask);
+    plan.setReducer(reducer);
+    reduceSinkDesc desc = (reduceSinkDesc)op.getConf();
+    
+    plan.setNumReduceTasks(desc.getNumReducers());
+    
+    if (reducer.getClass() == JoinOperator.class)
+      plan.setNeedsTagging(true);
+    
+    ctx.setCurrTask(currTask);
+    ctx.setCurrTopOp(null);
+    ctx.setCurrAliasId(null);
+    return null;
+  }
+}

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java Tue Apr  7 09:53:46 2009
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
 import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
 import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcFactory;
+import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
 import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext.UnionParseContext;
 
 /**
@@ -68,30 +69,22 @@
     UnionOperator union = (UnionOperator)nd;
     GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
     ParseContext parseCtx = ctx.getParseCtx();
-    UnionParseContext uPrsCtx = parseCtx.getUCtx().getUnionParseContext(union);
-    assert uPrsCtx != null;
-
-    // The plan needs to be broken only if one of the sub-queries involve a map-reduce job
-    int numInputs = uPrsCtx.getNumInputs();
-    boolean mapOnly = true;
-    int pos = 0;
-    for (pos = 0; pos < numInputs; pos++) {
-      if (!uPrsCtx.getMapOnlySubq(pos)) {
-        mapOnly = false;
-        break;
-      }
-    }
+    UnionProcContext uCtx = parseCtx.getUCtx();
 
     // Map-only subqueries can be optimized in future to not write to a file in future
     Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
 
-    if (mapOnly) {
+    // The plan needs to be broken only if one of the sub-queries involve a map-reduce job
+    if (uCtx.isMapOnlySubq()) {
       mapCurrCtx.put((Operator<? extends Serializable>)nd, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId()));
       return null;
     }
 
+    UnionParseContext uPrsCtx = uCtx.getUnionParseContext(union);
+    assert uPrsCtx != null;
+
     Task<? extends Serializable> currTask = ctx.getCurrTask();
-    pos = UnionProcFactory.getPositionParent(union, stack);
+    int pos = UnionProcFactory.getPositionParent(union, stack);
 
     // is the current task a root task
     if (uPrsCtx.getRootTask(pos) && (!ctx.getRootTasks().contains(currTask)))

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java Tue Apr  7 09:53:46 2009
@@ -30,10 +30,10 @@
     transient private boolean[] mapOnlySubq;
     transient private boolean[] rootTask;
     
-    // currently, union has 2 inputs, but that should change - merging of unions should be allowed
-    transient private int numInputs = 2;
+    transient private int numInputs;
 
-    public UnionParseContext() {
+    public UnionParseContext(int numInputs) {
+      this.numInputs = numInputs;
       mapOnlySubq = new boolean[numInputs];
       rootTask    = new boolean[numInputs];
     }
@@ -62,11 +62,29 @@
       this.numInputs = numInputs;
     }
   }
-   
+
+  // the subqueries are map-only jobs
+  private boolean mapOnlySubq;
+
+  /**
+   * @return the mapOnlySubq
+   */
+  public boolean isMapOnlySubq() {
+    return mapOnlySubq;
+  }
+
+  /**
+   * @param mapOnlySubq the mapOnlySubq to set
+   */
+  public void setMapOnlySubq(boolean mapOnlySubq) {
+    this.mapOnlySubq = mapOnlySubq;
+  }
+
   private Map<UnionOperator, UnionParseContext> uCtxMap;
 
   public UnionProcContext() {
     uCtxMap = new HashMap<UnionOperator, UnionParseContext>();
+    mapOnlySubq = true;
   }
 
   public void setUnionParseContext(UnionOperator u, UnionParseContext uCtx) { 
@@ -76,5 +94,4 @@
   public UnionParseContext getUnionParseContext(UnionOperator u) { 
     return uCtxMap.get(u);
   }
-
 }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java Tue Apr  7 09:53:46 2009
@@ -60,8 +60,9 @@
       int pos = getPositionParent(union, stack);
       UnionParseContext uCtx = ctx.getUnionParseContext(union);
       if (uCtx == null)
-        uCtx = new UnionParseContext();
+        uCtx = new UnionParseContext(union.getConf().getNumInputs());
 
+      ctx.setMapOnlySubq(false);
       uCtx.setMapOnlySubq(pos, false);
       uCtx.setRootTask(pos, false);
       ctx.setUnionParseContext(union, uCtx); 
@@ -84,7 +85,7 @@
       int pos = getPositionParent(union, stack);
       UnionParseContext uCtx = ctx.getUnionParseContext(union);
       if (uCtx == null)
-        uCtx = new UnionParseContext();
+        uCtx = new UnionParseContext(union.getConf().getNumInputs());
 
       uCtx.setMapOnlySubq(pos, true);
       uCtx.setRootTask(pos, true);
@@ -108,7 +109,7 @@
       int pos = getPositionParent(union, stack);
       UnionParseContext uCtx = ctx.getUnionParseContext(union);
       if (uCtx == null)
-        uCtx = new UnionParseContext();
+        uCtx = new UnionParseContext(union.getConf().getNumInputs());
 
       uCtx.setMapOnlySubq(pos, true);
       uCtx.setRootTask(pos, false);

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Apr  7 09:53:46 2009
@@ -76,6 +76,12 @@
 import org.apache.hadoop.hive.ql.optimizer.Optimizer;
 import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
 import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
+import org.apache.hadoop.hive.ql.optimizer.GenMRUnion1;
+import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3;
+import org.apache.hadoop.hive.ql.plan.*;
+import org.apache.hadoop.hive.ql.udf.UDFOPPositive;
+import org.apache.hadoop.hive.ql.exec.*;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.plan.PlanUtils;
 import org.apache.hadoop.hive.ql.plan.aggregationDesc;
 import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
@@ -3035,18 +3041,52 @@
       ColumnInfo lInfo = lEntry.getValue();
       unionoutRR.put(unionalias, field, lInfo);
     }
+
+    // If one of the children is a union, merge with it
+    // else create a new one
+    if ((leftOp instanceof UnionOperator) || (rightOp instanceof UnionOperator))
+    {
+      if (leftOp instanceof UnionOperator) {
+        // make left a child of right
+        List<Operator<? extends Serializable>> child = new ArrayList<Operator<? extends Serializable>>();
+        child.add(leftOp);
+        rightOp.setChildOperators(child);
+
+        List<Operator<? extends Serializable>> parent = leftOp.getParentOperators();
+        parent.add(rightOp);
+        
+        unionDesc uDesc = ((UnionOperator)leftOp).getConf();
+        uDesc.setNumInputs(uDesc.getNumInputs()+1);
+        return putOpInsertMap(leftOp, unionoutRR);
+      }
+      else {
+        // make right a child of left
+        List<Operator<? extends Serializable>> child = new ArrayList<Operator<? extends Serializable>>();
+        child.add(rightOp);
+        leftOp.setChildOperators(child);
+
+        List<Operator<? extends Serializable>> parent = rightOp.getParentOperators();
+        parent.add(leftOp);
+        unionDesc uDesc = ((UnionOperator)rightOp).getConf();
+        uDesc.setNumInputs(uDesc.getNumInputs()+1);
+        
+        return putOpInsertMap(rightOp, unionoutRR);
+      }
+    }
+
+    // Create a new union operator
     Operator<? extends Serializable> unionforward = 
       OperatorFactory.getAndMakeChild(new unionDesc(), new RowSchema(unionoutRR.getColumnInfos()));
-
-    // set forward operator as child of each of leftOp and rightOp
+    
+    // set union operator as child of each of leftOp and rightOp
     List<Operator<? extends Serializable>> child = new ArrayList<Operator<? extends Serializable>>();
     child.add(unionforward);
     rightOp.setChildOperators(child);
-
+    
     child = new ArrayList<Operator<? extends Serializable>>();
     child.add(unionforward);
     leftOp.setChildOperators(child);
-
+    
     List<Operator<? extends Serializable>> parent = new ArrayList<Operator<? extends Serializable>>();
     parent.add(leftOp);
     parent.add(rightOp);
@@ -3413,7 +3453,8 @@
     opRules.put(new RuleRegExp(new String("R2"), "TS%.*RS%"), new GenMRRedSink1());
     opRules.put(new RuleRegExp(new String("R3"), "RS%.*RS%"), new GenMRRedSink2());
     opRules.put(new RuleRegExp(new String("R4"), "FS%"), new GenMRFileSink1());
-    opRules.put(new RuleRegExp(new String("R4"), "UNION%"), new GenMRUnion1());
+    opRules.put(new RuleRegExp(new String("R5"), "UNION%"), new GenMRUnion1());
+    opRules.put(new RuleRegExp(new String("R6"), "UNION%.*RS%"), new GenMRRedSink3());
 
     // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
     Dispatcher disp = new DefaultRuleDispatcher(new GenMROperator(), opRules, procCtx);
@@ -3423,24 +3464,39 @@
     topNodes.addAll(this.topOps.values());
     ogw.startWalking(topNodes, null);
 
-    // reduce sink does not have any kids
-    breakOperatorTree(procCtx.getRootOps());
-  }
+    // reduce sink does not have any kids - since the plan by now has been broken up into multiple
+    // tasks, iterate over all tasks.
+    // For each task, go over all operators recursively
+    for(Task<? extends Serializable> rootTask: rootTasks)
+      breakTaskTree(rootTask);
+  }
+
+  // loop over all the tasks recursviely
+  private void breakTaskTree(Task<? extends Serializable> task) { 
+    if (task instanceof MapRedTask) {
+      HashMap<String, Operator<? extends Serializable>> opMap = ((MapRedTask)task).getWork().getAliasToWork();
+      if (!opMap.isEmpty())
+        for (Operator<? extends Serializable> op: opMap.values())
+          breakOperatorTree(op);
+    }
 
-  private void breakOperatorTree(Collection<Operator<? extends Serializable>> topOps) {
-    if (topOps == null) 
+    if (task.getChildTasks() == null)
       return;
-    Iterator<Operator<? extends Serializable>> topOpsIter = topOps.iterator();
-    while (topOpsIter.hasNext()) {
-      Operator<? extends Serializable> topOp = topOpsIter.next();
-      breakOperatorTree(topOp);
-    }
+    
+    for (Task<? extends Serializable> childTask :  task.getChildTasks())
+      breakTaskTree(childTask);
   }
-
+    
+  // loop over all the operators recursviely
   private void breakOperatorTree(Operator<? extends Serializable> topOp) {
-    breakOperatorTree(topOp.getChildOperators());
     if (topOp instanceof ReduceSinkOperator)
       topOp.setChildOperators(null);
+
+    if (topOp.getChildOperators() == null)
+      return;
+    
+    for (Operator<? extends Serializable> op: topOp.getChildOperators())
+      breakOperatorTree(op);
   }
 
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/unionDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/unionDesc.java?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/unionDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/unionDesc.java Tue Apr  7 09:53:46 2009
@@ -28,6 +28,24 @@
 public class unionDesc implements Serializable {
   private static final long serialVersionUID = 1L;
 
+  transient private int numInputs;
+  
   @SuppressWarnings("nls")
-  public unionDesc() { }
+  public unionDesc() { 
+    numInputs = 2;
+  }
+
+  /**
+   * @return the numInputs
+   */
+  public int getNumInputs() {
+    return numInputs;
+  }
+
+  /**
+   * @param numInputs the numInputs to set
+   */
+  public void setNumInputs(int numInputs) {
+    this.numInputs = numInputs;
+  }
 }

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/union14.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/union14.q?rev=762665&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/union14.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/union14.q Tue Apr  7 09:53:46 2009
@@ -0,0 +1,16 @@
+set hive.map.aggr = true;
+
+-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink
+
+explain 
+  select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2
+                                        UNION  ALL  
+                                      select 'tst1' as key, cast(count(1) as string) as value from src s1) 
+  unionsrc group by unionsrc.key;
+
+
+
+  select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2
+                                        UNION  ALL  
+                                      select 'tst1' as key, cast(count(1) as string) as value from src s1) 
+  unionsrc group by unionsrc.key;

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/union15.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/union15.q?rev=762665&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/union15.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/union15.q Tue Apr  7 09:53:46 2009
@@ -0,0 +1,18 @@
+set hive.map.aggr = true;
+
+-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink
+
+explain 
+  select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
+                                        UNION  ALL  
+                                            select s2.key as key, s2.value as value from src1 s2
+                                        UNION  ALL  
+                                            select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key;
+
+  select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
+                                        UNION  ALL  
+                                            select s2.key as key, s2.value as value from src1 s2
+                                        UNION  ALL  
+                                            select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key;
+
+

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map_skew.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map_skew.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map_skew.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map_skew.q.out Tue Apr  7 09:53:46 2009
@@ -66,7 +66,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/588492101/148869634.10002 
+        /data/users/njain/hive1/hive/build/ql/tmp/87821987/337667751.10002 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -127,7 +127,7 @@
   Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/588492101/148869634.10003 
+        /data/users/njain/hive1/hive/build/ql/tmp/87821987/337667751.10003 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -159,7 +159,7 @@
   Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/588492101/148869634.10004 
+        /data/users/njain/hive1/hive/build/ql/tmp/87821987/337667751.10004 
           Reduce Output Operator
             key expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8.q.out Tue Apr  7 09:53:46 2009
@@ -53,7 +53,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/56858921/8846383.10002 
+        /data/users/njain/hive1/hive/build/ql/tmp/376655103/656171317.10002 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -114,7 +114,7 @@
   Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/56858921/8846383.10003 
+        /data/users/njain/hive1/hive/build/ql/tmp/376655103/656171317.10003 
           Reduce Output Operator
             key expressions:
                   expr: key
@@ -147,7 +147,7 @@
   Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/56858921/8846383.10004 
+        /data/users/njain/hive1/hive/build/ql/tmp/376655103/656171317.10004 
           Reduce Output Operator
             key expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map_skew.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map_skew.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map_skew.q.out Tue Apr  7 09:53:46 2009
@@ -74,7 +74,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/169036351/65570237.10002 
+        /data/users/njain/hive1/hive/build/ql/tmp/1221208641/230657909.10002 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -135,7 +135,7 @@
   Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/169036351/65570237.10003 
+        /data/users/njain/hive1/hive/build/ql/tmp/1221208641/230657909.10003 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -171,7 +171,7 @@
   Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/zshao/sync/apache-trunk-HIVE-337-trunk/build/ql/tmp/169036351/65570237.10004 
+        /data/users/njain/hive1/hive/build/ql/tmp/1221208641/230657909.10004 
           Reduce Output Operator
             key expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out Tue Apr  7 09:53:46 2009
@@ -76,28 +76,6 @@
                   type: string
                   expr: 1
                   type: bigint
-            Join Operator
-              condition map:
-                   Outer Join 0 to 1
-              condition expressions:
-                0 {VALUE.0} {VALUE.1}
-                1 {VALUE.0} {VALUE.1}
-              Select Operator
-                expressions:
-                      expr: 0
-                      type: string
-                      expr: 1
-                      type: bigint
-                      expr: 2
-                      type: string
-                      expr: 3
-                      type: bigint
-                File Output Operator
-                  compressed: false
-                  GlobalTableId: 0
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
         $INTNAME 
           Reduce Output Operator
             key expressions:

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out Tue Apr  7 09:53:46 2009
@@ -289,30 +289,6 @@
                   type: string
                   expr: 7
                   type: string
-            Join Operator
-              condition map:
-                   Inner Join 0 to 1
-              condition expressions:
-                0 {VALUE.0} {VALUE.1} {VALUE.2} {VALUE.3} {VALUE.4} {VALUE.5} {VALUE.6} {VALUE.7}
-                1 {VALUE.0} {VALUE.1}
-              Select Operator
-                expressions:
-                      expr: 3
-                      type: string
-                      expr: 5
-                      type: string
-                      expr: 0
-                      type: string
-                      expr: 7
-                      type: string
-                      expr: 9
-                      type: string
-                File Output Operator
-                  compressed: false
-                  GlobalTableId: 0
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
       Reduce Operator Tree:
         Join Operator
           condition map:

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union10.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union10.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union10.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union10.q.out Tue Apr  7 09:53:46 2009
@@ -3,13 +3,11 @@
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-5
-  Stage-3 depends on stages: Stage-2, Stage-4
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
   Stage-4 is a root stage
-  Stage-3 depends on stages: Stage-2, Stage-4
-  Stage-5 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-5
-  Stage-3 depends on stages: Stage-2, Stage-4
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -50,29 +48,21 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1262385163/235073004.10005 
+        /data/users/njain/hive1/hive/build/ql/tmp/303193293/170242382.10004 
           Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-        /data/users/njain/hive1/hive/build/ql/tmp/1262385163/235073004.10002 
-          Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-
-  Stage: Stage-3
-    Map Reduce
-      Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1262385163/235073004.10004 
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: string
+                    expr: 1
+                    type: bigint
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+        /data/users/njain/hive1/hive/build/ql/tmp/303193293/170242382.10002 
           Union
             Select Operator
               expressions:
@@ -86,7 +76,7 @@
                 table:
                     input format: org.apache.hadoop.mapred.TextInputFormat
                     output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
-        /data/users/njain/hive1/hive/build/ql/tmp/1262385163/235073004.10003 
+        /data/users/njain/hive1/hive/build/ql/tmp/303193293/170242382.10003 
           Union
             Select Operator
               expressions:
@@ -101,7 +91,7 @@
                     input format: org.apache.hadoop.mapred.TextInputFormat
                     output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
 
-  Stage: Stage-4
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery1:unionsrc-subquery1:s1 
@@ -135,7 +125,7 @@
                   output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
                   name: binary_table
 
-  Stage: Stage-5
+  Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery2-subquery2:unionsrc-subquery2-subquery2:s3 
@@ -175,5 +165,5 @@
 
 
 tst2	500
-tst3	500
 tst1	500
+tst3	500

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union11.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union11.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union11.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union11.q.out Tue Apr  7 09:53:46 2009
@@ -3,16 +3,11 @@
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-6
-  Stage-3 depends on stages: Stage-2, Stage-5
-  Stage-4 depends on stages: Stage-3
-  Stage-5 is a root stage
-  Stage-3 depends on stages: Stage-2, Stage-5
-  Stage-4 depends on stages: Stage-3
-  Stage-6 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-6
-  Stage-3 depends on stages: Stage-2, Stage-5
-  Stage-4 depends on stages: Stage-3
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
+  Stage-4 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -53,29 +48,28 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1314958843/118504340.10006 
+        /data/users/njain/hive1/hive/build/ql/tmp/19233532/930077520.10002 
           Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-        /data/users/njain/hive1/hive/build/ql/tmp/1314958843/118504340.10002 
-          Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-
-  Stage: Stage-3
-    Map Reduce
-      Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1314958843/118504340.10005 
+            Group By Operator
+              aggregations:
+                    expr: count(1)
+              keys:
+                    expr: 0
+                    type: string
+              mode: hash
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+        /data/users/njain/hive1/hive/build/ql/tmp/19233532/930077520.10003 
           Union
             Group By Operator
               aggregations:
@@ -84,14 +78,19 @@
                     expr: 0
                     type: string
               mode: hash
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                    name: binary_table
-        /data/users/njain/hive1/hive/build/ql/tmp/1314958843/118504340.10003 
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+        /data/users/njain/hive1/hive/build/ql/tmp/19233532/930077520.10004 
           Union
             Group By Operator
               aggregations:
@@ -100,30 +99,18 @@
                     expr: 0
                     type: string
               mode: hash
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                    name: binary_table
-
-  Stage: Stage-4
-    Map Reduce
-      Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1314958843/118504340.10004 
-          Reduce Output Operator
-            key expressions:
-                  expr: 0
-                  type: string
-            sort order: +
-            Map-reduce partition columns:
-                  expr: 0
-                  type: string
-            tag: -1
-            value expressions:
-                  expr: 1
-                  type: bigint
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
       Reduce Operator Tree:
         Group By Operator
           aggregations:
@@ -145,7 +132,7 @@
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
 
-  Stage: Stage-5
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery1:unionsrc-subquery1:s1 
@@ -179,7 +166,7 @@
                   output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
                   name: binary_table
 
-  Stage: Stage-6
+  Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery2-subquery2:unionsrc-subquery2-subquery2:s3 

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union12.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union12.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union12.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union12.q.out Tue Apr  7 09:53:46 2009
@@ -3,13 +3,11 @@
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-5
-  Stage-3 depends on stages: Stage-2, Stage-4
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
   Stage-4 is a root stage
-  Stage-3 depends on stages: Stage-2, Stage-4
-  Stage-5 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-5
-  Stage-3 depends on stages: Stage-2, Stage-4
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -50,29 +48,21 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/63508421/145942527.10002 
+        /data/users/njain/hive1/hive/build/ql/tmp/215277211/396097616.10003 
           Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-        /data/users/njain/hive1/hive/build/ql/tmp/63508421/145942527.10005 
-          Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-
-  Stage: Stage-3
-    Map Reduce
-      Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/63508421/145942527.10003 
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: string
+                    expr: 1
+                    type: bigint
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+        /data/users/njain/hive1/hive/build/ql/tmp/215277211/396097616.10004 
           Union
             Select Operator
               expressions:
@@ -86,7 +76,7 @@
                 table:
                     input format: org.apache.hadoop.mapred.TextInputFormat
                     output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
-        /data/users/njain/hive1/hive/build/ql/tmp/63508421/145942527.10004 
+        /data/users/njain/hive1/hive/build/ql/tmp/215277211/396097616.10002 
           Union
             Select Operator
               expressions:
@@ -101,7 +91,7 @@
                     input format: org.apache.hadoop.mapred.TextInputFormat
                     output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
 
-  Stage: Stage-4
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery1:unionsrc-subquery1:s1 
@@ -135,7 +125,7 @@
                   output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
                   name: binary_table
 
-  Stage: Stage-5
+  Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery2-subquery2:unionsrc-subquery2-subquery2:s3 
@@ -175,5 +165,5 @@
 
 
 tst2	25
-tst3	1000
 tst1	500
+tst3	1000

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/union14.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union14.q.out?rev=762665&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union14.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union14.q.out Tue Apr  7 09:53:46 2009
@@ -0,0 +1,151 @@
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF src1 s2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF s2 key) key) (TOK_SELEXPR (TOK_COLREF s2 value) value)))) (TOK_QUERY (TOK_FROM (TOK_TABREF src s1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR 'tst1' key) (TOK_SELEXPR (TOK_FUNCTION TOK_STRING (TOK_FUNCTION count 1)) value))))) unionsrc)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF unionsrc key)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_COLREF unionsrc key))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery1:unionsrc-subquery1:s2 
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+                    name: binary_table
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        /data/users/njain/hive1/hive/build/ql/tmp/816515925/162973516.10002 
+          Union
+            Group By Operator
+              aggregations:
+                    expr: count(1)
+              keys:
+                    expr: 0
+                    type: string
+              mode: hash
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+        /data/users/njain/hive1/hive/build/ql/tmp/816515925/162973516.10003 
+          Union
+            Group By Operator
+              aggregations:
+                    expr: count(1)
+              keys:
+                    expr: 0
+                    type: string
+              mode: hash
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE.0)
+          keys:
+                expr: KEY.0
+                type: string
+          mode: mergepartial
+          Select Operator
+            expressions:
+                  expr: 0
+                  type: string
+                  expr: 1
+                  type: bigint
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+
+  Stage: Stage-3
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery2:unionsrc-subquery2:s1 
+            Select Operator
+              Group By Operator
+                aggregations:
+                      expr: count(1)
+                mode: hash
+                Reduce Output Operator
+                  sort order: 
+                  tag: -1
+                  value expressions:
+                        expr: 0
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE.0)
+          mode: mergepartial
+          Select Operator
+            expressions:
+                  expr: 'tst1'
+                  type: string
+                  expr: UDFToString(0)
+                  type: string
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+                  name: binary_table
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+	10
+128	1
+146	1
+150	1
+213	1
+224	1
+238	1
+255	1
+273	1
+278	1
+311	1
+369	1
+401	1
+406	1
+66	1
+98	1
+tst1	1

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/union15.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union15.q.out?rev=762665&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union15.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union15.q.out Tue Apr  7 09:53:46 2009
@@ -0,0 +1,192 @@
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF src s1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR 'tst1' key) (TOK_SELEXPR (TOK_FUNCTION TOK_STRING (TOK_FUNCTION count 1)) value)))) (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF src1 s2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF s2 key) key) (TOK_SELEXPR (TOK_COLREF s2 value) value)))) (TOK_QUERY (TOK_FROM (TOK_TABREF src1 s3)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF s3 key) key) (TOK_SELEXPR (TOK_COLREF s3 value) value)))))) unionsrc)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF unionsrc key)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_COLREF unionsrc key))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
+  Stage-4 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3, Stage-4
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery2-subquery1:unionsrc-subquery2-subquery1:s2 
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+                    name: binary_table
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        /data/users/njain/hive1/hive/build/ql/tmp/1083494579/195750824.10002 
+          Union
+            Group By Operator
+              aggregations:
+                    expr: count(1)
+              keys:
+                    expr: 0
+                    type: string
+              mode: hash
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+        /data/users/njain/hive1/hive/build/ql/tmp/1083494579/195750824.10004 
+          Union
+            Group By Operator
+              aggregations:
+                    expr: count(1)
+              keys:
+                    expr: 0
+                    type: string
+              mode: hash
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+        /data/users/njain/hive1/hive/build/ql/tmp/1083494579/195750824.10003 
+          Union
+            Group By Operator
+              aggregations:
+                    expr: count(1)
+              keys:
+                    expr: 0
+                    type: string
+              mode: hash
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE.0)
+          keys:
+                expr: KEY.0
+                type: string
+          mode: mergepartial
+          Select Operator
+            expressions:
+                  expr: 0
+                  type: string
+                  expr: 1
+                  type: bigint
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+
+  Stage: Stage-3
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery1:unionsrc-subquery1:s1 
+            Select Operator
+              Group By Operator
+                aggregations:
+                      expr: count(1)
+                mode: hash
+                Reduce Output Operator
+                  sort order: 
+                  tag: -1
+                  value expressions:
+                        expr: 0
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE.0)
+          mode: mergepartial
+          Select Operator
+            expressions:
+                  expr: 'tst1'
+                  type: string
+                  expr: UDFToString(0)
+                  type: string
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+                  name: binary_table
+
+  Stage: Stage-4
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery2-subquery2:unionsrc-subquery2-subquery2:s3 
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+                    name: binary_table
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+	20
+128	2
+146	2
+150	2
+213	2
+224	2
+238	2
+255	2
+273	2
+278	2
+311	2
+369	2
+401	2
+406	2
+66	2
+98	2
+tst1	1

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union3.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union3.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union3.q.out Tue Apr  7 09:53:46 2009
@@ -4,19 +4,14 @@
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-2 depends on stages: Stage-1
-  Stage-3 depends on stages: Stage-2, Stage-6
-  Stage-4 depends on stages: Stage-3, Stage-8
+  Stage-3 depends on stages: Stage-2, Stage-4, Stage-6, Stage-7
+  Stage-4 is a root stage
+  Stage-3 depends on stages: Stage-2, Stage-4, Stage-6, Stage-7
   Stage-5 is a root stage
-  Stage-6 depends on stages: Stage-5, Stage-9
-  Stage-3 depends on stages: Stage-2, Stage-6
-  Stage-4 depends on stages: Stage-3, Stage-8
+  Stage-6 depends on stages: Stage-5
+  Stage-3 depends on stages: Stage-2, Stage-4, Stage-6, Stage-7
   Stage-7 is a root stage
-  Stage-8 depends on stages: Stage-7
-  Stage-4 depends on stages: Stage-3, Stage-8
-  Stage-9 is a root stage
-  Stage-6 depends on stages: Stage-5, Stage-9
-  Stage-3 depends on stages: Stage-2, Stage-6
-  Stage-4 depends on stages: Stage-3, Stage-8
+  Stage-3 depends on stages: Stage-2, Stage-4, Stage-6, Stage-7
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -57,7 +52,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1348616199/203957602.10002 
+        /data/users/njain/hive1/hive/build/ql/tmp/525256518/528364249.10002 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -83,29 +78,31 @@
   Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1348616199/203957602.10003 
+        /data/users/njain/hive1/hive/build/ql/tmp/525256518/528364249.10004 
           Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-        /data/users/njain/hive1/hive/build/ql/tmp/1348616199/203957602.10006 
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: int
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+        /data/users/njain/hive1/hive/build/ql/tmp/525256518/528364249.10003 
           Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-
-  Stage: Stage-4
-    Map Reduce
-      Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1348616199/203957602.10004 
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: int
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+        /data/users/njain/hive1/hive/build/ql/tmp/525256518/528364249.10006 
           Union
             Select Operator
               expressions:
@@ -117,7 +114,7 @@
                 table:
                     input format: org.apache.hadoop.mapred.TextInputFormat
                     output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
-        /data/users/njain/hive1/hive/build/ql/tmp/1348616199/203957602.10008 
+        /data/users/njain/hive1/hive/build/ql/tmp/525256518/528364249.10007 
           Union
             Select Operator
               expressions:
@@ -130,7 +127,7 @@
                     input format: org.apache.hadoop.mapred.TextInputFormat
                     output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
 
-  Stage: Stage-5
+  Stage: Stage-4
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery2-subquery2-subquery2:a-subquery2-subquery2-subquery2:s2:src 
@@ -164,29 +161,7 @@
                     output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
                     name: binary_table
 
-  Stage: Stage-6
-    Map Reduce
-      Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1348616199/203957602.10005 
-          Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-        /data/users/njain/hive1/hive/build/ql/tmp/1348616199/203957602.10009 
-          Union
-            File Output Operator
-              compressed: false
-              GlobalTableId: 0
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                  name: binary_table
-
-  Stage: Stage-7
+  Stage: Stage-5
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery1:a-subquery1:s1:src 
@@ -220,10 +195,10 @@
                     output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
                     name: binary_table
 
-  Stage: Stage-8
+  Stage: Stage-6
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1348616199/203957602.10007 
+        /data/users/njain/hive1/hive/build/ql/tmp/525256518/528364249.10005 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -246,7 +221,7 @@
                 output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
                 name: binary_table
 
-  Stage: Stage-9
+  Stage: Stage-7
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery2-subquery2-subquery1:a-subquery2-subquery2-subquery1:s2:src 

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union5.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union5.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union5.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union5.q.out Tue Apr  7 09:53:46 2009
@@ -3,11 +3,9 @@
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-4
-  Stage-3 depends on stages: Stage-2
-  Stage-4 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-4
-  Stage-3 depends on stages: Stage-2
+  Stage-2 depends on stages: Stage-1, Stage-3
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -48,7 +46,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1182578141/25944894.10004 
+        /data/users/njain/hive1/hive/build/ql/tmp/99224757/1406619949.10003 
           Union
             Group By Operator
               aggregations:
@@ -57,14 +55,19 @@
                     expr: 0
                     type: string
               mode: hash
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                    name: binary_table
-        /data/users/njain/hive1/hive/build/ql/tmp/1182578141/25944894.10002 
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+        /data/users/njain/hive1/hive/build/ql/tmp/99224757/1406619949.10002 
           Union
             Group By Operator
               aggregations:
@@ -73,30 +76,18 @@
                     expr: 0
                     type: string
               mode: hash
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                    name: binary_table
-
-  Stage: Stage-3
-    Map Reduce
-      Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1182578141/25944894.10003 
-          Reduce Output Operator
-            key expressions:
-                  expr: 0
-                  type: string
-            sort order: +
-            Map-reduce partition columns:
-                  expr: 0
-                  type: string
-            tag: -1
-            value expressions:
-                  expr: 1
-                  type: bigint
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
       Reduce Operator Tree:
         Group By Operator
           aggregations:
@@ -118,7 +109,7 @@
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
 
-  Stage: Stage-4
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery2:unionsrc-subquery2:s2 

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union7.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union7.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union7.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union7.q.out Tue Apr  7 09:53:46 2009
@@ -3,11 +3,9 @@
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-4
-  Stage-3 depends on stages: Stage-2
-  Stage-4 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-4
-  Stage-3 depends on stages: Stage-2
+  Stage-2 depends on stages: Stage-1, Stage-3
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -48,7 +46,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/5099413/43325262.10004 
+        /data/users/njain/hive1/hive/build/ql/tmp/542153838/197914295.10003 
           Union
             Group By Operator
               aggregations:
@@ -57,14 +55,19 @@
                     expr: 0
                     type: string
               mode: hash
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                    name: binary_table
-        /data/users/njain/hive1/hive/build/ql/tmp/5099413/43325262.10002 
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
+        /data/users/njain/hive1/hive/build/ql/tmp/542153838/197914295.10002 
           Union
             Group By Operator
               aggregations:
@@ -73,30 +76,18 @@
                     expr: 0
                     type: string
               mode: hash
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
-                    name: binary_table
-
-  Stage: Stage-3
-    Map Reduce
-      Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/5099413/43325262.10003 
-          Reduce Output Operator
-            key expressions:
-                  expr: 0
-                  type: string
-            sort order: +
-            Map-reduce partition columns:
-                  expr: 0
-                  type: string
-            tag: -1
-            value expressions:
-                  expr: 1
-                  type: bigint
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 1
+                      type: bigint
       Reduce Operator Tree:
         Group By Operator
           aggregations:
@@ -118,7 +109,7 @@
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
 
-  Stage: Stage-4
+  Stage: Stage-3
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery2:unionsrc-subquery2:s2 

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union8.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union8.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union8.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union8.q.out Tue Apr  7 09:53:46 2009
@@ -17,19 +17,18 @@
                     expr: value
                     type: string
               Union
-                Union
-                  Select Operator
-                    expressions:
-                          expr: 0
-                          type: string
-                          expr: 1
-                          type: string
-                    File Output Operator
-                      compressed: false
-                      GlobalTableId: 0
-                      table:
-                          input format: org.apache.hadoop.mapred.TextInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+                Select Operator
+                  expressions:
+                        expr: 0
+                        type: string
+                        expr: 1
+                        type: string
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
         null-subquery1:unionsrc-subquery1:s1 
             Select Operator
               expressions:
@@ -58,19 +57,18 @@
                     expr: value
                     type: string
               Union
-                Union
-                  Select Operator
-                    expressions:
-                          expr: 0
-                          type: string
-                          expr: 1
-                          type: string
-                    File Output Operator
-                      compressed: false
-                      GlobalTableId: 0
-                      table:
-                          input format: org.apache.hadoop.mapred.TextInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+                Select Operator
+                  expressions:
+                        expr: 0
+                        type: string
+                        expr: 1
+                        type: string
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
 
   Stage: Stage-0
     Fetch Operator

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union9.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union9.q.out?rev=762665&r1=762664&r2=762665&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union9.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union9.q.out Tue Apr  7 09:53:46 2009
@@ -17,17 +17,16 @@
                     expr: value
                     type: string
               Union
-                Union
-                  Group By Operator
-                    aggregations:
-                          expr: count(1)
-                    mode: hash
-                    Reduce Output Operator
-                      sort order: 
-                      tag: -1
-                      value expressions:
-                            expr: 0
-                            type: bigint
+                Group By Operator
+                  aggregations:
+                        expr: count(1)
+                  mode: hash
+                  Reduce Output Operator
+                    sort order: 
+                    tag: -1
+                    value expressions:
+                          expr: 0
+                          type: bigint
         null-subquery1:unionsrc-subquery1:s1 
             Select Operator
               expressions:
@@ -54,17 +53,16 @@
                     expr: value
                     type: string
               Union
-                Union
-                  Group By Operator
-                    aggregations:
-                          expr: count(1)
-                    mode: hash
-                    Reduce Output Operator
-                      sort order: 
-                      tag: -1
-                      value expressions:
-                            expr: 0
-                            type: bigint
+                Group By Operator
+                  aggregations:
+                        expr: count(1)
+                  mode: hash
+                  Reduce Output Operator
+                    sort order: 
+                    tag: -1
+                    value expressions:
+                          expr: 0
+                          type: bigint
       Reduce Operator Tree:
         Group By Operator
           aggregations:



Mime
View raw message