hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1237253 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/optimizer/ java/org/apache/hadoop/hive/ql/optimizer/unionproc/ test/queries/clientpositive/ test/results/clientpositive/
Date Sun, 29 Jan 2012 11:45:47 GMT
Author: namit
Date: Sun Jan 29 11:45:47 2012
New Revision: 1237253

URL: http://svn.apache.org/viewvc?rev=1237253&view=rev
Log:
HIVE-2755 union follwowed by union_subq does not work if the subquery union
has reducers (He Yongqiang via namit)


Added:
    hive/trunk/ql/src/test/queries/clientpositive/union28.q
    hive/trunk/ql/src/test/queries/clientpositive/union29.q
    hive/trunk/ql/src/test/queries/clientpositive/union30.q
    hive/trunk/ql/src/test/results/clientpositive/union28.q.out
    hive/trunk/ql/src/test/results/clientpositive/union29.q.out
    hive/trunk/ql/src/test/results/clientpositive/union30.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=1237253&r1=1237252&r2=1237253&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Sun Jan 29 11:45:47 2012
@@ -653,11 +653,19 @@ public class GenMRFileSink1 implements N
           seenOps.add(currTopOp);
           GenMapRedUtils.setTaskPlan(currAliasId, currTopOp,
               (MapredWork) mapTask.getWork(), false, ctx);
+        } else {
+          UnionOperator currUnionOp = ctx.getCurrUnionOp();
+          if (currUnionOp != null) {
+            opTaskMap.put(null, currTask);
+            ctx.setCurrTopOp(null);
+            GenMapRedUtils.initUnionPlan(ctx, currTask, false);
+            return dest;
+          }
         }
         // mapTask and currTask should be merged by and join/union operator
         // (e.g., GenMRUnion1j) which has multiple topOps.
-        assert mapTask == currTask : "mapTask.id = " + mapTask.getId()
-            + "; currTask.id = " + currTask.getId();
+        // assert mapTask == currTask : "mapTask.id = " + mapTask.getId()
+        // + "; currTask.id = " + currTask.getId();
       }
 
       return dest;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java?rev=1237253&r1=1237252&r2=1237253&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java Sun Jan 29 11:45:47 2012
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.optimi
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Stack;
@@ -68,12 +67,17 @@ public class GenMRUnion1 implements Node
    */
   private Object processMapOnlyUnion(UnionOperator union, Stack<Node> stack,
       GenMRProcContext ctx, UnionProcContext uCtx) throws SemanticException {
+
     // merge currTask from multiple topOps
     GenMRUnionCtx uCtxTask = ctx.getUnionTask(union);
     if (uCtxTask != null) {
       // get task associated with this union
       Task<? extends Serializable> uTask = ctx.getUnionTask(union).getUTask();
       if (uTask != null) {
+        if (ctx.getCurrTask() != null && ctx.getCurrTask() != uTask) {
+          // if ctx.getCurrTask() is in rootTasks, should be removed
+          ctx.getRootTasks().remove(ctx.getCurrTask());
+        }
         ctx.setCurrTask(uTask);
       }
     }
@@ -276,8 +280,8 @@ public class GenMRUnion1 implements Node
     }
 
     // Copy into the current union task plan if
-    if (uPrsCtx.getMapOnlySubq(pos) && uPrsCtx.getRootTask(pos)
-        && !uPrsCtx.getMapJoinSubq(pos)) {
+    if (uPrsCtx.getMapOnlySubq(pos)
+        && !uPrsCtx.getMapJoinSubq(pos) && uPrsCtx.getRootTask(pos)) {
       processSubQueryUnionMerge(ctx, uCtxTask, union, stack);
     }
     // If it a map-reduce job, create a temporary file
@@ -296,6 +300,7 @@ public class GenMRUnion1 implements Node
       //the currAliasId and CurrTopOp is not valid any more
       ctx.setCurrAliasId(null);
       ctx.setCurrTopOp(null);
+      ctx.getOpTaskMap().put(null, uTask);
     }
 
     ctx.setCurrTask(uTask);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java?rev=1237253&r1=1237252&r2=1237253&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcContext.java Sun Jan 29 11:45:47 2012
@@ -36,6 +36,7 @@ public class UnionProcContext implements
    */
   public static class UnionParseContext {
     private final transient boolean[] mapOnlySubq;
+    private final transient boolean[] mapOnlySubqSet;
     private final transient boolean[] rootTask;
     private final transient boolean[] mapJoinSubq;
 
@@ -47,6 +48,7 @@ public class UnionProcContext implements
       mapOnlySubq = new boolean[numInputs];
       rootTask = new boolean[numInputs];
       mapJoinSubq = new boolean[numInputs];
+      mapOnlySubqSet = new boolean[numInputs];
     }
 
     public boolean getMapOnlySubq(int pos) {
@@ -55,6 +57,7 @@ public class UnionProcContext implements
 
     public void setMapOnlySubq(int pos, boolean mapOnlySubq) {
       this.mapOnlySubq[pos] = mapOnlySubq;
+      this.mapOnlySubqSet[pos] = true;
     }
 
     public boolean getMapJoinSubq(int pos) {
@@ -99,6 +102,17 @@ public class UnionProcContext implements
       return true;
     }
 
+    public boolean allMapOnlySubQSet() {
+      if (mapOnlySubqSet != null) {
+        for (boolean mapOnlySet : mapOnlySubqSet) {
+          if (!mapOnlySet) {
+            return false;
+          }
+        }
+      }
+      return true;
+    }
+
   }
 
   // the subqueries are map-only jobs

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java?rev=1237253&r1=1237252&r2=1237253&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java Sun Jan 29 11:45:47 2012
@@ -142,8 +142,30 @@ public final class UnionProcFactory {
       if (uCtx == null) {
         uCtx = new UnionParseContext(union.getConf().getNumInputs());
       }
+      int start = stack.size() - 2;
+      UnionOperator parentUnionOperator = null;
+      while (start >= 0) {
+        Operator<? extends Serializable> parent =
+            (Operator<? extends Serializable>) stack.get(start);
+        if (parent instanceof UnionOperator) {
+          parentUnionOperator = (UnionOperator) parent;
+          break;
+        }
+        start--;
+      }
+
+      // default to false
+      boolean mapOnly = false;
+      if (parentUnionOperator != null) {
+        UnionParseContext parentUCtx =
+            ctx.getUnionParseContext(parentUnionOperator);
+        if (parentUCtx != null && parentUCtx.allMapOnlySubQSet()) {
+          mapOnly = parentUCtx.allMapOnlySubQ();
+        }
+      }
+
+      uCtx.setMapOnlySubq(pos, mapOnly);
 
-      uCtx.setMapOnlySubq(pos, true);
       uCtx.setRootTask(pos, false);
       ctx.setUnionParseContext(union, uCtx);
       return null;

Added: hive/trunk/ql/src/test/queries/clientpositive/union28.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union28.q?rev=1237253&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/union28.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/union28.q Sun Jan 29 11:45:47 2012
@@ -0,0 +1,30 @@
+create table union_subq_union(key int, value string);
+
+explain
+insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+;
+
+insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+;
+
+select * from union_subq_union order by key, value limit 20;

Added: hive/trunk/ql/src/test/queries/clientpositive/union29.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union29.q?rev=1237253&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/union29.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/union29.q Sun Jan 29 11:45:47 2012
@@ -0,0 +1,30 @@
+create table union_subq_union(key int, value string);
+
+explain
+insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value from src 
+    union all
+    select key, value from src
+  ) subq
+) a
+;
+
+insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value from src 
+    union all
+    select key, value from src
+  ) subq
+) a
+;
+
+select * from union_subq_union order by key, value limit 20;

Added: hive/trunk/ql/src/test/queries/clientpositive/union30.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union30.q?rev=1237253&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/union30.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/union30.q Sun Jan 29 11:45:47 2012
@@ -0,0 +1,44 @@
+create table union_subq_union(key int, value string);
+
+explain
+insert overwrite table union_subq_union 
+select * from (
+
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+
+union all
+
+select key, value from src
+) aa
+;
+
+insert overwrite table union_subq_union 
+select * from (
+
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+
+union all
+
+select key, value from src
+) aa
+;
+
+select * from union_subq_union order by key, value limit 20;

Added: hive/trunk/ql/src/test/results/clientpositive/union28.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union28.q.out?rev=1237253&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union28.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/union28.q.out Sun Jan 29 11:45:47 2012
@@ -0,0 +1,376 @@
+PREHOOK: query: create table union_subq_union(key int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table union_subq_union(key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@union_subq_union
+PREHOOK: query: explain
+insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))))) (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value))))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL ke
 y)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME union_subq_union))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+  Stage-7 is a root stage
+  Stage-8 depends on stages: Stage-7, Stage-9
+  Stage-2 depends on stages: Stage-8
+  Stage-6 depends on stages: Stage-2 , consists of Stage-5, Stage-4
+  Stage-5
+  Stage-0 depends on stages: Stage-5, Stage-4
+  Stage-3 depends on stages: Stage-0
+  Stage-4
+  Stage-9 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-7
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery2:a-subquery2-subquery1:subq-subquery1:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: key, value
+              Group By Operator
+                aggregations:
+                      expr: count(1)
+                bucketGroup: false
+                keys:
+                      expr: key
+                      type: string
+                      expr: value
+                      type: string
+                mode: hash
+                outputColumnNames: _col0, _col1, _col2
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  sort order: ++
+                  Map-reduce partition columns:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  tag: -1
+                  value expressions:
+                        expr: _col2
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          bucketGroup: false
+          keys:
+                expr: KEY._col0
+                type: string
+                expr: KEY._col1
+                type: string
+          mode: mergepartial
+          outputColumnNames: _col0, _col1, _col2
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: string
+                  expr: _col1
+                  type: string
+                  expr: _col2
+                  type: bigint
+            outputColumnNames: _col0, _col1, _col2
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-8
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+          TableScan
+            Union
+              Select Operator
+                expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+                outputColumnNames: _col0, _col1
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+#### A masked pattern was here ####
+          TableScan
+            Union
+              Select Operator
+                expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+                outputColumnNames: _col0, _col1
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+          TableScan
+            Union
+              Select Operator
+                expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+                outputColumnNames: _col0, _col1
+                Select Operator
+                  expressions:
+                        expr: UDFToInteger(_col0)
+                        type: int
+                        expr: _col1
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 1
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        name: default.union_subq_union
+        null-subquery1:a-subquery1:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Union
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  Select Operator
+                    expressions:
+                          expr: UDFToInteger(_col0)
+                          type: int
+                          expr: _col1
+                          type: string
+                    outputColumnNames: _col0, _col1
+                    File Output Operator
+                      compressed: false
+                      GlobalTableId: 1
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                          name: default.union_subq_union
+
+  Stage: Stage-6
+    Conditional Operator
+
+  Stage: Stage-5
+    Move Operator
+      files:
+          hdfs directory: true
+#### A masked pattern was here ####
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.union_subq_union
+
+  Stage: Stage-3
+    Stats-Aggr Operator
+
+  Stage: Stage-4
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  name: default.union_subq_union
+
+  Stage: Stage-9
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery2:a-subquery2-subquery2:subq-subquery2:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: key, value
+              Group By Operator
+                aggregations:
+                      expr: count(1)
+                bucketGroup: false
+                keys:
+                      expr: key
+                      type: string
+                      expr: value
+                      type: string
+                mode: hash
+                outputColumnNames: _col0, _col1, _col2
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  sort order: ++
+                  Map-reduce partition columns:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  tag: -1
+                  value expressions:
+                        expr: _col2
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          bucketGroup: false
+          keys:
+                expr: KEY._col0
+                type: string
+                expr: KEY._col1
+                type: string
+          mode: mergepartial
+          outputColumnNames: _col0, _col1, _col2
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: string
+                  expr: _col1
+                  type: string
+                  expr: _col2
+                  type: bigint
+            outputColumnNames: _col0, _col1, _col2
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+
+PREHOOK: query: insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@union_subq_union
+POSTHOOK: query: insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@union_subq_union
+POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from union_subq_union order by key, value limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_subq_union
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_subq_union order by key, value limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_subq_union
+#### A masked pattern was here ####
+POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+2	val_2
+2	val_2
+2	val_2
+4	val_4
+4	val_4
+4	val_4
+5	val_5
+5	val_5
+5	val_5
+5	val_5
+5	val_5
+8	val_8
+8	val_8
+8	val_8
+9	val_9

Added: hive/trunk/ql/src/test/results/clientpositive/union29.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union29.q.out?rev=1237253&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union29.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/union29.q.out Sun Jan 29 11:45:47 2012
@@ -0,0 +1,258 @@
+PREHOOK: query: create table union_subq_union(key int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table union_subq_union(key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@union_subq_union
+PREHOOK: query: explain
+insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value from src 
+    union all
+    select key, value from src
+  ) subq
+) a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value from src 
+    union all
+    select key, value from src
+  ) subq
+) a
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))))) (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)))))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME union_subq_union))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3
+  Stage-4
+  Stage-0 depends on stages: Stage-4, Stage-3
+  Stage-2 depends on stages: Stage-0
+  Stage-3
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery1:a-subquery1:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Union
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  Select Operator
+                    expressions:
+                          expr: UDFToInteger(_col0)
+                          type: int
+                          expr: _col1
+                          type: string
+                    outputColumnNames: _col0, _col1
+                    File Output Operator
+                      compressed: false
+                      GlobalTableId: 1
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                          name: default.union_subq_union
+        null-subquery2:a-subquery2-subquery1:subq-subquery1:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Union
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  Union
+                    Select Operator
+                      expressions:
+                            expr: _col0
+                            type: string
+                            expr: _col1
+                            type: string
+                      outputColumnNames: _col0, _col1
+                      Select Operator
+                        expressions:
+                              expr: UDFToInteger(_col0)
+                              type: int
+                              expr: _col1
+                              type: string
+                        outputColumnNames: _col0, _col1
+                        File Output Operator
+                          compressed: false
+                          GlobalTableId: 1
+                          table:
+                              input format: org.apache.hadoop.mapred.TextInputFormat
+                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                              name: default.union_subq_union
+        null-subquery2:a-subquery2-subquery2:subq-subquery2:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Union
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  Union
+                    Select Operator
+                      expressions:
+                            expr: _col0
+                            type: string
+                            expr: _col1
+                            type: string
+                      outputColumnNames: _col0, _col1
+                      Select Operator
+                        expressions:
+                              expr: UDFToInteger(_col0)
+                              type: int
+                              expr: _col1
+                              type: string
+                        outputColumnNames: _col0, _col1
+                        File Output Operator
+                          compressed: false
+                          GlobalTableId: 1
+                          table:
+                              input format: org.apache.hadoop.mapred.TextInputFormat
+                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                              name: default.union_subq_union
+
+  Stage: Stage-5
+    Conditional Operator
+
+  Stage: Stage-4
+    Move Operator
+      files:
+          hdfs directory: true
+#### A masked pattern was here ####
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.union_subq_union
+
+  Stage: Stage-2
+    Stats-Aggr Operator
+
+  Stage: Stage-3
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  name: default.union_subq_union
+
+
+PREHOOK: query: insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value from src 
+    union all
+    select key, value from src
+  ) subq
+) a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@union_subq_union
+POSTHOOK: query: insert overwrite table union_subq_union 
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value from src 
+    union all
+    select key, value from src
+  ) subq
+) a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@union_subq_union
+POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from union_subq_union order by key, value limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_subq_union
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_subq_union order by key, value limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_subq_union
+#### A masked pattern was here ####
+POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+2	val_2
+2	val_2
+2	val_2
+4	val_4
+4	val_4
+4	val_4
+5	val_5
+5	val_5
+5	val_5
+5	val_5
+5	val_5

Added: hive/trunk/ql/src/test/results/clientpositive/union30.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union30.q.out?rev=1237253&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union30.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/union30.q.out Sun Jan 29 11:45:47 2012
@@ -0,0 +1,449 @@
+PREHOOK: query: create table union_subq_union(key int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table union_subq_union(key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@union_subq_union
+PREHOOK: query: explain
+insert overwrite table union_subq_union 
+select * from (
+
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+
+union all
+
+select key, value from src
+) aa
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+insert overwrite table union_subq_union 
+select * from (
+
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+
+union all
+
+select key, value from src
+) aa
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))))) (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value))))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
  (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)))))) aa)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME union_subq_union))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-9
+  Stage-3 depends on stages: Stage-2
+  Stage-4 depends on stages: Stage-3
+  Stage-8 depends on stages: Stage-4 , consists of Stage-7, Stage-6
+  Stage-7
+  Stage-0 depends on stages: Stage-7, Stage-6
+  Stage-5 depends on stages: Stage-0
+  Stage-6
+  Stage-9 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery1:aa-subquery1-subquery2:a-subquery2-subquery1:subq-subquery1:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: key, value
+              Group By Operator
+                aggregations:
+                      expr: count(1)
+                bucketGroup: false
+                keys:
+                      expr: key
+                      type: string
+                      expr: value
+                      type: string
+                mode: hash
+                outputColumnNames: _col0, _col1, _col2
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  sort order: ++
+                  Map-reduce partition columns:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  tag: -1
+                  value expressions:
+                        expr: _col2
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          bucketGroup: false
+          keys:
+                expr: KEY._col0
+                type: string
+                expr: KEY._col1
+                type: string
+          mode: mergepartial
+          outputColumnNames: _col0, _col1, _col2
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: string
+                  expr: _col1
+                  type: string
+                  expr: _col2
+                  type: bigint
+            outputColumnNames: _col0, _col1, _col2
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+          TableScan
+            Union
+              Select Operator
+                expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+                outputColumnNames: _col0, _col1
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+#### A masked pattern was here ####
+          TableScan
+            Union
+              Select Operator
+                expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+                outputColumnNames: _col0, _col1
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-3
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+          TableScan
+            Union
+              Select Operator
+                expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+                outputColumnNames: _col0, _col1
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+        null-subquery1:aa-subquery1-subquery1:a-subquery1:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Union
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-4
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+          TableScan
+            Union
+              Select Operator
+                expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+                outputColumnNames: _col0, _col1
+                Select Operator
+                  expressions:
+                        expr: UDFToInteger(_col0)
+                        type: int
+                        expr: _col1
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 1
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        name: default.union_subq_union
+        null-subquery2:aa-subquery2:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Union
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  Select Operator
+                    expressions:
+                          expr: UDFToInteger(_col0)
+                          type: int
+                          expr: _col1
+                          type: string
+                    outputColumnNames: _col0, _col1
+                    File Output Operator
+                      compressed: false
+                      GlobalTableId: 1
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                          name: default.union_subq_union
+
+  Stage: Stage-8
+    Conditional Operator
+
+  Stage: Stage-7
+    Move Operator
+      files:
+          hdfs directory: true
+#### A masked pattern was here ####
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.union_subq_union
+
+  Stage: Stage-5
+    Stats-Aggr Operator
+
+  Stage: Stage-6
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  name: default.union_subq_union
+
+  Stage: Stage-9
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery1:aa-subquery1-subquery2:a-subquery2-subquery2:subq-subquery2:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: key, value
+              Group By Operator
+                aggregations:
+                      expr: count(1)
+                bucketGroup: false
+                keys:
+                      expr: key
+                      type: string
+                      expr: value
+                      type: string
+                mode: hash
+                outputColumnNames: _col0, _col1, _col2
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  sort order: ++
+                  Map-reduce partition columns:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                  tag: -1
+                  value expressions:
+                        expr: _col2
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          bucketGroup: false
+          keys:
+                expr: KEY._col0
+                type: string
+                expr: KEY._col1
+                type: string
+          mode: mergepartial
+          outputColumnNames: _col0, _col1, _col2
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: string
+                  expr: _col1
+                  type: string
+                  expr: _col2
+                  type: bigint
+            outputColumnNames: _col0, _col1, _col2
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+
+PREHOOK: query: insert overwrite table union_subq_union 
+select * from (
+
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+
+union all
+
+select key, value from src
+) aa
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@union_subq_union
+POSTHOOK: query: insert overwrite table union_subq_union 
+select * from (
+
+select * from (
+  select key, value from src 
+  union all 
+  select key, value from 
+  (
+    select key, value, count(1) from src group by key, value
+    union all
+    select key, value, count(1) from src group by key, value
+  ) subq
+) a
+
+union all
+
+select key, value from src
+) aa
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@union_subq_union
+POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from union_subq_union order by key, value limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_subq_union
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_subq_union order by key, value limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_subq_union
+#### A masked pattern was here ####
+POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+0	val_0
+2	val_2
+2	val_2
+2	val_2
+2	val_2
+4	val_4
+4	val_4
+4	val_4
+4	val_4
+5	val_5
+5	val_5
+5	val_5
+5	val_5



Mime
View raw message