hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1635245 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/parse/ test/results/clientpositive/
Date Wed, 29 Oct 2014 18:17:38 GMT
Author: hashutosh
Date: Wed Oct 29 18:17:37 2014
New Revision: 1635245

URL: http://svn.apache.org/r1635245
Log:
HIVE-8625 : Some union queries result in plans with many unions with CBO on ( Jesús Camacho
Rodríguez via Ashutosh Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/trunk/ql/src/test/results/clientpositive/union29.q.out
    hive/trunk/ql/src/test/results/clientpositive/union30.q.out
    hive/trunk/ql/src/test/results/clientpositive/union34.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_remove_11.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java Wed Oct 29 18:17:37
2014
@@ -143,4 +143,8 @@ public class OperatorUtils {
       }
     }
   }
+
+  public static boolean sameRowSchema(Operator<?> operator1, Operator<?> operator2)
{
+	  return operator1.getSchema().equals(operator2.getSchema());
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java Wed Oct 29 18:17:37
2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Iterator;
 
 /**
  * RowSchema Implementation.
@@ -49,6 +50,51 @@ public class RowSchema implements Serial
   }
 
   @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof RowSchema) || (obj == null)) {
+      return false;
+    }
+    if(this == obj) {
+      return true;
+    }
+
+    RowSchema dest = (RowSchema)obj;
+    if(this.signature == null && dest.getSignature() == null) {
+      return true;
+    }
+    if((this.signature == null && dest.getSignature() != null) ||
+        (this.signature != null && dest.getSignature() == null) ) {
+      return false;
+    }
+
+    if(this.signature.size() != dest.getSignature().size()) {
+      return false;
+    }
+
+    Iterator<ColumnInfo> origIt = this.signature.iterator();
+    Iterator<ColumnInfo> destIt = dest.getSignature().iterator();
+    while(origIt.hasNext()) {
+      ColumnInfo origColumn = origIt.next();
+      ColumnInfo destColumn = destIt.next();
+
+      if(origColumn == null && destColumn == null) {
+        continue;
+      }
+
+      if((origColumn == null && destColumn != null) ||
+          (origColumn != null && destColumn == null) ) {
+        return false;
+      }
+
+      if(!origColumn.equals(destColumn)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append('(');

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java Wed Oct 29 18:17:37
2014
@@ -125,4 +125,31 @@ public class SelectOperator extends Oper
   public boolean acceptLimitPushdown() {
     return true;
   }
+
+  /**
+   * Checks whether this select operator does something to the
+   * input tuples.
+   *
+   * @return if it is an identity select operator or not
+   */
+  public boolean isIdentitySelect() {
+    //Safety check
+    if(this.getNumParent() != 1) {
+      return false;
+    }
+
+    //Select *
+    if(this.getConf().isSelStarNoCompute() ||
+        this.getConf().isSelectStar()) {
+      return true;
+    }
+
+    //Check whether the have the same schema
+    if(!OperatorUtils.sameRowSchema(this, this.getParentOperators().get(0))) {
+      return false;
+    }
+
+    return true;
+  }
+
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Oct 29
18:17:37 2014
@@ -93,6 +93,7 @@ import org.apache.hadoop.hive.ql.exec.Re
 import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
 import org.apache.hadoop.hive.ql.exec.RowSchema;
 import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -9053,38 +9054,65 @@ public class SemanticAnalyzer extends Ba
       rightOp = genInputSelectForUnion(rightOp, rightmap, rightalias, unionoutRR, unionalias);
     }
 
-    // If one of the children is a union, merge with it
+    // If one of the children (left or right) is:
+    // (i) a union, or
+    // (ii) an identity projection followed by a union,
+    // merge with it
     // else create a new one
-    if ((leftOp instanceof UnionOperator) || (rightOp instanceof UnionOperator)) {
-      if (leftOp instanceof UnionOperator) {
-        // make left a child of right
-        List<Operator<? extends OperatorDesc>> child =
-            new ArrayList<Operator<? extends OperatorDesc>>();
-        child.add(leftOp);
-        rightOp.setChildOperators(child);
-
-        List<Operator<? extends OperatorDesc>> parent = leftOp
-            .getParentOperators();
-        parent.add(rightOp);
-
-        UnionDesc uDesc = ((UnionOperator) leftOp).getConf();
-        uDesc.setNumInputs(uDesc.getNumInputs() + 1);
-        return putOpInsertMap(leftOp, unionoutRR);
-      } else {
-        // make right a child of left
-        List<Operator<? extends OperatorDesc>> child =
-            new ArrayList<Operator<? extends OperatorDesc>>();
-        child.add(rightOp);
-        leftOp.setChildOperators(child);
-
-        List<Operator<? extends OperatorDesc>> parent = rightOp
-            .getParentOperators();
-        parent.add(leftOp);
-        UnionDesc uDesc = ((UnionOperator) rightOp).getConf();
-        uDesc.setNumInputs(uDesc.getNumInputs() + 1);
+    if (leftOp instanceof UnionOperator ||
+        (leftOp instanceof SelectOperator &&
+         leftOp.getParentOperators() != null &&
+         !leftOp.getParentOperators().isEmpty() &&
+         leftOp.getParentOperators().get(0) instanceof UnionOperator &&
+         ((SelectOperator)leftOp).isIdentitySelect()) ) {
+
+      if(!(leftOp instanceof UnionOperator)) {
+        Operator oldChild = leftOp;
+        leftOp = (Operator) leftOp.getParentOperators().get(0);
+        leftOp.removeChildAndAdoptItsChildren(oldChild);
+      }
+
+      // make left a child of right
+      List<Operator<? extends OperatorDesc>> child =
+          new ArrayList<Operator<? extends OperatorDesc>>();
+      child.add(leftOp);
+      rightOp.setChildOperators(child);
+
+      List<Operator<? extends OperatorDesc>> parent = leftOp
+          .getParentOperators();
+      parent.add(rightOp);
+
+      UnionDesc uDesc = ((UnionOperator) leftOp).getConf();
+      uDesc.setNumInputs(uDesc.getNumInputs() + 1);
+      return putOpInsertMap(leftOp, unionoutRR);
+    }
+
+    if (rightOp instanceof UnionOperator ||
+        (rightOp instanceof SelectOperator &&
+         rightOp.getParentOperators() != null &&
+         !rightOp.getParentOperators().isEmpty() &&
+         rightOp.getParentOperators().get(0) instanceof UnionOperator &&
+         ((SelectOperator)rightOp).isIdentitySelect()) ) {
+
+      if(!(rightOp instanceof UnionOperator)) {
+        Operator oldChild = rightOp;
+        rightOp = (Operator) rightOp.getParentOperators().get(0);
+        rightOp.removeChildAndAdoptItsChildren(oldChild);
+      }
+
+      // make right a child of left
+      List<Operator<? extends OperatorDesc>> child =
+          new ArrayList<Operator<? extends OperatorDesc>>();
+      child.add(rightOp);
+      leftOp.setChildOperators(child);
+
+      List<Operator<? extends OperatorDesc>> parent = rightOp
+          .getParentOperators();
+      parent.add(leftOp);
+      UnionDesc uDesc = ((UnionOperator) rightOp).getConf();
+      uDesc.setNumInputs(uDesc.getNumInputs() + 1);
 
-        return putOpInsertMap(rightOp, unionoutRR);
-      }
+      return putOpInsertMap(rightOp, unionoutRR);
     }
 
     // Create a new union operator

Modified: hive/trunk/ql/src/test/results/clientpositive/union29.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union29.q.out?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union29.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/union29.q.out Wed Oct 29 18:17:37 2014
@@ -75,25 +75,19 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
               Union
-                Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
                 Select Operator
-                  expressions: _col0 (type: string), _col1 (type: string)
+                  expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
-                  Union
+                  Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
+                  File Output Operator
+                    compressed: false
                     Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
-                    Select Operator
-                      expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
-                      File Output Operator
-                        compressed: false
-                        Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE
Column stats: NONE
-                        table:
-                            input format: org.apache.hadoop.mapred.TextInputFormat
-                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                            name: default.union_subq_union
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        name: default.union_subq_union
           TableScan
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
@@ -102,25 +96,19 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
               Union
-                Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
                 Select Operator
-                  expressions: _col0 (type: string), _col1 (type: string)
+                  expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
-                  Union
+                  Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
+                  File Output Operator
+                    compressed: false
                     Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
-                    Select Operator
-                      expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
-                      File Output Operator
-                        compressed: false
-                        Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE
Column stats: NONE
-                        table:
-                            input format: org.apache.hadoop.mapred.TextInputFormat
-                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                            name: default.union_subq_union
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        name: default.union_subq_union
 
   Stage: Stage-7
     Conditional Operator

Modified: hive/trunk/ql/src/test/results/clientpositive/union30.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union30.q.out?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union30.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/union30.q.out Wed Oct 29 18:17:37 2014
@@ -48,17 +48,16 @@ select key, value from src
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1, Stage-11
+  Stage-2 depends on stages: Stage-1, Stage-10
   Stage-3 depends on stages: Stage-2
-  Stage-4 depends on stages: Stage-3
-  Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8
-  Stage-7
-  Stage-0 depends on stages: Stage-7, Stage-6, Stage-9
-  Stage-5 depends on stages: Stage-0
+  Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7
   Stage-6
-  Stage-8
-  Stage-9 depends on stages: Stage-8
-  Stage-11 is a root stage
+  Stage-0 depends on stages: Stage-6, Stage-5, Stage-8
+  Stage-4 depends on stages: Stage-0
+  Stage-5
+  Stage-7
+  Stage-8 depends on stages: Stage-7
+  Stage-10 is a root stage
 
 STAGE PLANS:
   Stage: Stage-1
@@ -135,42 +134,6 @@ STAGE PLANS:
     Map Reduce
       Map Operator Tree:
           TableScan
-            alias: src
-            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
-            Select Operator
-              expressions: key (type: string), value (type: string)
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
-              Union
-                Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
-                Select Operator
-                  expressions: _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
-                  File Output Operator
-                    compressed: false
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-          TableScan
-            Union
-              Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats:
NONE
-              Select Operator
-                expressions: _col0 (type: string), _col1 (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
-                File Output Operator
-                  compressed: false
-                  table:
-                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
-  Stage: Stage-4
-    Map Reduce
-      Map Operator Tree:
-          TableScan
             Union
               Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
@@ -206,11 +169,32 @@ STAGE PLANS:
                         output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                         serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                         name: default.union_subq_union
+          TableScan
+            alias: src
+            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+            Select Operator
+              expressions: key (type: string), value (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+              Union
+                Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
+                Select Operator
+                  expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column
stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        name: default.union_subq_union
 
-  Stage: Stage-10
+  Stage: Stage-9
     Conditional Operator
 
-  Stage: Stage-7
+  Stage: Stage-6
     Move Operator
       files:
           hdfs directory: true
@@ -226,10 +210,10 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: default.union_subq_union
 
-  Stage: Stage-5
+  Stage: Stage-4
     Stats-Aggr Operator
 
-  Stage: Stage-6
+  Stage: Stage-5
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -241,7 +225,7 @@ STAGE PLANS:
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                   name: default.union_subq_union
 
-  Stage: Stage-8
+  Stage: Stage-7
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -253,13 +237,13 @@ STAGE PLANS:
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                   name: default.union_subq_union
 
-  Stage: Stage-9
+  Stage: Stage-8
     Move Operator
       files:
           hdfs directory: true
 #### A masked pattern was here ####
 
-  Stage: Stage-11
+  Stage: Stage-10
     Map Reduce
       Map Operator Tree:
           TableScan

Modified: hive/trunk/ql/src/test/results/clientpositive/union34.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union34.q.out?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union34.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/union34.q.out Wed Oct 29 18:17:37 2014
@@ -121,22 +121,16 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats:
NONE
               Union
-                Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats:
NONE
+                Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats:
NONE
                 Select Operator
                   expressions: _col0 (type: string), _col1 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats:
NONE
-                  Union
+                  Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats:
NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string)
+                    sort order: +
                     Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                    Select Operator
-                      expressions: _col0 (type: string), _col1 (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                      Reduce Output Operator
-                        key expressions: _col0 (type: string)
-                        sort order: +
-                        Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                        value expressions: _col1 (type: string)
+                    value expressions: _col1 (type: string)
           TableScan
             alias: src10_4
             Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE
@@ -145,22 +139,16 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats:
NONE
               Union
-                Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats:
NONE
+                Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats:
NONE
                 Select Operator
                   expressions: _col0 (type: string), _col1 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats:
NONE
-                  Union
+                  Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats:
NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string)
+                    sort order: +
                     Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                    Select Operator
-                      expressions: _col0 (type: string), _col1 (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                      Reduce Output Operator
-                        key expressions: _col0 (type: string)
-                        sort order: +
-                        Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                        value expressions: _col1 (type: string)
+                    value expressions: _col1 (type: string)
           TableScan
             alias: src10_2
             Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE
@@ -375,22 +363,16 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats:
NONE
               Union
-                Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats:
NONE
+                Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats:
NONE
                 Select Operator
                   expressions: _col0 (type: string), _col1 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats:
NONE
-                  Union
+                  Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats:
NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string)
+                    sort order: +
                     Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                    Select Operator
-                      expressions: _col0 (type: string), _col1 (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                      Reduce Output Operator
-                        key expressions: _col0 (type: string)
-                        sort order: +
-                        Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                        value expressions: _col1 (type: string)
+                    value expressions: _col1 (type: string)
           TableScan
             alias: src10_4
             Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE
@@ -399,22 +381,16 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats:
NONE
               Union
-                Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats:
NONE
+                Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats:
NONE
                 Select Operator
                   expressions: _col0 (type: string), _col1 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats:
NONE
-                  Union
+                  Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats:
NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string)
+                    sort order: +
                     Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                    Select Operator
-                      expressions: _col0 (type: string), _col1 (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                      Reduce Output Operator
-                        key expressions: _col0 (type: string)
-                        sort order: +
-                        Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column
stats: NONE
-                        value expressions: _col1 (type: string)
+                    value expressions: _col1 (type: string)
       Reduce Operator Tree:
         Select Operator
           expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)

Modified: hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out Wed Oct 29 18:17:37
2014
@@ -78,13 +78,13 @@ select * FROM (
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
-  Stage-7 depends on stages: Stage-2, Stage-8 , consists of Stage-4, Stage-3, Stage-5
-  Stage-4
-  Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+  Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4
   Stage-3
-  Stage-5
-  Stage-6 depends on stages: Stage-5
+  Stage-0 depends on stages: Stage-3, Stage-2, Stage-5
+  Stage-2
+  Stage-4
+  Stage-5 depends on stages: Stage-4
+  Stage-7 is a root stage
   Stage-8 is a root stage
 
 STAGE PLANS:
@@ -123,55 +123,17 @@ STAGE PLANS:
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             File Output Operator
               compressed: false
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                  serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                  input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+                  name: default.outputtbl1
 
-  Stage: Stage-2
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            Union
-              Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
-              Select Operator
-                expressions: _col0 (type: string), _col1 (type: bigint)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats:
NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats:
NONE
-                  table:
-                      input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-                      name: default.outputtbl1
-          TableScan
-            alias: inputtbl1
-            Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
-            Select Operator
-              expressions: key (type: string), UDFToLong(2) (type: bigint)
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
-              Union
-                Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats:
NONE
-                Select Operator
-                  expressions: _col0 (type: string), _col1 (type: bigint)
-                  outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats:
NONE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats:
NONE
-                    table:
-                        input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-                        name: default.outputtbl1
-
-  Stage: Stage-7
+  Stage: Stage-6
     Conditional Operator
 
-  Stage: Stage-4
+  Stage: Stage-3
     Move Operator
       files:
           hdfs directory: true
@@ -187,27 +149,27 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
               name: default.outputtbl1
 
-  Stage: Stage-3
+  Stage: Stage-2
     Merge File Operator
       Map Operator Tree:
           RCFile Merge Operator
       merge level: block
       input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
 
-  Stage: Stage-5
+  Stage: Stage-4
     Merge File Operator
       Map Operator Tree:
           RCFile Merge Operator
       merge level: block
       input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
 
-  Stage: Stage-6
+  Stage: Stage-5
     Move Operator
       files:
           hdfs directory: true
 #### A masked pattern was here ####
 
-  Stage: Stage-8
+  Stage: Stage-7
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -226,6 +188,25 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
                     name: default.outputtbl1
 
+  Stage: Stage-8
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: inputtbl1
+            Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
+            Select Operator
+              expressions: key (type: string), UDFToLong(2) (type: bigint)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
+              File Output Operator
+                compressed: false
+                Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats:
NONE
+                table:
+                    input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+                    name: default.outputtbl1
+
 PREHOOK: query: insert overwrite table outputTbl1
 SELECT * FROM
 (
@@ -255,7 +236,6 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@inputtbl1
 POSTHOOK: Output: default@outputtbl1
 POSTHOOK: Lineage: outputtbl1.key EXPRESSION [(inputtbl1)inputtbl1.FieldSchema(name:key,
type:string, comment:null), (inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null),
(inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), ]
-POSTHOOK: Lineage: outputtbl1.values EXPRESSION [(inputtbl1)inputtbl1.null, ]
 PREHOOK: query: desc formatted outputTbl1
 PREHOOK: type: DESCTABLE
 PREHOOK: Input: default@outputtbl1

Modified: hive/trunk/ql/src/test/results/clientpositive/union_remove_11.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_remove_11.q.out?rev=1635245&r1=1635244&r2=1635245&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union_remove_11.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/union_remove_11.q.out Wed Oct 29 18:17:37
2014
@@ -97,25 +97,19 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
               Union
-                Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats:
NONE
+                Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats:
NONE
                 Select Operator
-                  expressions: _col0 (type: string), _col1 (type: int)
+                  expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats:
NONE
-                  Union
+                  Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats:
NONE
+                  File Output Operator
+                    compressed: false
                     Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats:
NONE
-                    Select Operator
-                      expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats:
NONE
-                      File Output Operator
-                        compressed: false
-                        Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column
stats: NONE
-                        table:
-                            input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-                            output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
-                            serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-                            name: default.outputtbl1
+                    table:
+                        input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+                        name: default.outputtbl1
           TableScan
             alias: inputtbl1
             Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
@@ -145,25 +139,19 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
               Union
-                Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats:
NONE
+                Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats:
NONE
                 Select Operator
-                  expressions: _col0 (type: string), _col1 (type: int)
+                  expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats:
NONE
-                  Union
+                  Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats:
NONE
+                  File Output Operator
+                    compressed: false
                     Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats:
NONE
-                    Select Operator
-                      expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats:
NONE
-                      File Output Operator
-                        compressed: false
-                        Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column
stats: NONE
-                        table:
-                            input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-                            output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
-                            serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-                            name: default.outputtbl1
+                    table:
+                        input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+                        name: default.outputtbl1
 
   Stage: Stage-6
     Conditional Operator



Mime
View raw message