hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [16/50] [abbrv] hive git commit: HIVE-11416: CBO: Calcite Operator To Hive Operator (Calcite Return Path): Groupby Optimizer assumes the schema can match after removing RS and GBY (reviewed by Jesus Camacho Rodriguez)
Date Mon, 17 Aug 2015 22:00:46 GMT
HIVE-11416: CBO: Calcite Operator To Hive Operator (Calcite Return Path): Groupby Optimizer
assumes the schema can match after removing RS and GBY (reviewed by Jesus Camacho Rodriguez)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/763cb02b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/763cb02b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/763cb02b

Branch: refs/heads/hbase-metastore
Commit: 763cb02b5eafb0ecd3fd0eb512636a1b092df671
Parents: 57ba795
Author: Pengcheng Xiong <pxiong@apache.org>
Authored: Tue Aug 11 11:26:48 2015 -0700
Committer: Pengcheng Xiong <pxiong@apache.org>
Committed: Tue Aug 11 11:26:48 2015 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/exec/Operator.java    | 25 ---------
 .../hive/ql/optimizer/GroupByOptimizer.java     | 58 +++++++++++++++++++-
 2 files changed, 57 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/763cb02b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index 0f02737..acbe504 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@ -769,31 +769,6 @@ public abstract class Operator<T extends OperatorDesc> implements
Serializable,C
     }
   }
 
-  // Remove the operators till a certain depth.
-  // Return true if the remove was successful, false otherwise
-  public boolean removeChildren(int depth) {
-    Operator<? extends OperatorDesc> currOp = this;
-    for (int i = 0; i < depth; i++) {
-      // If there are more than 1 children at any level, don't do anything
-      if ((currOp.getChildOperators() == null) || (currOp.getChildOperators().isEmpty())
||
-          (currOp.getChildOperators().size() > 1)) {
-        return false;
-      }
-      currOp = currOp.getChildOperators().get(0);
-    }
-
-    setChildOperators(currOp.getChildOperators());
-
-    List<Operator<? extends OperatorDesc>> parentOps =
-      new ArrayList<Operator<? extends OperatorDesc>>();
-    parentOps.add(this);
-
-    for (Operator<? extends OperatorDesc> op : currOp.getChildOperators()) {
-      op.setParentOperators(parentOps);
-    }
-    return true;
-  }
-
   /**
    * Replace one parent with another at the same position. Chilren of the new
    * parent are not updated

http://git-wip-us.apache.org/repos/asf/hive/blob/763cb02b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
index af54286..ce3f59a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.optimizer;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -31,9 +32,12 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.GroupByOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorFactory;
 import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.RowSchema;
 import org.apache.hadoop.hive.ql.exec.SelectOperator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -520,12 +524,64 @@ public class GroupByOptimizer implements Transform {
         return;
       }
 
-      if (groupByOp.removeChildren(depth)) {
+      if (removeChildren(groupByOp, depth)) {
         // Use bucketized hive input format - that makes sure that one mapper reads the entire
file
         groupByOp.setUseBucketizedHiveInputFormat(true);
         groupByOp.getConf().setMode(GroupByDesc.Mode.FINAL);
       }
     }
+
+    // Remove the operators till a certain depth.
+    // Return true if the remove was successful, false otherwise
+    public boolean removeChildren(Operator<? extends OperatorDesc> currOp, int depth)
{
+      Operator<? extends OperatorDesc> inputOp = currOp;
+      for (int i = 0; i < depth; i++) {
+        // If there are more than 1 children at any level, don't do anything
+        if ((currOp.getChildOperators() == null) || (currOp.getChildOperators().isEmpty())
+            || (currOp.getChildOperators().size() > 1)) {
+          return false;
+        }
+        currOp = currOp.getChildOperators().get(0);
+      }
+
+      // add selectOp to match the schema
+      // after that, inputOp is the parent of selOp.
+      for (Operator<? extends OperatorDesc> op : inputOp.getChildOperators()) {
+        op.getParentOperators().clear();
+      }
+      inputOp.getChildOperators().clear();
+      Operator<? extends OperatorDesc> selOp = genOutputSelectForGroupBy(inputOp, currOp);
+
+      // update the childOp of selectOp
+      selOp.setChildOperators(currOp.getChildOperators());
+
+      // update the parentOp
+      for (Operator<? extends OperatorDesc> op : currOp.getChildOperators()) {
+        op.replaceParent(currOp, selOp);
+      }
+      return true;
+    }
+
+    private Operator<? extends OperatorDesc> genOutputSelectForGroupBy(
+        Operator<? extends OperatorDesc> parentOp, Operator<? extends OperatorDesc>
currOp) {
+      assert (parentOp.getSchema().getSignature().size() == currOp.getSchema().getSignature().size());
+      Iterator<ColumnInfo> pIter = parentOp.getSchema().getSignature().iterator();
+      Iterator<ColumnInfo> cIter = currOp.getSchema().getSignature().iterator();
+      List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>();
+      List<String> colName = new ArrayList<String>();
+      Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>();
+      while (pIter.hasNext()) {
+        ColumnInfo pInfo = pIter.next();
+        ColumnInfo cInfo = cIter.next();
+        ExprNodeDesc column = new ExprNodeColumnDesc(pInfo.getType(), pInfo.getInternalName(),
+            pInfo.getTabAlias(), pInfo.getIsVirtualCol(), pInfo.isSkewedCol());
+        columns.add(column);
+        colName.add(cInfo.getInternalName());
+        columnExprMap.put(cInfo.getInternalName(), column);
+      }
+      return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(currOp
+          .getSchema().getSignature()), columnExprMap, parentOp);
+    }
   }
 
   /**


Mime
View raw message