hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [64/70] [abbrv] hive git commit: HIVE-15808: Remove semijoin reduction branch if it is on bigtable along with hash join (Deepak Jaiswal, reviewed by Jason Dere)
Date Tue, 07 Feb 2017 20:59:32 GMT
HIVE-15808: Remove semijoin reduction branch if it is on bigtable along with hash join (Deepak
Jaiswal, reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f6cdbc87
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f6cdbc87
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f6cdbc87

Branch: refs/heads/hive-14535
Commit: f6cdbc87955aa5cdb83f174a73db9a7d8071f78b
Parents: 3ed7dc2
Author: Gunther Hagleitner <gunther@apache.org>
Authored: Tue Feb 7 11:11:09 2017 -0800
Committer: Gunther Hagleitner <gunther@apache.org>
Committed: Tue Feb 7 11:11:09 2017 -0800

----------------------------------------------------------------------
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   | 64 +++++++++++---------
 .../hadoop/hive/ql/parse/GenTezUtils.java       |  8 +--
 2 files changed, 39 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f6cdbc87/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index 0f9e86b..e3b293a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -775,51 +775,57 @@ public class ConvertJoinMapJoin implements NodeProcessor {
     return mapJoinOp;
   }
 
-  // Remove any semijoin branch associated with mapjoin's parent's operator
-  // pipeline which can cause a cycle after mapjoin optimization.
+  // Remove any semijoin branch associated with hashjoin's parent's operator
+  // pipeline which can cause a cycle after hashjoin optimization.
   private void removeCycleCreatingSemiJoinOps(MapJoinOperator mapjoinOp,
                                               Operator<?> parentSelectOpOfBigTable,
                                               ParseContext parseContext) throws SemanticException
{
-    boolean semiJoinCycle = false;
-    ReduceSinkOperator rs = null;
-    TableScanOperator ts = null;
+    Map<ReduceSinkOperator, TableScanOperator> semiJoinMap =
+            new HashMap<ReduceSinkOperator, TableScanOperator>();
     for (Operator<?> op : parentSelectOpOfBigTable.getChildOperators()) {
       if (!(op instanceof SelectOperator)) {
         continue;
       }
 
-      while (op.getChildOperators().size() > 0 ) {
+      while (op.getChildOperators().size() > 0) {
         op = op.getChildOperators().get(0);
-        if (!(op instanceof ReduceSinkOperator)) {
-          continue;
-        }
-        rs = (ReduceSinkOperator) op;
-        ts = parseContext.getRsOpToTsOpMap().get(rs);
-        if (ts == null) {
+      }
+
+      // If not ReduceSink Op, skip
+      if (!(op instanceof ReduceSinkOperator)) {
+        continue;
+      }
+
+      ReduceSinkOperator rs = (ReduceSinkOperator) op;
+      TableScanOperator ts = parseContext.getRsOpToTsOpMap().get(rs);
+      if (ts == null) {
+        // skip, no semijoin branch
+        continue;
+      }
+
+      // Found a semijoin branch.
+      for (Operator<?> parent : mapjoinOp.getParentOperators()) {
+        if (!(parent instanceof ReduceSinkOperator)) {
           continue;
         }
-        for (Operator<?> parent : mapjoinOp.getParentOperators()) {
-          if (!(parent instanceof ReduceSinkOperator)) {
-            continue;
-          }
 
-          Set<TableScanOperator> tsOps = OperatorUtils.findOperatorsUpstream(parent,
-                  TableScanOperator.class);
-          for (TableScanOperator parentTS : tsOps) {
-            // If the parent is same as the ts, then we have a cycle.
-            if (ts == parentTS) {
-              semiJoinCycle = true;
-              break;
-            }
+        Set<TableScanOperator> tsOps = OperatorUtils.findOperatorsUpstream(parent,
+                TableScanOperator.class);
+        for (TableScanOperator parentTS : tsOps) {
+          // If the parent is same as the ts, then we have a cycle.
+          if (ts == parentTS) {
+            semiJoinMap.put(rs, ts);
+            break;
           }
         }
       }
     }
-
-    // By design there can be atmost 1 such cycle.
-    if (semiJoinCycle) {
-      GenTezUtils.removeBranch(rs);
-      GenTezUtils.removeSemiJoinOperator(parseContext, rs, ts);
+    if (semiJoinMap.size() > 0) {
+      for (ReduceSinkOperator rs : semiJoinMap.keySet()) {
+        GenTezUtils.removeBranch(rs);
+        GenTezUtils.removeSemiJoinOperator(parseContext, rs,
+                semiJoinMap.get(rs));
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/f6cdbc87/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index aee74ad..7f5fdff 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -265,7 +265,6 @@ public class GenTezUtils {
           for (ReduceSinkOperator rs : rsOpToTsOpMap.keySet()) {
             if (rsOpToTsOpMap.get(rs) == orig) {
               rsOpToTsOpMap.put(rs, (TableScanOperator) newRoot);
-              break;
             }
           }
         }
@@ -569,7 +568,8 @@ public class GenTezUtils {
             TypeInfoFactory.booleanTypeInfo, Boolean.TRUE);
     DynamicValuePredicateContext filterDynamicValuePredicatesCollection =
             new DynamicValuePredicateContext();
-    collectDynamicValuePredicates(((FilterOperator)(ts.getChildOperators().get(0))).getConf().getPredicate(),
+    FilterDesc filterDesc = ((FilterOperator)(ts.getChildOperators().get(0))).getConf();
+    collectDynamicValuePredicates(filterDesc.getPredicate(),
             filterDynamicValuePredicatesCollection);
     for (ExprNodeDesc nodeToRemove : filterDynamicValuePredicatesCollection
             .childParentMapping.keySet()) {
@@ -594,8 +594,8 @@ public class GenTezUtils {
         ExprNodeDesc nodeParent = filterDynamicValuePredicatesCollection
                 .childParentMapping.get(nodeToRemove);
         if (nodeParent == null) {
-          // This was the only predicate, set filter expression to null
-          ts.getConf().setFilterExpr(null);
+          // This was the only predicate, set filter expression to const
+          filterDesc.setPredicate(constNode);
         } else {
           int i = nodeParent.getChildren().indexOf(nodeToRemove);
           nodeParent.getChildren().remove(i);


Mime
View raw message