hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject [1/2] hive git commit: HIVE-12664 : Bug in reduce deduplication optimization causing ArrayOutOfBoundException (Johan Gustavsson via Ashutosh Chauhan)
Date Thu, 07 Jan 2016 16:28:05 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 3133ac5a3 -> 81617ae83
  refs/heads/master cb1745619 -> a6debb95d


HIVE-12664 : Bug in reduce deduplication optimization causing ArrayOutOfBoundException (Johan
Gustavsson via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/81617ae8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/81617ae8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/81617ae8

Branch: refs/heads/branch-2.0
Commit: 81617ae83e46088d4186b0112858ad305a0dda6d
Parents: 3133ac5
Author: Johan Gustavsson <johan+apache+jira@treasure-data.com>
Authored: Mon Jan 4 18:01:00 2016 -0800
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Thu Jan 7 08:16:58 2016 -0800

----------------------------------------------------------------------
 .../correlation/ReduceSinkDeDuplication.java    | 39 ++++++++++----------
 1 file changed, 20 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/81617ae8/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
index 8da9ad5..1746dd3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
@@ -186,22 +186,24 @@ public class ReduceSinkDeDuplication extends Transform {
       List<Operator<?>> parents = pJoin.getParentOperators();
       ReduceSinkOperator[] pRSs = parents.toArray(new ReduceSinkOperator[parents.size()]);
       ReduceSinkDesc cRSc = cRS.getConf();
-      ReduceSinkDesc pRS0c = pRSs[0].getConf();
-      if (cRSc.getKeyCols().size() < pRS0c.getKeyCols().size()) {
-        return false;
-      }
-      if (cRSc.getPartitionCols().size() != pRS0c.getPartitionCols().size()) {
-        return false;
-      }
-      Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRS0c.getNumReducers());
-      if (moveReducerNumTo == null ||
-          moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) {
-        return false;
-      }
+      for (ReduceSinkOperator pRSNs : pRSs) {
+        ReduceSinkDesc pRSNc = pRSNs.getConf();
+        if (cRSc.getKeyCols().size() < pRSNc.getKeyCols().size()) {
+          return false;
+        }
+        if (cRSc.getPartitionCols().size() != pRSNc.getPartitionCols().size()) {
+          return false;
+        }
+        Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRSNc.getNumReducers());
+        if (moveReducerNumTo == null ||
+            moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) {
+          return false;
+        }
 
-      Integer moveRSOrderTo = checkOrder(cRSc.getOrder(), pRS0c.getOrder());
-      if (moveRSOrderTo == null) {
-        return false;
+        Integer moveRSOrderTo = checkOrder(cRSc.getOrder(), pRSNc.getOrder());
+        if (moveRSOrderTo == null) {
+          return false;
+        }
       }
 
       boolean[] sorted = CorrelationUtilities.getSortedTags(pJoin);
@@ -231,11 +233,10 @@ public class ReduceSinkDeDuplication extends Transform {
         }
       }
 
-      if (moveReducerNumTo > 0) {
-        for (ReduceSinkOperator pRS : pRSs) {
-          pRS.getConf().setNumReducers(cRS.getConf().getNumReducers());
-        }
+      for (ReduceSinkOperator pRS : pRSs) {
+        pRS.getConf().setNumReducers(cRS.getConf().getNumReducers());
       }
+
       return true;
     }
 


Mime
View raw message