hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject hive git commit: HIVE-14200: Tez: disable auto-reducer parallelism when reducer-count * min.partition.factor < 1.0 (Gopal V, reviewed by Gunther Hagleitner)
Date Mon, 11 Jul 2016 23:23:38 GMT
Repository: hive
Updated Branches:
  refs/heads/master c790391cc -> a61c35184


HIVE-14200: Tez: disable auto-reducer parallelism when reducer-count * min.partition.factor
< 1.0 (Gopal V, reviewed by Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a61c3518
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a61c3518
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a61c3518

Branch: refs/heads/master
Commit: a61c351848ba358b3834d9ace934f01a60d8a021
Parents: c790391
Author: Gopal V <gopalv@apache.org>
Authored: Mon Jul 11 16:22:59 2016 -0700
Committer: Gopal V <gopalv@apache.org>
Committed: Mon Jul 11 16:22:59 2016 -0700

----------------------------------------------------------------------
 .../hadoop/hive/ql/parse/GenTezUtils.java       | 28 +++++++++++++-------
 1 file changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a61c3518/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index d5a2eca..32bf24d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -107,23 +107,31 @@ public class GenTezUtils {
     reduceWork.setNumReduceTasks(reduceSink.getConf().getNumReducers());
 
     if (isAutoReduceParallelism && reduceSink.getConf().getReducerTraits().contains(AUTOPARALLEL))
{
-      reduceWork.setAutoReduceParallelism(true);
 
       // configured limit for reducers
-      int maxReducers = context.conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS);
+      final int maxReducers = context.conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS);
+      // estimated number of reducers
+      final int nReducers = reduceSink.getConf().getNumReducers();
 
       // min we allow tez to pick
-      int minPartition = Math.max(1, (int) (reduceSink.getConf().getNumReducers()
-        * minPartitionFactor));
+      int minPartition = Math.max(1, (int) (nReducers * minPartitionFactor));
       minPartition = (minPartition > maxReducers) ? maxReducers : minPartition;
 
       // max we allow tez to pick
-      int maxPartition = (int) (reduceSink.getConf().getNumReducers() * maxPartitionFactor);
-      maxPartition = Math.max(1, (maxPartition > maxReducers) ? maxReducers :
-          maxPartition);
-
-      reduceWork.setMinReduceTasks(minPartition);
-      reduceWork.setMaxReduceTasks(maxPartition);
+      int maxPartition = Math.max(1, (int) (nReducers * maxPartitionFactor));
+      maxPartition = (maxPartition > maxReducers) ? maxReducers : maxPartition;
+
+      // reduce only if the parameters are significant
+      if (minPartition < maxPartition &&
+          nReducers * minPartitionFactor >= 1.0) {
+        reduceWork.setAutoReduceParallelism(true);
+
+        reduceWork.setMinReduceTasks(minPartition);
+        reduceWork.setMaxReduceTasks(maxPartition);
+      } else if (nReducers < maxPartition) {
+        // the max is good, the min is too low
+        reduceWork.setNumReduceTasks(maxPartition);
+      }
     }
 
     setupReduceSink(context, reduceWork, reduceSink);


Mime
View raw message