kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject kylin git commit: KYLIN-1657 add configuration kylin.job.mapreduce.min.reducer.number
Date Fri, 27 May 2016 05:53:05 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 86f7149fc -> 3bca0bfe4


KYLIN-1657 add configuration kylin.job.mapreduce.min.reducer.number

Signed-off-by: lidongsjtu <lidong@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3bca0bfe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3bca0bfe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3bca0bfe

Branch: refs/heads/master
Commit: 3bca0bfe4d5fe9f9080a6bc2e38d6fc8e3c2069b
Parents: 86f7149
Author: gaodayue <gaodayue@meituan.com>
Authored: Thu Apr 28 10:51:46 2016 +0800
Committer: lidongsjtu <lidong@apache.org>
Committed: Fri May 27 13:52:19 2016 +0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
 .../main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java  | 6 +++---
 .../java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java  | 6 +++---
 3 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/3bca0bfe/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2ffb219..b2294b0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -307,6 +307,10 @@ abstract public class KylinConfigBase implements Serializable {
         return Double.parseDouble(getOptional("kylin.job.mapreduce.default.reduce.count.ratio",
"1.0"));
     }
 
+    public int getHadoopJobMinReducerNumber() {
+        return Integer.parseInt(getOptional("kylin.job.mapreduce.min.reducer.number", "1"));
+    }
+
     public int getHadoopJobMaxReducerNumber() {
         return Integer.parseInt(getOptional("kylin.job.mapreduce.max.reducer.number", "500"));
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3bca0bfe/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index ce353f7..a445f71 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -203,9 +203,9 @@ public class CuboidJob extends AbstractHadoopJob {
             numReduceTasks = numReduceTasks * 4;
         }
 
-        // at least 1 reducer
-        numReduceTasks = Math.max(1, numReduceTasks);
-        // no more than 5000 reducer by default
+        // at least 1 reducer by default
+        numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
+        // no more than 500 reducer by default
         numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
 
         jobConf.setInt(MAPRED_REDUCE_TASKS, numReduceTasks);

http://git-wip-us.apache.org/repos/asf/kylin/blob/3bca0bfe/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
index e7bbdf1..258ec95 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
@@ -158,9 +158,9 @@ public class InMemCuboidJob extends AbstractHadoopJob {
         // number of reduce tasks
         int numReduceTasks = (int) Math.round(totalSizeInM / perReduceInputMB);
 
-        // at least 1 reducer
-        numReduceTasks = Math.max(1, numReduceTasks);
-        // no more than 5000 reducer by default
+        // at least 1 reducer by default
+        numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
+        // no more than 500 reducer by default
         numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
 
         logger.info("Having total map input MB " + Math.round(totalSizeInM));


Mime
View raw message