kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-1226 apply in-mem conf only to in-mem mr job
Date Fri, 11 Mar 2016 07:05:06 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 5e13bba08 -> 0232f17ed


KYLIN-1226 apply in-mem conf only to in-mem mr job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0232f17e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0232f17e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0232f17e

Branch: refs/heads/master
Commit: 0232f17ed74a3354c7ffd6647d159a8581626861
Parents: 5e13bba
Author: Li Yang <liyang@apache.org>
Authored: Fri Mar 11 15:03:40 2016 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Fri Mar 11 15:04:19 2016 +0800

----------------------------------------------------------------------
 build/conf/kylin_job_conf.xml                   | 14 -------------
 .../apache/kylin/common/KylinConfigBase.java    | 21 +++++++++++++++++++-
 .../kylin/engine/mr/steps/InMemCuboidJob.java   | 11 ++++++++++
 3 files changed, 31 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/build/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 052021c..3ecaeda 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -55,20 +55,6 @@
         <description>Block replication</description>
     </property>
 
-    <!--Properties for calculating cube by splits (in-mem), with which each Mapper need
more mem to hold a full cube segment -->
-    <property>
-        <name>mapreduce.map.java.opts</name>
-        <value>-Xmx2500m</value>
-    </property>
-    <property>
-        <name>mapreduce.map.memory.mb</name>
-        <value>3072</value>
-    </property>
-    <property>
-        <name>mapreduce.task.io.sort.mb</name>
-        <value>200</value>
-    </property>
-    
     <property>
         <name>mapred.task.timeout</name>
         <value>3600000</value>

http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 487f78e..6a47321 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Serializable;
+import java.util.Map;
 import java.util.Properties;
 import java.util.SortedSet;
 import java.util.regex.Matcher;
@@ -33,6 +34,7 @@ import org.apache.kylin.common.util.CliCommandExecutor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
 @SuppressWarnings("serial")
@@ -476,11 +478,28 @@ public class KylinConfigBase implements Serializable {
     }
 
     public int getCubingInMemSamplingPercent() {
-        int percent = Integer.parseInt(this.getOptional("kylin.job.cubing.inMem.sampling.percent",
"30"));
+        int percent = Integer.parseInt(this.getOptional("kylin.job.cubing.inmem.sampling.percent",
"30"));
         percent = Math.max(percent, 1);
         percent = Math.min(percent, 100);
         return percent;
     }
+    
+    public Map<String, String> getCubingInMemMRJobConfOverride() {
+        String str = getOptional("kylin.job.cubing.inmem.mrjob_conf_override", //
+                "mapreduce.map.java.opts=-Xmx2700m;  mapreduce.map.memory.mb=3072;  mapreduce.task.io.sort.mb=200");
+        Map<String, String> result = Maps.newHashMap();
+        for (String pair : str.split(";")) {
+            int cut = pair.indexOf('=');
+            if (cut < 0)
+                continue;
+            String k = pair.substring(0, cut).trim();
+            String v = pair.substring(cut + 1).trim();
+            if (k.isEmpty() || v.isEmpty())
+                continue;
+            result.put(k, v);
+        }
+        return result;
+    }
 
     public String getHbaseDefaultCompressionCodec() {
         return getOptional("kylin.hbase.default.compression.codec", "");

http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
index d197252..2319451 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
@@ -20,8 +20,10 @@ package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
@@ -97,6 +99,9 @@ public class InMemCuboidJob extends AbstractHadoopJob {
 
             job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
             logger.info("Starting: " + job.getJobName());
+            
+            // some special tuning for in-mem MR job
+            overrideJobConf(job.getConfiguration(), config);
 
             setJobClasspath(job);
 
@@ -143,6 +148,12 @@ public class InMemCuboidJob extends AbstractHadoopJob {
         }
     }
 
+    private void overrideJobConf(Configuration jobConf, KylinConfig kylinConfig) {
+        for (Entry<String, String> entry : kylinConfig.getCubingInMemMRJobConfOverride().entrySet())
{
+            jobConf.set(entry.getKey(), entry.getValue());
+        }
+    }
+
     private int calculateReducerNum(CubeSegment cubeSeg) throws IOException {
         KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 


Mime
View raw message