kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [5/7] kylin git commit: minor, adjust out-of-box kylin.properties for sparking cubing
Date Fri, 24 Feb 2017 08:38:08 GMT
minor, adjust out-of-box kylin.properties for sparking cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ff78bce
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ff78bce
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ff78bce

Branch: refs/heads/master-hbase0.98
Commit: 9ff78bce72a56b63dc9e1c8f0b4cd59d0c01b011
Parents: d59b8ae
Author: Li Yang <liyang@apache.org>
Authored: Fri Feb 24 16:15:52 2017 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Fri Feb 24 16:20:33 2017 +0800

----------------------------------------------------------------------
 build/conf/kylin.properties                        | 17 ++++++++---------
 .../apache/kylin/engine/spark/SparkExecutable.java |  4 ++--
 webapp/app/js/model/cubeConfig.js                  |  2 +-
 3 files changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index d7ba0b3..46fe86b 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -203,8 +203,9 @@ kylin.security.saml.context-path=/kylin
 
 ### Spark Engine Configs ###
 
-# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
-#kylin.env.hadoop-conf-dir=/etc/hive/conf
+# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" to run spark-submit
+# This must contain site xmls of core, yarn, hive, and hbase in one folder
+#kylin.env.hadoop-conf-dir=/etc/hadoop/conf
 
 # Estimate the RDD partition numbers
 kylin.engine.spark.rdd-partition-cut-mb=10
@@ -215,19 +216,17 @@ kylin.engine.spark.min-partition=1
 # Max partition numbers of rdd
 kylin.engine.spark.max-partition=5000
 
-### Spark conf (default is in spark/conf/spark-defaults.conf)
+## Spark conf (default is in spark/conf/spark-defaults.conf)
 kylin.engine.spark-conf.spark.master=yarn
 kylin.engine.spark-conf.spark.submit.deployMode=cluster
-kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=1024
-kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=384
 kylin.engine.spark-conf.spark.yarn.queue=default
-kylin.engine.spark-conf.spark.executor.memory=4G
-kylin.engine.spark-conf.spark.executor.cores=4
-kylin.engine.spark-conf.spark.executor.instances=8
-kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
+kylin.engine.spark-conf.spark.executor.memory=1G
+kylin.engine.spark-conf.spark.executor.cores=2
+kylin.engine.spark-conf.spark.executor.instances=1
 kylin.engine.spark-conf.spark.eventLog.enabled=true
 kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
 kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+
 ## manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly
uploading jar at runtime
 #kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 #kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index cf7438c..5ad21cf 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -71,7 +71,7 @@ public class SparkExecutable extends AbstractExecutable {
     @Override
     protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
         final KylinConfig config = context.getConfig();
-        if (config.getSparkHome() == null) {
+        if (KylinConfig.getSparkHome() == null) {
             throw new NullPointerException();
         }
         if (config.getKylinJobJarPath() == null) {
@@ -117,7 +117,7 @@ public class SparkExecutable extends AbstractExecutable {
 
         stringBuilder.append("--files %s --jars %s %s %s");
         try {
-            String cmd = String.format(stringBuilder.toString(), hadoopConf, config.getSparkHome(),
hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs());
+            String cmd = String.format(stringBuilder.toString(), hadoopConf, KylinConfig.getSparkHome(),
hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs());
             logger.info("cmd:" + cmd);
             final StringBuilder output = new StringBuilder();
             CliCommandExecutor exec = new CliCommandExecutor();

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index 47d0f46..eec3f10 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -25,7 +25,7 @@ KylinApp.constant('cubeConfig', {
   cubePartitionTypes: ['APPEND'],
   engineType:[
     {name:'MapReduce',value: 2},
-    {name:'Spark(Beta)',value: 4}
+    {name:'Spark (Beta)',value: 4}
   ],
   joinTypes: [
     {name: 'Left', value: 'left'},


Mime
View raw message