eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject incubator-eagle git commit: [MINOR] add jobConf.additional.info in spark history job app
Date Thu, 17 Nov 2016 05:19:00 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master ce8e1c508 -> 4874433a5


[MINOR] add jobConf.additional.info in spark history job app

A quick fixed bug

Author: Zhao, Qingwen <qingwzhao@apache.org>

Closes #662 from qingwen220/minor.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4874433a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4874433a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4874433a

Branch: refs/heads/master
Commit: 4874433a59c59e599b73a1235819db92e3248ce1
Parents: ce8e1c5
Author: Zhao, Qingwen <qingwzhao@apache.org>
Authored: Thu Nov 17 13:18:55 2016 +0800
Committer: Zhao, Qingwen <qingwzhao@apache.org>
Committed: Thu Nov 17 13:18:55 2016 +0800

----------------------------------------------------------------------
 .../eagle/jpm/spark/history/crawl/JHFSparkEventReader.java    | 7 ++++++-
 ...che.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml | 6 ++++++
 .../src/main/resources/application.conf                       | 1 +
 3 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
index a4ac77e..82e8a41 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
@@ -115,10 +115,15 @@ public class JHFSparkEventReader {
         app.setConfig(new JobConfig());
         JSONObject sparkProps = (JSONObject) event.get("Spark Properties");
 
-        String[] additionalJobConf = conf.getString("basic.jobConf.additional.info").split(",\\s*");
         String[] props = {"spark.yarn.app.id", "spark.executor.memory", "spark.driver.host",
"spark.driver.port",
             "spark.driver.memory", "spark.scheduler.pool", "spark.executor.cores", "spark.yarn.am.memory",
             "spark.yarn.am.cores", "spark.yarn.executor.memoryOverhead", "spark.yarn.driver.memoryOverhead",
"spark.yarn.am.memoryOverhead", "spark.master"};
+
+        String[] additionalJobConf = null;
+        if (conf.hasPath("spark.jobConf.additional.info")) {
+            additionalJobConf = conf.getString("spark.jobConf.additional.info").split(",\\s*");
+        }
+
         String[] jobConf = (String[]) ArrayUtils.addAll(additionalJobConf, props);
         for (String prop : jobConf) {
             if (sparkProps.containsKey(prop)) {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
index c68d4e8..17a3a4a 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
@@ -90,6 +90,12 @@
         </property>
 
         <property>
+            <name>spark.jobConf.additional.info</name>
+            <displayName>additional spark job config info</displayName>
+            <description>additional job config list</description>
+            <value></value>
+        </property>
+        <property>
             <name>spark.defaultVal.spark.executor.memory</name>
             <displayName>spark.executor.memory</displayName>
             <value>1g</value>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf
index a51abc9..2839915 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf
@@ -57,6 +57,7 @@
     numOfParserBoltTasks: 4
   },
   "spark":{
+    jobConf.additional.info: "",
     "defaultVal": {
       spark.executor.memory:"1g",
       spark.driver.memory: "1g",


Mime
View raw message