zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject zeppelin git commit: ZEPPELIN-3556. set spark.yarn.submit.waitAppCompletion to false in yarn-cluster mode
Date Tue, 26 Jun 2018 00:25:44 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master adf83a3f5 -> 31696b788


ZEPPELIN-3556. set spark.yarn.submit.waitAppCompletion to false in yarn-cluster mode

### What is this PR for?

Set `spark.yarn.submit.waitAppCompletion` to false in yarn-cluster mode, so that in the interpreter
log we won't see many logs of monitor yarn app status which is useless.

### What type of PR is it?
[Improvement]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3556

### How should this be tested?
* CI pass

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Jeff Zhang <zjffdu@apache.org>

Closes #3038 from zjffdu/ZEPPELIN-3556 and squashes the following commits:

7474987a6 [Jeff Zhang] ZEPPELIN-3556. set spark.yarn.submit.waitAppCompletion to false in
yarn-cluster mode


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/31696b78
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/31696b78
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/31696b78

Branch: refs/heads/master
Commit: 31696b7889799d47eb36160fa3288ed05cb690a8
Parents: adf83a3
Author: Jeff Zhang <zjffdu@apache.org>
Authored: Wed Jun 20 14:56:11 2018 +0800
Committer: Jeff Zhang <zjffdu@apache.org>
Committed: Tue Jun 26 08:25:36 2018 +0800

----------------------------------------------------------------------
 .../zeppelin/interpreter/launcher/SparkInterpreterLauncher.java  | 1 +
 .../interpreter/launcher/SparkInterpreterLauncherTest.java       | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31696b78/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
index 688d95f..ff65e0d 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
@@ -58,6 +58,7 @@ public class SparkInterpreterLauncher extends ShellScriptLauncher {
     setupPropertiesForSparkR(sparkProperties);
     if (isYarnMode() && getDeployMode().equals("cluster")) {
       env.put("ZEPPELIN_SPARK_YARN_CLUSTER", "true");
+      sparkProperties.setProperty("spark.yarn.submit.waitAppCompletion", "false");
     }
 
     StringBuilder sparkConfBuilder = new StringBuilder();

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31696b78/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
index 7d8b002..c2abd60 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
@@ -159,7 +159,7 @@ public class SparkInterpreterLauncherTest {
     assertTrue(interpreterProcess.getEnv().size() >= 3);
     assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME"));
     assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
-    assertEquals(" --master yarn-cluster --files .//conf/log4j_yarn_cluster.properties --conf
spark.files='file_1' --conf spark.jars='jar_1' --conf spark.yarn.isPython=true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
+    assertEquals(" --master yarn-cluster --files .//conf/log4j_yarn_cluster.properties --conf
spark.files='file_1' --conf spark.jars='jar_1' --conf spark.yarn.isPython=true --conf spark.yarn.submit.waitAppCompletion=false",
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
   }
 
   @Test
@@ -187,6 +187,6 @@ public class SparkInterpreterLauncherTest {
     assertTrue(interpreterProcess.getEnv().size() >= 3);
     assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME"));
     assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
-    assertEquals(" --master yarn --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1'
--conf spark.jars='jar_1' --conf spark.submit.deployMode='cluster' --conf spark.yarn.isPython=true
--proxy-user user1", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
+    assertEquals(" --master yarn --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1'
--conf spark.jars='jar_1' --conf spark.submit.deployMode='cluster' --conf spark.yarn.isPython=true
--conf spark.yarn.submit.waitAppCompletion=false --proxy-user user1", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
   }
 }


Mime
View raw message