spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sro...@apache.org
Subject spark git commit: [SPARK-6372] [core] Propagate --conf to child processes.
Date Wed, 18 Mar 2015 13:07:11 GMT
Repository: spark
Updated Branches:
  refs/heads/master 357900311 -> 6205a255a


[SPARK-6372] [core] Propagate --conf to child processes.

And add unit test.

Author: Marcelo Vanzin <vanzin@cloudera.com>

Closes #5057 from vanzin/SPARK-6372 and squashes the following commits:

b33728b [Marcelo Vanzin] [SPARK-6372] [core] Propagate --conf to child processes.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6205a255
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6205a255
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6205a255

Branch: refs/heads/master
Commit: 6205a255aea0652dddadf953771f5405065b5bec
Parents: 3579003
Author: Marcelo Vanzin <vanzin@cloudera.com>
Authored: Wed Mar 18 09:06:57 2015 -0400
Committer: Sean Owen <sowen@cloudera.com>
Committed: Wed Mar 18 09:06:57 2015 -0400

----------------------------------------------------------------------
 .../apache/spark/launcher/SparkSubmitCommandBuilder.java  | 10 +---------
 .../spark/launcher/SparkSubmitCommandBuilderSuite.java    |  4 ++++
 2 files changed, 5 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/6205a255/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
index 6ffdff6..91dcf70 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
@@ -253,12 +253,6 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
 
   private class OptionParser extends SparkSubmitOptionParser {
 
-    private final List<String> driverJvmKeys = Arrays.asList(
-      SparkLauncher.DRIVER_EXTRA_CLASSPATH,
-      SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
-      SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH,
-      SparkLauncher.DRIVER_MEMORY);
-
     @Override
     protected boolean handle(String opt, String value) {
       if (opt.equals(MASTER)) {
@@ -278,9 +272,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
       } else if (opt.equals(CONF)) {
         String[] setConf = value.split("=", 2);
         checkArgument(setConf.length == 2, "Invalid argument to %s: %s", CONF, value);
-        if (driverJvmKeys.contains(setConf[0])) {
-          conf.put(setConf[0], setConf[1]);
-        }
+        conf.put(setConf[0], setConf[1]);
       } else if (opt.equals(CLASS)) {
         // The special classes require some special command line handling, since they allow
         // mixing spark-submit arguments with arguments that should be propagated to the
shell

http://git-wip-us.apache.org/repos/asf/spark/blob/6205a255/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
----------------------------------------------------------------------
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
index 815edc4..626116a 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
@@ -68,6 +68,8 @@ public class SparkSubmitCommandBuilderSuite {
       parser.DRIVER_JAVA_OPTIONS,
       "extraJavaOpt",
       parser.CONF,
+      "spark.randomOption=foo",
+      parser.CONF,
       SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH + "=/driverLibPath");
     Map<String, String> env = new HashMap<String, String>();
     List<String> cmd = buildCommand(sparkSubmitArgs, env);
@@ -77,6 +79,8 @@ public class SparkSubmitCommandBuilderSuite {
     assertTrue(findInStringList(findArgValue(cmd, "-cp"), File.pathSeparator, "/driverCp"));
     assertTrue("Driver -Xms should be configured.", cmd.contains("-Xms42g"));
     assertTrue("Driver -Xmx should be configured.", cmd.contains("-Xmx42g"));
+    assertTrue("Command should contain user-defined conf.",
+      Collections.indexOfSubList(cmd, Arrays.asList(parser.CONF, "spark.randomOption=foo"))
> 0);
   }
 
   @Test


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message