spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From andrewo...@apache.org
Subject spark git commit: [SPARK-15067][YARN] YARN executors are launched with fixed perm gen size
Date Mon, 09 May 2016 18:10:54 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-2.0 8f0ed2891 -> 3c6f686f9


[SPARK-15067][YARN] YARN executors are launched with fixed perm gen size

## What changes were proposed in this pull request?

Look for MaxPermSize arguments anywhere in an arg, to account for quoted args. See JIRA for
discussion.

## How was this patch tested?

Jenkins tests

Author: Sean Owen <sowen@cloudera.com>

Closes #12985 from srowen/SPARK-15067.

(cherry picked from commit 6747171eb19dec57c4076cab32580e42ffeb4f51)
Signed-off-by: Andrew Or <andrew@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3c6f686f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3c6f686f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3c6f686f

Branch: refs/heads/branch-2.0
Commit: 3c6f686f966f594347f45e676f162da4f797786d
Parents: 8f0ed28
Author: Sean Owen <sowen@cloudera.com>
Authored: Mon May 9 11:10:36 2016 -0700
Committer: Andrew Or <andrew@databricks.com>
Committed: Mon May 9 11:10:47 2016 -0700

----------------------------------------------------------------------
 .../spark/launcher/CommandBuilderUtils.java     |  2 +-
 .../launcher/CommandBuilderUtilsSuite.java      | 40 +++++++++++++++++++-
 2 files changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/3c6f686f/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
index 91586aa..62a2200 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
@@ -325,7 +325,7 @@ class CommandBuilderUtils {
       return;
     }
     for (String arg : cmd) {
-      if (arg.startsWith("-XX:MaxPermSize=")) {
+      if (arg.contains("-XX:MaxPermSize=")) {
         return;
       }
     }

http://git-wip-us.apache.org/repos/asf/spark/blob/3c6f686f/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
----------------------------------------------------------------------
diff --git a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
index 4fafc43..caeeea5 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
@@ -99,12 +99,48 @@ public class CommandBuilderUtilsSuite {
     assertEquals(10, javaMajorVersion("10"));
   }
 
-  private void testOpt(String opts, List<String> expected) {
+  @Test
+  public void testAddPermGenSizeOpt() {
+    List<String> cmd = new ArrayList<>();
+
+    if (javaMajorVersion(System.getProperty("java.version")) > 7) {
+      // Does nothing in Java 8
+      addPermGenSizeOpt(cmd);
+      assertEquals(0, cmd.size());
+      cmd.clear();
+
+    } else {
+      addPermGenSizeOpt(cmd);
+      assertEquals(1, cmd.size());
+      assertTrue(cmd.get(0).startsWith("-XX:MaxPermSize="));
+      cmd.clear();
+
+      cmd.add("foo");
+      addPermGenSizeOpt(cmd);
+      assertEquals(2, cmd.size());
+      assertTrue(cmd.get(1).startsWith("-XX:MaxPermSize="));
+      cmd.clear();
+
+      cmd.add("-XX:MaxPermSize=512m");
+      addPermGenSizeOpt(cmd);
+      assertEquals(1, cmd.size());
+      assertEquals("-XX:MaxPermSize=512m", cmd.get(0));
+      cmd.clear();
+
+      cmd.add("'-XX:MaxPermSize=512m'");
+      addPermGenSizeOpt(cmd);
+      assertEquals(1, cmd.size());
+      assertEquals("'-XX:MaxPermSize=512m'", cmd.get(0));
+      cmd.clear();
+    }
+  }
+
+  private static void testOpt(String opts, List<String> expected) {
     assertEquals(String.format("test string failed to parse: [[ %s ]]", opts),
         expected, parseOptionString(opts));
   }
 
-  private void testInvalidOpt(String opts) {
+  private static void testInvalidOpt(String opts) {
     try {
       parseOptionString(opts);
       fail("Expected exception for invalid option string.");


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message