Repository: spark Updated Branches: refs/heads/branch-2.0 8f0ed2891 -> 3c6f686f9
[SPARK-15067][YARN] YARN executors are launched with fixed perm gen size ## What changes were proposed in this pull request? Look for MaxPermSize arguments anywhere in an arg, to account for quoted args. See JIRA for discussion. ## How was this patch tested? Jenkins tests Author: Sean Owen <so...@cloudera.com> Closes #12985 from srowen/SPARK-15067. (cherry picked from commit 6747171eb19dec57c4076cab32580e42ffeb4f51) Signed-off-by: Andrew Or <and...@databricks.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3c6f686f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3c6f686f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3c6f686f Branch: refs/heads/branch-2.0 Commit: 3c6f686f966f594347f45e676f162da4f797786d Parents: 8f0ed28 Author: Sean Owen <so...@cloudera.com> Authored: Mon May 9 11:10:36 2016 -0700 Committer: Andrew Or <and...@databricks.com> Committed: Mon May 9 11:10:47 2016 -0700 ---------------------------------------------------------------------- .../spark/launcher/CommandBuilderUtils.java | 2 +- .../launcher/CommandBuilderUtilsSuite.java | 40 +++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/3c6f686f/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java ---------------------------------------------------------------------- diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java index 91586aa..62a2200 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java +++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java @@ -325,7 +325,7 @@ class CommandBuilderUtils { return; } for (String arg : cmd) { - if (arg.startsWith("-XX:MaxPermSize=")) { + if (arg.contains("-XX:MaxPermSize=")) { return; } } http://git-wip-us.apache.org/repos/asf/spark/blob/3c6f686f/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java ---------------------------------------------------------------------- diff --git a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java index 4fafc43..caeeea5 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java @@ -99,12 +99,48 @@ public class CommandBuilderUtilsSuite { assertEquals(10, javaMajorVersion("10")); } - private void testOpt(String opts, List<String> expected) { + @Test + public void testAddPermGenSizeOpt() { + List<String> cmd = new ArrayList<>(); + + if (javaMajorVersion(System.getProperty("java.version")) > 7) { + // Does nothing in Java 8 + addPermGenSizeOpt(cmd); + assertEquals(0, cmd.size()); + cmd.clear(); + + } else { + addPermGenSizeOpt(cmd); + assertEquals(1, cmd.size()); + assertTrue(cmd.get(0).startsWith("-XX:MaxPermSize=")); + cmd.clear(); + + cmd.add("foo"); + addPermGenSizeOpt(cmd); + assertEquals(2, cmd.size()); + assertTrue(cmd.get(1).startsWith("-XX:MaxPermSize=")); + cmd.clear(); + + cmd.add("-XX:MaxPermSize=512m"); + addPermGenSizeOpt(cmd); + assertEquals(1, cmd.size()); + assertEquals("-XX:MaxPermSize=512m", cmd.get(0)); + cmd.clear(); + + cmd.add("'-XX:MaxPermSize=512m'"); + addPermGenSizeOpt(cmd); + assertEquals(1, cmd.size()); + assertEquals("'-XX:MaxPermSize=512m'", cmd.get(0)); + cmd.clear(); + } + } + + private static void testOpt(String opts, List<String> expected) { assertEquals(String.format("test string failed to parse: [[ %s ]]", opts), expected, parseOptionString(opts)); } - private void testInvalidOpt(String opts) { + private static void testInvalidOpt(String opts) { try { parseOptionString(opts); fail("Expected exception for invalid option string."); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
