Repository: spark Updated Branches: refs/heads/branch-2.0 396be560d -> bcf0c51b6
[SPARK-15665][CORE] spark-submit --kill and --status are not working ## What changes were proposed in this pull request? --kill and --status were not considered while handling in OptionParser and due to that it was failing. Now handling the --kill and --status options as part of OptionParser.handle. ## How was this patch tested? Added a test org.apache.spark.launcher.SparkSubmitCommandBuilderSuite.testCliKillAndStatus() and also I have verified these manually by running --kill and --status commands. Author: Devaraj K <deva...@apache.org> Closes #13407 from devaraj-kavali/SPARK-15665. (cherry picked from commit efd3b11a47ec553f68ab6aa714167f3aee2ecf56) Signed-off-by: Marcelo Vanzin <van...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bcf0c51b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bcf0c51b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bcf0c51b Branch: refs/heads/branch-2.0 Commit: bcf0c51b6852a237a5b7dbe0dcb6904431668074 Parents: 396be56 Author: Devaraj K <deva...@apache.org> Authored: Fri Jun 3 13:03:28 2016 -0700 Committer: Marcelo Vanzin <van...@cloudera.com> Committed: Fri Jun 3 13:03:38 2016 -0700 ---------------------------------------------------------------------- .../launcher/SparkSubmitCommandBuilder.java | 26 +++++++++++--------- .../SparkSubmitCommandBuilderSuite.java | 14 +++++++++++ 2 files changed, 29 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/bcf0c51b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java ---------------------------------------------------------------------- diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 824500d..b3ccc48 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -89,7 +89,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { } final List<String> sparkArgs; - private final boolean printInfo; + private final boolean isAppResourceReq; private final boolean isExample; /** @@ -101,7 +101,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { SparkSubmitCommandBuilder() { this.sparkArgs = new ArrayList<>(); - this.printInfo = false; + this.isAppResourceReq = true; this.isExample = false; } @@ -133,19 +133,19 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { this.isExample = isExample; OptionParser parser = new OptionParser(); parser.parse(submitArgs); - this.printInfo = parser.infoRequested; + this.isAppResourceReq = parser.isAppResourceReq; } else { this.isExample = isExample; - this.printInfo = true; + this.isAppResourceReq = false; } } @Override public List<String> buildCommand(Map<String, String> env) throws IOException, IllegalArgumentException { - if (PYSPARK_SHELL.equals(appResource) && !printInfo) { + if (PYSPARK_SHELL.equals(appResource) && isAppResourceReq) { return buildPySparkShellCommand(env); - } else if (SPARKR_SHELL.equals(appResource) && !printInfo) { + } else if (SPARKR_SHELL.equals(appResource) && isAppResourceReq) { return buildSparkRCommand(env); } else { return buildSparkSubmitCommand(env); @@ -156,7 +156,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { List<String> args = new ArrayList<>(); SparkSubmitOptionParser parser = new SparkSubmitOptionParser(); - if (!allowsMixedArguments && !printInfo) { + if (!allowsMixedArguments && isAppResourceReq) { checkArgument(appResource != null, "Missing application resource."); } @@ -208,7 +208,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { args.add(join(",", pyFiles)); } - if (!printInfo) { + if (isAppResourceReq) { checkArgument(!isExample || mainClass != null, "Missing example class name."); } if (mainClass != null) { @@ -388,7 +388,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { private class OptionParser extends SparkSubmitOptionParser { - boolean infoRequested = false; + boolean isAppResourceReq = true; @Override protected boolean handle(String opt, String value) { @@ -420,11 +420,15 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { allowsMixedArguments = true; appResource = specialClasses.get(value); } + } else if (opt.equals(KILL_SUBMISSION) || opt.equals(STATUS)) { + isAppResourceReq = false; + sparkArgs.add(opt); + sparkArgs.add(value); } else if (opt.equals(HELP) || opt.equals(USAGE_ERROR)) { - infoRequested = true; + isAppResourceReq = false; sparkArgs.add(opt); } else if (opt.equals(VERSION)) { - infoRequested = true; + isAppResourceReq = false; sparkArgs.add(opt); } else { sparkArgs.add(opt); http://git-wip-us.apache.org/repos/asf/spark/blob/bcf0c51b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java ---------------------------------------------------------------------- diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index d451651..16e5a22 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -73,6 +73,12 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite { } @Test + public void testCliKillAndStatus() throws Exception { + testCLIOpts(parser.STATUS); + testCLIOpts(parser.KILL_SUBMISSION); + } + + @Test public void testCliParser() throws Exception { List<String> sparkSubmitArgs = Arrays.asList( parser.MASTER, @@ -326,4 +332,12 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite { return newCommandBuilder(args).buildCommand(env); } + private void testCLIOpts(String opt) throws Exception { + List<String> helpArgs = Arrays.asList(opt, "driver-20160531171222-0000"); + Map<String, String> env = new HashMap<>(); + List<String> cmd = buildCommand(helpArgs, env); + assertTrue(opt + " should be contained in the final cmd.", + cmd.contains(opt)); + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
