Repository: spark Updated Branches: refs/heads/branch-1.1 457ef5955 -> eb62094e8
[SPARK-3877][YARN] Throw an exception when application is not successful so that the exit code wil be set to 1 (for branch-1.1) This is a patch to fix SPARK-3877 in branch-1.1. See also #2732 Author: zsxwing <zsxw...@gmail.com> Closes #2748 from zsxwing/SPARK-3877-branch-1.1 and squashes the following commits: 3701984 [zsxwing] Remove System.exit from Client.scala 8681881 [zsxwing] [SPARK-3877] Throw an exception when application is not successful so that the exit code wil be set to 1 Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eb62094e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eb62094e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eb62094e Branch: refs/heads/branch-1.1 Commit: eb62094e820992355e4cf8ba0e881417f8c6bbfa Parents: 457ef59 Author: zsxwing <zsxw...@gmail.com> Authored: Wed Oct 22 15:08:28 2014 -0700 Committer: Andrew Or <andrewo...@gmail.com> Committed: Wed Oct 22 15:08:28 2014 -0700 ---------------------------------------------------------------------- .../org/apache/spark/deploy/yarn/Client.scala | 27 ++++++++------------ .../org/apache/spark/deploy/yarn/Client.scala | 27 ++++++++------------ 2 files changed, 22 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/eb62094e/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala ---------------------------------------------------------------------- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 3607eed..c51ce62 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{Apps, Records} -import org.apache.spark.{Logging, SparkConf} +import org.apache.spark.{Logging, SparkConf, SparkException} /** * Version of [[org.apache.spark.deploy.yarn.ClientBase]] tailored to YARN's alpha API. @@ -84,7 +84,9 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa def run() { val appId = runApp() - monitorApplication(appId) + if (!monitorApplication(appId)) { + throw new SparkException("Application is not successful") + } } def logClusterResourceDetails() { @@ -138,10 +140,12 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa ) val state = report.getYarnApplicationState() - if (state == YarnApplicationState.FINISHED || - state == YarnApplicationState.FAILED || + if (state == YarnApplicationState.FINISHED) { + return report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED + } + if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) { - return true + return false } } true @@ -162,16 +166,7 @@ object Client { val sparkConf = new SparkConf - try { - val args = new ClientArguments(argStrings, sparkConf) - new Client(args, sparkConf).run() - } catch { - case e: Exception => { - Console.err.println(e.getMessage) - System.exit(1) - } - } - - System.exit(0) + val args = new ClientArguments(argStrings, sparkConf) + new Client(args, sparkConf).run() } } http://git-wip-us.apache.org/repos/asf/spark/blob/eb62094e/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala ---------------------------------------------------------------------- diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 0b55282..da05f7d 100644 --- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.Records -import org.apache.spark.{Logging, SparkConf} +import org.apache.spark.{Logging, SparkConf, SparkException} /** @@ -94,7 +94,9 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa def run() { val appId = runApp() - monitorApplication(appId) + if (!monitorApplication(appId)) { + throw new SparkException("Application is not successful") + } } def logClusterResourceDetails() { @@ -144,10 +146,12 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa ) val state = report.getYarnApplicationState() - if (state == YarnApplicationState.FINISHED || - state == YarnApplicationState.FAILED || + if (state == YarnApplicationState.FINISHED) { + return report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED + } + if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) { - return true + return false } } true @@ -168,17 +172,8 @@ object Client { System.setProperty("SPARK_YARN_MODE", "true") val sparkConf = new SparkConf() - try { - val args = new ClientArguments(argStrings, sparkConf) - new Client(args, sparkConf).run() - } catch { - case e: Exception => { - Console.err.println(e.getMessage) - System.exit(1) - } - } - - System.exit(0) + val args = new ClientArguments(argStrings, sparkConf) + new Client(args, sparkConf).run() } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
