Repository: spark
Updated Branches:
refs/heads/master 3528c08be -> 8a7872dc2
[SPARK-25636][CORE] spark-submit cuts off the failure reason when there is an
error connecting to master
## What changes were proposed in this pull request?
Cause of the error is wrapped with SparkException, now finding the cause from
the wrapped exception and throwing the cause instead of the wrapped exception.
## How was this patch tested?
Verified it manually by checking the cause of the error, it gives the error as
shown below.
### Without the PR change
```
[apache-spark]$ ./bin/spark-submit --verbose --master spark://******
....
Error: Exception thrown in awaitResult:
Run with --help for usage help or --verbose for debug output
```
### With the PR change
```
[apache-spark]$ ./bin/spark-submit --verbose --master spark://******
....
Exception in thread "main" org.apache.spark.SparkException: Exception thrown in
awaitResult:
at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:226)
at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
....
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.io.IOException: Failed to connect to devaraj-pc1/10.3.66.65:7077
at
org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:245)
....
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException:
Connection refused: devaraj-pc1/10.3.66.65:7077
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
....
at
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
... 1 more
Caused by: java.net.ConnectException: Connection refused
... 11 more
```
Closes #22623 from devaraj-kavali/SPARK-25636.
Authored-by: Devaraj K <deva...@apache.org>
Signed-off-by: Marcelo Vanzin <van...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8a7872dc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8a7872dc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8a7872dc
Branch: refs/heads/master
Commit: 8a7872dc254710f9b29fdfdb2915a949ef606871
Parents: 3528c08
Author: Devaraj K <deva...@apache.org>
Authored: Wed Oct 10 09:24:36 2018 -0700
Committer: Marcelo Vanzin <van...@cloudera.com>
Committed: Wed Oct 10 09:24:36 2018 -0700
----------------------------------------------------------------------
.../org/apache/spark/deploy/SparkSubmit.scala | 2 --
.../org/apache/spark/deploy/SparkSubmitSuite.scala | 17 +++++++++++------
2 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8a7872dc/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index d5f2865..61b379f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -927,8 +927,6 @@ object SparkSubmit extends CommandLineUtils with Logging {
} catch {
case e: SparkUserAppException =>
exitFn(e.exitCode)
- case e: SparkException =>
- printErrorAndExit(e.getMessage())
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/8a7872dc/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 9eae360..652c36f 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -74,20 +74,25 @@ trait TestPrematureExit {
@volatile var exitedCleanly = false
mainObject.exitFn = (_) => exitedCleanly = true
+ @volatile var exception: Exception = null
val thread = new Thread {
override def run() = try {
mainObject.main(input)
} catch {
- // If exceptions occur after the "exit" has happened, fine to ignore
them.
- // These represent code paths not reachable during normal execution.
- case e: Exception => if (!exitedCleanly) throw e
+ // Capture the exception to check whether the exception contains
searchString or not
+ case e: Exception => exception = e
}
}
thread.start()
thread.join()
- val joined = printStream.lineBuffer.mkString("\n")
- if (!joined.contains(searchString)) {
- fail(s"Search string '$searchString' not found in $joined")
+ if (exitedCleanly) {
+ val joined = printStream.lineBuffer.mkString("\n")
+ assert(joined.contains(searchString))
+ } else {
+ assert(exception != null)
+ if (!exception.getMessage.contains(searchString)) {
+ throw exception
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
