This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c5b6ec734bd0 [SPARK-47441][YARN] Do not add log link for unmanaged AM
in Spark UI
c5b6ec734bd0 is described below
commit c5b6ec734bd0c47551b59f9de13c6323b80974b2
Author: Yuming Wang <yumw...@ebay.com>
AuthorDate: Fri May 10 08:22:03 2024 -0700
[SPARK-47441][YARN] Do not add log link for unmanaged AM in Spark UI
### What changes were proposed in this pull request?
This PR makes it do not add log link for unmanaged AM in Spark UI.
### Why are the changes needed?
Avoid start driver error messages:
```
24/03/18 04:58:25,022 ERROR [spark-listener-group-appStatus]
scheduler.AsyncEventQueue:97 : Listener AppStatusListener threw an exception
java.lang.NumberFormatException: For input string: "null"
at
java.lang.NumberFormatException.forInputString(NumberFormatException.java:67)
~[?:?]
at java.lang.Integer.parseInt(Integer.java:668) ~[?:?]
at java.lang.Integer.parseInt(Integer.java:786) ~[?:?]
at
scala.collection.immutable.StringLike.toInt(StringLike.scala:310)
~[scala-library-2.12.18.jar:?]
at
scala.collection.immutable.StringLike.toInt$(StringLike.scala:310)
~[scala-library-2.12.18.jar:?]
at scala.collection.immutable.StringOps.toInt(StringOps.scala:33)
~[scala-library-2.12.18.jar:?]
at org.apache.spark.util.Utils$.parseHostPort(Utils.scala:1105)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.status.ProcessSummaryWrapper.<init>(storeTypes.scala:609)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.status.LiveMiscellaneousProcess.doUpdate(LiveEntity.scala:1045)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at org.apache.spark.status.LiveEntity.write(LiveEntity.scala:50)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.status.AppStatusListener.update(AppStatusListener.scala:1233)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.status.AppStatusListener.onMiscellaneousProcessAdded(AppStatusListener.scala:1445)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.status.AppStatusListener.onOtherEvent(AppStatusListener.scala:113)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.scheduler.SparkListenerBus.doPostEvent(SparkListenerBus.scala:100)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.scheduler.SparkListenerBus.doPostEvent$(SparkListenerBus.scala:28)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.util.ListenerBus.postToAll(ListenerBus.scala:117)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.util.ListenerBus.postToAll$(ListenerBus.scala:101)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.scheduler.AsyncEventQueue.super$postToAll(AsyncEventQueue.scala:105)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.scheduler.AsyncEventQueue.$anonfun$dispatch$1(AsyncEventQueue.scala:105)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
~[scala-library-2.12.18.jar:?]
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
~[scala-library-2.12.18.jar:?]
at
org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$dispatch(AsyncEventQueue.scala:100)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.scheduler.AsyncEventQueue$$anon$2.$anonfun$run$1(AsyncEventQueue.scala:96)
~[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1356)
[spark-core_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.scheduler.AsyncEventQueue$$anon$2.run(AsyncEventQueue.scala:96)
[spark-core_2.12-3.5.1.jar:3.5.1]
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manual testing:
```shell
bin/spark-sql --master yarn --conf spark.yarn.unmanagedAM.enabled=true
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #45565 from wangyum/SPARK-47441.
Authored-by: Yuming Wang <yumw...@ebay.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index dffb05e196d7..8f20f6602ec5 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -793,9 +793,9 @@ private[spark] class ApplicationMaster(
override def onStart(): Unit = {
driver.send(RegisterClusterManager(self))
- // if deployment mode for yarn Application is client
+ // if deployment mode for yarn Application is managed client
// then send the AM Log Info to spark driver
- if (!isClusterMode) {
+ if (!isClusterMode && !sparkConf.get(YARN_UNMANAGED_AM)) {
val hostPort = YarnContainerInfoHelper.getNodeManagerHttpAddress(None)
val yarnAMID = "yarn-am"
val info = new MiscellaneousProcessDetails(hostPort,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
