This is an automated email from the ASF dual-hosted git repository. sunchao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 87ccfc24023 [SPARK-43196][YARN] Replace reflection w/ direct calling for `ContainerLaunchContext#setTokensConf` 87ccfc24023 is described below commit 87ccfc2402394d168a90c8df494482c7a1d6a552 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Wed Apr 19 21:09:18 2023 -0700 [SPARK-43196][YARN] Replace reflection w/ direct calling for `ContainerLaunchContext#setTokensConf` ### What changes were proposed in this pull request? This pr replace reflection with direct calling for `ContainerLaunchContext#setTokensConf` in `o.a.s.deploy.yarn.Client#setTokenConf` function. ### Why are the changes needed? SPARK-37205 uses reflection to call `ContainerLaunchContext#setTokensConf` for compatibility with Hadoop 2.7, since SPARK-42452 removed support for Hadoop2, we can call it directly instead of using reflection. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GA. Closes #40855 from LuciferYang/SPARK-43196. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Chao Sun <sunc...@apple.com> --- .../main/scala/org/apache/spark/deploy/yarn/Client.scala | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 06d8b4e2509..7010067e1ae 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -77,10 +77,6 @@ private[spark] class Client( private val isClusterMode = sparkConf.get(SUBMIT_DEPLOY_MODE) == "cluster" - // ContainerLaunchContext.setTokensConf is only available in Hadoop 2.9+ and 3.x, so here we use - // reflection to avoid compilation for Hadoop 2.7 profile. - private val SET_TOKENS_CONF_METHOD = "setTokensConf" - private val isClientUnmanagedAMEnabled = sparkConf.get(YARN_UNMANAGED_AM) && !isClusterMode private var appMaster: ApplicationMaster = _ private var stagingDirPath: Path = _ @@ -382,16 +378,7 @@ private[spark] class Client( } copy.write(dob); - // since this method was added in Hadoop 2.9 and 3.0, we use reflection here to avoid - // compilation error for Hadoop 2.7 profile. - val setTokensConfMethod = try { - amContainer.getClass.getMethod(SET_TOKENS_CONF_METHOD, classOf[ByteBuffer]) - } catch { - case _: NoSuchMethodException => - throw new SparkException(s"Cannot find setTokensConf method in ${amContainer.getClass}." + - s" Please check YARN version and make sure it is 2.9+ or 3.x") - } - setTokensConfMethod.invoke(amContainer, ByteBuffer.wrap(dob.getData)) + amContainer.setTokensConf(ByteBuffer.wrap(dob.getData)) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org