This is an automated email from the ASF dual-hosted git repository. yangjie01 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new bffffee1bceb [SPARK-47474][CORE] Revert SPARK-47461 and add some comments bffffee1bceb is described below commit bffffee1bcebdad218a4151ad192d4893ff0fed9 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Thu Mar 21 13:58:39 2024 +0800 [SPARK-47474][CORE] Revert SPARK-47461 and add some comments ### What changes were proposed in this pull request? This pr revert the change of SPARK-47461 and add some comments to `ExecutorAllocationManager#totalRunningTasksPerResourceProfile` to clarify that the tests in `ExecutorAllocationManagerSuite` need to call `listener.totalRunningTasksPerResourceProfile` with `synchronized`. ### Why are the changes needed? `ExecutorAllocationManagerSuite` need to call `listener.totalRunningTasksPerResourceProfile` with `synchronized`. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GitHub Actions ### Was this patch authored or co-authored using generative AI tooling? No Closes #45602 from LuciferYang/SPARK-47474. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: yangjie01 <yangji...@baidu.com> --- .../src/main/scala/org/apache/spark/ExecutorAllocationManager.scala | 6 ++++++ .../scala/org/apache/spark/ExecutorAllocationManagerSuite.scala | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala index cdd1aecf4a22..94927caff1d7 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala @@ -320,6 +320,12 @@ private[spark] class ExecutorAllocationManager( } } + // Please do not delete this function, the tests in `ExecutorAllocationManagerSuite` + // need to access `listener.totalRunningTasksPerResourceProfile` with `synchronized`. + private def totalRunningTasksPerResourceProfile(id: Int): Int = synchronized { + listener.totalRunningTasksPerResourceProfile(id) + } + /** * This is called at a fixed interval to regulate the number of pending executor requests * and number of executors running. diff --git a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala index aeb3cf53ff1a..e1da2b6dd9d6 100644 --- a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala @@ -1934,6 +1934,8 @@ private object ExecutorAllocationManagerSuite extends PrivateMethodTester { PrivateMethod[Map[Int, Map[String, Int]]](Symbol("rpIdToHostToLocalTaskCount")) private val _onSpeculativeTaskSubmitted = PrivateMethod[Unit](Symbol("onSpeculativeTaskSubmitted")) + private val _totalRunningTasksPerResourceProfile = + PrivateMethod[Int](Symbol("totalRunningTasksPerResourceProfile")) private val defaultProfile = ResourceProfile.getOrCreateDefaultProfile(new SparkConf) @@ -2041,7 +2043,7 @@ private object ExecutorAllocationManagerSuite extends PrivateMethodTester { } private def totalRunningTasksPerResourceProfile(manager: ExecutorAllocationManager): Int = { - manager.listener.totalRunningTasksPerResourceProfile(defaultProfile.id) + manager invokePrivate _totalRunningTasksPerResourceProfile(defaultProfile.id) } private def hostToLocalTaskCount( --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org