This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new fdcf85ee263 [SPARK-42336][CORE] Use `getOrElse()` instead of `contains()` in ResourceAllocator fdcf85ee263 is described below commit fdcf85ee263f406597aa99a2f9a187bb9cd1ef76 Author: smallzhongfeng <zhongjingxi...@didiglobal.com> AuthorDate: Sun Feb 5 20:54:56 2023 -0600 [SPARK-42336][CORE] Use `getOrElse()` instead of `contains()` in ResourceAllocator ### What changes were proposed in this pull request? Use `.getOrElse(address, throw new SparkException(...))` instead of one `contains` in `addressAvailabilityMap`‘s `require` and `release`. ### Why are the changes needed? Improving performance. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Origin uts. Closes #39879 from smallzhongfeng/SPARK-42336. Authored-by: smallzhongfeng <zhongjingxi...@didiglobal.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- .../org/apache/spark/resource/ResourceAllocator.scala | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala b/core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala index 10cf0402d5f..7b97d970428 100644 --- a/core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala +++ b/core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala @@ -38,8 +38,6 @@ private[spark] trait ResourceAllocator { * For task resources ([[org.apache.spark.scheduler.ExecutorResourceInfo]]), this value * can be a multiple, such that each address can be allocated up to [[slotsPerAddress]] * times. - * - * TODO Use [[org.apache.spark.util.collection.OpenHashMap]] instead to gain better performance. */ private lazy val addressAvailabilityMap = { mutable.HashMap(resourceAddresses.map(_ -> slotsPerAddress): _*) @@ -76,11 +74,9 @@ private[spark] trait ResourceAllocator { */ def acquire(addrs: Seq[String]): Unit = { addrs.foreach { address => - if (!addressAvailabilityMap.contains(address)) { + val isAvailable = addressAvailabilityMap.getOrElse(address, throw new SparkException(s"Try to acquire an address that doesn't exist. $resourceName " + - s"address $address doesn't exist.") - } - val isAvailable = addressAvailabilityMap(address) + s"address $address doesn't exist.")) if (isAvailable > 0) { addressAvailabilityMap(address) -= 1 } else { @@ -97,11 +93,9 @@ private[spark] trait ResourceAllocator { */ def release(addrs: Seq[String]): Unit = { addrs.foreach { address => - if (!addressAvailabilityMap.contains(address)) { + val isAvailable = addressAvailabilityMap.getOrElse(address, throw new SparkException(s"Try to release an address that doesn't exist. $resourceName " + - s"address $address doesn't exist.") - } - val isAvailable = addressAvailabilityMap(address) + s"address $address doesn't exist.")) if (isAvailable < slotsPerAddress) { addressAvailabilityMap(address) += 1 } else { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org