This is an automated email from the ASF dual-hosted git repository.
holden pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a101c48dd965 [SPARK-44953][CORE] Log a warning when shuffle tracking
is enabled along side another DA supported mechanism
a101c48dd965 is described below
commit a101c48dd9650d2bca2047b91f9e2a3ba90f142d
Author: zwangsheng <binjiey...@apache.org>
AuthorDate: Mon May 13 13:33:34 2024 -0700
[SPARK-44953][CORE] Log a warning when shuffle tracking is enabled along
side another DA supported mechanism
### What changes were proposed in this pull request?
Log a warning when shuffle tracking is enabled along side another DA
supported mechanism
### Why are the changes needed?
Some users enable both shuffle tracking and another mechanism (like
migration) and then are confused when their jobs don't scale down.
https://issues.apache.org/jira/browse/SPARK-44953
### Does this PR introduce _any_ user-facing change?
Yes, user can find the warning log when enabled both shuffle tracking and
another DA supported mechanism(shuffle decommission).
### How was this patch tested?
No
### Was this patch authored or co-authored using generative AI tooling?
NO
Closes #45454 from zwangsheng/SPARK-44953.
Authored-by: zwangsheng <binjiey...@apache.org>
Signed-off-by: Holden Karau <hka...@netflix.com>
---
.../scala/org/apache/spark/ExecutorAllocationManager.scala | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index 3bfa1ae0d4dc..1fe02eec3a07 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -206,11 +206,13 @@ private[spark] class ExecutorAllocationManager(
throw new SparkException(
s"s${DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT.key} must be >
0!")
}
+ val shuffleTrackingEnabled =
conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)
+ val shuffleDecommissionEnabled = decommissionEnabled &&
+ conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED)
if (!conf.get(config.SHUFFLE_SERVICE_ENABLED) && !reliableShuffleStorage) {
- if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)) {
+ if (shuffleTrackingEnabled) {
logInfo("Dynamic allocation is enabled without a shuffle service.")
- } else if (decommissionEnabled &&
- conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED)) {
+ } else if (shuffleDecommissionEnabled) {
logInfo("Shuffle data decommission is enabled without a shuffle
service.")
} else if (!testing) {
throw new SparkException("Dynamic allocation of executors requires one
of the " +
@@ -224,6 +226,12 @@ private[spark] class ExecutorAllocationManager(
}
}
+ if (shuffleTrackingEnabled && (shuffleDecommissionEnabled ||
reliableShuffleStorage)) {
+ logWarning("You are enabling both shuffle tracking and other DA
supported mechanism, " +
+ "which will cause idle executors not to be released in a timely, " +
+ "please check the configurations.")
+ }
+
if (executorAllocationRatio > 1.0 || executorAllocationRatio <= 0.0) {
throw new SparkException(
s"${DYN_ALLOCATION_EXECUTOR_ALLOCATION_RATIO.key} must be > 0 and <=
1.0")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
