tgravescs commented on a change in pull request #24615: [SPARK-27488][CORE]
Driver interface to support GPU resources
URL: https://github.com/apache/spark/pull/24615#discussion_r286181293
##########
File path: core/src/main/scala/org/apache/spark/SparkConf.scala
##########
@@ -587,6 +594,30 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable
with Logging with Seria
require(executorTimeoutThresholdMs > executorHeartbeatIntervalMs, "The
value of " +
s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be no less than
the value of " +
s"spark.executor.heartbeatInterval=${executorHeartbeatIntervalMs}ms.")
+
+ // Make sure the executor resources were specified and are large enough if
+ // any task resources were specified.
+ val taskResourcesAndCount =
+ getAllWithPrefixAndSuffix(SPARK_TASK_RESOURCE_PREFIX,
SPARK_RESOURCE_COUNT_SUFFIX).toMap
+ val executorResourcesAndCounts =
+ getAllWithPrefixAndSuffix(SPARK_EXECUTOR_RESOURCE_PREFIX,
SPARK_RESOURCE_COUNT_SUFFIX).toMap
+
+ taskResourcesAndCount.foreach { case (rName, taskCount) =>
+ val execCount = executorResourcesAndCounts.get(rName).getOrElse(
+ throw new SparkException(
+ s"The executor resource config: " +
+ s"${SPARK_EXECUTOR_RESOURCE_PREFIX + rName +
SPARK_RESOURCE_COUNT_SUFFIX} " +
+ "needs to be specified since a task requirement config: " +
+ s"${SPARK_TASK_RESOURCE_PREFIX + rName +
SPARK_RESOURCE_COUNT_SUFFIX} was specified")
+ )
+ if (execCount.toLong < taskCount.toLong) {
Review comment:
that should be in the scheduler PR
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
