This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new a566099133ff [SPARK-45756][CORE] Support `spark.master.useAppNameAsAppId.enabled` a566099133ff is described below commit a566099133ff38cd1b2cd2fe64879bf0ba75fa9b Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Thu Nov 9 18:34:43 2023 -0800 [SPARK-45756][CORE] Support `spark.master.useAppNameAsAppId.enabled` ### What changes were proposed in this pull request? This PR aims to support `spark.master.useAppNameAsAppId.enabled` as an experimental feature in Spark Standalone cluster. ### Why are the changes needed? This allows the users to control the appID completely. <img width="359" alt="Screenshot 2023-11-09 at 5 33 45 PM" src="https://github.com/apache/spark/assets/9700541/ad2b89ce-9d7d-4144-bd52-f29b94051103"> ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual tests with the following procedure. ``` $ SPARK_MASTER_OPTS="-Dspark.master.useAppNameAsAppId.enabled=true" sbin/start-master.sh $ bin/spark-shell --master spark://max.local:7077 ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43743 from dongjoon-hyun/SPARK-45756. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../scala/org/apache/spark/deploy/master/Master.scala | 7 ++++++- .../scala/org/apache/spark/internal/config/package.scala | 8 ++++++++ .../org/apache/spark/deploy/master/MasterSuite.scala | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index dbb647252c5f..b3fbec1830e4 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -120,6 +120,7 @@ private[deploy] class Master( private val defaultCores = conf.get(DEFAULT_CORES) val reverseProxy = conf.get(UI_REVERSE_PROXY) val historyServerUrl = conf.get(MASTER_UI_HISTORY_SERVER_URL) + val useAppNameAsAppId = conf.get(MASTER_USE_APP_NAME_AS_APP_ID) // Alternative application submission gateway that is stable across Spark versions private val restServerEnabled = conf.get(MASTER_REST_SERVER_ENABLED) @@ -1041,7 +1042,11 @@ private[deploy] class Master( ApplicationInfo = { val now = System.currentTimeMillis() val date = new Date(now) - val appId = newApplicationId(date) + val appId = if (useAppNameAsAppId) { + desc.name.toLowerCase().replaceAll("\\s+", "") + } else { + newApplicationId(date) + } new ApplicationInfo(now, appId, desc, date, driver, defaultCores) } diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index bbadf91fc41c..b2bf30863a91 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -1846,6 +1846,14 @@ package object config { .stringConf .createOptional + private[spark] val MASTER_USE_APP_NAME_AS_APP_ID = + ConfigBuilder("spark.master.useAppNameAsAppId.enabled") + .internal() + .doc("(Experimental) If true, Spark master uses the user-provided appName for appId.") + .version("4.0.0") + .booleanConf + .createWithDefault(false) + private[spark] val IO_COMPRESSION_SNAPPY_BLOCKSIZE = ConfigBuilder("spark.io.compression.snappy.blockSize") .doc("Block size in bytes used in Snappy compression, in the case when " + diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala index 4f8457f930e4..2e54673649c7 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala @@ -804,6 +804,7 @@ class MasterSuite extends SparkFunSuite private val _state = PrivateMethod[RecoveryState.Value](Symbol("state")) private val _newDriverId = PrivateMethod[String](Symbol("newDriverId")) private val _newApplicationId = PrivateMethod[String](Symbol("newApplicationId")) + private val _createApplication = PrivateMethod[ApplicationInfo](Symbol("createApplication")) private val workerInfo = makeWorkerInfo(4096, 10) private val workerInfos = Array(workerInfo, workerInfo, workerInfo) @@ -1275,6 +1276,21 @@ class MasterSuite extends SparkFunSuite assert(master.invokePrivate(_newApplicationId(submitDate)) === s"${i % 1000}") } } + + test("SPARK-45756: Use appName for appId") { + val conf = new SparkConf() + .set(MASTER_USE_APP_NAME_AS_APP_ID, true) + val master = makeMaster(conf) + val desc = new ApplicationDescription( + name = " spark - 45756 ", + maxCores = None, + command = null, + appUiUrl = "", + defaultProfile = DeployTestUtils.defaultResourceProfile, + eventLogDir = None, + eventLogCodec = None) + assert(master.invokePrivate(_createApplication(desc, null)).id === "spark-45756") + } } private class FakeRecoveryModeFactory(conf: SparkConf, ser: serializer.Serializer) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org