This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 68abf77 [SPARK-27184][CORE] Avoid hardcoded 'spark.jars', 'spark.files', 'spark.submit.pyFiles' and 'spark.submit.deployMode' 68abf77 is described below commit 68abf77b1ad8da7916a9dc5fa8bb350b64479410 Author: hehuiyuan <hehuiyuan@ZBMAC-C02WD3K5H.local> AuthorDate: Sat Mar 23 09:43:00 2019 +0900 [SPARK-27184][CORE] Avoid hardcoded 'spark.jars', 'spark.files', 'spark.submit.pyFiles' and 'spark.submit.deployMode' ## What changes were proposed in this pull request? For [SPARK-27184](https://issues.apache.org/jira/browse/SPARK-27184) In the `org.apache.spark.internal.config`, we define the variables of `FILES` and `JARS`, we can use them instead of "spark.jars" and "spark.files". ```scala private[spark] val JARS = ConfigBuilder("spark.jars") .stringConf .toSequence .createWithDefault(Nil) ``` ```scala private[spark] val FILES = ConfigBuilder("spark.files") .stringConf .toSequence .createWithDefault(Nil) ``` Other : In the `org.apache.spark.internal.config`, we define the variables of `SUBMIT_PYTHON_FILES ` and `SUBMIT_DEPLOY_MODE `, we can use them instead of "spark.submit.pyFiles" and "spark.submit.deployMode". ```scala private[spark] val SUBMIT_PYTHON_FILES = ConfigBuilder("spark.submit.pyFiles") .stringConf .toSequence .createWithDefault(Nil) ``` ```scala private[spark] val SUBMIT_DEPLOY_MODE = ConfigBuilder("spark.submit.deployMode") .stringConf .createWithDefault("client") ``` Closes #24123 from hehuiyuan/hehuiyuan-patch-6. Authored-by: hehuiyuan <hehuiyuan@ZBMAC-C02WD3K5H.local> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- core/src/main/scala/org/apache/spark/SparkContext.scala | 2 +- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 6 +++--- .../main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala | 8 ++++---- .../src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 5cd6c2b..4abb18d 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -394,7 +394,7 @@ class SparkContext(config: SparkConf) extends Logging { _conf.set(EXECUTOR_ID, SparkContext.DRIVER_IDENTIFIER) _jars = Utils.getUserJars(_conf) - _files = _conf.getOption("spark.files").map(_.split(",")).map(_.filter(_.nonEmpty)) + _files = _conf.getOption(FILES.key).map(_.split(",")).map(_.filter(_.nonEmpty)) .toSeq.flatten _eventLogDir = diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index b6673e4..b4d7462 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -542,10 +542,10 @@ private[spark] class SparkSubmit extends Logging { OptionAssigner(args.totalExecutorCores, STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES, confKey = CORES_MAX.key), OptionAssigner(args.files, LOCAL | STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES, - confKey = "spark.files"), - OptionAssigner(args.jars, LOCAL, CLIENT, confKey = "spark.jars"), + confKey = FILES.key), + OptionAssigner(args.jars, LOCAL, CLIENT, confKey = JARS.key), OptionAssigner(args.jars, STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES, - confKey = "spark.jars"), + confKey = JARS.key), OptionAssigner(args.driverMemory, STANDALONE | MESOS | YARN | KUBERNETES, CLUSTER, confKey = DRIVER_MEMORY.key), OptionAssigner(args.driverCores, STANDALONE | MESOS | YARN | KUBERNETES, CLUSTER, diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index f8c5330..e7954d1 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -183,9 +183,9 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S .orElse(sparkProperties.get(config.CORES_MAX.key)) .orNull name = Option(name).orElse(sparkProperties.get("spark.app.name")).orNull - jars = Option(jars).orElse(sparkProperties.get("spark.jars")).orNull - files = Option(files).orElse(sparkProperties.get("spark.files")).orNull - pyFiles = Option(pyFiles).orElse(sparkProperties.get("spark.submit.pyFiles")).orNull + jars = Option(jars).orElse(sparkProperties.get(config.JARS.key)).orNull + files = Option(files).orElse(sparkProperties.get(config.FILES.key)).orNull + pyFiles = Option(pyFiles).orElse(sparkProperties.get(config.SUBMIT_PYTHON_FILES.key)).orNull ivyRepoPath = sparkProperties.get("spark.jars.ivy").orNull ivySettingsPath = sparkProperties.get("spark.jars.ivySettings") packages = Option(packages).orElse(sparkProperties.get("spark.jars.packages")).orNull @@ -194,7 +194,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S repositories = Option(repositories) .orElse(sparkProperties.get("spark.jars.repositories")).orNull deployMode = Option(deployMode) - .orElse(sparkProperties.get("spark.submit.deployMode")) + .orElse(sparkProperties.get(config.SUBMIT_DEPLOY_MODE.key)) .orElse(env.get("DEPLOY_MODE")) .orNull numExecutors = Option(numExecutors) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 4b414f7..3051d65 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -681,7 +681,7 @@ class SparkSubmitSuite appArgs.jars should be(Utils.resolveURIs(jars)) appArgs.files should be(Utils.resolveURIs(files)) conf.get(JARS) should be(Utils.resolveURIs(jars + ",thejar.jar").split(",").toSeq) - conf.get("spark.files") should be(Utils.resolveURIs(files)) + conf.get(FILES) should be(Utils.resolveURIs(files).split(",").toSeq) // Test files and archives (Yarn) val clArgs2 = Seq( --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org