This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new f5deb7c9af6 [SPARK-42424][YARN] Remove unused declarations from Yarn module f5deb7c9af6 is described below commit f5deb7c9af618494adecfeb103f94219e27fcf70 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Thu Feb 16 09:25:47 2023 -0600 [SPARK-42424][YARN] Remove unused declarations from Yarn module ### What changes were proposed in this pull request? This pr cleans up unused declarations in the Yarn module: - `YarnSparkHadoopUtil#setEnvFromInputString`: this method Introduced by SPARK-3477 and becomes a unused and no test coverage method after SPARK-17979 - `YarnSparkHadoopUtil#environmentVariableRegex`: this `val` is only used by `YarnSparkHadoopUtil#setEnvFromInputString` - `ApplicationMasterArguments.DEFAULT_NUMBER_EXECUTORS`: this `val` Introduced by SPARK-1946 and replaced by `YarnSparkHadoopUtil.DEFAULT_NUMBER_EXECUTORS` in SPARK-4138, and It was eventually replaced by `config#EXECUTOR_INSTANCES` - `ApplicationMaster.EXIT_SECURITY`: this `val` introduced by SPARK-3627 and it is used to represent the exit code of errors related to `System.setSecurityManager`, SPARK-4584 deleted the use of `SecurityManager` and this val is useless. ### Why are the changes needed? Code clean up. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GitHub Actions Closes #39997 from LuciferYang/SPARK-42424. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- .../spark/deploy/yarn/ApplicationMaster.scala | 1 - .../deploy/yarn/ApplicationMasterArguments.scala | 4 -- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 50 ---------------------- 3 files changed, 55 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 9815fa6df8a..252c84a1cd4 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -891,7 +891,6 @@ object ApplicationMaster extends Logging { private val EXIT_MAX_EXECUTOR_FAILURES = 11 private val EXIT_REPORTER_FAILURE = 12 private val EXIT_SC_NOT_INITED = 13 - private val EXIT_SECURITY = 14 private val EXIT_EXCEPTION_USER_CLASS = 15 private val EXIT_EARLY = 16 private val EXIT_DISCONNECTED = 17 diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala index d2275980814..821a31502af 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala @@ -102,7 +102,3 @@ class ApplicationMasterArguments(val args: Array[String]) { System.exit(exitCode) } } - -object ApplicationMasterArguments { - val DEFAULT_NUMBER_EXECUTORS = 2 -} diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala index 1869c739e48..4d050b91a85 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala @@ -17,8 +17,6 @@ package org.apache.spark.deploy.yarn -import java.util.regex.{Matcher, Pattern} - import scala.collection.immutable.{Map => IMap} import scala.collection.mutable.{HashMap, ListBuffer} import scala.util.matching.Regex @@ -59,41 +57,6 @@ object YarnSparkHadoopUtil { env.put(key, newValue) } - /** - * Set zero or more environment variables specified by the given input string. - * The input string is expected to take the form "KEY1=VAL1,KEY2=VAL2,KEY3=VAL3". - */ - def setEnvFromInputString(env: HashMap[String, String], inputString: String): Unit = { - if (inputString != null && inputString.length() > 0) { - val childEnvs = inputString.split(",") - val p = Pattern.compile(environmentVariableRegex) - for (cEnv <- childEnvs) { - val parts = cEnv.split("=") // split on '=' - val m = p.matcher(parts(1)) - val sb = new StringBuffer - while (m.find()) { - val variable = m.group(1) - var replace = "" - if (env.contains(variable)) { - replace = env(variable) - } else { - // if this key is not configured for the child .. get it from the env - replace = System.getenv(variable) - if (replace == null) { - // the env key is note present anywhere .. simply set it - replace = "" - } - } - m.appendReplacement(sb, Matcher.quoteReplacement(replace)) - } - m.appendTail(sb) - // This treats the environment variable as path variable delimited by `File.pathSeparator` - // This is kept for backward compatibility and consistency with Hadoop's behavior - addPathToEnvironment(env, parts(0), sb.toString) - } - } - } - /** * Regex pattern to match the name of an environment variable. Note that Unix variable naming * conventions (alphanumeric plus underscore, case-sensitive, can't start with a digit) @@ -102,19 +65,6 @@ object YarnSparkHadoopUtil { */ private val envVarNameRegex: String = "[A-Za-z_][A-Za-z0-9_]*" - /** - * Note that this regex only supports the `$VAR_NAME` and `%VAR_NAME%` syntax, for Unix and - * Windows respectively, and does not perform any handling of escapes. The Unix `${VAR_NAME}` - * syntax is not supported. - */ - private val environmentVariableRegex: String = { - if (Utils.isWindows) { - s"%($envVarNameRegex)%" - } else { - s"\\$$($envVarNameRegex)" - } - } - // scalastyle:off line.size.limit /** * Replace environment variables in a string according to the same rules as --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org