Repository: spark Updated Branches: refs/heads/branch-1.5 c8392cd5b -> 6847be6d1
[SPARK-10901] [YARN] spark.yarn.user.classpath.first doesn't work This should go into 1.5.2 also. The issue is we were no longer adding the __app__.jar to the system classpath. Author: Thomas Graves <tgra...@staydecay.corp.gq1.yahoo.com> Author: Tom Graves <tgra...@yahoo-inc.com> Closes #8959 from tgravescs/SPARK-10901. (cherry picked from commit e9783601599758df87418bf61a7b4636f06714fa) Signed-off-by: Marcelo Vanzin <van...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6847be6d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6847be6d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6847be6d Branch: refs/heads/branch-1.5 Commit: 6847be6d1e600439d19e747f1779aab855ac5a02 Parents: c8392cd Author: Thomas Graves <tgra...@staydecay.corp.gq1.yahoo.com> Authored: Tue Oct 6 10:18:50 2015 -0700 Committer: Marcelo Vanzin <van...@cloudera.com> Committed: Tue Oct 6 10:19:05 2015 -0700 ---------------------------------------------------------------------- .../org/apache/spark/deploy/yarn/Client.scala | 39 ++++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/6847be6d/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala ---------------------------------------------------------------------- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 7cedcbc..28228c2 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -1134,13 +1134,24 @@ object Client extends Logging { } if (sparkConf.getBoolean("spark.yarn.user.classpath.first", false)) { - val userClassPath = + // in order to properly add the app jar when user classpath is first + // we have to do the mainJar separate in order to send the right thing + // into addFileToClasspath + val mainJar = if (args != null) { - getUserClasspath(Option(args.userJar), Option(args.addJars)) + getMainJarUri(Option(args.userJar)) } else { - getUserClasspath(sparkConf) + getMainJarUri(sparkConf.getOption(CONF_SPARK_USER_JAR)) } - userClassPath.foreach { x => + mainJar.foreach(addFileToClasspath(sparkConf, _, APP_JAR, env)) + + val secondaryJars = + if (args != null) { + getSecondaryJarUris(Option(args.addJars)) + } else { + getSecondaryJarUris(sparkConf.getOption(CONF_SPARK_YARN_SECONDARY_JARS)) + } + secondaryJars.foreach { x => addFileToClasspath(sparkConf, x, null, env) } } @@ -1157,16 +1168,20 @@ object Client extends Logging { * @param conf Spark configuration. */ def getUserClasspath(conf: SparkConf): Array[URI] = { - getUserClasspath(conf.getOption(CONF_SPARK_USER_JAR), - conf.getOption(CONF_SPARK_YARN_SECONDARY_JARS)) + val mainUri = getMainJarUri(conf.getOption(CONF_SPARK_USER_JAR)) + val secondaryUris = getSecondaryJarUris(conf.getOption(CONF_SPARK_YARN_SECONDARY_JARS)) + (mainUri ++ secondaryUris).toArray } - private def getUserClasspath( - mainJar: Option[String], - secondaryJars: Option[String]): Array[URI] = { - val mainUri = mainJar.orElse(Some(APP_JAR)).map(new URI(_)) - val secondaryUris = secondaryJars.map(_.split(",")).toSeq.flatten.map(new URI(_)) - (mainUri ++ secondaryUris).toArray + private def getMainJarUri(mainJar: Option[String]): Option[URI] = { + mainJar.flatMap { path => + val uri = new URI(path) + if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None + }.orElse(Some(new URI(APP_JAR))) + } + + private def getSecondaryJarUris(secondaryJars: Option[String]): Seq[URI] = { + secondaryJars.map(_.split(",")).toSeq.flatten.map(new URI(_)) } /** --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
