Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/18962#discussion_r134561742
--- Diff: core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala ---
@@ -484,15 +489,28 @@ object SparkSubmit extends CommandLineUtils {
sysProp = "spark.driver.cores"),
OptionAssigner(args.supervise.toString, STANDALONE | MESOS, CLUSTER,
sysProp = "spark.driver.supervise"),
- OptionAssigner(args.ivyRepoPath, STANDALONE, CLUSTER, sysProp =
"spark.jars.ivy")
+ OptionAssigner(args.ivyRepoPath, STANDALONE, CLUSTER, sysProp =
"spark.jars.ivy"),
+
+ // An internal option used only for spark-shell to add user jars to
repl's classloader,
+ // previously it uses "spark.jars" or "spark.yarn.dist.jars" which
now may be pointed to
+ // remote jars, so adding a new option to only specify local jars
for spark-shell internally.
+ OptionAssigner(localJars, ALL_CLUSTER_MGRS, CLIENT, sysProp =
"spark.repl.local.jars")
)
// In client mode, launch the application main class directly
// In addition, add the main application jar and any added jars (if
any) to the classpath
- // Also add the main application jar and any added jars to classpath
in case YARN client
- // requires these jars.
- if (deployMode == CLIENT || isYarnCluster) {
+ if (deployMode == CLIENT) {
childMainClass = args.mainClass
+ if (localPrimaryResource != null && isUserJar(localPrimaryResource))
{
+ childClasspath += localPrimaryResource
+ }
+ if (localJars != null) { childClasspath ++= localJars.split(",") }
+ }
+ // Add the main application jar and any added jars to classpath in
case YARN client
+ // requires these jars.
+ // This assumes both primaryResource and user jars are local jars,
eitherwise it will not be
--- End diff --
otherwise
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
