git commit: [SPARK-1870] Make spark-submit --jars work in yarn-cluster mode.
Repository: spark
Updated Branches:
refs/heads/branch-1.0 75af8bd33 - 79cd26c52
[SPARK-1870] Make spark-submit --jars work in yarn-cluster mode.
Sent secondary jars to distributed cache of all containers and add the cached
jars to classpath before executors start. Tested on a YARN cluster (CDH-5.0).
`spark-submit --jars` also works in standalone server and `yarn-client`. Thanks
for @andrewor14 for testing!
I removed Doesn't work for drivers in standalone mode with cluster deploy
mode. from `spark-submit`'s help message, though we haven't tested mesos yet.
CC: @dbtsai @sryza
Author: Xiangrui Meng m...@databricks.com
Closes #848 from mengxr/yarn-classpath and squashes the following commits:
23e7df4 [Xiangrui Meng] rename spark.jar to __spark__.jar and app.jar to
__app__.jar to avoid confliction apped $CWD/ and $CWD/* to the classpath remove
unused methods
a40f6ed [Xiangrui Meng] standalone - cluster
65e04ad [Xiangrui Meng] update spark-submit help message and add a comment for
yarn-client
11e5354 [Xiangrui Meng] minor changes
3e7e1c4 [Xiangrui Meng] use sparkConf instead of hadoop conf
dc3c825 [Xiangrui Meng] add secondary jars to classpath in yarn
(cherry picked from commit dba314029b4c9d72d7e48a2093b39edd01931f57)
Signed-off-by: Tathagata Das tathagata.das1...@gmail.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/79cd26c5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/79cd26c5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/79cd26c5
Branch: refs/heads/branch-1.0
Commit: 79cd26c528902369b9a2354ba0a0d033e0cf5950
Parents: 75af8bd
Author: Xiangrui Meng m...@databricks.com
Authored: Thu May 22 01:52:50 2014 -0700
Committer: Tathagata Das tathagata.das1...@gmail.com
Committed: Thu May 22 01:53:45 2014 -0700
--
.../spark/deploy/SparkSubmitArguments.scala | 3 +-
.../apache/spark/deploy/yarn/ClientBase.scala | 69 +---
.../cluster/YarnClientSchedulerBackend.scala| 2 +-
3 files changed, 19 insertions(+), 55 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/79cd26c5/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 264d454..0cc05fb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -326,8 +326,7 @@ private[spark] class SparkSubmitArguments(args:
Seq[String]) {
| --class CLASS_NAME Your application's main class (for Java
/ Scala apps).
| --name NAME A name of your application.
| --jars JARS Comma-separated list of local jars to
include on the driver
-| and executor classpaths. Doesn't work
for drivers in
-| standalone mode with cluster deploy
mode.
+| and executor classpaths.
| --py-files PY_FILES Comma-separated list of .zip or .egg
files to place on the
| PYTHONPATH for Python apps.
| --files FILES Comma-separated list of files to be
placed in the working
http://git-wip-us.apache.org/repos/asf/spark/blob/79cd26c5/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 27a518c..aeb3f00 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -44,7 +44,7 @@ import org.apache.spark.{Logging, SparkConf, SparkContext}
* Client submits an application to the YARN ResourceManager.
*
* Depending on the deployment mode this will launch one of two application
master classes:
- * 1. In standalone mode, it will launch an
[[org.apache.spark.deploy.yarn.ApplicationMaster]]
+ * 1. In cluster mode, it will launch an
[[org.apache.spark.deploy.yarn.ApplicationMaster]]
* which launches a driver program inside of the cluster.
* 2. In client mode, it will launch an
[[org.apache.spark.deploy.yarn.ExecutorLauncher]] to
* request executors on behalf of a driver running outside of the cluster.
@@ -220,10 +220,11 @@ trait ClientBase extends Logging {
}
}
+var cachedSecondaryJarLinks = ListBuffer.empty[String]
git commit: Fix UISuite unit test that fails under Jenkins contention
Repository: spark
Updated Branches:
refs/heads/master dba314029 - f9f5fd5f4
Fix UISuite unit test that fails under Jenkins contention
Due to perhaps zombie processes on Jenkins, it seems that at least 10
Spark ports are in use. It also doesn't matter that the port increases
when used, it could in fact go down -- the only part that matters is
that it selects a different port rather than failing to bind.
Changed test to match this.
Thanks to @andrewor14 for helping diagnose this.
Author: Aaron Davidson aa...@databricks.com
Closes #857 from aarondav/tiny and squashes the following commits:
c199ec8 [Aaron Davidson] Fix UISuite unit test that fails under Jenkins
contention
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f9f5fd5f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f9f5fd5f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f9f5fd5f
Branch: refs/heads/master
Commit: f9f5fd5f4e81828a3e0c391892e0f28751568843
Parents: dba3140
Author: Aaron Davidson aa...@databricks.com
Authored: Thu May 22 15:11:05 2014 -0700
Committer: Reynold Xin r...@apache.org
Committed: Thu May 22 15:11:05 2014 -0700
--
core/src/test/scala/org/apache/spark/ui/UISuite.scala | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f9f5fd5f/core/src/test/scala/org/apache/spark/ui/UISuite.scala
--
diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala
b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
index fff8020..038746d 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
@@ -94,7 +94,7 @@ class UISuite extends FunSuite {
}
}
- test(jetty port increases under contention) {
+ test(jetty selects different port under contention) {
val startPort = 4040
val server = new Server(startPort)
@@ -110,8 +110,9 @@ class UISuite extends FunSuite {
// Allow some wiggle room in case ports on the machine are under contention
val boundPort1 = serverInfo1.boundPort
val boundPort2 = serverInfo2.boundPort
-assert(boundPort1 startPort boundPort1 startPort + 10)
-assert(boundPort2 boundPort1 boundPort2 boundPort1 + 10)
+assert(boundPort1 != startPort)
+assert(boundPort2 != startPort)
+assert(boundPort1 != boundPort2)
}
test(jetty binds to port 0 correctly) {
git commit: Fix UISuite unit test that fails under Jenkins contention
Repository: spark
Updated Branches:
refs/heads/branch-1.0 79cd26c52 - a56621622
Fix UISuite unit test that fails under Jenkins contention
Due to perhaps zombie processes on Jenkins, it seems that at least 10
Spark ports are in use. It also doesn't matter that the port increases
when used, it could in fact go down -- the only part that matters is
that it selects a different port rather than failing to bind.
Changed test to match this.
Thanks to @andrewor14 for helping diagnose this.
Author: Aaron Davidson aa...@databricks.com
Closes #857 from aarondav/tiny and squashes the following commits:
c199ec8 [Aaron Davidson] Fix UISuite unit test that fails under Jenkins
contention
(cherry picked from commit f9f5fd5f4e81828a3e0c391892e0f28751568843)
Signed-off-by: Reynold Xin r...@apache.org
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a5662162
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a5662162
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a5662162
Branch: refs/heads/branch-1.0
Commit: a56621622fdfe3aa691d13574d4831c0053a7059
Parents: 79cd26c
Author: Aaron Davidson aa...@databricks.com
Authored: Thu May 22 15:11:05 2014 -0700
Committer: Reynold Xin r...@apache.org
Committed: Thu May 22 15:11:12 2014 -0700
--
core/src/test/scala/org/apache/spark/ui/UISuite.scala | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a5662162/core/src/test/scala/org/apache/spark/ui/UISuite.scala
--
diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala
b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
index fff8020..038746d 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
@@ -94,7 +94,7 @@ class UISuite extends FunSuite {
}
}
- test(jetty port increases under contention) {
+ test(jetty selects different port under contention) {
val startPort = 4040
val server = new Server(startPort)
@@ -110,8 +110,9 @@ class UISuite extends FunSuite {
// Allow some wiggle room in case ports on the machine are under contention
val boundPort1 = serverInfo1.boundPort
val boundPort2 = serverInfo2.boundPort
-assert(boundPort1 startPort boundPort1 startPort + 10)
-assert(boundPort2 boundPort1 boundPort2 boundPort1 + 10)
+assert(boundPort1 != startPort)
+assert(boundPort2 != startPort)
+assert(boundPort1 != boundPort2)
}
test(jetty binds to port 0 correctly) {
git commit: [SPARK-1897] Respect spark.jars (and --jars) in spark-shell
Repository: spark
Updated Branches:
refs/heads/branch-1.0 a56621622 - 23cc40e39
[SPARK-1897] Respect spark.jars (and --jars) in spark-shell
Spark shell currently overwrites `spark.jars` with `ADD_JARS`. In all modes
except yarn-cluster, this means the `--jar` flag passed to `bin/spark-shell` is
also discarded. However, in the
[docs](http://people.apache.org/~pwendell/spark-1.0.0-rc7-docs/scala-programming-guide.html#initializing-spark),
we explicitly tell the users to add the jars this way.
Author: Andrew Or andrewo...@gmail.com
Closes #849 from andrewor14/shell-jars and squashes the following commits:
928a7e6 [Andrew Or] ',' - , (minor)
afc357c [Andrew Or] Handle spark.jars == in SparkILoop, not SparkSubmit
c6da113 [Andrew Or] Do not set spark.jars to
d8549f7 [Andrew Or] Respect spark.jars and --jars in spark-shell
(cherry picked from commit 8edbee7d1b4afc192d97ba192a5526affc464205)
Signed-off-by: Tathagata Das tathagata.das1...@gmail.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/23cc40e3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/23cc40e3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/23cc40e3
Branch: refs/heads/branch-1.0
Commit: 23cc40e39acc598816ac46b36463e13587a0dd60
Parents: a566216
Author: Andrew Or andrewo...@gmail.com
Authored: Thu May 22 20:25:41 2014 -0700
Committer: Tathagata Das tathagata.das1...@gmail.com
Committed: Thu May 22 20:25:53 2014 -0700
--
repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/23cc40e3/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
--
diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
index 296da74..55684e9 100644
--- a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
+++ b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
@@ -993,7 +993,13 @@ object SparkILoop {
implicit def loopToInterpreter(repl: SparkILoop): SparkIMain = repl.intp
private def echo(msg: String) = Console println msg
- def getAddedJars: Array[String] =
Option(System.getenv(ADD_JARS)).map(_.split(',')).getOrElse(new
Array[String](0))
+ def getAddedJars: Array[String] = {
+val envJars = sys.env.get(ADD_JARS)
+val propJars = sys.props.get(spark.jars).flatMap { p =
+ if (p == ) None else Some(p)
+}
+propJars.orElse(envJars).map(_.split(,)).getOrElse(Array.empty)
+ }
// Designed primarily for use by test code: take a String with a
// bunch of code, and prints out a transcript of what it would look
git commit: [SPARK-1896] Respect spark.master (and --master) before MASTER in spark-shell
Repository: spark
Updated Branches:
refs/heads/master 8edbee7d1 - cce77457e
[SPARK-1896] Respect spark.master (and --master) before MASTER in spark-shell
The hierarchy for configuring the Spark master in the shell is as follows:
```
MASTER --master spark.master (spark-defaults.conf)
```
This is inconsistent with the way we run normal applications, which is:
```
--master spark.master (spark-defaults.conf) MASTER
```
I was trying to run a shell locally on a standalone cluster launched through
the ec2 scripts, which automatically set `MASTER` in spark-env.sh. It was
surprising to me that `--master` didn't take effect, considering that this is
the way we tell users to set their masters
[here](http://people.apache.org/~pwendell/spark-1.0.0-rc7-docs/scala-programming-guide.html#initializing-spark).
Author: Andrew Or andrewo...@gmail.com
Closes #846 from andrewor14/shell-master and squashes the following commits:
2cb81c9 [Andrew Or] Respect spark.master before MASTER in REPL
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cce77457
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cce77457
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cce77457
Branch: refs/heads/master
Commit: cce77457e00aa5f1f4db3d50454cf257efb156ed
Parents: 8edbee7
Author: Andrew Or andrewo...@gmail.com
Authored: Thu May 22 20:32:27 2014 -0700
Committer: Tathagata Das tathagata.das1...@gmail.com
Committed: Thu May 22 20:32:27 2014 -0700
--
repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/cce77457/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
--
diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
index 55684e9..5f34362 100644
--- a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
+++ b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
@@ -962,11 +962,10 @@ class SparkILoop(in0: Option[BufferedReader], protected
val out: JPrintWriter,
private def getMaster(): String = {
val master = this.master match {
case Some(m) = m
- case None = {
+ case None =
val envMaster = sys.env.get(MASTER)
val propMaster = sys.props.get(spark.master)
-envMaster.orElse(propMaster).getOrElse(local[*])
- }
+propMaster.orElse(envMaster).getOrElse(local[*])
}
master
}
git commit: Updated scripts for auditing releases
Repository: spark
Updated Branches:
refs/heads/master cce77457e - b2bdd0e50
Updated scripts for auditing releases
- Added script to automatically generate change list CHANGES.txt
- Added test for verifying linking against maven distributions of `spark-sql`
and `spark-hive`
- Added SBT projects for testing functionality of `spark-sql` and `spark-hive`
- Fixed issues in existing tests that might have come up because of changes in
Spark 1.0
Author: Tathagata Das tathagata.das1...@gmail.com
Closes #844 from tdas/update-dev-scripts and squashes the following commits:
25090ba [Tathagata Das] Added missing license
e2e20b3 [Tathagata Das] Updated tests for auditing releases.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b2bdd0e5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b2bdd0e5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b2bdd0e5
Branch: refs/heads/master
Commit: b2bdd0e505f1ae3d39c46139f17bd43779ece635
Parents: cce7745
Author: Tathagata Das tathagata.das1...@gmail.com
Authored: Thu May 22 20:48:55 2014 -0700
Committer: Tathagata Das tathagata.das1...@gmail.com
Committed: Thu May 22 20:48:55 2014 -0700
--
dev/audit-release/audit_release.py | 11 +-
.../maven_app_core/src/main/java/SimpleApp.java | 1 +
.../sbt_app_core/src/main/scala/SparkApp.scala | 2 +-
dev/audit-release/sbt_app_hive/build.sbt| 29 +++
dev/audit-release/sbt_app_hive/data.txt | 9 +
.../src/main/resources/hive-site.xml| 213 +++
.../sbt_app_hive/src/main/scala/HiveApp.scala | 57 +
dev/audit-release/sbt_app_sql/build.sbt | 29 +++
.../sbt_app_sql/src/main/scala/SqlApp.scala | 57 +
.../src/main/scala/StreamingApp.scala | 1 -
dev/create-release/generate-changelist.py | 144 +
11 files changed, 547 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/audit_release.py
--
diff --git a/dev/audit-release/audit_release.py
b/dev/audit-release/audit_release.py
index 4a816d4..8c7573b 100755
--- a/dev/audit-release/audit_release.py
+++ b/dev/audit-release/audit_release.py
@@ -93,9 +93,12 @@ original_dir = os.getcwd()
# For each of these modules, we'll test an 'empty' application in sbt and
# maven that links against them. This will catch issues with messed up
# dependencies within those projects.
-modules = [spark-core, spark-bagel, spark-mllib, spark-streaming,
spark-repl,
- spark-graphx, spark-streaming-flume, spark-streaming-kafka,
- spark-streaming-mqtt, spark-streaming-twitter,
spark-streaming-zeromq]
+modules = [
+ spark-core, spark-bagel, spark-mllib, spark-streaming, spark-repl,
+ spark-graphx, spark-streaming-flume, spark-streaming-kafka,
+ spark-streaming-mqtt, spark-streaming-twitter, spark-streaming-zeromq,
+ spark-catalyst, spark-sql, spark-hive
+]
modules = map(lambda m: %s_%s % (m, SCALA_BINARY_VERSION), modules)
# Check for directories that might interfere with tests
@@ -122,7 +125,7 @@ for module in modules:
os.chdir(original_dir)
# SBT application tests
-for app in [sbt_app_core, sbt_app_graphx, sbt_app_streaming]:
+for app in [sbt_app_core, sbt_app_graphx, sbt_app_streaming,
sbt_app_sql, sbt_app_hive]:
os.chdir(app)
ret = run_cmd(sbt clean run, exit_on_failure=False)
test(ret == 0, sbt application (%s) % app)
http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
--
diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
index 6b65dda..5217689 100644
--- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
+++ b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
@@ -37,5 +37,6 @@ public class SimpleApp {
System.exit(-1);
}
System.out.println(Test succeeded);
+ sc.stop();
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
--
diff --git a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
index a89b0d7..77bbd16 100644
--- a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
+++ b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
@@ -19,6 +19,7 @@ package main.scala
import scala.util.Try
+import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
@@
git commit: Updated scripts for auditing releases
Repository: spark
Updated Branches:
refs/heads/branch-1.0 c3b406518 - 6541ca24b
Updated scripts for auditing releases
- Added script to automatically generate change list CHANGES.txt
- Added test for verifying linking against maven distributions of `spark-sql`
and `spark-hive`
- Added SBT projects for testing functionality of `spark-sql` and `spark-hive`
- Fixed issues in existing tests that might have come up because of changes in
Spark 1.0
Author: Tathagata Das tathagata.das1...@gmail.com
Closes #844 from tdas/update-dev-scripts and squashes the following commits:
25090ba [Tathagata Das] Added missing license
e2e20b3 [Tathagata Das] Updated tests for auditing releases.
(cherry picked from commit b2bdd0e505f1ae3d39c46139f17bd43779ece635)
Signed-off-by: Tathagata Das tathagata.das1...@gmail.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6541ca24
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6541ca24
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6541ca24
Branch: refs/heads/branch-1.0
Commit: 6541ca24b2cf79d6914b9d0bcf7b51bff405369e
Parents: c3b4065
Author: Tathagata Das tathagata.das1...@gmail.com
Authored: Thu May 22 20:48:55 2014 -0700
Committer: Tathagata Das tathagata.das1...@gmail.com
Committed: Thu May 22 20:49:25 2014 -0700
--
dev/audit-release/audit_release.py | 11 +-
.../maven_app_core/src/main/java/SimpleApp.java | 1 +
.../sbt_app_core/src/main/scala/SparkApp.scala | 2 +-
dev/audit-release/sbt_app_hive/build.sbt| 29 +++
dev/audit-release/sbt_app_hive/data.txt | 9 +
.../src/main/resources/hive-site.xml| 213 +++
.../sbt_app_hive/src/main/scala/HiveApp.scala | 57 +
dev/audit-release/sbt_app_sql/build.sbt | 29 +++
.../sbt_app_sql/src/main/scala/SqlApp.scala | 57 +
.../src/main/scala/StreamingApp.scala | 1 -
dev/create-release/generate-changelist.py | 144 +
11 files changed, 547 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/audit_release.py
--
diff --git a/dev/audit-release/audit_release.py
b/dev/audit-release/audit_release.py
index 4a816d4..8c7573b 100755
--- a/dev/audit-release/audit_release.py
+++ b/dev/audit-release/audit_release.py
@@ -93,9 +93,12 @@ original_dir = os.getcwd()
# For each of these modules, we'll test an 'empty' application in sbt and
# maven that links against them. This will catch issues with messed up
# dependencies within those projects.
-modules = [spark-core, spark-bagel, spark-mllib, spark-streaming,
spark-repl,
- spark-graphx, spark-streaming-flume, spark-streaming-kafka,
- spark-streaming-mqtt, spark-streaming-twitter,
spark-streaming-zeromq]
+modules = [
+ spark-core, spark-bagel, spark-mllib, spark-streaming, spark-repl,
+ spark-graphx, spark-streaming-flume, spark-streaming-kafka,
+ spark-streaming-mqtt, spark-streaming-twitter, spark-streaming-zeromq,
+ spark-catalyst, spark-sql, spark-hive
+]
modules = map(lambda m: %s_%s % (m, SCALA_BINARY_VERSION), modules)
# Check for directories that might interfere with tests
@@ -122,7 +125,7 @@ for module in modules:
os.chdir(original_dir)
# SBT application tests
-for app in [sbt_app_core, sbt_app_graphx, sbt_app_streaming]:
+for app in [sbt_app_core, sbt_app_graphx, sbt_app_streaming,
sbt_app_sql, sbt_app_hive]:
os.chdir(app)
ret = run_cmd(sbt clean run, exit_on_failure=False)
test(ret == 0, sbt application (%s) % app)
http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
--
diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
index 6b65dda..5217689 100644
--- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
+++ b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
@@ -37,5 +37,6 @@ public class SimpleApp {
System.exit(-1);
}
System.out.println(Test succeeded);
+ sc.stop();
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
--
diff --git a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
index a89b0d7..77bbd16 100644
--- a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
+++ b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
@@ -19,6 +19,7 @@ package main.scala
import
