Fwd: SparkILoop doesn't run
I am trying to use SparkILoop to write some tests(shown below) but the test
hangs with the following stack trace. Any idea what is going on?
import org.apache.log4j.{Level, LogManager}
import org.apache.spark.repl.SparkILoop
import org.scalatest.{BeforeAndAfterAll, FunSuite}
class SparkReplSpec extends FunSuite with BeforeAndAfterAll {
override def beforeAll(): Unit = {
}
override def afterAll(): Unit = {
}
test("yay!") {
val rootLogger = LogManager.getRootLogger
val logLevel = rootLogger.getLevel
rootLogger.setLevel(Level.ERROR)
val output = SparkILoop.run(
"""
|println("hello")
""".stripMargin)
println(s" $output ")
}
}
/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/bin/java
-Dspark.master=local[*] -Didea.launcher.port=7532
"-Didea.launcher.bin.path=/Applications/IntelliJ IDEA CE.app/Contents/bin"
-Dfile.encoding=UTF-8 -classpath "/Users/mohit/Library/Application
Support/IdeaIC2016.2/Scala/lib/scala-plugin-runners.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/
charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/jre/lib/deploy.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/
ext/cldrdata.jar:/Library/Java/JavaVirtualMachines/jdk1.
8.0_66.jdk/Contents/Home/jre/lib/ext/dnsns.jar:/Library/
Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/
lib/ext/jaccess.jar:/Library/Java/JavaVirtualMachines/jdk1.
8.0_66.jdk/Contents/Home/jre/lib/ext/jfxrt.jar:/Library/
Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/
lib/ext/localedata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/jre/lib/ext/nashorn.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/
ext/sunec.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/jre/lib/ext/sunjce_provider.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/
ext/sunpkcs11.jar:/Library/Java/JavaVirtualMachines/jdk1.
8.0_66.jdk/Contents/Home/jre/lib/ext/zipfs.jar:/Library/
Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/
lib/javaws.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/
jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/jre/lib/jfxswt.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/
jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/jre/lib/management-agent.jar:/Library/
Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/
lib/plugin.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/
rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/lib/ant-javafx.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/dt.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/
javafx-mx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/lib/jconsole.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/
packager.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_
66.jdk/Contents/Home/lib/sa-jdi.jar:/Library/Java/
JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/
tools.jar:/Users/mohit/code/datagears-play/target/scala-2.
11/test-classes:/Users/mohit/code/datagears-play/target/
scala-2.11/classes:/Users/mohit/code/datagears-play/
macros/target/scala-2.11/classes:/Users/mohit/.ivy2/cache/org.xerial.snappy/
snappy-java/bundles/snappy-java-1.1.2.4.jar:/Users/mohit/
.ivy2/cache/org.apache.spark/spark-unsafe_2.11/jars/spark-
unsafe_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.
spark/spark-tags_2.11/jars/spark-tags_2.11-2.0.0.jar:/
Users/mohit/.ivy2/cache/org.apache.spark/spark-streaming_
2.11/jars/spark-streaming_2.11-2.0.0.jar:/Users/mohit/.
ivy2/cache/org.apache.spark/spark-sql_2.11/jars/spark-sql_
2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/
spark-sketch_2.11/jars/spark-sketch_2.11-2.0.0.jar:/Users/
mohit/.ivy2/cache/org.apache.spark/spark-repl_2.11/jars/
spark-repl_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.
apache.spark/spark-network-shuffle_2.11/jars/spark-
network-shuffle_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/
org.apache.spark/spark-network-common_2.11/jars/
spark-network-common_2.11-2.0.0.jar:/Users/mohit/.ivy2/
cache/org.apache.spark/spark-mllib_2.11/jars/spark-mllib_2.
11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/
spark-mllib-local_2.11/jars/spark-mllib-local_2.11-2.0.0.
jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-
launcher_2.11/jars/spark-launcher_2.11-2.0.0.jar:/
Users/mohit/.ivy2/cache/org.apache.spark/spark-graphx_2.
11/jars/spark-graphx_2.11-2.0.0.jar:/Users/mohit/.ivy2/
cache/org.apache.spark/spark-core_2.11/jars/spark-core_2.
11-2.0.0.jar:/Users/mohit/.ivy2/cache/org
SparkILoop doesn't run
I am trying to use SparkILoop to write some tests(shown below) but the test
hangs with the following stack trace. Any idea what is going on?
import org.apache.log4j.{Level, LogManager}
import org.apache.spark.repl.SparkILoop
import org.scalatest.{BeforeAndAfterAll, FunSuite}
class SparkReplSpec extends FunSuite with BeforeAndAfterAll {
override def beforeAll(): Unit = {
}
override def afterAll(): Unit = {
}
test("yay!") {
val rootLogger = LogManager.getRootLogger
val logLevel = rootLogger.getLevel
rootLogger.setLevel(Level.ERROR)
val output = SparkILoop.run(
"""
|println("hello")
""".stripMargin)
println(s" $output ")
}
}
/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/bin/java
-Dspark.master=local[*] -Didea.launcher.port=7532
"-Didea.launcher.bin.path=/Applications/IntelliJ IDEA CE.app/Contents/bin"
-Dfile.encoding=UTF-8 -classpath "/Users/mohit/Library/Application
Support/IdeaIC2016.2/Scala/lib/scala-plugin-runners.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/deploy.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/cldrdata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/dnsns.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/jaccess.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/jfxrt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/localedata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/nashorn.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/sunec.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/sunjce_provider.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/sunpkcs11.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/ext/zipfs.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/javaws.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/jfxswt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/management-agent.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/plugin.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/ant-javafx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/dt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/javafx-mx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/jconsole.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/packager.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/sa-jdi.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home/lib/tools.jar:/Users/mohit/code/datagears-play/target/scala-2.11/test-classes:/Users/mohit/code/datagears-play/target/scala-2.11/classes:/Users/mohit/code/datagears-play/macros/target/scala-2.11/classes:/Users/mohit/.ivy2/cache/org.xerial.snappy/snappy-java/bundles/snappy-java-1.1.2.4.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-unsafe_2.11/jars/spark-unsafe_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-tags_2.11/jars/spark-tags_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-streaming_2.11/jars/spark-streaming_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-sql_2.11/jars/spark-sql_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-sketch_2.11/jars/spark-sketch_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-repl_2.11/jars/spark-repl_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-network-shuffle_2.11/jars/spark-network-shuffle_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-network-common_2.11/jars/spark-network-common_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-mllib_2.11/jars/spark-mllib_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-mllib-local_2.11/jars/spark-mllib-local_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-launcher_2.11/jars/spark-launcher_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-graphx_2.11/jars/spark-graphx_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-core_2.11/jars/spark-core_2.11-2.0.0.jar:/Users/mohit/.ivy2/cache/org.apache.spark/spark-catalyst_2.11/jars/spark-catalyst_2.11-2.0.0.
