LIVY-315. Fixed SparkRInterpreter to respect config enableHiveContext. (#293)
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/ed1c86e8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/ed1c86e8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/ed1c86e8 Branch: refs/heads/master Commit: ed1c86e874bee2d4323d6267cdf2a8d14b7db571 Parents: f7c434c Author: Jeff Zhang <zjf...@gmail.com> Authored: Sat Feb 18 07:50:05 2017 +0800 Committer: Alex Man <tc.technet...@gmail.com> Committed: Fri Feb 17 15:50:05 2017 -0800 ---------------------------------------------------------------------- .../cloudera/livy/repl/ProcessInterpreter.scala | 1 + .../cloudera/livy/repl/SparkRInterpreter.scala | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/ed1c86e8/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala ---------------------------------------------------------------------- diff --git a/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala index 78826e3..25a8654 100644 --- a/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala +++ b/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala @@ -130,6 +130,7 @@ abstract class ProcessInterpreter(process: Process) val exitCode = process.waitFor() if (exitCode != 0) { error(f"Process has died with $exitCode") + error(stderrLines.mkString("\n")) } } } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/ed1c86e8/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala ---------------------------------------------------------------------- diff --git a/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala index b3d3b7e..6c42fe3 100644 --- a/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala +++ b/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala @@ -116,7 +116,8 @@ object SparkRInterpreter { builder.redirectErrorStream(true) val process = builder.start() new SparkRInterpreter(process, backendInstance, backendThread, - conf.get("spark.livy.spark_major_version", "1")) + conf.get("spark.livy.spark_major_version", "1"), + conf.getBoolean("spark.repl.enableHiveContext", false)) } catch { case e: Exception => if (backendThread != null) { @@ -130,7 +131,8 @@ object SparkRInterpreter { class SparkRInterpreter(process: Process, backendInstance: Any, backendThread: Thread, - val sparkMajorVersion: String) + val sparkMajorVersion: String, + hiveEnabled: Boolean) extends ProcessInterpreter(process) { import SparkRInterpreter._ @@ -145,15 +147,22 @@ class SparkRInterpreter(process: Process, sendRequest("options(error = dump.frames)") if (!ClientConf.TEST_MODE) { sendRequest("library(SparkR)") - if (sparkMajorVersion >= "2") { - sendRequest("spark <- SparkR::sparkR.session()") + if (hiveEnabled) { + sendRequest("spark <- SparkR::sparkR.session()") + } else { + sendRequest("spark <- SparkR::sparkR.session(enableHiveSupport=FALSE)") + } sendRequest( """sc <- SparkR:::callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", spark)""") } else { sendRequest("sc <- sparkR.init()") - sendRequest("sqlContext <- sparkRSQL.init(sc)") + if (hiveEnabled) { + sendRequest("sqlContext <- sparkRHive.init(sc)") + } else { + sendRequest("sqlContext <- sparkRSQL.init(sc)") + } } }