Bringing zeppelin.spark.useHiveContext option back with default value true
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/1012fb46 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/1012fb46 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/1012fb46 Branch: refs/heads/master Commit: 1012fb46988f1f06d3e5342b8fbd144766fcbd13 Parents: c6350b6 Author: Lee moon soo <[email protected]> Authored: Mon Mar 16 15:31:44 2015 +0900 Committer: Lee moon soo <[email protected]> Committed: Mon Mar 16 15:31:44 2015 +0900 ---------------------------------------------------------------------- .../nflabs/zeppelin/spark/SparkInterpreter.java | 36 +++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/1012fb46/spark/src/main/java/com/nflabs/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/com/nflabs/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/com/nflabs/zeppelin/spark/SparkInterpreter.java index 8473a24..5a28bdb 100644 --- a/spark/src/main/java/com/nflabs/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/com/nflabs/zeppelin/spark/SparkInterpreter.java @@ -88,6 +88,8 @@ public class SparkInterpreter extends Interpreter { getSystemDefault("SPARK_YARN_JAR", "spark.yarn.jar", ""), "The location of the Spark jar file. If you use yarn as a cluster, " + "we should set this value") + .add("zeppelin.spark.useHiveContext", "true", + "Use HiveContext instead of SQLContext if it is true.") .add("args", "", "spark commandline args").build()); } @@ -139,21 +141,29 @@ public class SparkInterpreter extends Interpreter { return pl; } + private boolean useHiveContext() { + return Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext")); + } + public SQLContext getSQLContext() { if (sqlc == null) { - String name = "org.apache.spark.sql.hive.HiveContext"; - Constructor<?> hc; - try { - hc = getClass().getClassLoader().loadClass(name) - .getConstructor(SparkContext.class); - sqlc = (SQLContext) hc.newInstance(getSparkContext()); - } catch (NoSuchMethodException | SecurityException - | ClassNotFoundException | InstantiationException - | IllegalAccessException | IllegalArgumentException - | InvocationTargetException e) { - - // when hive dependency is not loaded, it'll fail. - // in this case SQLContext can be used. + if (useHiveContext()) { + String name = "org.apache.spark.sql.hive.HiveContext"; + Constructor<?> hc; + try { + hc = getClass().getClassLoader().loadClass(name) + .getConstructor(SparkContext.class); + sqlc = (SQLContext) hc.newInstance(getSparkContext()); + } catch (NoSuchMethodException | SecurityException + | ClassNotFoundException | InstantiationException + | IllegalAccessException | IllegalArgumentException + | InvocationTargetException e) { + logger.warn("Can't create HiveContext. Fallback to SQLContext", e); + // when hive dependency is not loaded, it'll fail. + // in this case SQLContext can be used. + sqlc = new SQLContext(getSparkContext()); + } + } else { sqlc = new SQLContext(getSparkContext()); } }
