Repository: incubator-zeppelin Updated Branches: refs/heads/master c79d4379c -> 26a2d641c
[ZEPPELIN-767] HBase interpreter does not work with HBase on a remote cluster ### What is this PR for? HBase interpreter fails with message "ERROR: KeeperErrorCode = ConnectionLoss for /hbase" when connecting to a remote HBASE (for instance, HBase running in CDH cluster) Initially it's thought that zkquoram setttings are not getting applied, but deeper investigations reveal that hbase-site.xml cannot be loaded. HBASE_HOME or HBASE_CONF_DIR is set by `hbase` script when running hbase shell - interpreter will need to at minimum replicate that behavior to add the directory with hbase-site.xml to CLASS_PATH in order to fix this issue. ### What type of PR is it? Bug Fix ### Todos * [x] - Bug fix * [x] - Update documentation ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-767 ### How should this be tested? (tested) Run HBase locally (standalone: https://hbase.apache.org/book.html#quickstart) (tested) Set HBASE_HOME in env and work with HBASE on a Hadoop cluster (tested) Set HBASE_CONF_DIR in env and work with HBASE on a Hadoop cluster ### Screenshots (if appropriate) N/A ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? Added Author: Felix Cheung <[email protected]> Closes #799 from felixcheung/hbaseconf and squashes the following commits: ae90626 [Felix Cheung] fix test a82c2a6 [Felix Cheung] fix bug, add doc, update text eeb341f [Felix Cheung] set hbase conf dir to classpath Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/26a2d641 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/26a2d641 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/26a2d641 Branch: refs/heads/master Commit: 26a2d641c70b1a96b9276ea5e48504e0911f62cf Parents: c79d437 Author: Felix Cheung <[email protected]> Authored: Sun Mar 27 22:04:23 2016 -0700 Committer: Felix Cheung <[email protected]> Committed: Fri Apr 1 14:53:04 2016 -0700 ---------------------------------------------------------------------- bin/interpreter.sh | 8 ++++ conf/zeppelin-env.sh.template | 6 +++ docs/interpreter/hbase.md | 23 ++++++++++- .../apache/zeppelin/hbase/HbaseInterpreter.java | 43 +++++++++++++++----- .../zeppelin/hbase/HbaseInterpreterTest.java | 4 +- 5 files changed, 69 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/bin/interpreter.sh ---------------------------------------------------------------------- diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 18e1540..69c94f6 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -132,6 +132,14 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}" fi +elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then + if [[ -n "${HBASE_CONF_DIR}" ]]; then + ZEPPELIN_CLASSPATH+=":${HBASE_CONF_DIR}" + elif [[ -n "${HBASE_HOME}" ]]; then + ZEPPELIN_CLASSPATH+=":${HBASE_HOME}/conf" + else + echo "HBASE_HOME and HBASE_CONF_DIR are not set, configuration might not be loaded" + fi fi addJarInDir "${LOCAL_INTERPRETER_REPO}" http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/conf/zeppelin-env.sh.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 638162c..6279de7 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -64,3 +64,9 @@ # export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default. # export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000 +#### HBase interpreter configuration #### + +## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set + +# export HBASE_HOME= # (require) Under which HBase scripts and configuration should be +# export HBASE_CONF_DIR= # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/docs/interpreter/hbase.md ---------------------------------------------------------------------- diff --git a/docs/interpreter/hbase.md b/docs/interpreter/hbase.md index 7766b02..2eaa915 100644 --- a/docs/interpreter/hbase.md +++ b/docs/interpreter/hbase.md @@ -34,7 +34,7 @@ mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr <tr> <td>hbase.home</td> <td>/usr/lib/hbase</td> - <td>Installation directory of Hbase</td> + <td>Installation directory of HBase, defaults to HBASE_HOME in environment</td> </tr> <tr> <td>hbase.ruby.sources</td> @@ -42,12 +42,31 @@ mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr <td>Path to Ruby scripts relative to 'hbase.home'</td> </tr> <tr> - <td>hbase.test.mode</td> + <td>zeppelin.hbase.test.mode</td> <td>false</td> <td>Disable checks for unit and manual tests</td> </tr> </table> +If you want to connect to HBase running on a cluster, you'll need to follow the next step. + +### Export HBASE_HOME +In **conf/zeppelin-env.sh**, export `HBASE_HOME` environment variable with your HBase installation path. This ensures `hbase-site.xml` can be loaded. + +for example + +```bash +export HBASE_HOME=/usr/lib/hbase +``` + +or, when running with CDH + +```bash +export HBASE_HOME="/opt/cloudera/parcels/CDH/lib/hbase" +``` + +You can optionally export `HBASE_CONF_DIR` instead of `HBASE_HOME` should you have custom HBase configurations. + ## Enabling the HBase Shell Interpreter In a notebook, to enable the **HBase Shell** interpreter, click the **Gear** icon and select **HBase Shell**. http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index dbcb33d..84e4105 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -37,21 +37,20 @@ import java.util.List; import java.util.Properties; /** - * Support for Hbase Shell. All the commands documented here + * Support for HBase Shell. All the commands documented here * http://hbase.apache.org/book.html#shell is supported. * * Requirements: - * Hbase Shell should be installed on the same machine. To be more specific, the following dir. + * HBase Shell should be installed on the same machine. To be more specific, the following dir. * should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby - * Hbase Shell should be able to connect to the Hbase cluster from terminal. This makes sure + * HBase Shell should be able to connect to the HBase cluster from terminal. This makes sure * that the client is configured properly. * * The interpreter takes 3 config parameters: - * hbase.home: Root dir. where hbase is installed. Default is /usr/lib/hbase/ + * hbase.home: Root directory where HBase is installed. Default is /usr/lib/hbase/ * hbase.ruby.sources: Dir where shell ruby code is installed. * Path is relative to hbase.home. Default: lib/ruby - * hbase.irb.load: (Testing only) Default is true. - * Whether to load irb in the interpreter. + * zeppelin.hbase.test.mode: (Testing only) Disable checks for unit and manual tests. Default: false */ public class HbaseInterpreter extends Interpreter { private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class); @@ -62,11 +61,13 @@ public class HbaseInterpreter extends Interpreter { static { Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(), new InterpreterPropertyBuilder() - .add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase") + .add("hbase.home", + getSystemDefault("HBASE_HOME", "hbase.home", "/usr/lib/hbase/"), + "Installation directory of HBase") .add("hbase.ruby.sources", "lib/ruby", "Path to Ruby scripts relative to 'hbase.home'") - .add("hbase.test.mode", "false", "Disable checks for unit and manual tests") - .build()); + .add("zeppelin.hbase.test.mode", "false", "Disable checks for unit and manual tests") + .build()); } public HbaseInterpreter(Properties property) { @@ -79,7 +80,7 @@ public class HbaseInterpreter extends Interpreter { this.writer = new StringWriter(); scriptingContainer.setOutput(this.writer); - if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) { + if (!Boolean.parseBoolean(getProperty("zeppelin.hbase.test.mode"))) { String hbase_home = getProperty("hbase.home"); String ruby_src = getProperty("hbase.ruby.sources"); Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath(); @@ -89,7 +90,7 @@ public class HbaseInterpreter extends Interpreter { File f = abs_ruby_src.toFile(); if (!f.exists() || !f.isDirectory()) { - throw new InterpreterException("hbase ruby sources is not available at '" + abs_ruby_src + throw new InterpreterException("HBase ruby sources is not available at '" + abs_ruby_src + "'"); } @@ -155,4 +156,24 @@ public class HbaseInterpreter extends Interpreter { return null; } + private static String getSystemDefault( + String envName, + String propertyName, + String defaultValue) { + + if (envName != null && !envName.isEmpty()) { + String envValue = System.getenv().get(envName); + if (envValue != null) { + return envValue; + } + } + + if (propertyName != null && !propertyName.isEmpty()) { + String propValue = System.getProperty(propertyName); + if (propValue != null) { + return propValue; + } + } + return defaultValue; + } } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/26a2d641/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java ---------------------------------------------------------------------- diff --git a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java index e218070..8c5e29c 100644 --- a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java +++ b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java @@ -40,7 +40,7 @@ public class HbaseInterpreterTest { Properties properties = new Properties(); properties.put("hbase.home", ""); properties.put("hbase.ruby.sources", ""); - properties.put("hbase.test.mode", "true"); + properties.put("zeppelin.hbase.test.mode", "true"); hbaseInterpreter = new HbaseInterpreter(properties); hbaseInterpreter.open(); @@ -72,4 +72,4 @@ public class HbaseInterpreterTest { assertEquals(InterpreterResult.Code.ERROR, result.code()); assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message()); } -} \ No newline at end of file +}
