ZEPPELIN-3196. Plugin framework for Zeppelin Engine ### What is this PR for? More details is in the design doc. https://docs.google.com/document/d/1QRgznfDKsi6XxR9FTJtUuiGAvgQvmc12FdIbHrmOZzc/edit?usp=sharing
### What type of PR is it? [ Improvement | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3196 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <zjf...@apache.org> Closes #2760 from zjffdu/ZEPPELIN-3196 and squashes the following commits: 639c599 [Jeff Zhang] ZEPPELIN-3196. Plugin framework for Zeppelin Engine Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/3eea57ab Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/3eea57ab Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/3eea57ab Branch: refs/heads/master Commit: 3eea57ab26e8c4a8811bde64854b95f706858f10 Parents: db716c8 Author: Jeff Zhang <zjf...@apache.org> Authored: Wed Jan 31 19:46:44 2018 +0800 Committer: Jeff Zhang <zjf...@apache.org> Committed: Thu Mar 22 21:19:25 2018 +0800 ---------------------------------------------------------------------- .travis.yml | 23 +- pom.xml | 11 + .../src/assemble/distribution.xml | 3 + .../zeppelin/conf/ZeppelinConfiguration.java | 9 + zeppelin-plugins/notebookrepo/azure/pom.xml | 71 +++ .../notebook/repo/AzureNotebookRepo.java | 216 +++++++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + .../notebookrepo/filesystem/pom.xml | 236 ++++++++++ .../notebook/repo/FileSystemNotebookRepo.java | 103 +++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + .../repo/FileSystemNotebookRepoTest.java | 102 +++++ zeppelin-plugins/notebookrepo/gcs/pom.xml | 141 ++++++ .../zeppelin/notebook/repo/GCSNotebookRepo.java | 214 +++++++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + .../notebook/repo/GCSNotebookRepoTest.java | 235 ++++++++++ zeppelin-plugins/notebookrepo/git/pom.xml | 71 +++ .../zeppelin/notebook/repo/GitNotebookRepo.java | 205 +++++++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + .../notebook/repo/GitNotebookRepoTest.java | 385 ++++++++++++++++ .../NotebookRepoSyncInitializationTest.java | 161 +++++++ .../notebook/repo/NotebookRepoSyncTest.java | 444 +++++++++++++++++++ .../notebook/repo/mock/VFSNotebookRepoMock.java | 36 ++ .../git/src/test/resources/2A94M5J1Z/note.json | 341 ++++++++++++++ .../git/src/test/resources/2A94M5J2Z/note.json | 87 ++++ .../git/src/test/resources/log4j.properties | 50 +++ zeppelin-plugins/notebookrepo/github/pom.xml | 50 +++ .../notebook/repo/GitHubNotebookRepo.java | 126 ++++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + .../notebook/repo/GitHubNotebookRepoTest.java | 207 +++++++++ .../src/test/resources/2A94M5J1Z/note.json | 341 ++++++++++++++ .../src/test/resources/2A94M5J2Z/note.json | 87 ++++ zeppelin-plugins/notebookrepo/mongodb/pom.xml | 55 +++ .../notebook/repo/MongoNotebookRepo.java | 222 ++++++++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + zeppelin-plugins/notebookrepo/s3/pom.xml | 51 +++ .../zeppelin/notebook/repo/S3NotebookRepo.java | 294 ++++++++++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + zeppelin-plugins/notebookrepo/vfs/pom.xml | 58 +++ .../zeppelin/notebook/repo/VFSNotebookRepo.java | 286 ++++++++++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + .../notebook/repo/TestVFSNotebookRepo.java | 113 +++++ .../notebookrepo/zeppelin-hub/pom.xml | 81 ++++ .../repo/zeppelinhub/ZeppelinHubRepo.java | 385 ++++++++++++++++ ...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 + .../repo/zeppelinhub/ZeppelinHubRepoTest.java | 155 +++++++ .../websocket/ZeppelinClientTest.java | 127 ++++++ .../websocket/ZeppelinhubClientTest.java | 72 +++ .../websocket/mock/MockEchoWebsocketServer.java | 46 ++ .../websocket/mock/MockEventServlet.java | 14 + .../websocket/mock/MockEventSocket.java | 38 ++ .../protocol/ZeppelinhubMessageTest.java | 43 ++ zeppelin-plugins/pom.xml | 124 ++++++ zeppelin-server/pom.xml | 36 -- .../org/apache/zeppelin/realm/PamRealm.java | 2 +- .../apache/zeppelin/realm/ZeppelinHubRealm.java | 26 +- .../apache/zeppelin/socket/NotebookServer.java | 5 +- .../src/test/resources/log4j.properties | 3 +- zeppelin-zengine/pom.xml | 298 +------------ .../recovery/FileSystemRecoveryStorage.java | 11 - .../java/org/apache/zeppelin/notebook/Note.java | 5 + .../org/apache/zeppelin/notebook/Notebook.java | 2 +- .../notebook/repo/AzureNotebookRepo.java | 209 --------- .../notebook/repo/FileSystemNotebookRepo.java | 99 ----- .../zeppelin/notebook/repo/GCSNotebookRepo.java | 208 --------- .../notebook/repo/GitHubNotebookRepo.java | 126 ------ .../zeppelin/notebook/repo/GitNotebookRepo.java | 190 -------- .../notebook/repo/MongoNotebookRepo.java | 217 --------- .../zeppelin/notebook/repo/NotebookRepo.java | 4 + .../notebook/repo/NotebookRepoSync.java | 45 +- .../zeppelin/notebook/repo/S3NotebookRepo.java | 290 ------------ .../zeppelin/notebook/repo/VFSNotebookRepo.java | 280 ------------ .../repo/zeppelinhub/ZeppelinHubRepo.java | 376 ---------------- .../websocket/ZeppelinhubClient.java | 4 +- .../apache/zeppelin/plugin/PluginManager.java | 95 ++++ .../helium/HeliumApplicationFactoryTest.java | 7 +- .../apache/zeppelin/notebook/NotebookTest.java | 324 ++++++++++---- .../repo/FileSystemNotebookRepoTest.java | 101 ----- .../notebook/repo/GCSNotebookRepoTest.java | 235 ---------- .../notebook/repo/GitHubNotebookRepoTest.java | 207 --------- .../notebook/repo/GitNotebookRepoTest.java | 385 ---------------- .../NotebookRepoSyncInitializationTest.java | 159 ------- .../notebook/repo/NotebookRepoSyncTest.java | 437 ------------------ .../notebook/repo/VFSNotebookRepoTest.java | 167 ------- .../notebook/repo/mock/VFSNotebookRepoMock.java | 42 -- .../repo/zeppelinhub/ZeppelinHubRepoTest.java | 154 ------- .../websocket/ZeppelinClientTest.java | 127 ------ .../websocket/ZeppelinhubClientTest.java | 72 --- .../websocket/mock/MockEchoWebsocketServer.java | 46 -- .../websocket/mock/MockEventServlet.java | 14 - .../websocket/mock/MockEventSocket.java | 38 -- .../protocol/ZeppelinhubMessageTest.java | 43 -- .../src/test/resources/2A94M5J1Z/note.json | 341 -------------- .../src/test/resources/2A94M5J2Z/note.json | 87 ---- .../src/test/resources/log4j.properties | 3 +- 94 files changed, 6651 insertions(+), 5150 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index e113ae3..5a81441 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ matrix: # Test License compliance using RAT tool - jdk: "openjdk7" dist: trusty - env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS="" + env: BUILD_PLUGINS="false" SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS="" # Run e2e tests (in zeppelin-web) # chrome dropped the support for precise (ubuntu 12.04), so need to use trusty @@ -53,7 +53,7 @@ matrix: sudo: false dist: trusty jdk: "oraclejdk8" - env: CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e" + env: BUILD_PLUGINS="false" CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e" addons: apt: packages: @@ -66,19 +66,19 @@ matrix: - sudo: required jdk: "oraclejdk8" dist: trusty - env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" + env: BUILD_PLUGINS="true" PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="install -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk8" dist: trusty addons: firefox: "31.0" - env: CI="true" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false" + env: BUILD_PLUGINS="true" CI="true" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false" # Test interpreter modules - jdk: "openjdk7" dist: trusty - env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" + env: BUILD_PLUGINS="false" PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" # Run ZeppelinSparkClusterTest & SparkIntegrationTest in one build would exceed the time limitation of travis, so running them separately @@ -86,35 +86,35 @@ matrix: - sudo: required jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: BUILD_PLUGINS="true" PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Integration test of spark interpreter with different spark versions under python3, only run SparkIntegrationTest. Also run spark unit test of spark 1.6 in this build. - sudo: required jdk: "oraclejdk8" dist: trusty - env: PYTHON="3" PROFILE="-Pspark-1.6" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: BUILD_PLUGINS="true" PYTHON="3" PROFILE="-Pspark-1.6" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 2.1.0 with scala 2.11 - jdk: "openjdk7" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 2.0.2 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test python/pyspark with python 2, livy 0.5 - sudo: required dist: trusty jdk: "openjdk7" - env: PYTHON="2" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="" + env: BUILD_PLUGINS="false" PYTHON="2" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="" # Test livy 0.5 with spark 2.2.0 under python3 - sudo: required dist: trusty jdk: "openjdk8" - env: PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="" + env: BUILD_PLUGINS="false" PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="" before_install: # check files included in commit range, clear bower_components if a bower.json file has changed. @@ -136,6 +136,7 @@ before_install: install: - echo "mvn $BUILD_FLAG $MODULES $PROFILE -B" - mvn $BUILD_FLAG $MODULES $PROFILE -B + - if [ $BUILD_PLUGINS == "true" ]; then echo "mvn clean package -pl zeppelin-plugins -amd -B"; mvn clean package -pl zeppelin-plugins -amd -B; fi before_script: - if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ca6c409..a2b5553 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,7 @@ <module>zeppelin-web</module> <module>zeppelin-server</module> <module>zeppelin-jupyter</module> + <module>zeppelin-plugins</module> <module>zeppelin-distribution</module> </modules> @@ -115,6 +116,7 @@ <commons.logging.version>1.1.1</commons.logging.version> <commons.cli.version>1.3.1</commons.cli.version> <shiro.version>1.3.2</shiro.version> + <joda.version>2.9.9</joda.version> <!-- test library versions --> <junit.version>4.12</junit.version> @@ -247,6 +249,12 @@ <version>${commons.cli.version}</version> </dependency> + <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + <version>${joda.version}</version> + </dependency> + <!-- Apache Shiro --> <dependency> <groupId>org.apache.shiro</groupId> @@ -592,6 +600,9 @@ <version>${plugin.surefire.version}</version> <configuration combine.children="append"> <argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine> + <environmentVariables> + <IS_ZEPPELIN_TEST>true</IS_ZEPPELIN_TEST> + </environmentVariables> <excludes> <exclude>${tests.to.exclude}</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-distribution/src/assemble/distribution.xml ---------------------------------------------------------------------- diff --git a/zeppelin-distribution/src/assemble/distribution.xml b/zeppelin-distribution/src/assemble/distribution.xml index 5c369e2..068f85b 100644 --- a/zeppelin-distribution/src/assemble/distribution.xml +++ b/zeppelin-distribution/src/assemble/distribution.xml @@ -92,6 +92,9 @@ <directory>../notebook</directory> </fileSet> <fileSet> + <directory>../plugins</directory> + </fileSet> + <fileSet> <outputDirectory>/lib/interpreter</outputDirectory> <directory>../zeppelin-interpreter/target/lib</directory> </fileSet> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index db6e150..2960cd0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -347,10 +347,18 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR); } + public String getPluginsDir() { + return getRelativeDir(getString(ConfVars.ZEPPELIN_PLUGINS_DIR)); + } + public String getRecoveryDir() { return getRelativeDir(ConfVars.ZEPPELIN_RECOVERY_DIR); } + public String getNotebookStorageClass() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE); + } + public String getRecoveryStorageClass() { return getString(ConfVars.ZEPPELIN_RECOVERY_STORAGE_CLASS); } @@ -711,6 +719,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_RECOVERY_DIR("zeppelin.recovery.dir", "recovery"), ZEPPELIN_RECOVERY_STORAGE_CLASS("zeppelin.recovery.storage.class", "org.apache.zeppelin.interpreter.recovery.NullRecoveryStorage"), + ZEPPELIN_PLUGINS_DIR("zeppelin.plugins.dir", "plugins"), // use specified notebook (id) as homescreen ZEPPELIN_NOTEBOOK_HOMESCREEN("zeppelin.notebook.homescreen", null), http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/azure/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/azure/pom.xml b/zeppelin-plugins/notebookrepo/azure/pom.xml new file mode 100644 index 0000000..f663de6 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/azure/pom.xml @@ -0,0 +1,71 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-azure</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin AzureNotebookRepo</name> + <description>NotebookRepo implementation based on Azure</description> + + <properties> + <adl.sdk.version>2.1.4</adl.sdk.version> + <azure.storage.version>4.0.0</azure.storage.version> + <plugin.name>NotebookRepo/AzureNotebookRepo</plugin.name> + </properties> + + <dependencies> + <dependency> + <groupId>com.microsoft.azure</groupId> + <artifactId>azure-data-lake-store-sdk</artifactId> + <version>${adl.sdk.version}</version> + </dependency> + <dependency> + <groupId>com.microsoft.azure</groupId> + <artifactId>azure-storage</artifactId> + <version>${azure.storage.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java new file mode 100644 index 0000000..12dbd90 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.file.CloudFile; +import com.microsoft.azure.storage.file.CloudFileClient; +import com.microsoft.azure.storage.file.CloudFileDirectory; +import com.microsoft.azure.storage.file.CloudFileShare; +import com.microsoft.azure.storage.file.ListFileItem; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.net.URISyntaxException; +import java.security.InvalidKeyException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Azure storage backend for notebooks + */ +public class AzureNotebookRepo implements NotebookRepo { + private static final Logger LOG = LoggerFactory.getLogger(AzureNotebookRepo.class); + + private ZeppelinConfiguration conf; + private String user; + private String shareName; + private CloudFileDirectory rootDir; + + public AzureNotebookRepo() { + + } + + public void init(ZeppelinConfiguration conf) throws IOException { + this.conf = conf; + user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER); + shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE); + + try { + CloudStorageAccount account = CloudStorageAccount.parse( + conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING)); + CloudFileClient client = account.createCloudFileClient(); + CloudFileShare share = client.getShareReference(shareName); + share.createIfNotExists(); + + CloudFileDirectory userDir = StringUtils.isBlank(user) ? + share.getRootDirectoryReference() : + share.getRootDirectoryReference().getDirectoryReference(user); + userDir.createIfNotExists(); + + rootDir = userDir.getDirectoryReference("notebook"); + rootDir.createIfNotExists(); + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + List<NoteInfo> infos = new LinkedList<>(); + NoteInfo info = null; + + for (ListFileItem item : rootDir.listFilesAndDirectories()) { + if (item.getClass() == CloudFileDirectory.class) { + CloudFileDirectory dir = (CloudFileDirectory) item; + + try { + if (dir.getFileReference("note.json").exists()) { + info = new NoteInfo(getNote(dir.getName())); + + if (info != null) { + infos.add(info); + } + } + } catch (StorageException | URISyntaxException e) { + String msg = "Error enumerating notebooks from Azure storage"; + LOG.error(msg, e); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + } + + return infos; + } + + private Note getNote(String noteId) throws IOException { + InputStream ins = null; + + try { + CloudFileDirectory dir = rootDir.getDirectoryReference(noteId); + CloudFile file = dir.getFileReference("note.json"); + + ins = file.openRead(); + } catch (URISyntaxException | StorageException e) { + String msg = String.format("Error reading notebook %s from Azure storage", noteId); + + LOG.error(msg, e); + + throw new IOException(msg, e); + } + + String json = IOUtils.toString(ins, + conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING)); + ins.close(); + return Note.fromJson(json); + } + + @Override + public Note get(String noteId, AuthenticationInfo subject) throws IOException { + return getNote(noteId); + } + + @Override + public void save(Note note, AuthenticationInfo subject) throws IOException { + String json = note.toJson(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Writer writer = new OutputStreamWriter(output); + writer.write(json); + writer.close(); + output.close(); + + byte[] buffer = output.toByteArray(); + + try { + CloudFileDirectory dir = rootDir.getDirectoryReference(note.getId()); + dir.createIfNotExists(); + + CloudFile cloudFile = dir.getFileReference("note.json"); + cloudFile.uploadFromByteArray(buffer, 0, buffer.length); + } catch (URISyntaxException | StorageException e) { + String msg = String.format("Error saving notebook %s to Azure storage", note.getId()); + + LOG.error(msg, e); + + throw new IOException(msg, e); + } + } + + // unfortunately, we need to use a recursive delete here + private void delete(ListFileItem item) throws StorageException { + if (item.getClass() == CloudFileDirectory.class) { + CloudFileDirectory dir = (CloudFileDirectory) item; + + for (ListFileItem subItem : dir.listFilesAndDirectories()) { + delete(subItem); + } + + dir.deleteIfExists(); + } else if (item.getClass() == CloudFile.class) { + CloudFile file = (CloudFile) item; + + file.deleteIfExists(); + } + } + + @Override + public void remove(String noteId, AuthenticationInfo subject) throws IOException { + try { + CloudFileDirectory dir = rootDir.getDirectoryReference(noteId); + + delete(dir); + } catch (URISyntaxException | StorageException e) { + String msg = String.format("Error deleting notebook %s from Azure storage", noteId); + + LOG.error(msg, e); + + throw new IOException(msg, e); + } + } + + @Override + public void close() { + } + + @Override + public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { + LOG.warn("Method not implemented"); + return Collections.emptyList(); + } + + @Override + public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { + LOG.warn("Method not implemented"); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..b028d54 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.AzureNotebookRepo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/filesystem/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/pom.xml b/zeppelin-plugins/notebookrepo/filesystem/pom.xml new file mode 100644 index 0000000..422b7b6 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/filesystem/pom.xml @@ -0,0 +1,236 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-filesystem</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin FileSystemNotebookRepo</name> + <description>NotebookRepo implementation based on Hadoop FileSystem</description> + + <properties> + <adl.sdk.version>2.1.4</adl.sdk.version> + <plugin.name>NotebookRepo/FileSystemNotebookRepo</plugin.name> + </properties> + + <profiles> + + <profile> + <id>hadoop2-azure</id> + <properties> + <hadoop.version>2.7.3</hadoop.version> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-azure</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </exclusion> + <exclusion> + <groupId>com.jcraf</groupId> + <artifactId>jsch</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.microsoft.azure</groupId> + <artifactId>azure-data-lake-store-sdk</artifactId> + <version>${adl.sdk.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </profile> + + <profile> + <id>hadoop2-aws</id> + <properties> + <hadoop.version>2.7.3</hadoop.version> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-aws</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + <exclusion> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </profile> + + <profile> + <id>hadoop3-azure</id> + <properties> + <hadoop.version>3.0.0</hadoop.version> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-azure</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>com.jcraft</groupId> + <artifactId>jsch</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>com.nimbusds</groupId> + <artifactId>nimbus-jose-jwt</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-azure-datalake</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </profile> + + <profile> + <id>hadoop3-aws</id> + <properties> + <hadoop.version>3.0.0</hadoop.version> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-aws</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + <exclusion> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java new file mode 100644 index 0000000..5d9c85c --- /dev/null +++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java @@ -0,0 +1,103 @@ +package org.apache.zeppelin.notebook.repo; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.FileSystemStorage; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * NotebookRepos for hdfs. + * + * Assume the notebook directory structure is as following + * - notebookdir + * - noteId/note.json + * - noteId/note.json + * - noteId/note.json + */ +public class FileSystemNotebookRepo implements NotebookRepo { + private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemNotebookRepo.class); + + private FileSystemStorage fs; + private Path notebookDir; + + public FileSystemNotebookRepo() { + + } + + public void init(ZeppelinConfiguration zConf) throws IOException { + this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir()); + LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() + + " for Zeppelin Notebook."); + this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir())); + LOGGER.info("Using folder {} to store notebook", notebookDir); + this.fs.tryMkDir(notebookDir); + } + + @Override + public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + List<Path> notePaths = fs.list(new Path(notebookDir, "*/note.json")); + List<NoteInfo> noteInfos = new ArrayList<>(); + for (Path path : notePaths) { + NoteInfo noteInfo = new NoteInfo(path.getParent().getName(), "", null); + noteInfos.add(noteInfo); + } + return noteInfos; + } + + @Override + public Note get(final String noteId, AuthenticationInfo subject) throws IOException { + String content = this.fs.readFile( + new Path(notebookDir.toString() + "/" + noteId + "/note.json")); + return Note.fromJson(content); + } + + @Override + public void save(final Note note, AuthenticationInfo subject) throws IOException { + this.fs.writeFile(note.toJson(), + new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"), + true); + } + + @Override + public void remove(final String noteId, AuthenticationInfo subject) throws IOException { + this.fs.delete(new Path(notebookDir.toString() + "/" + noteId)); + } + + @Override + public void close() { + LOGGER.warn("close is not implemented for HdfsNotebookRepo"); + } + + @Override + public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { + LOGGER.warn("getSettings is not implemented for HdfsNotebookRepo"); + return null; + } + + @Override + public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { + LOGGER.warn("updateSettings is not implemented for HdfsNotebookRepo"); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..06ef085 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java new file mode 100644 index 0000000..5fd8becc --- /dev/null +++ b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java @@ -0,0 +1,102 @@ +package org.apache.zeppelin.notebook.repo; + + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class FileSystemNotebookRepoTest { + + private ZeppelinConfiguration zConf; + private Configuration hadoopConf; + private FileSystem fs; + private FileSystemNotebookRepo hdfsNotebookRepo; + private String notebookDir; + private AuthenticationInfo authInfo = AuthenticationInfo.ANONYMOUS; + + @Before + public void setUp() throws IOException { + notebookDir = Files.createTempDirectory("FileSystemNotebookRepoTest").toFile().getAbsolutePath(); + zConf = new ZeppelinConfiguration(); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir); + hadoopConf = new Configuration(); + fs = FileSystem.get(hadoopConf); + hdfsNotebookRepo = new FileSystemNotebookRepo(); + hdfsNotebookRepo.init(zConf); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(new File(notebookDir)); + } + + @Test + public void testBasics() throws IOException { + assertEquals(0, hdfsNotebookRepo.list(authInfo).size()); + + // create a new note + Note note = new Note(); + note.setName("title_1"); + + Map<String, Object> config = new HashMap<>(); + config.put("config_1", "value_1"); + note.setConfig(config); + hdfsNotebookRepo.save(note, authInfo); + assertEquals(1, hdfsNotebookRepo.list(authInfo).size()); + + // read this note from hdfs + Note note_copy = hdfsNotebookRepo.get(note.getId(), authInfo); + assertEquals(note.getName(), note_copy.getName()); + assertEquals(note.getConfig(), note_copy.getConfig()); + + // update this note + note.setName("title_2"); + hdfsNotebookRepo.save(note, authInfo); + assertEquals(1, hdfsNotebookRepo.list(authInfo).size()); + note_copy = hdfsNotebookRepo.get(note.getId(), authInfo); + assertEquals(note.getName(), note_copy.getName()); + assertEquals(note.getConfig(), note_copy.getConfig()); + + // delete this note + hdfsNotebookRepo.remove(note.getId(), authInfo); + assertEquals(0, hdfsNotebookRepo.list(authInfo).size()); + } + + @Test + public void testComplicatedScenarios() throws IOException { + // scenario_1: notebook_dir is not clean. There're some unrecognized dir and file under notebook_dir + fs.mkdirs(new Path(notebookDir, "1/2")); + OutputStream out = fs.create(new Path(notebookDir, "1/a.json")); + out.close(); + + assertEquals(0, hdfsNotebookRepo.list(authInfo).size()); + + // scenario_2: note_folder is existed. + // create a new note + Note note = new Note(); + note.setName("title_1"); + Map<String, Object> config = new HashMap<>(); + config.put("config_1", "value_1"); + note.setConfig(config); + + fs.mkdirs(new Path(notebookDir, note.getId())); + hdfsNotebookRepo.save(note, authInfo); + assertEquals(1, hdfsNotebookRepo.list(authInfo).size()); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/gcs/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/pom.xml b/zeppelin-plugins/notebookrepo/gcs/pom.xml new file mode 100644 index 0000000..15aa7c7 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/gcs/pom.xml @@ -0,0 +1,141 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-gcs</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin GCSNotebookRepo</name> + <description>NotebookRepo implementation based on Google Cloud Storage</description> + + <properties> + <gcs.storage.version>1.14.0</gcs.storage.version> + <google.testing.nio.version>0.32.0-alpha</google.testing.nio.version> + <google.truth.version>0.27</google.truth.version> + <plugin.name>NotebookRepo/GCSNotebookRepo</plugin.name> + </properties> + + <dependencies> + <dependency> + <groupId>com.google.cloud</groupId> + <artifactId>google-cloud-storage</artifactId> + <version>${gcs.storage.version}</version> + <exclusions> + <exclusion> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.api</groupId> + <artifactId>api-common</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.http-client</groupId> + <artifactId>google-http-client-jackson2</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.http-client</groupId> + <artifactId>google-http-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.google.cloud</groupId> + <artifactId>google-cloud-nio</artifactId> + <version>${google.testing.nio.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.google.truth</groupId> + <artifactId>truth</artifactId> + <version>${google.truth.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.google.api</groupId> + <artifactId>api-common</artifactId> + <version>1.2.0</version> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.google.http-client</groupId> + <artifactId>google-http-client-jackson2</artifactId> + <version>1.23.0</version> + <exclusions> + <exclusion> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java new file mode 100644 index 0000000..ad99aba --- /dev/null +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.gson.JsonParseException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A NotebookRepo implementation for storing notebooks in Google Cloud Storage. + * + * Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path + * must be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an + * object store, so this "directory" should not itself be an object. Instead, it represents the base + * path for the note.json files. + * + * Authentication is provided by google-auth-library-java. + * @see <a href="https://github.com/google/google-auth-library-java";> + * google-auth-library-java</a>. + */ +public class GCSNotebookRepo implements NotebookRepo { + + private static final Logger LOG = LoggerFactory.getLogger(GCSNotebookRepo.class); + private String encoding; + private String bucketName; + private Optional<String> basePath; + private Pattern noteNamePattern; + private Storage storage; + + public GCSNotebookRepo() { + } + + @VisibleForTesting + public GCSNotebookRepo(ZeppelinConfiguration zConf, Storage storage) throws IOException { + init(zConf); + this.storage = storage; + } + + @Override + public void init(ZeppelinConfiguration zConf) throws IOException { + this.encoding = zConf.getString(ConfVars.ZEPPELIN_ENCODING); + + String gcsStorageDir = zConf.getGCSStorageDir(); + if (gcsStorageDir.isEmpty()) { + throw new IOException("GCS storage directory must be set using 'zeppelin.notebook.gcs.dir'"); + } + if (!gcsStorageDir.startsWith("gs://")) { + throw new IOException(String.format( + "GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir)); + } + String storageDirWithoutScheme = gcsStorageDir.substring("gs://".length()); + + // pathComponents excludes empty string if trailing slash is present + List<String> pathComponents = Arrays.asList(storageDirWithoutScheme.split("/")); + if (pathComponents.size() < 1) { + throw new IOException(String.format( + "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir", + gcsStorageDir)); + } + this.bucketName = pathComponents.get(0); + if (pathComponents.size() > 1) { + this.basePath = Optional.of(StringUtils.join( + pathComponents.subList(1, pathComponents.size()), "/")); + } else { + this.basePath = Optional.absent(); + } + + // Notes are stored at gs://bucketName/basePath/<note-id>/note.json + if (basePath.isPresent()) { + this.noteNamePattern = Pattern.compile( + "^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$"); + } else { + this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$"); + } + + this.storage = StorageOptions.getDefaultInstance().getService(); + } + + private BlobId makeBlobId(String noteId) { + if (basePath.isPresent()) { + return BlobId.of(bucketName, basePath.get() + "/" + noteId + "/note.json"); + } else { + return BlobId.of(bucketName, noteId + "/note.json"); + } + } + + @Override + public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + try { + List<NoteInfo> infos = new ArrayList<>(); + Iterable<Blob> blobsUnderDir; + if (basePath.isPresent()) { + blobsUnderDir = storage + .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/")) + .iterateAll(); + } else { + blobsUnderDir = storage + .list(bucketName) + .iterateAll(); + } + for (Blob b : blobsUnderDir) { + Matcher matcher = noteNamePattern.matcher(b.getName()); + if (matcher.matches()) { + // Callers only use the id field, so do not fetch each note + // This matches the implementation in FileSystemNoteRepo#list + infos.add(new NoteInfo(matcher.group(1), "", null)); + } + } + return infos; + } catch (StorageException se) { + throw new IOException("Could not list GCS directory: " + se.getMessage(), se); + } + } + + @Override + public Note get(String noteId, AuthenticationInfo subject) throws IOException { + BlobId blobId = makeBlobId(noteId); + byte[] contents; + try { + contents = storage.readAllBytes(blobId); + } catch (StorageException se) { + throw new IOException("Could not read " + blobId.toString() + ": " + se.getMessage(), se); + } + + try { + return Note.fromJson(new String(contents, encoding)); + } catch (JsonParseException jpe) { + throw new IOException( + "Could note parse as json " + blobId.toString() + jpe.getMessage(), jpe); + } + } + + @Override + public void save(Note note, AuthenticationInfo subject) throws IOException { + BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) + .setContentType("application/json") + .build(); + try { + storage.create(info, note.toJson().getBytes("UTF-8")); + } catch (StorageException se) { + throw new IOException("Could not write " + info.toString() + ": " + se.getMessage(), se); + } + } + + @Override + public void remove(String noteId, AuthenticationInfo subject) throws IOException { + Preconditions.checkArgument(!Strings.isNullOrEmpty(noteId)); + BlobId blobId = makeBlobId(noteId); + try { + boolean deleted = storage.delete(blobId); + if (!deleted) { + throw new IOException("Tried to remove nonexistent blob " + blobId.toString()); + } + } catch (StorageException se) { + throw new IOException("Could not remove " + blobId.toString() + ": " + se.getMessage(), se); + } + } + + @Override + public void close() { + //no-op + } + + @Override + public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { + LOG.warn("getSettings is not implemented for GCSNotebookRepo"); + return Collections.emptyList(); + } + + @Override + public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { + LOG.warn("updateSettings is not implemented for GCSNotebookRepo"); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..784027b --- /dev/null +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.GCSNotebookRepo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java new file mode 100644 index 0000000..c1fae67 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.fail; + +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class GCSNotebookRepoTest { + private static final AuthenticationInfo AUTH_INFO = AuthenticationInfo.ANONYMOUS; + + private GCSNotebookRepo notebookRepo; + private Storage storage; + + @Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + { "bucketname", Optional.absent(), "gs://bucketname" }, + { "bucketname-with-slash", Optional.absent(), "gs://bucketname-with-slash/" }, + { "bucketname", Optional.of("path/to/dir"), "gs://bucketname/path/to/dir" }, + { "bucketname", Optional.of("trailing/slash"), "gs://bucketname/trailing/slash/" } + }); + } + + @Parameter(0) + public String bucketName; + + @Parameter(1) + public Optional<String> basePath; + + @Parameter(2) + public String uriPath; + + private Note runningNote; + + @Before + public void setUp() throws Exception { + this.runningNote = makeRunningNote(); + + this.storage = LocalStorageHelper.getOptions().getService(); + + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), uriPath); + this.notebookRepo = new GCSNotebookRepo(new ZeppelinConfiguration(), storage); + } + + private static Note makeRunningNote() { + Note note = new Note(); + note.setConfig(ImmutableMap.<String, Object>of("key", "value")); + + Paragraph p = new Paragraph(note, null, null); + p.setText("text"); + p.setStatus(Status.RUNNING); + note.addParagraph(p); + return note; + } + + @Test + public void testList_nonexistent() throws Exception { + assertThat(notebookRepo.list(AUTH_INFO)).isEmpty(); + } + + @Test + public void testList() throws Exception { + createAt(runningNote, "note.json"); + createAt(runningNote, "/note.json"); + createAt(runningNote, "validid/note.json"); + createAt(runningNote, "validid-2/note.json"); + createAt(runningNote, "cannot-be-dir/note.json/foo"); + createAt(runningNote, "cannot/be/nested/note.json"); + + List<NoteInfo> infos = notebookRepo.list(AUTH_INFO); + List<String> noteIds = new ArrayList<>(); + for (NoteInfo info : infos) { + noteIds.add(info.getId()); + } + // Only valid paths are gs://bucketname/path/<noteid>/note.json + assertThat(noteIds).containsExactlyElementsIn(ImmutableList.of("validid", "validid-2")); + } + + @Test + public void testGet_nonexistent() throws Exception { + try { + notebookRepo.get("id", AUTH_INFO); + fail(); + } catch (IOException e) {} + } + + @Test + public void testGet() throws Exception { + create(runningNote); + + // Status of saved running note is removed in get() + Note got = notebookRepo.get(runningNote.getId(), AUTH_INFO); + assertThat(got.getLastParagraph().getStatus()).isEqualTo(Status.ABORT); + + // But otherwise equal + got.getLastParagraph().setStatus(Status.RUNNING); + assertThat(got).isEqualTo(runningNote); + } + + @Test + public void testGet_malformed() throws Exception { + createMalformed("id"); + try { + notebookRepo.get("id", AUTH_INFO); + fail(); + } catch (IOException e) {} + } + + @Test + public void testSave_create() throws Exception { + notebookRepo.save(runningNote, AUTH_INFO); + // Output is saved + assertThat(storage.readAllBytes(makeBlobId(runningNote.getId()))) + .isEqualTo(runningNote.toJson().getBytes("UTF-8")); + } + + @Test + public void testSave_update() throws Exception { + notebookRepo.save(runningNote, AUTH_INFO); + // Change name of runningNote + runningNote.setName("new-name"); + notebookRepo.save(runningNote, AUTH_INFO); + assertThat(storage.readAllBytes(makeBlobId(runningNote.getId()))) + .isEqualTo(runningNote.toJson().getBytes("UTF-8")); + } + + @Test + public void testRemove_nonexistent() throws Exception { + try { + notebookRepo.remove("id", AUTH_INFO); + fail(); + } catch (IOException e) {} + } + + @Test + public void testRemove() throws Exception { + create(runningNote); + notebookRepo.remove(runningNote.getId(), AUTH_INFO); + assertThat(storage.get(makeBlobId(runningNote.getId()))).isNull(); + } + + private String makeName(String relativePath) { + if (basePath.isPresent()) { + return basePath.get() + "/" + relativePath; + } else { + return relativePath; + } + } + + private BlobId makeBlobId(String noteId) { + return BlobId.of(bucketName, makeName(noteId + "/note.json")); + } + + private void createAt(Note note, String relativePath) throws IOException { + BlobId id = BlobId.of(bucketName, makeName(relativePath)); + BlobInfo info = BlobInfo.newBuilder(id).setContentType("application/json").build(); + storage.create(info, note.toJson().getBytes("UTF-8")); + } + + private void create(Note note) throws IOException { + BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) + .setContentType("application/json") + .build(); + storage.create(info, note.toJson().getBytes("UTF-8")); + } + + private void createMalformed(String noteId) throws IOException { + BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId)) + .setContentType("application/json") + .build(); + storage.create(info, "{ invalid-json }".getBytes("UTF-8")); + } + + /* These tests test path parsing for illegal paths, and do not use the parameterized vars */ + + @Test + public void testInitialization_pathNotSet() throws Exception { + try { + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), ""); + new GCSNotebookRepo(new ZeppelinConfiguration(), storage); + fail(); + } catch (IOException e) {} + } + + @Test + public void testInitialization_malformedPath() throws Exception { + try { + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "foo"); + new GCSNotebookRepo(new ZeppelinConfiguration(), storage); + fail(); + } catch (IOException e) {} + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/pom.xml b/zeppelin-plugins/notebookrepo/git/pom.xml new file mode 100644 index 0000000..6f4ce8c --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/pom.xml @@ -0,0 +1,71 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-git</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin GitNotebookRepo</name> + <description>NotebookRepo implementation based on Git</description> + + <properties> + <eclipse.jgit.version>4.5.4.201711221230-r</eclipse.jgit.version> + <google.truth.version>0.27</google.truth.version> + <plugin.name>NotebookRepo/GitNotebookRepo</plugin.name> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-vfs</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.eclipse.jgit</groupId> + <artifactId>org.eclipse.jgit</artifactId> + <version>${eclipse.jgit.version}</version> + </dependency> + + <dependency> + <groupId>com.google.truth</groupId> + <artifactId>truth</artifactId> + <version>${google.truth.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> +</project>
