http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java new file mode 100644 index 0000000..04a59ad --- /dev/null +++ b/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + + +import com.google.common.base.Joiner; +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import static org.mockito.Mockito.mock; + +/** + * This tests the remote Git tracking for notebooks. The tests uses two local Git repositories created locally + * to handle the tracking of Git actions (pushes and pulls). The repositories are: + * 1. The first repository is considered as a remote that mimics a remote GitHub directory + * 2. The second repository is considered as the local notebook repository + */ +public class GitHubNotebookRepoTest { + private static final Logger LOG = LoggerFactory.getLogger(GitHubNotebookRepoTest.class); + + private static final String TEST_NOTE_ID = "2A94M5J1Z"; + + private File remoteZeppelinDir; + private File localZeppelinDir; + private String localNotebooksDir; + private String remoteNotebooksDir; + private ZeppelinConfiguration conf; + private GitHubNotebookRepo gitHubNotebookRepo; + private RevCommit firstCommitRevision; + private Git remoteGit; + + @Before + public void setUp() throws Exception { + conf = ZeppelinConfiguration.create(); + + String remoteRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTestRemote_" + + System.currentTimeMillis(); + String localRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + + System.currentTimeMillis(); + + // Create a fake remote notebook Git repository locally in another directory + remoteZeppelinDir = new File(remoteRepositoryPath); + remoteZeppelinDir.mkdirs(); + + // Create a local repository for notebooks + localZeppelinDir = new File(localRepositoryPath); + localZeppelinDir.mkdirs(); + + // Notebooks directory (for both the remote and local directories) + localNotebooksDir = Joiner.on(File.separator).join(localRepositoryPath, "notebook"); + remoteNotebooksDir = Joiner.on(File.separator).join(remoteRepositoryPath, "notebook"); + + File notebookDir = new File(localNotebooksDir); + notebookDir.mkdirs(); + + // Copy the test notebook directory from the test/resources/2A94M5J1Z folder to the fake remote Git directory + String remoteTestNoteDir = Joiner.on(File.separator).join(remoteNotebooksDir, TEST_NOTE_ID); + FileUtils.copyDirectory( + new File( + GitHubNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID) + ).getFile() + ), new File(remoteTestNoteDir) + ); + + // Create the fake remote Git repository + Repository remoteRepository = new FileRepository(Joiner.on(File.separator).join(remoteNotebooksDir, ".git")); + remoteRepository.create(); + + remoteGit = new Git(remoteRepository); + remoteGit.add().addFilepattern(".").call(); + firstCommitRevision = remoteGit.commit().setMessage("First commit from remote repository").call(); + + // Set the Git and Git configurations + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), remoteZeppelinDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); + + // Set the GitHub configurations + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), + "org.apache.zeppelin.notebook.repo.GitHubNotebookRepo"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL.getVarName(), + remoteNotebooksDir + File.separator + ".git"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME.getVarName(), "token"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN.getVarName(), + "access-token"); + + // Create the Notebook repository (configured for the local repository) + gitHubNotebookRepo = new GitHubNotebookRepo(conf); + } + + @After + public void tearDown() throws Exception { + // Cleanup the temporary folders uses as Git repositories + File[] temporaryFolders = { remoteZeppelinDir, localZeppelinDir }; + + for(File temporaryFolder : temporaryFolders) { + if (!FileUtils.deleteQuietly(temporaryFolder)) + LOG.error("Failed to delete {} ", temporaryFolder.getName()); + } + } + + @Test + /** + * Test the case when the Notebook repository is created, it pulls the latest changes from the remote repository + */ + public void pullChangesFromRemoteRepositoryOnLoadingNotebook() throws IOException, GitAPIException { + NotebookRepoWithVersionControl.Revision firstHistoryRevision = gitHubNotebookRepo.revisionHistory(TEST_NOTE_ID, null).get(0); + + assert(this.firstCommitRevision.getName().equals(firstHistoryRevision.id)); + } + + @Test + /** + * Test the case when the check-pointing (add new files and commit) it also pulls the latest changes from the + * remote repository + */ + public void pullChangesFromRemoteRepositoryOnCheckpointing() throws GitAPIException, IOException { + // Create a new commit in the remote repository + RevCommit secondCommitRevision = remoteGit.commit().setMessage("Second commit from remote repository").call(); + + // Add a new paragraph to the local repository + addParagraphToNotebook(TEST_NOTE_ID); + + // Commit and push the changes to remote repository + NotebookRepoWithVersionControl.Revision thirdCommitRevision = gitHubNotebookRepo.checkpoint( + TEST_NOTE_ID, "Third commit from local repository", null); + + // Check all the commits as seen from the local repository. The commits are ordered chronologically. The last + // commit is the first in the commit logs. + Iterator<RevCommit> revisions = gitHubNotebookRepo.getGit().log().all().call().iterator(); + + revisions.next(); // The Merge `master` commit after pushing to the remote repository + + assert(thirdCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph + + // The second commit done on the remote repository + assert(secondCommitRevision.getName().equals(revisions.next().getName())); + + // The first commit done on the remote repository + assert(firstCommitRevision.getName().equals(revisions.next().getName())); + } + + @Test + /** + * Test the case when the check-pointing (add new files and commit) it pushes the local commits to the remote + * repository + */ + public void pushLocalChangesToRemoteRepositoryOnCheckpointing() throws IOException, GitAPIException { + // Add a new paragraph to the local repository + addParagraphToNotebook(TEST_NOTE_ID); + + // Commit and push the changes to remote repository + NotebookRepoWithVersionControl.Revision secondCommitRevision = gitHubNotebookRepo.checkpoint( + TEST_NOTE_ID, "Second commit from local repository", null); + + // Check all the commits as seen from the remote repository. The commits are ordered chronologically. The last + // commit is the first in the commit logs. + Iterator<RevCommit> revisions = remoteGit.log().all().call().iterator(); + + assert(secondCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph + + // The first commit done on the remote repository + assert(firstCommitRevision.getName().equals(revisions.next().getName())); + } + + private void addParagraphToNotebook(String noteId) throws IOException { + Note note = gitHubNotebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + Paragraph paragraph = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + paragraph.setText("%md text"); + gitHubNotebookRepo.save(note, null); + } +}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json b/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json new file mode 100644 index 0000000..785ccea --- /dev/null +++ b/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json @@ -0,0 +1,341 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836981412_-1007008116", + "id": "20150213-231621_168813393", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:16:21 PM", + "dateStarted": "Apr 1, 2015 9:11:09 PM", + "dateFinished": "Apr 1, 2015 9:11:10 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "title": "Load data into table", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\";),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "title": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500779206_-1502780787", + "id": "20150210-015259_1403135953", + "result": { + "code": "SUCCESS", + "type": "TEXT", + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n" + }, + "dateCreated": "Feb 10, 2015 1:52:59 AM", + "dateStarted": "Jul 3, 2015 1:43:40 PM", + "dateFinished": "Jul 3, 2015 1:43:45 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500782552_-1439281894", + "id": "20150210-015302_1492795503", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n" + }, + "dateCreated": "Feb 10, 2015 1:53:02 AM", + "dateStarted": "Jul 3, 2015 1:43:17 PM", + "dateFinished": "Jul 3, 2015 1:43:23 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "maxAge": "35" + }, + "forms": { + "maxAge": { + "name": "maxAge", + "defaultValue": "30", + "hidden": false + } + } + }, + "jobName": "paragraph_1423720444030_-1424110477", + "id": "20150212-145404_867439529", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n" + }, + "dateCreated": "Feb 12, 2015 2:54:04 PM", + "dateStarted": "Jul 3, 2015 1:43:28 PM", + "dateFinished": "Jul 3, 2015 1:43:29 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "marital": "single" + }, + "forms": { + "marital": { + "name": "marital", + "defaultValue": "single", + "options": [ + { + "value": "single" + }, + { + "value": "divorced" + }, + { + "value": "married" + } + ], + "hidden": false + } + } + }, + "jobName": "paragraph_1423836262027_-210588283", + "id": "20150213-230422_1600658137", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n" + }, + "dateCreated": "Feb 13, 2015 11:04:22 PM", + "dateStarted": "Jul 3, 2015 1:43:33 PM", + "dateFinished": "Jul 3, 2015 1:43:34 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Apr 1, 2015 9:12:18 PM", + "dateFinished": "Apr 1, 2015 9:12:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```";, + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"; + }, + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "dateStarted": "Jul 3, 2015 1:44:56 PM", + "dateFinished": "Jul 3, 2015 1:44:56 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1435955447812_-158639899", + "id": "20150703-133047_853701097", + "dateCreated": "Jul 3, 2015 1:30:47 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Zeppelin Tutorial", + "id": "2A94M5J1Z", + "angularObjects": {}, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json b/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json new file mode 100644 index 0000000..79fe35c --- /dev/null +++ b/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json @@ -0,0 +1,87 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Apr 1, 2015 9:12:18 PM", + "dateFinished": "Apr 1, 2015 9:12:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```";, + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"; + }, + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "dateStarted": "Jul 3, 2015 1:44:56 PM", + "dateFinished": "Jul 3, 2015 1:44:56 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1435955447812_-158639899", + "id": "20150703-133047_853701097", + "dateCreated": "Jul 3, 2015 1:30:47 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Sample note - excerpt from Zeppelin Tutorial", + "id": "2A94M5J2Z", + "angularObjects": {}, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/mongodb/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/mongodb/pom.xml b/zeppelin-plugins/notebookrepo/mongodb/pom.xml new file mode 100644 index 0000000..2e0f90f --- /dev/null +++ b/zeppelin-plugins/notebookrepo/mongodb/pom.xml @@ -0,0 +1,55 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-mongodb</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin MongoNotebookRepo</name> + <description>NotebookRepo implementation based on Mongodb</description> + + <properties> + <plugin.name>NotebookRepo/MongoNotebookRepo</plugin.name> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-vfs</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.mongodb</groupId> + <artifactId>mongo-java-driver</artifactId> + <version>3.4.1</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java new file mode 100644 index 0000000..618568d --- /dev/null +++ b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java @@ -0,0 +1,222 @@ +package org.apache.zeppelin.notebook.repo; + +import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.client.model.Filters.in; +import static com.mongodb.client.model.Filters.type; + +import com.mongodb.MongoBulkWriteException; +import com.mongodb.MongoClient; +import com.mongodb.MongoClientURI; +import com.mongodb.bulk.BulkWriteError; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.InsertManyOptions; +import com.mongodb.client.model.UpdateOptions; +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.bson.BsonType; +import org.bson.Document; +import org.bson.types.ObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Backend for storing Notebook on MongoDB + */ +public class MongoNotebookRepo implements NotebookRepo { + private static final Logger LOG = LoggerFactory.getLogger(MongoNotebookRepo.class); + + private ZeppelinConfiguration conf; + private MongoClient mongo; + private MongoDatabase db; + private MongoCollection<Document> coll; + + public MongoNotebookRepo() { + + } + + public void init(ZeppelinConfiguration conf) throws IOException { + this.conf = conf; + + mongo = new MongoClient(new MongoClientURI(conf.getMongoUri())); + db = mongo.getDatabase(conf.getMongoDatabase()); + coll = db.getCollection(conf.getMongoCollection()); + + if (conf.getMongoAutoimport()) { + // import local notes into MongoDB + insertFileSystemNotes(); + } + } + + /** + * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true, + * this method will insert local notes into MongoDB on startup. + * If a note already exists in MongoDB, skip it. + */ + private void insertFileSystemNotes() throws IOException { + LinkedList<Document> docs = new LinkedList<>(); // docs to be imported + NotebookRepo vfsRepo = new VFSNotebookRepo(); + vfsRepo.init(conf); + List<NoteInfo> infos = vfsRepo.list(null); + // collect notes to be imported + for (NoteInfo info : infos) { + Note note = vfsRepo.get(info.getId(), null); + Document doc = noteToDocument(note); + docs.add(doc); + } + + /* + * 'ordered(false)' option allows to proceed bulk inserting even though + * there are duplicated documents. The duplicated documents will be skipped + * and print a WARN log. + */ + try { + coll.insertMany(docs, new InsertManyOptions().ordered(false)); + } catch (MongoBulkWriteException e) { + printDuplicatedException(e); //print duplicated document warning log + } + + vfsRepo.close(); // it does nothing for now but maybe in the future... + } + + /** + * MongoBulkWriteException contains error messages that inform + * which documents were duplicated. This method catches those ID and print them. + * @param e + */ + private void printDuplicatedException(MongoBulkWriteException e) { + List<BulkWriteError> errors = e.getWriteErrors(); + for (BulkWriteError error : errors) { + String msg = error.getMessage(); + Pattern pattern = Pattern.compile("[A-Z0-9]{9}"); // regex for note ID + Matcher matcher = pattern.matcher(msg); + if (matcher.find()) { // if there were a note ID + String noteId = matcher.group(); + LOG.warn("Note " + noteId + " not inserted since already exists in MongoDB"); + } + } + } + + @Override + public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + syncId(); + + List<NoteInfo> infos = new LinkedList<>(); + MongoCursor<Document> cursor = coll.find().iterator(); + + while (cursor.hasNext()) { + Document doc = cursor.next(); + Note note = documentToNote(doc); + NoteInfo info = new NoteInfo(note); + infos.add(info); + } + + cursor.close(); + + return infos; + } + + /** + * Find documents of which type of _id is object ID, and change it to note ID. + * Since updating _id field is not allowed, remove original documents and insert + * new ones with string _id(note ID) + */ + private void syncId() { + // find documents whose id type is object id + MongoCursor<Document> cursor = coll.find(type("_id", BsonType.OBJECT_ID)).iterator(); + // if there is no such document, exit + if (!cursor.hasNext()) + return; + + List<ObjectId> oldDocIds = new LinkedList<>(); // document ids need to update + List<Document> updatedDocs = new LinkedList<>(); // new documents to be inserted + + while (cursor.hasNext()) { + Document doc = cursor.next(); + // store original _id + ObjectId oldId = doc.getObjectId("_id"); + oldDocIds.add(oldId); + // store the document with string _id (note id) + String noteId = doc.getString("id"); + doc.put("_id", noteId); + updatedDocs.add(doc); + } + + coll.insertMany(updatedDocs); + coll.deleteMany(in("_id", oldDocIds)); + + cursor.close(); + } + + /** + * Convert document to note + */ + private Note documentToNote(Document doc) { + // document to JSON + String json = doc.toJson(); + // JSON to note + return Note.fromJson(json); + } + + /** + * Convert note to document + */ + private Document noteToDocument(Note note) { + // note to JSON + String json = note.toJson(); + // JSON to document + Document doc = Document.parse(json); + // set object id as note id + doc.put("_id", note.getId()); + return doc; + } + + @Override + public Note get(String noteId, AuthenticationInfo subject) throws IOException { + Document doc = coll.find(eq("_id", noteId)).first(); + + if (doc == null) { + throw new IOException("Note " + noteId + "not found"); + } + + return documentToNote(doc); + } + + @Override + public void save(Note note, AuthenticationInfo subject) throws IOException { + Document doc = noteToDocument(note); + coll.replaceOne(eq("_id", note.getId()), doc, new UpdateOptions().upsert(true)); + } + + @Override + public void remove(String noteId, AuthenticationInfo subject) throws IOException { + coll.deleteOne(eq("_id", noteId)); + } + + @Override + public void close() { + mongo.close(); + } + + @Override + public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { + LOG.warn("Method not implemented"); + return Collections.emptyList(); + } + + @Override + public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { + LOG.warn("Method not implemented"); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..e1943b7 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.MongoNotebookRepo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/s3/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/s3/pom.xml b/zeppelin-plugins/notebookrepo/s3/pom.xml new file mode 100644 index 0000000..ee533f2 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/s3/pom.xml @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-s3</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin S3NotebookRepo</name> + <description>NotebookRepo implementation based on S3</description> + + <properties> + <aws.sdk.s3.version>1.10.62</aws.sdk.s3.version> + <plugin.name>NotebookRepo/S3NotebookRepo</plugin.name> + </properties> + + <dependencies> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-s3</artifactId> + <version>${aws.sdk.s3.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java new file mode 100644 index 0000000..364943c --- /dev/null +++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.ClientConfigurationFactory; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * Backend for storing Notebooks on S3 + */ +public class S3NotebookRepo implements NotebookRepo { + private static final Logger LOG = LoggerFactory.getLogger(S3NotebookRepo.class); + + // Use a credential provider chain so that instance profiles can be utilized + // on an EC2 instance. The order of locations where credentials are searched + // is documented here + // + // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ + // auth/DefaultAWSCredentialsProviderChain.html + // + // In summary, the order is: + // + // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY + // 2. Java System Properties - aws.accessKeyId and aws.secretKey + // 3. Credential profiles file at the default location (~/.aws/credentials) + // shared by all AWS SDKs and the AWS CLI + // 4. Instance profile credentials delivered through the Amazon EC2 metadata service + private AmazonS3 s3client; + private String bucketName; + private String user; + private boolean useServerSideEncryption; + private ZeppelinConfiguration conf; + + public S3NotebookRepo() { + + } + + public void init(ZeppelinConfiguration conf) throws IOException { + this.conf = conf; + bucketName = conf.getS3BucketName(); + user = conf.getS3User(); + useServerSideEncryption = conf.isS3ServerSideEncryption(); + + // always use the default provider chain + AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + CryptoConfiguration cryptoConf = new CryptoConfiguration(); + String keyRegion = conf.getS3KMSKeyRegion(); + + if (StringUtils.isNotBlank(keyRegion)) { + cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion))); + } + + ClientConfiguration cliConf = createClientConfiguration(); + + // see if we should be encrypting data in S3 + String kmsKeyID = conf.getS3KMSKeyID(); + if (kmsKeyID != null) { + // use the AWS KMS to encrypt data + KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID); + this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf); + } + else if (conf.getS3EncryptionMaterialsProviderClass() != null) { + // use a custom encryption materials provider class + EncryptionMaterialsProvider emp = createCustomProvider(conf); + this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf); + } + else { + // regular S3 + this.s3client = new AmazonS3Client(credentialsProvider, cliConf); + } + + // set S3 endpoint to use + s3client.setEndpoint(conf.getS3Endpoint()); + } + + /** + * Create an instance of a custom encryption materials provider class + * which supplies encryption keys to use when reading/writing data in S3. + */ + private EncryptionMaterialsProvider createCustomProvider(ZeppelinConfiguration conf) + throws IOException { + // use a custom encryption materials provider class + String empClassname = conf.getS3EncryptionMaterialsProviderClass(); + EncryptionMaterialsProvider emp; + try { + Object empInstance = Class.forName(empClassname).newInstance(); + if (empInstance instanceof EncryptionMaterialsProvider) { + emp = (EncryptionMaterialsProvider) empInstance; + } + else { + throw new IOException("Class " + empClassname + " does not implement " + + EncryptionMaterialsProvider.class.getName()); + } + } + catch (Exception e) { + throw new IOException("Unable to instantiate encryption materials provider class " + + empClassname + ": " + e, e); + } + + return emp; + } + + /** + * Create AWS client configuration and return it. + * @return AWS client configuration + */ + private ClientConfiguration createClientConfiguration() { + ClientConfigurationFactory configFactory = new ClientConfigurationFactory(); + ClientConfiguration config = configFactory.getConfig(); + + String s3SignerOverride = conf.getS3SignerOverride(); + if (StringUtils.isNotBlank(s3SignerOverride)) { + config.setSignerOverride(s3SignerOverride); + } + + return config; + } + + @Override + public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + List<NoteInfo> infos = new LinkedList<>(); + NoteInfo info; + try { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withPrefix(user + "/" + "notebook"); + ObjectListing objectListing; + do { + objectListing = s3client.listObjects(listObjectsRequest); + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + if (objectSummary.getKey().endsWith("note.json")) { + info = getNoteInfo(objectSummary.getKey()); + if (info != null) { + infos.add(info); + } + } + } + listObjectsRequest.setMarker(objectListing.getNextMarker()); + } while (objectListing.isTruncated()); + } catch (AmazonClientException ace) { + throw new IOException("Unable to list objects in S3: " + ace, ace); + } + return infos; + } + + private Note getNote(String key) throws IOException { + S3Object s3object; + try { + s3object = s3client.getObject(new GetObjectRequest(bucketName, key)); + } + catch (AmazonClientException ace) { + throw new IOException("Unable to retrieve object from S3: " + ace, ace); + } + + try (InputStream ins = s3object.getObjectContent()) { + String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); + return Note.fromJson(json); + } + } + + private NoteInfo getNoteInfo(String key) throws IOException { + Note note = getNote(key); + return new NoteInfo(note); + } + + @Override + public Note get(String noteId, AuthenticationInfo subject) throws IOException { + return getNote(user + "/" + "notebook" + "/" + noteId + "/" + "note.json"); + } + + @Override + public void save(Note note, AuthenticationInfo subject) throws IOException { + String json = note.toJson(); + String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json"; + + File file = File.createTempFile("note", "json"); + try { + Writer writer = new OutputStreamWriter(new FileOutputStream(file)); + writer.write(json); + writer.close(); + + PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file); + + if (useServerSideEncryption) { + // Request server-side encryption. + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + putRequest.setMetadata(objectMetadata); + } + + s3client.putObject(putRequest); + } + catch (AmazonClientException ace) { + throw new IOException("Unable to store note in S3: " + ace, ace); + } + finally { + FileUtils.deleteQuietly(file); + } + } + + @Override + public void remove(String noteId, AuthenticationInfo subject) throws IOException { + String key = user + "/" + "notebook" + "/" + noteId; + final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName).withPrefix(key); + + try { + ObjectListing objects = s3client.listObjects(listObjectsRequest); + do { + for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { + s3client.deleteObject(bucketName, objectSummary.getKey()); + } + objects = s3client.listNextBatchOfObjects(objects); + } while (objects.isTruncated()); + } + catch (AmazonClientException ace) { + throw new IOException("Unable to remove note in S3: " + ace, ace); + } + } + + @Override + public void close() { + //no-op + } + + @Override + public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { + LOG.warn("Method not implemented"); + return Collections.emptyList(); + } + + @Override + public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { + LOG.warn("Method not implemented"); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..790bbdf --- /dev/null +++ b/zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.S3NotebookRepo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/vfs/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/vfs/pom.xml b/zeppelin-plugins/notebookrepo/vfs/pom.xml new file mode 100644 index 0000000..ab414f6 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/vfs/pom.xml @@ -0,0 +1,58 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-vfs</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin VFSNotebookRepo</name> + <description>NotebookRepo implementation based on VFS</description> + + <properties> + <commons.vfs2.version>2.2</commons.vfs2.version> + <plugin.name>NotebookRepo/VFSNotebookRepo</plugin.name> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-vfs2</artifactId> + <version>${commons.vfs2.version}</version> + <exclusions> + <exclusion> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java new file mode 100644 index 0000000..4294b86 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import com.google.common.collect.Lists; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.vfs2.FileContent; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemManager; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.NameScope; +import org.apache.commons.vfs2.Selectors; +import org.apache.commons.vfs2.VFS; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** +* +*/ +public class VFSNotebookRepo implements NotebookRepo { + private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepo.class); + + private FileSystemManager fsManager; + private URI filesystemRoot; + protected ZeppelinConfiguration conf; + + public VFSNotebookRepo() { + + } + + @Override + public void init(ZeppelinConfiguration conf) throws IOException { + this.conf = conf; + setNotebookDirectory(conf.getNotebookDir()); + } + + protected void setNotebookDirectory(String notebookDirPath) throws IOException { + try { + LOG.info("Using notebookDir: " + notebookDirPath); + if (conf.isWindowsPath(notebookDirPath)) { + filesystemRoot = new File(notebookDirPath).toURI(); + } else { + filesystemRoot = new URI(notebookDirPath); + } + } catch (URISyntaxException e1) { + throw new IOException(e1); + } + + if (filesystemRoot.getScheme() == null) { // it is local path + File f = new File(conf.getRelativeDir(filesystemRoot.getPath())); + this.filesystemRoot = f.toURI(); + } + + fsManager = VFS.getManager(); + FileObject file = fsManager.resolveFile(filesystemRoot.getPath()); + if (!file.exists()) { + LOG.info("Notebook dir doesn't exist, create on is {}.", file.getName()); + file.createFolder(); + } + } + + private String getNotebookDirPath() { + return filesystemRoot.getPath().toString(); + } + + private String getPath(String path) { + if (path == null || path.trim().length() == 0) { + return filesystemRoot.toString(); + } + if (path.startsWith("/")) { + return filesystemRoot.toString() + path; + } else { + return filesystemRoot.toString() + "/" + path; + } + } + + private boolean isDirectory(FileObject fo) throws IOException { + if (fo == null) return false; + if (fo.getType() == FileType.FOLDER) { + return true; + } else { + return false; + } + } + + @Override + public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + FileObject rootDir = getRootDir(); + + FileObject[] children = rootDir.getChildren(); + + List<NoteInfo> infos = new LinkedList<>(); + for (FileObject f : children) { + String fileName = f.getName().getBaseName(); + if (f.isHidden() + || fileName.startsWith(".") + || fileName.startsWith("#") + || fileName.startsWith("~")) { + // skip hidden, temporary files + continue; + } + + if (!isDirectory(f)) { + // currently single note is saved like, [NOTE_ID]/note.json. + // so it must be a directory + continue; + } + + NoteInfo info = null; + + try { + info = getNoteInfo(f); + if (info != null) { + infos.add(info); + } + } catch (Exception e) { + LOG.error("Can't read note " + f.getName().toString(), e); + } + } + + return infos; + } + + private Note getNote(FileObject noteDir) throws IOException { + if (!isDirectory(noteDir)) { + throw new IOException(noteDir.getName().toString() + " is not a directory"); + } + + FileObject noteJson = noteDir.resolveFile("note.json", NameScope.CHILD); + if (!noteJson.exists()) { + throw new IOException(noteJson.getName().toString() + " not found"); + } + + FileContent content = noteJson.getContent(); + InputStream ins = content.getInputStream(); + String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); + ins.close(); + + return Note.fromJson(json); + } + + private NoteInfo getNoteInfo(FileObject noteDir) throws IOException { + Note note = getNote(noteDir); + return new NoteInfo(note); + } + + @Override + public Note get(String noteId, AuthenticationInfo subject) throws IOException { + FileObject rootDir = fsManager.resolveFile(getPath("/")); + FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD); + + return getNote(noteDir); + } + + protected FileObject getRootDir() throws IOException { + FileObject rootDir = fsManager.resolveFile(getPath("/")); + + if (!rootDir.exists()) { + throw new IOException("Root path does not exists"); + } + + if (!isDirectory(rootDir)) { + throw new IOException("Root path is not a directory"); + } + + return rootDir; + } + + @Override + public synchronized void save(Note note, AuthenticationInfo subject) throws IOException { + LOG.info("Saving note:" + note.getId()); + String json = note.toJson(); + + FileObject rootDir = getRootDir(); + + FileObject noteDir = rootDir.resolveFile(note.getId(), NameScope.CHILD); + + if (!noteDir.exists()) { + noteDir.createFolder(); + } + if (!isDirectory(noteDir)) { + throw new IOException(noteDir.getName().toString() + " is not a directory"); + } + + FileObject noteJson = noteDir.resolveFile(".note.json", NameScope.CHILD); + // false means not appending. creates file if not exists + OutputStream out = noteJson.getContent().getOutputStream(false); + out.write(json.getBytes(conf.getString(ConfVars.ZEPPELIN_ENCODING))); + out.close(); + noteJson.moveTo(noteDir.resolveFile("note.json", NameScope.CHILD)); + } + + @Override + public void remove(String noteId, AuthenticationInfo subject) throws IOException { + FileObject rootDir = fsManager.resolveFile(getPath("/")); + FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD); + + if (!noteDir.exists()) { + // nothing to do + return; + } + + if (!isDirectory(noteDir)) { + // it is not look like zeppelin note savings + throw new IOException("Can not remove " + noteDir.getName().toString()); + } + + noteDir.delete(Selectors.SELECT_SELF_AND_CHILDREN); + } + + @Override + public void close() { + //no-op + } + + @Override + public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { + NotebookRepoSettingsInfo repoSetting = NotebookRepoSettingsInfo.newInstance(); + List<NotebookRepoSettingsInfo> settings = new ArrayList<>(); + repoSetting.name = "Notebook Path"; + repoSetting.type = NotebookRepoSettingsInfo.Type.INPUT; + repoSetting.value = Collections.emptyList(); + repoSetting.selected = getNotebookDirPath(); + + settings.add(repoSetting); + return settings; + } + + @Override + public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { + if (settings == null || settings.isEmpty()) { + LOG.error("Cannot update {} with empty settings", this.getClass().getName()); + return; + } + String newNotebookDirectotyPath = StringUtils.EMPTY; + if (settings.containsKey("Notebook Path")) { + newNotebookDirectotyPath = settings.get("Notebook Path"); + } + + if (StringUtils.isBlank(newNotebookDirectotyPath)) { + LOG.error("Notebook path is invalid"); + return; + } + LOG.warn("{} will change notebook dir from {} to {}", + subject.getUser(), getNotebookDirPath(), newNotebookDirectotyPath); + try { + setNotebookDirectory(newNotebookDirectotyPath); + } catch (IOException e) { + LOG.error("Cannot update notebook directory", e); + } + } + +} + http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..ed95232 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.VFSNotebookRepo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java b/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java new file mode 100644 index 0000000..452adc0 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import com.google.common.collect.ImmutableMap; +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class TestVFSNotebookRepo { + + private ZeppelinConfiguration zConf; + private VFSNotebookRepo notebookRepo; + private String notebookDir = "/tmp/zeppelin/vfs_notebookrepo/"; + + @Before + public void setUp() throws IOException { + notebookRepo = new VFSNotebookRepo(); + FileUtils.forceMkdir(new File(notebookDir)); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir); + zConf = new ZeppelinConfiguration(); + notebookRepo.init(zConf); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(new File(notebookDir)); + } + + @Test + public void testBasics() throws IOException { + assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + + Note note1 = new Note(); + Paragraph p1 = note1.insertNewParagraph(0, AuthenticationInfo.ANONYMOUS); + p1.setText("%md hello world"); + p1.setTitle("my title"); + notebookRepo.save(note1, AuthenticationInfo.ANONYMOUS); + + assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + Note note2 = notebookRepo.get(note1.getId(), AuthenticationInfo.ANONYMOUS); + assertEquals(note1.getParagraphCount(), note2.getParagraphCount()); + + Paragraph p2 = note2.getParagraph(p1.getId()); + assertEquals(p1.getText(), p2.getText()); + assertEquals(p1.getTitle(), p2.getTitle()); + + notebookRepo.remove(note1.getId(), AuthenticationInfo.ANONYMOUS); + assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + } + + @Test + public void testInvalidJson() throws IOException { + assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + + // invalid note will be ignored + createNewNote("invalid_content", "id_1"); + assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + + // only valid note will be fetched + createNewNote("{}", "id_2"); + assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + } + + @Test + public void testUpdateSettings() throws IOException { + List<NotebookRepoSettingsInfo> repoSettings = notebookRepo.getSettings(AuthenticationInfo.ANONYMOUS); + assertEquals(1, repoSettings.size()); + NotebookRepoSettingsInfo settingInfo = repoSettings.get(0); + assertEquals("Notebook Path", settingInfo.name); + assertEquals(notebookDir, settingInfo.selected); + + createNewNote("{}", "id_2"); + assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + + String newNotebookDir = "/tmp/zeppelin/vfs_notebookrepo2"; + FileUtils.forceMkdir(new File(newNotebookDir)); + Map<String, String> newSettings = ImmutableMap.of("Notebook Path", newNotebookDir); + notebookRepo.updateSettings(newSettings, AuthenticationInfo.ANONYMOUS); + assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + } + + private void createNewNote(String content, String noteId) throws IOException { + FileUtils.writeStringToFile(new File(notebookDir + "/" + noteId, "note.json"), content); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml b/zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml new file mode 100644 index 0000000..af2c73e --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml @@ -0,0 +1,81 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-zeppelin-hub</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin ZeppelinHubRepo</name> + <description>NotebookRepo implementation based on Zeppelin Hub</description> + + <properties> + <jetty.version>9.2.15.v20160210</jetty.version> + <google.truth.version>0.27</google.truth.version> + <plugin.name>NotebookRepo/ZeppelinHubRepo</plugin.name> + </properties> + + <dependencies> + + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.eclipse.jetty.websocket</groupId> + <artifactId>websocket-server</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.google.truth</groupId> + <artifactId>truth</artifactId> + <version>${google.truth.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> +</project>
