http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java index 8d1bf4d..5d9c85c 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java @@ -1,10 +1,12 @@ package org.apache.zeppelin.notebook.repo; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.FileSystemStorage; import org.apache.zeppelin.notebook.Note; @@ -13,11 +15,25 @@ import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + /** * NotebookRepos for hdfs. * - * <p>Assume the notebook directory structure is as following - notebookdir - noteId/note.json - - * noteId/note.json - noteId/note.json + * Assume the notebook directory structure is as following + * - notebookdir + * - noteId/note.json + * - noteId/note.json + * - noteId/note.json */ public class FileSystemNotebookRepo implements NotebookRepo { private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemNotebookRepo.class); @@ -25,12 +41,14 @@ public class FileSystemNotebookRepo implements NotebookRepo { private FileSystemStorage fs; private Path notebookDir; - public FileSystemNotebookRepo() {} + public FileSystemNotebookRepo() { + + } public void init(ZeppelinConfiguration zConf) throws IOException { this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir()); - LOGGER.info( - "Creating FileSystem: " + this.fs.getFs().getClass().getName() + " for Zeppelin Notebook."); + LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() + + " for Zeppelin Notebook."); this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir())); LOGGER.info("Using folder {} to store notebook", notebookDir); this.fs.tryMkDir(notebookDir); @@ -49,15 +67,16 @@ public class FileSystemNotebookRepo implements NotebookRepo { @Override public Note get(final String noteId, AuthenticationInfo subject) throws IOException { - String content = - this.fs.readFile(new Path(notebookDir.toString() + "/" + noteId + "/note.json")); + String content = this.fs.readFile( + new Path(notebookDir.toString() + "/" + noteId + "/note.json")); return Note.fromJson(content); } @Override public void save(final Note note, AuthenticationInfo subject) throws IOException { - this.fs.writeFile( - note.toJson(), new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"), true); + this.fs.writeFile(note.toJson(), + new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"), + true); } @Override @@ -80,4 +99,5 @@ public class FileSystemNotebookRepo implements NotebookRepo { public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { LOGGER.warn("updateSettings is not implemented for HdfsNotebookRepo"); } + }
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java index cbbe5a6..5fd8becc 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java @@ -1,13 +1,6 @@ package org.apache.zeppelin.notebook.repo; -import static org.junit.Assert.assertEquals; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -19,6 +12,15 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + public class FileSystemNotebookRepoTest { private ZeppelinConfiguration zConf; @@ -30,11 +32,9 @@ public class FileSystemNotebookRepoTest { @Before public void setUp() throws IOException { - notebookDir = - Files.createTempDirectory("FileSystemNotebookRepoTest").toFile().getAbsolutePath(); + notebookDir = Files.createTempDirectory("FileSystemNotebookRepoTest").toFile().getAbsolutePath(); zConf = new ZeppelinConfiguration(); - System.setProperty( - ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir); hadoopConf = new Configuration(); fs = FileSystem.get(hadoopConf); hdfsNotebookRepo = new FileSystemNotebookRepo(); @@ -80,8 +80,7 @@ public class FileSystemNotebookRepoTest { @Test public void testComplicatedScenarios() throws IOException { - // scenario_1: notebook_dir is not clean. There're some unrecognized dir and file under - // notebook_dir + // scenario_1: notebook_dir is not clean. There're some unrecognized dir and file under notebook_dir fs.mkdirs(new Path(notebookDir, "1/2")); OutputStream out = fs.create(new Path(notebookDir, "1/a.json")); out.close(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java index 7647fbc..ad99aba 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java @@ -49,14 +49,14 @@ import org.slf4j.LoggerFactory; /** * A NotebookRepo implementation for storing notebooks in Google Cloud Storage. * - * <p>Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path must - * be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an object - * store, so this "directory" should not itself be an object. Instead, it represents the base path - * for the note.json files. + * Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path + * must be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an + * object store, so this "directory" should not itself be an object. Instead, it represents the base + * path for the note.json files. * - * <p>Authentication is provided by google-auth-library-java. - * - * @see <a href="https://github.com/google/google-auth-library-java";>google-auth-library-java</a>. + * Authentication is provided by google-auth-library-java. + * @see <a href="https://github.com/google/google-auth-library-java";> + * google-auth-library-java</a>. */ public class GCSNotebookRepo implements NotebookRepo { @@ -67,7 +67,8 @@ public class GCSNotebookRepo implements NotebookRepo { private Pattern noteNamePattern; private Storage storage; - public GCSNotebookRepo() {} + public GCSNotebookRepo() { + } @VisibleForTesting public GCSNotebookRepo(ZeppelinConfiguration zConf, Storage storage) throws IOException { @@ -77,38 +78,37 @@ public class GCSNotebookRepo implements NotebookRepo { @Override public void init(ZeppelinConfiguration zConf) throws IOException { - this.encoding = zConf.getString(ConfVars.ZEPPELIN_ENCODING); + this.encoding = zConf.getString(ConfVars.ZEPPELIN_ENCODING); String gcsStorageDir = zConf.getGCSStorageDir(); if (gcsStorageDir.isEmpty()) { throw new IOException("GCS storage directory must be set using 'zeppelin.notebook.gcs.dir'"); } if (!gcsStorageDir.startsWith("gs://")) { - throw new IOException( - String.format("GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir)); + throw new IOException(String.format( + "GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir)); } String storageDirWithoutScheme = gcsStorageDir.substring("gs://".length()); // pathComponents excludes empty string if trailing slash is present List<String> pathComponents = Arrays.asList(storageDirWithoutScheme.split("/")); if (pathComponents.size() < 1) { - throw new IOException( - String.format( - "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir", - gcsStorageDir)); + throw new IOException(String.format( + "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir", + gcsStorageDir)); } this.bucketName = pathComponents.get(0); if (pathComponents.size() > 1) { - this.basePath = - Optional.of(StringUtils.join(pathComponents.subList(1, pathComponents.size()), "/")); + this.basePath = Optional.of(StringUtils.join( + pathComponents.subList(1, pathComponents.size()), "/")); } else { this.basePath = Optional.absent(); } // Notes are stored at gs://bucketName/basePath/<note-id>/note.json if (basePath.isPresent()) { - this.noteNamePattern = - Pattern.compile("^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$"); + this.noteNamePattern = Pattern.compile( + "^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$"); } else { this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$"); } @@ -130,10 +130,13 @@ public class GCSNotebookRepo implements NotebookRepo { List<NoteInfo> infos = new ArrayList<>(); Iterable<Blob> blobsUnderDir; if (basePath.isPresent()) { - blobsUnderDir = - storage.list(bucketName, BlobListOption.prefix(this.basePath.get() + "/")).iterateAll(); + blobsUnderDir = storage + .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/")) + .iterateAll(); } else { - blobsUnderDir = storage.list(bucketName).iterateAll(); + blobsUnderDir = storage + .list(bucketName) + .iterateAll(); } for (Blob b : blobsUnderDir) { Matcher matcher = noteNamePattern.matcher(b.getName()); @@ -169,8 +172,9 @@ public class GCSNotebookRepo implements NotebookRepo { @Override public void save(Note note, AuthenticationInfo subject) throws IOException { - BlobInfo info = - BlobInfo.newBuilder(makeBlobId(note.getId())).setContentType("application/json").build(); + BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) + .setContentType("application/json") + .build(); try { storage.create(info, note.toJson().getBytes("UTF-8")); } catch (StorageException se) { @@ -194,7 +198,7 @@ public class GCSNotebookRepo implements NotebookRepo { @Override public void close() { - // no-op + //no-op } @Override http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java index 2c603ad..c1fae67 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java @@ -55,13 +55,12 @@ public class GCSNotebookRepoTest { @Parameters public static Collection<Object[]> data() { - return Arrays.asList( - new Object[][] { - {"bucketname", Optional.absent(), "gs://bucketname"}, - {"bucketname-with-slash", Optional.absent(), "gs://bucketname-with-slash/"}, - {"bucketname", Optional.of("path/to/dir"), "gs://bucketname/path/to/dir"}, - {"bucketname", Optional.of("trailing/slash"), "gs://bucketname/trailing/slash/"} - }); + return Arrays.asList(new Object[][] { + { "bucketname", Optional.absent(), "gs://bucketname" }, + { "bucketname-with-slash", Optional.absent(), "gs://bucketname-with-slash/" }, + { "bucketname", Optional.of("path/to/dir"), "gs://bucketname/path/to/dir" }, + { "bucketname", Optional.of("trailing/slash"), "gs://bucketname/trailing/slash/" } + }); } @Parameter(0) @@ -124,8 +123,7 @@ public class GCSNotebookRepoTest { try { notebookRepo.get("id", AUTH_INFO); fail(); - } catch (IOException e) { - } + } catch (IOException e) {} } @Test @@ -147,8 +145,7 @@ public class GCSNotebookRepoTest { try { notebookRepo.get("id", AUTH_INFO); fail(); - } catch (IOException e) { - } + } catch (IOException e) {} } @Test @@ -174,8 +171,7 @@ public class GCSNotebookRepoTest { try { notebookRepo.remove("id", AUTH_INFO); fail(); - } catch (IOException e) { - } + } catch (IOException e) {} } @Test @@ -204,14 +200,16 @@ public class GCSNotebookRepoTest { } private void create(Note note) throws IOException { - BlobInfo info = - BlobInfo.newBuilder(makeBlobId(note.getId())).setContentType("application/json").build(); + BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) + .setContentType("application/json") + .build(); storage.create(info, note.toJson().getBytes("UTF-8")); } private void createMalformed(String noteId) throws IOException { - BlobInfo info = - BlobInfo.newBuilder(makeBlobId(noteId)).setContentType("application/json").build(); + BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId)) + .setContentType("application/json") + .build(); storage.create(info, "{ invalid-json }".getBytes("UTF-8")); } @@ -223,8 +221,7 @@ public class GCSNotebookRepoTest { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), ""); new GCSNotebookRepo(new ZeppelinConfiguration(), storage); fail(); - } catch (IOException e) { - } + } catch (IOException e) {} } @Test @@ -233,7 +230,6 @@ public class GCSNotebookRepoTest { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "foo"); new GCSNotebookRepo(new ZeppelinConfiguration(), storage); fail(); - } catch (IOException e) { - } + } catch (IOException e) {} } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java index 29ed2f4..7729d52 100644 --- a/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java @@ -20,10 +20,6 @@ package org.apache.zeppelin.notebook.repo; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.Lists; -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.List; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.user.AuthenticationInfo; @@ -41,14 +37,20 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; + /** * NotebookRepo that hosts all the notebook FS in a single Git repo * - * <p>This impl intended to be simple and straightforward: - does not handle branches - only basic - * local git file repo, no remote Github push\pull. GitHub integration is implemented in @see {@link - * org.apache.zeppelin.notebook.repo.GitNotebookRepo} + * This impl intended to be simple and straightforward: + * - does not handle branches + * - only basic local git file repo, no remote Github push\pull. GitHub integration is + * implemented in @see {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} * - * <p>TODO(bzz): add default .gitignore + * TODO(bzz): add default .gitignore */ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWithVersionControl { private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class); @@ -68,8 +70,8 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith @Override public void init(ZeppelinConfiguration conf) throws IOException { - // TODO(zjffdu), it is weird that I can not call super.init directly here, as it would cause - // AbstractMethodError + //TODO(zjffdu), it is weird that I can not call super.init directly here, as it would cause + //AbstractMethodError this.conf = conf; setNotebookDirectory(conf.getNotebookDir()); @@ -115,8 +117,11 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith } /** - * the idea is to: 1. stash current changes 2. remember head commit and checkout to the desired - * revision 3. get note and checkout back to the head 4. apply stash on top and remove it + * the idea is to: + * 1. stash current changes + * 2. remember head commit and checkout to the desired revision + * 3. get note and checkout back to the head + * 4. apply stash on top and remove it */ @Override public synchronized Note get(String noteId, String revId, AuthenticationInfo subject) @@ -144,10 +149,7 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith ObjectId applied = git.stashApply().setStashRef(stash.getName()).call(); ObjectId dropped = git.stashDrop().setStashRef(0).call(); Collection<RevCommit> stashes = git.stashList().call(); - LOG.debug( - "Stash applied as : {}, and dropped : {}, stash size: {}", - applied, - dropped, + LOG.debug("Stash applied as : {}, and dropped : {}, stash size: {}", applied, dropped, stashes.size()); } } catch (GitAPIException e) { @@ -162,12 +164,12 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith LOG.debug("Listing history for {}:", noteId); try { Iterable<RevCommit> logs = git.log().addPath(noteId).call(); - for (RevCommit log : logs) { + for (RevCommit log: logs) { history.add(new Revision(log.getName(), log.getShortMessage(), log.getCommitTime())); LOG.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage()); } } catch (NoHeadException e) { - // when no initial commit exists + //when no initial commit exists LOG.warn("No Head found for {}, {}", noteId, e.getMessage()); } catch (GitAPIException e) { LOG.error("Failed to get logs for {}", noteId, e); @@ -184,13 +186,13 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith } return revisionNote; } - + @Override public void close() { git.getRepository().close(); } - // DI replacements for Tests + //DI replacements for Tests protected Git getGit() { return git; } @@ -198,4 +200,6 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith void setGit(Git git) { this.git = git; } + } + http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java index 7de299d..58d7cc1 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java @@ -20,11 +20,11 @@ package org.apache.zeppelin.notebook.repo; import static com.google.common.truth.Truth.assertThat; import static org.mockito.Mockito.mock; -import com.google.common.base.Joiner; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -44,6 +44,8 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Joiner; + public class GitNotebookRepoTest { private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepoTest.class); @@ -57,8 +59,7 @@ public class GitNotebookRepoTest { @Before public void setUp() throws Exception { - String zpath = - System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + System.currentTimeMillis(); + String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + System.currentTimeMillis(); zeppelinDir = new File(zpath); zeppelinDir.mkdirs(); new File(zeppelinDir, "conf").mkdirs(); @@ -70,23 +71,24 @@ public class GitNotebookRepoTest { String testNoteDir = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID); String testNoteDir2 = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID2); FileUtils.copyDirectory( - new File( - GitNotebookRepoTest.class - .getResource(Joiner.on(File.separator).join("", TEST_NOTE_ID)) - .getFile()), - new File(testNoteDir)); + new File( + GitNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID) + ).getFile() + ), + new File(testNoteDir)); FileUtils.copyDirectory( - new File( - GitNotebookRepoTest.class - .getResource(Joiner.on(File.separator).join("", TEST_NOTE_ID2)) - .getFile()), - new File(testNoteDir2)); + new File( + GitNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID2) + ).getFile() + ), + new File(testNoteDir2) + ); System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), zeppelinDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); - System.setProperty( - ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), - "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); conf = ZeppelinConfiguration.create(); } @@ -100,14 +102,14 @@ public class GitNotebookRepoTest { @Test public void initNonemptyNotebookDir() throws IOException, GitAPIException { - // given - .git does not exit + //given - .git does not exit File dotGit = new File(Joiner.on(File.separator).join(notebooksDir, ".git")); assertThat(dotGit.exists()).isEqualTo(false); - // when + //when notebookRepo = new GitNotebookRepo(conf); - // then + //then Git git = notebookRepo.getGit(); assertThat(git).isNotNull(); @@ -121,21 +123,21 @@ public class GitNotebookRepoTest { @Test public void showNotebookHistoryEmptyTest() throws GitAPIException, IOException { - // given + //given notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); - // when + //when List<Revision> testNotebookHistory = notebookRepo.revisionHistory(TEST_NOTE_ID, null); - // then - // no initial commit, empty history + //then + //no initial commit, empty history assertThat(testNotebookHistory).isEmpty(); } @Test public void showNotebookHistoryMultipleNotesTest() throws IOException { - // initial checks + //initial checks notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); @@ -143,13 +145,13 @@ public class GitNotebookRepoTest { assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null)).isEmpty(); - // add commit to both notes + //add commit to both notes notebookRepo.checkpoint(TEST_NOTE_ID, "first commit, note1", null); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); - // modify, save and checkpoint first note + //modify, save and checkpoint first note Note note = notebookRepo.get(TEST_NOTE_ID, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -162,10 +164,10 @@ public class GitNotebookRepoTest { assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null)) - .isEqualTo(Revision.EMPTY); + .isEqualTo(Revision.EMPTY); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); - // modify, save and checkpoint second note + //modify, save and checkpoint second note note = notebookRepo.get(TEST_NOTE_ID2, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -211,7 +213,7 @@ public class GitNotebookRepoTest { } private boolean containsNote(List<NoteInfo> notes, String noteId) { - for (NoteInfo note : notes) { + for (NoteInfo note: notes) { if (note.getId().equals(noteId)) { return true; } @@ -321,7 +323,7 @@ public class GitNotebookRepoTest { @Test public void setRevisionTest() throws IOException { - // create repo and check that note doesn't contain revisions + //create repo and check that note doesn't contain revisions notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); @@ -335,7 +337,7 @@ public class GitNotebookRepoTest { // checkpoint revision1 Revision revision1 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: first commit", null); - // TODO(khalid): change to EMPTY after rebase + //TODO(khalid): change to EMPTY after rebase assertThat(revision1).isNotNull(); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); @@ -352,7 +354,7 @@ public class GitNotebookRepoTest { // checkpoint revision2 Revision revision2 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: second commit", null); - // TODO(khalid): change to EMPTY after rebase + //TODO(khalid): change to EMPTY after rebase assertThat(revision2).isNotNull(); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java index bf35f67..738d708 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java @@ -17,11 +17,6 @@ package org.apache.zeppelin.notebook.repo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock; @@ -31,31 +26,33 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -// TODO(zjffdu) move it to zeppelin-zengine +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +//TODO(zjffdu) move it to zeppelin-zengine public class NotebookRepoSyncInitializationTest { - private static final Logger LOG = - LoggerFactory.getLogger(NotebookRepoSyncInitializationTest.class); + private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncInitializationTest.class); private String validFirstStorageClass = "org.apache.zeppelin.notebook.repo.VFSNotebookRepo"; - private String validSecondStorageClass = - "org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"; + private String validSecondStorageClass = "org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"; private String invalidStorageClass = "org.apache.zeppelin.notebook.repo.DummyNotebookRepo"; private String validOneStorageConf = validFirstStorageClass; private String validTwoStorageConf = validFirstStorageClass + "," + validSecondStorageClass; private String invalidTwoStorageConf = validFirstStorageClass + "," + invalidStorageClass; - private String unsupportedStorageConf = - validFirstStorageClass + "," + validSecondStorageClass + "," + validSecondStorageClass; + private String unsupportedStorageConf = validFirstStorageClass + "," + validSecondStorageClass + "," + validSecondStorageClass; private String emptyStorageConf = ""; @Before - public void setUp() { - System.setProperty( - ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); - // setup routine + public void setUp(){ + System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); + //setup routine } @After public void tearDown() { - // tear-down routine + //tear-down routine } @Test @@ -74,12 +71,11 @@ public class NotebookRepoSyncInitializationTest { @Test public void validInitTwoStorageTest() throws IOException { // initialize folders for each storage - String zpath = - System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); + String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); File mainZepDir = new File(zpath); mainZepDir.mkdirs(); new File(mainZepDir, "conf").mkdirs(); - String mainNotePath = zpath + "/notebook"; + String mainNotePath = zpath+"/notebook"; String secNotePath = mainNotePath + "_secondary"; File mainNotebookDir = new File(mainNotePath); File secNotebookDir = new File(secNotePath); @@ -88,8 +84,7 @@ public class NotebookRepoSyncInitializationTest { // set confs System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); - System.setProperty( - ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), validTwoStorageConf); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo @@ -116,12 +111,11 @@ public class NotebookRepoSyncInitializationTest { @Test public void initUnsupportedNumberStoragesTest() throws IOException { // initialize folders for each storage, currently for 2 only - String zpath = - System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); + String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); File mainZepDir = new File(zpath); mainZepDir.mkdirs(); new File(mainZepDir, "conf").mkdirs(); - String mainNotePath = zpath + "/notebook"; + String mainNotePath = zpath+"/notebook"; String secNotePath = mainNotePath + "_secondary"; File mainNotebookDir = new File(mainNotePath); File secNotebookDir = new File(secNotePath); @@ -130,8 +124,7 @@ public class NotebookRepoSyncInitializationTest { // set confs System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); - System.setProperty( - ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), unsupportedStorageConf); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo @@ -156,7 +149,7 @@ public class NotebookRepoSyncInitializationTest { @Test public void initOneDummyStorageTest() throws IOException { - // set confs + // set confs System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), invalidStorageClass); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo @@ -165,4 +158,4 @@ public class NotebookRepoSyncInitializationTest { assertEquals(notebookRepoSync.getRepoCount(), 1); assertTrue(notebookRepoSync.getRepo(0) instanceof NotebookRepo); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index a012f72..68e2615 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -17,17 +17,6 @@ package org.apache.zeppelin.notebook.repo; -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -import java.io.File; -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; @@ -43,6 +32,7 @@ import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.NotebookAuthorization; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.ParagraphJobListener; +import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.SearchService; @@ -56,7 +46,20 @@ import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -// TODO(zjffdu) move it to zeppelin-zengine +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + + +//TODO(zjffdu) move it to zeppelin-zengine public class NotebookRepoSyncTest implements ParagraphJobListener { private File mainZepDir; @@ -77,12 +80,11 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { @Before public void setUp() throws Exception { - String zpath = - System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); + String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis(); mainZepDir = new File(zpath); mainZepDir.mkdirs(); new File(mainZepDir, "conf").mkdirs(); - String mainNotePath = zpath + "/notebook"; + String mainNotePath = zpath+"/notebook"; String secNotePath = mainNotePath + "_secondary"; mainNotebookDir = new File(mainNotePath); secNotebookDir = new File(secNotePath); @@ -90,16 +92,11 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { secNotebookDir.mkdirs(); System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); - System.setProperty( - ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); - System.setProperty( - ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), - "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "false"); - System.setProperty( - ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), mainZepDir.getAbsolutePath() + "/conf"); - System.setProperty( - ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), mainZepDir.getAbsolutePath() + "/conf"); + System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); LOG.info("main Note dir : " + mainNotePath); LOG.info("secondary note dir : " + secNotePath); @@ -110,29 +107,16 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { this.schedulerFactory = SchedulerFactory.singleton(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - interpreterSettingManager = - new InterpreterSettingManager( - conf, - mock(AngularObjectRegistryListener.class), - mock(RemoteInterpreterProcessListener.class), - mock(ApplicationEventListener.class)); + interpreterSettingManager = new InterpreterSettingManager(conf, + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); factory = new InterpreterFactory(interpreterSettingManager); search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); notebookAuthorization = NotebookAuthorization.init(conf); credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath(), null); - notebookSync = - new Notebook( - conf, - notebookRepoSync, - schedulerFactory, - factory, - interpreterSettingManager, - this, - search, - notebookAuthorization, - credentials); + notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, + notebookAuthorization, credentials); anonymous = new AuthenticationInfo("anonymous"); } @@ -159,9 +143,7 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { // check that automatically saved on both storages assertEquals(1, notebookRepoSync.list(0, anonymous).size()); assertEquals(1, notebookRepoSync.list(1, anonymous).size()); - assertEquals( - notebookRepoSync.list(0, anonymous).get(0).getId(), - notebookRepoSync.list(1, anonymous).get(0).getId()); + assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId()); notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous); } @@ -178,9 +160,7 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { /* check that created in both storage systems */ assertEquals(1, notebookRepoSync.list(0, anonymous).size()); assertEquals(1, notebookRepoSync.list(1, anonymous).size()); - assertEquals( - notebookRepoSync.list(0, anonymous).get(0).getId(), - notebookRepoSync.list(1, anonymous).get(0).getId()); + assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId()); /* remove Note */ notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous); @@ -188,6 +168,7 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { /* check that deleted in both storages */ assertEquals(0, notebookRepoSync.list(0, anonymous).size()); assertEquals(0, notebookRepoSync.list(1, anonymous).size()); + } @Test @@ -205,58 +186,30 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { assertEquals(1, note.getParagraphs().size()); /* new paragraph not yet saved into storages */ - assertEquals( - 0, - notebookRepoSync - .get(0, notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous) - .getParagraphs() - .size()); - assertEquals( - 0, - notebookRepoSync - .get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous) - .getParagraphs() - .size()); + assertEquals(0, notebookRepoSync.get(0, + notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + assertEquals(0, notebookRepoSync.get(1, + notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); /* save to storage under index 0 (first storage) */ notebookRepoSync.save(0, note, anonymous); /* check paragraph saved to first storage */ - assertEquals( - 1, - notebookRepoSync - .get(0, notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous) - .getParagraphs() - .size()); + assertEquals(1, notebookRepoSync.get(0, + notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size()); /* check paragraph isn't saved to second storage */ - assertEquals( - 0, - notebookRepoSync - .get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous) - .getParagraphs() - .size()); + assertEquals(0, notebookRepoSync.get(1, + notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); /* apply sync */ notebookRepoSync.sync(null); /* check whether added to second storage */ - assertEquals( - 1, - notebookRepoSync - .get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous) - .getParagraphs() - .size()); + assertEquals(1, notebookRepoSync.get(1, + notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); /* check whether same paragraph id */ - assertEquals( - p1.getId(), - notebookRepoSync - .get(0, notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous) - .getLastParagraph() - .getId()); - assertEquals( - p1.getId(), - notebookRepoSync - .get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous) - .getLastParagraph() - .getId()); + assertEquals(p1.getId(), notebookRepoSync.get(0, + notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getLastParagraph().getId()); + assertEquals(p1.getId(), notebookRepoSync.get(1, + notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getLastParagraph().getId()); notebookRepoSync.remove(note.getId(), anonymous); } @@ -287,22 +240,12 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { @Test public void testOneWaySyncOnReloadedList() throws IOException, SchedulerException { - System.setProperty( - ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "true"); conf = ZeppelinConfiguration.create(); notebookRepoSync = new NotebookRepoSync(conf); - notebookSync = - new Notebook( - conf, - notebookRepoSync, - schedulerFactory, - factory, - interpreterSettingManager, - this, - search, - notebookAuthorization, - credentials); + notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, + notebookAuthorization, credentials); // check that both storage repos are empty assertTrue(notebookRepoSync.getRepoCount() > 1); @@ -345,23 +288,12 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { @Test public void testCheckpointOneStorage() throws IOException, SchedulerException { - System.setProperty( - ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), - "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); ZeppelinConfiguration vConf = ZeppelinConfiguration.create(); NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf); - Notebook vNotebookSync = - new Notebook( - vConf, - vRepoSync, - schedulerFactory, - factory, - interpreterSettingManager, - this, - search, - notebookAuthorization, - credentials); + Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, + notebookAuthorization, credentials); // one git versioned storage initialized assertThat(vRepoSync.getRepoCount()).isEqualTo(1); @@ -422,30 +354,18 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { notebookRepoSync.save(1, note, null); /* check paragraph isn't saved into first storage */ - assertEquals( - 0, - notebookRepoSync - .get(0, notebookRepoSync.list(0, null).get(0).getId(), null) - .getParagraphs() - .size()); + assertEquals(0, notebookRepoSync.get(0, + notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size()); /* check paragraph is saved into second storage */ - assertEquals( - 1, - notebookRepoSync - .get(1, notebookRepoSync.list(1, null).get(0).getId(), null) - .getParagraphs() - .size()); + assertEquals(1, notebookRepoSync.get(1, + notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size()); /* now sync by user1 */ notebookRepoSync.sync(user1); /* check that note updated and acl are same on main storage*/ - assertEquals( - 1, - notebookRepoSync - .get(0, notebookRepoSync.list(0, null).get(0).getId(), null) - .getParagraphs() - .size()); + assertEquals(1, notebookRepoSync.get(0, + notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size()); assertEquals(true, authInfo.isOwner(note.getId(), entity)); assertEquals(1, authInfo.getOwners(note.getId()).size()); assertEquals(0, authInfo.getReaders(note.getId()).size()); @@ -477,31 +397,44 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { assertEquals(true, authInfo.isWriter(note.getId(), entity)); } - static void delete(File file) { - if (file.isFile()) file.delete(); - else if (file.isDirectory()) { - File[] files = file.listFiles(); - if (files != null && files.length > 0) { - for (File f : files) { - delete(f); + static void delete(File file){ + if(file.isFile()) file.delete(); + else if(file.isDirectory()){ + File [] files = file.listFiles(); + if(files!=null && files.length>0){ + for(File f : files){ + delete(f); + } } + file.delete(); } - file.delete(); - } } + + @Override - public void onOutputAppend(Paragraph paragraph, int idx, String output) {} + public void onOutputAppend(Paragraph paragraph, int idx, String output) { + + } @Override - public void onOutputUpdate(Paragraph paragraph, int idx, InterpreterResultMessage msg) {} + public void onOutputUpdate(Paragraph paragraph, int idx, InterpreterResultMessage msg) { + + } @Override - public void onOutputUpdateAll(Paragraph paragraph, List<InterpreterResultMessage> msgs) {} + public void onOutputUpdateAll(Paragraph paragraph, List<InterpreterResultMessage> msgs) { + + } @Override - public void onProgressUpdate(Paragraph paragraph, int progress) {} + public void onProgressUpdate(Paragraph paragraph, int progress) { + } @Override - public void onStatusChange(Paragraph paragraph, Status before, Status after) {} + public void onStatusChange(Paragraph paragraph, Status before, Status after) { + + } + + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java index 4d8bf52..7f450df 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java @@ -16,11 +16,12 @@ */ package org.apache.zeppelin.notebook.repo.mock; -import java.io.IOException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; +import java.io.IOException; + public class VFSNotebookRepoMock extends VFSNotebookRepo { public VFSNotebookRepoMock() { @@ -31,4 +32,5 @@ public class VFSNotebookRepoMock extends VFSNotebookRepo { public void init(ZeppelinConfiguration conf) throws IOException { super.init(conf); } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java index 19da9d5..7d0415b 100644 --- a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java @@ -17,8 +17,6 @@ package org.apache.zeppelin.notebook.repo; -import java.io.IOException; -import java.net.URISyntaxException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.user.AuthenticationInfo; import org.eclipse.jgit.api.Git; @@ -31,18 +29,22 @@ import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.net.URISyntaxException; + /** - * GitHub integration to store notebooks in a GitHub repository. It uses the same simple logic - * implemented in @see {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} + * GitHub integration to store notebooks in a GitHub repository. + * It uses the same simple logic implemented in @see + * {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} * - * <p>The logic for updating the local repository from the remote repository is the following: - - * When the <code>GitHubNotebookRepo</code> is initialized - When pushing the changes to the remote - * repository + * The logic for updating the local repository from the remote repository is the following: + * - When the <code>GitHubNotebookRepo</code> is initialized + * - When pushing the changes to the remote repository * - * <p>The logic for updating the remote repository on GitHub from local repository is the following: + * The logic for updating the remote repository on GitHub from local repository is the following: * - When commit the changes (saving the notebook) * - * <p>You should be able to use this integration with all remote git repositories that accept + * You should be able to use this integration with all remote git repositories that accept * username + password authentication, not just GitHub. */ public class GitHubNotebookRepo extends GitNotebookRepo { @@ -96,9 +98,11 @@ public class GitHubNotebookRepo extends GitNotebookRepo { LOG.debug("Pulling latest changes from remote stream"); PullCommand pullCommand = git.pull(); pullCommand.setCredentialsProvider( - new UsernamePasswordCredentialsProvider( - zeppelinConfiguration.getZeppelinNotebookGitUsername(), - zeppelinConfiguration.getZeppelinNotebookGitAccessToken())); + new UsernamePasswordCredentialsProvider( + zeppelinConfiguration.getZeppelinNotebookGitUsername(), + zeppelinConfiguration.getZeppelinNotebookGitAccessToken() + ) + ); pullCommand.call(); @@ -112,9 +116,11 @@ public class GitHubNotebookRepo extends GitNotebookRepo { LOG.debug("Pushing latest changes to remote stream"); PushCommand pushCommand = git.push(); pushCommand.setCredentialsProvider( - new UsernamePasswordCredentialsProvider( - zeppelinConfiguration.getZeppelinNotebookGitUsername(), - zeppelinConfiguration.getZeppelinNotebookGitAccessToken())); + new UsernamePasswordCredentialsProvider( + zeppelinConfiguration.getZeppelinNotebookGitUsername(), + zeppelinConfiguration.getZeppelinNotebookGitAccessToken() + ) + ); pushCommand.call(); } catch (GitAPIException e) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java index de59e94..e331714 100644 --- a/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java @@ -1,3 +1,4 @@ + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -17,12 +18,8 @@ package org.apache.zeppelin.notebook.repo; -import static org.mockito.Mockito.mock; import com.google.common.base.Joiner; -import java.io.File; -import java.io.IOException; -import java.util.Iterator; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterFactory; @@ -40,11 +37,17 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import static org.mockito.Mockito.mock; + /** - * This tests the remote Git tracking for notebooks. The tests uses two local Git repositories - * created locally to handle the tracking of Git actions (pushes and pulls). The repositories are: - * 1. The first repository is considered as a remote that mimics a remote GitHub directory 2. The - * second repository is considered as the local notebook repository + * This tests the remote Git tracking for notebooks. The tests uses two local Git repositories created locally + * to handle the tracking of Git actions (pushes and pulls). The repositories are: + * 1. The first repository is considered as a remote that mimics a remote GitHub directory + * 2. The second repository is considered as the local notebook repository */ public class GitHubNotebookRepoTest { private static final Logger LOG = LoggerFactory.getLogger(GitHubNotebookRepoTest.class); @@ -64,10 +67,10 @@ public class GitHubNotebookRepoTest { public void setUp() throws Exception { conf = ZeppelinConfiguration.create(); - String remoteRepositoryPath = - System.getProperty("java.io.tmpdir") + "/ZeppelinTestRemote_" + System.currentTimeMillis(); - String localRepositoryPath = - System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + System.currentTimeMillis(); + String remoteRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTestRemote_" + + System.currentTimeMillis(); + String localRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + + System.currentTimeMillis(); // Create a fake remote notebook Git repository locally in another directory remoteZeppelinDir = new File(remoteRepositoryPath); @@ -84,46 +87,37 @@ public class GitHubNotebookRepoTest { File notebookDir = new File(localNotebooksDir); notebookDir.mkdirs(); - // Copy the test notebook directory from the test/resources/2A94M5J1Z folder to the fake remote - // Git directory + // Copy the test notebook directory from the test/resources/2A94M5J1Z folder to the fake remote Git directory String remoteTestNoteDir = Joiner.on(File.separator).join(remoteNotebooksDir, TEST_NOTE_ID); FileUtils.copyDirectory( - new File( - GitHubNotebookRepoTest.class - .getResource(Joiner.on(File.separator).join("", TEST_NOTE_ID)) - .getFile()), - new File(remoteTestNoteDir)); + new File( + GitHubNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID) + ).getFile() + ), new File(remoteTestNoteDir) + ); // Create the fake remote Git repository - Repository remoteRepository = - new FileRepository(Joiner.on(File.separator).join(remoteNotebooksDir, ".git")); + Repository remoteRepository = new FileRepository(Joiner.on(File.separator).join(remoteNotebooksDir, ".git")); remoteRepository.create(); remoteGit = new Git(remoteRepository); remoteGit.add().addFilepattern(".").call(); - firstCommitRevision = - remoteGit.commit().setMessage("First commit from remote repository").call(); + firstCommitRevision = remoteGit.commit().setMessage("First commit from remote repository").call(); // Set the Git and Git configurations - System.setProperty( - ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), - remoteZeppelinDir.getAbsolutePath()); - System.setProperty( - ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), - notebookDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), remoteZeppelinDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); // Set the GitHub configurations System.setProperty( - ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), - "org.apache.zeppelin.notebook.repo.GitHubNotebookRepo"); - System.setProperty( - ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL.getVarName(), - remoteNotebooksDir + File.separator + ".git"); - System.setProperty( - ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME.getVarName(), "token"); - System.setProperty( - ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN.getVarName(), - "access-token"); + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), + "org.apache.zeppelin.notebook.repo.GitHubNotebookRepo"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL.getVarName(), + remoteNotebooksDir + File.separator + ".git"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME.getVarName(), "token"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN.getVarName(), + "access-token"); // Create the Notebook repository (configured for the local repository) gitHubNotebookRepo = new GitHubNotebookRepo(); @@ -133,9 +127,9 @@ public class GitHubNotebookRepoTest { @After public void tearDown() throws Exception { // Cleanup the temporary folders uses as Git repositories - File[] temporaryFolders = {remoteZeppelinDir, localZeppelinDir}; + File[] temporaryFolders = { remoteZeppelinDir, localZeppelinDir }; - for (File temporaryFolder : temporaryFolders) { + for(File temporaryFolder : temporaryFolders) { if (!FileUtils.deleteQuietly(temporaryFolder)) LOG.error("Failed to delete {} ", temporaryFolder.getName()); } @@ -143,75 +137,66 @@ public class GitHubNotebookRepoTest { @Test /** - * Test the case when the Notebook repository is created, it pulls the latest changes from the - * remote repository + * Test the case when the Notebook repository is created, it pulls the latest changes from the remote repository */ - public void pullChangesFromRemoteRepositoryOnLoadingNotebook() - throws IOException, GitAPIException { - NotebookRepoWithVersionControl.Revision firstHistoryRevision = - gitHubNotebookRepo.revisionHistory(TEST_NOTE_ID, null).get(0); + public void pullChangesFromRemoteRepositoryOnLoadingNotebook() throws IOException, GitAPIException { + NotebookRepoWithVersionControl.Revision firstHistoryRevision = gitHubNotebookRepo.revisionHistory(TEST_NOTE_ID, null).get(0); - assert (this.firstCommitRevision.getName().equals(firstHistoryRevision.id)); + assert(this.firstCommitRevision.getName().equals(firstHistoryRevision.id)); } @Test /** - * Test the case when the check-pointing (add new files and commit) it also pulls the latest - * changes from the remote repository + * Test the case when the check-pointing (add new files and commit) it also pulls the latest changes from the + * remote repository */ public void pullChangesFromRemoteRepositoryOnCheckpointing() throws GitAPIException, IOException { // Create a new commit in the remote repository - RevCommit secondCommitRevision = - remoteGit.commit().setMessage("Second commit from remote repository").call(); + RevCommit secondCommitRevision = remoteGit.commit().setMessage("Second commit from remote repository").call(); // Add a new paragraph to the local repository addParagraphToNotebook(TEST_NOTE_ID); // Commit and push the changes to remote repository - NotebookRepoWithVersionControl.Revision thirdCommitRevision = - gitHubNotebookRepo.checkpoint(TEST_NOTE_ID, "Third commit from local repository", null); + NotebookRepoWithVersionControl.Revision thirdCommitRevision = gitHubNotebookRepo.checkpoint( + TEST_NOTE_ID, "Third commit from local repository", null); - // Check all the commits as seen from the local repository. The commits are ordered - // chronologically. The last + // Check all the commits as seen from the local repository. The commits are ordered chronologically. The last // commit is the first in the commit logs. Iterator<RevCommit> revisions = gitHubNotebookRepo.getGit().log().all().call().iterator(); revisions.next(); // The Merge `master` commit after pushing to the remote repository - assert (thirdCommitRevision.id.equals( - revisions.next().getName())); // The local commit after adding the paragraph + assert(thirdCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph // The second commit done on the remote repository - assert (secondCommitRevision.getName().equals(revisions.next().getName())); + assert(secondCommitRevision.getName().equals(revisions.next().getName())); // The first commit done on the remote repository - assert (firstCommitRevision.getName().equals(revisions.next().getName())); + assert(firstCommitRevision.getName().equals(revisions.next().getName())); } @Test /** - * Test the case when the check-pointing (add new files and commit) it pushes the local commits to - * the remote repository + * Test the case when the check-pointing (add new files and commit) it pushes the local commits to the remote + * repository */ - public void pushLocalChangesToRemoteRepositoryOnCheckpointing() - throws IOException, GitAPIException { + public void pushLocalChangesToRemoteRepositoryOnCheckpointing() throws IOException, GitAPIException { // Add a new paragraph to the local repository addParagraphToNotebook(TEST_NOTE_ID); // Commit and push the changes to remote repository - NotebookRepoWithVersionControl.Revision secondCommitRevision = - gitHubNotebookRepo.checkpoint(TEST_NOTE_ID, "Second commit from local repository", null); + NotebookRepoWithVersionControl.Revision secondCommitRevision = gitHubNotebookRepo.checkpoint( + TEST_NOTE_ID, "Second commit from local repository", null); - // Check all the commits as seen from the remote repository. The commits are ordered - // chronologically. The last + // Check all the commits as seen from the remote repository. The commits are ordered chronologically. The last // commit is the first in the commit logs. Iterator<RevCommit> revisions = remoteGit.log().all().call().iterator(); - assert (secondCommitRevision.id.equals( - revisions.next().getName())); // The local commit after adding the paragraph + assert(secondCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph // The first commit done on the remote repository - assert (firstCommitRevision.getName().equals(revisions.next().getName())); + assert(firstCommitRevision.getName().equals(revisions.next().getName())); } private void addParagraphToNotebook(String noteId) throws IOException { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java index 0bc85fd..618568d 100644 --- a/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java @@ -30,7 +30,9 @@ import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Backend for storing Notebook on MongoDB */ +/** + * Backend for storing Notebook on MongoDB + */ public class MongoNotebookRepo implements NotebookRepo { private static final Logger LOG = LoggerFactory.getLogger(MongoNotebookRepo.class); @@ -39,7 +41,9 @@ public class MongoNotebookRepo implements NotebookRepo { private MongoDatabase db; private MongoCollection<Document> coll; - public MongoNotebookRepo() {} + public MongoNotebookRepo() { + + } public void init(ZeppelinConfiguration conf) throws IOException { this.conf = conf; @@ -55,14 +59,15 @@ public class MongoNotebookRepo implements NotebookRepo { } /** - * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true, this method will insert - * local notes into MongoDB on startup. If a note already exists in MongoDB, skip it. + * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true, + * this method will insert local notes into MongoDB on startup. + * If a note already exists in MongoDB, skip it. */ private void insertFileSystemNotes() throws IOException { LinkedList<Document> docs = new LinkedList<>(); // docs to be imported NotebookRepo vfsRepo = new VFSNotebookRepo(); vfsRepo.init(conf); - List<NoteInfo> infos = vfsRepo.list(null); + List<NoteInfo> infos = vfsRepo.list(null); // collect notes to be imported for (NoteInfo info : infos) { Note note = vfsRepo.get(info.getId(), null); @@ -78,16 +83,15 @@ public class MongoNotebookRepo implements NotebookRepo { try { coll.insertMany(docs, new InsertManyOptions().ordered(false)); } catch (MongoBulkWriteException e) { - printDuplicatedException(e); // print duplicated document warning log + printDuplicatedException(e); //print duplicated document warning log } - vfsRepo.close(); // it does nothing for now but maybe in the future... + vfsRepo.close(); // it does nothing for now but maybe in the future... } /** - * MongoBulkWriteException contains error messages that inform which documents were duplicated. - * This method catches those ID and print them. - * + * MongoBulkWriteException contains error messages that inform + * which documents were duplicated. This method catches those ID and print them. * @param e */ private void printDuplicatedException(MongoBulkWriteException e) { @@ -123,17 +127,19 @@ public class MongoNotebookRepo implements NotebookRepo { } /** - * Find documents of which type of _id is object ID, and change it to note ID. Since updating _id - * field is not allowed, remove original documents and insert new ones with string _id(note ID) + * Find documents of which type of _id is object ID, and change it to note ID. + * Since updating _id field is not allowed, remove original documents and insert + * new ones with string _id(note ID) */ private void syncId() { // find documents whose id type is object id - MongoCursor<Document> cursor = coll.find(type("_id", BsonType.OBJECT_ID)).iterator(); + MongoCursor<Document> cursor = coll.find(type("_id", BsonType.OBJECT_ID)).iterator(); // if there is no such document, exit - if (!cursor.hasNext()) return; + if (!cursor.hasNext()) + return; - List<ObjectId> oldDocIds = new LinkedList<>(); // document ids need to update - List<Document> updatedDocs = new LinkedList<>(); // new documents to be inserted + List<ObjectId> oldDocIds = new LinkedList<>(); // document ids need to update + List<Document> updatedDocs = new LinkedList<>(); // new documents to be inserted while (cursor.hasNext()) { Document doc = cursor.next(); @@ -152,7 +158,9 @@ public class MongoNotebookRepo implements NotebookRepo { cursor.close(); } - /** Convert document to note */ + /** + * Convert document to note + */ private Note documentToNote(Document doc) { // document to JSON String json = doc.toJson(); @@ -160,7 +168,9 @@ public class MongoNotebookRepo implements NotebookRepo { return Note.fromJson(json); } - /** Convert note to document */ + /** + * Convert note to document + */ private Document noteToDocument(Note note) { // note to JSON String json = note.toJson(); @@ -208,4 +218,5 @@ public class MongoNotebookRepo implements NotebookRepo { public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { LOG.warn("Method not implemented"); } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index 21363d8..364943c 100644 --- a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -17,26 +17,6 @@ package org.apache.zeppelin.notebook.repo; -import com.amazonaws.AmazonClientException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.ClientConfigurationFactory; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.AmazonS3EncryptionClient; -import com.amazonaws.services.s3.model.CryptoConfiguration; -import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -47,6 +27,7 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; + import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -54,11 +35,36 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Backend for storing Notebooks on S3 */ +import com.amazonaws.AmazonClientException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.ClientConfigurationFactory; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * Backend for storing Notebooks on S3 + */ public class S3NotebookRepo implements NotebookRepo { private static final Logger LOG = LoggerFactory.getLogger(S3NotebookRepo.class); @@ -82,7 +88,9 @@ public class S3NotebookRepo implements NotebookRepo { private boolean useServerSideEncryption; private ZeppelinConfiguration conf; - public S3NotebookRepo() {} + public S3NotebookRepo() { + + } public void init(ZeppelinConfiguration conf) throws IOException { this.conf = conf; @@ -100,18 +108,20 @@ public class S3NotebookRepo implements NotebookRepo { } ClientConfiguration cliConf = createClientConfiguration(); - + // see if we should be encrypting data in S3 String kmsKeyID = conf.getS3KMSKeyID(); if (kmsKeyID != null) { // use the AWS KMS to encrypt data KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID); this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf); - } else if (conf.getS3EncryptionMaterialsProviderClass() != null) { + } + else if (conf.getS3EncryptionMaterialsProviderClass() != null) { // use a custom encryption materials provider class EncryptionMaterialsProvider emp = createCustomProvider(conf); this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf); - } else { + } + else { // regular S3 this.s3client = new AmazonS3Client(credentialsProvider, cliConf); } @@ -121,8 +131,8 @@ public class S3NotebookRepo implements NotebookRepo { } /** - * Create an instance of a custom encryption materials provider class which supplies encryption - * keys to use when reading/writing data in S3. + * Create an instance of a custom encryption materials provider class + * which supplies encryption keys to use when reading/writing data in S3. */ private EncryptionMaterialsProvider createCustomProvider(ZeppelinConfiguration conf) throws IOException { @@ -133,17 +143,15 @@ public class S3NotebookRepo implements NotebookRepo { Object empInstance = Class.forName(empClassname).newInstance(); if (empInstance instanceof EncryptionMaterialsProvider) { emp = (EncryptionMaterialsProvider) empInstance; - } else { - throw new IOException( - "Class " - + empClassname - + " does not implement " + } + else { + throw new IOException("Class " + empClassname + " does not implement " + EncryptionMaterialsProvider.class.getName()); } - } catch (Exception e) { - throw new IOException( - "Unable to instantiate encryption materials provider class " + empClassname + ": " + e, - e); + } + catch (Exception e) { + throw new IOException("Unable to instantiate encryption materials provider class " + + empClassname + ": " + e, e); } return emp; @@ -151,7 +159,6 @@ public class S3NotebookRepo implements NotebookRepo { /** * Create AWS client configuration and return it. - * * @return AWS client configuration */ private ClientConfiguration createClientConfiguration() { @@ -171,8 +178,9 @@ public class S3NotebookRepo implements NotebookRepo { List<NoteInfo> infos = new LinkedList<>(); NoteInfo info; try { - ListObjectsRequest listObjectsRequest = - new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook"); + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withPrefix(user + "/" + "notebook"); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); @@ -196,7 +204,8 @@ public class S3NotebookRepo implements NotebookRepo { S3Object s3object; try { s3object = s3client.getObject(new GetObjectRequest(bucketName, key)); - } catch (AmazonClientException ace) { + } + catch (AmazonClientException ace) { throw new IOException("Unable to retrieve object from S3: " + ace, ace); } @@ -237,9 +246,11 @@ public class S3NotebookRepo implements NotebookRepo { } s3client.putObject(putRequest); - } catch (AmazonClientException ace) { + } + catch (AmazonClientException ace) { throw new IOException("Unable to store note in S3: " + ace, ace); - } finally { + } + finally { FileUtils.deleteQuietly(file); } } @@ -247,8 +258,8 @@ public class S3NotebookRepo implements NotebookRepo { @Override public void remove(String noteId, AuthenticationInfo subject) throws IOException { String key = user + "/" + "notebook" + "/" + noteId; - final ListObjectsRequest listObjectsRequest = - new ListObjectsRequest().withBucketName(bucketName).withPrefix(key); + final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName).withPrefix(key); try { ObjectListing objects = s3client.listObjects(listObjectsRequest); @@ -258,14 +269,15 @@ public class S3NotebookRepo implements NotebookRepo { } objects = s3client.listNextBatchOfObjects(objects); } while (objects.isTruncated()); - } catch (AmazonClientException ace) { + } + catch (AmazonClientException ace) { throw new IOException("Unable to remove note in S3: " + ace, ace); } } @Override public void close() { - // no-op + //no-op } @Override @@ -278,4 +290,5 @@ public class S3NotebookRepo implements NotebookRepo { public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { LOG.warn("Method not implemented"); } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 8577554..4294b86 100644 --- a/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.notebook.repo; +import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -45,7 +46,9 @@ import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** */ +/** +* +*/ public class VFSNotebookRepo implements NotebookRepo { private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepo.class); @@ -53,7 +56,9 @@ public class VFSNotebookRepo implements NotebookRepo { private URI filesystemRoot; protected ZeppelinConfiguration conf; - public VFSNotebookRepo() {} + public VFSNotebookRepo() { + + } @Override public void init(ZeppelinConfiguration conf) throws IOException { @@ -157,7 +162,7 @@ public class VFSNotebookRepo implements NotebookRepo { if (!noteJson.exists()) { throw new IOException(noteJson.getName().toString() + " not found"); } - + FileContent content = noteJson.getContent(); InputStream ins = content.getInputStream(); String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); @@ -237,7 +242,7 @@ public class VFSNotebookRepo implements NotebookRepo { @Override public void close() { - // no-op + //no-op } @Override @@ -268,15 +273,14 @@ public class VFSNotebookRepo implements NotebookRepo { LOG.error("Notebook path is invalid"); return; } - LOG.warn( - "{} will change notebook dir from {} to {}", - subject.getUser(), - getNotebookDirPath(), - newNotebookDirectotyPath); + LOG.warn("{} will change notebook dir from {} to {}", + subject.getUser(), getNotebookDirPath(), newNotebookDirectotyPath); try { setNotebookDirectory(newNotebookDirectotyPath); } catch (IOException e) { LOG.error("Cannot update notebook directory", e); } } + } +
