http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java new file mode 100644 index 0000000..7729d52 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.NoHeadException; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +/** + * NotebookRepo that hosts all the notebook FS in a single Git repo + * + * This impl intended to be simple and straightforward: + * - does not handle branches + * - only basic local git file repo, no remote Github push\pull. GitHub integration is + * implemented in @see {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} + * + * TODO(bzz): add default .gitignore + */ +public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWithVersionControl { + private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class); + + private String localPath; + private Git git; + + public GitNotebookRepo() { + super(); + } + + @VisibleForTesting + public GitNotebookRepo(ZeppelinConfiguration conf) throws IOException { + this(); + init(conf); + } + + @Override + public void init(ZeppelinConfiguration conf) throws IOException { + //TODO(zjffdu), it is weird that I can not call super.init directly here, as it would cause + //AbstractMethodError + this.conf = conf; + setNotebookDirectory(conf.getNotebookDir()); + + localPath = getRootDir().getName().getPath(); + LOG.info("Opening a git repo at '{}'", localPath); + Repository localRepo = new FileRepository(Joiner.on(File.separator).join(localPath, ".git")); + if (!localRepo.getDirectory().exists()) { + LOG.info("Git repo {} does not exist, creating a new one", localRepo.getDirectory()); + localRepo.create(); + } + git = new Git(localRepo); + } + + @Override + public synchronized void save(Note note, AuthenticationInfo subject) throws IOException { + super.save(note, subject); + } + + /* implemented as git add+commit + * @param pattern is the noteId + * @param commitMessage is a commit message (checkpoint message) + * (non-Javadoc) + * @see org.apache.zeppelin.notebook.repo.VFSNotebookRepo#checkpoint(String, String) + */ + @Override + public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) { + Revision revision = Revision.EMPTY; + try { + List<DiffEntry> gitDiff = git.diff().call(); + if (!gitDiff.isEmpty()) { + LOG.debug("Changes found for pattern '{}': {}", pattern, gitDiff); + DirCache added = git.add().addFilepattern(pattern).call(); + LOG.debug("{} changes are about to be commited", added.getEntryCount()); + RevCommit commit = git.commit().setMessage(commitMessage).call(); + revision = new Revision(commit.getName(), commit.getShortMessage(), commit.getCommitTime()); + } else { + LOG.debug("No changes found {}", pattern); + } + } catch (GitAPIException e) { + LOG.error("Failed to add+commit {} to Git", pattern, e); + } + return revision; + } + + /** + * the idea is to: + * 1. stash current changes + * 2. remember head commit and checkout to the desired revision + * 3. get note and checkout back to the head + * 4. apply stash on top and remove it + */ + @Override + public synchronized Note get(String noteId, String revId, AuthenticationInfo subject) + throws IOException { + Note note = null; + RevCommit stash = null; + try { + List<DiffEntry> gitDiff = git.diff().setPathFilter(PathFilter.create(noteId)).call(); + boolean modified = !gitDiff.isEmpty(); + if (modified) { + // stash changes + stash = git.stashCreate().call(); + Collection<RevCommit> stashes = git.stashList().call(); + LOG.debug("Created stash : {}, stash size : {}", stash, stashes.size()); + } + ObjectId head = git.getRepository().resolve(Constants.HEAD); + // checkout to target revision + git.checkout().setStartPoint(revId).addPath(noteId).call(); + // get the note + note = super.get(noteId, subject); + // checkout back to head + git.checkout().setStartPoint(head.getName()).addPath(noteId).call(); + if (modified && stash != null) { + // unstash changes + ObjectId applied = git.stashApply().setStashRef(stash.getName()).call(); + ObjectId dropped = git.stashDrop().setStashRef(0).call(); + Collection<RevCommit> stashes = git.stashList().call(); + LOG.debug("Stash applied as : {}, and dropped : {}, stash size: {}", applied, dropped, + stashes.size()); + } + } catch (GitAPIException e) { + LOG.error("Failed to return note from revision \"{}\"", revId, e); + } + return note; + } + + @Override + public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) { + List<Revision> history = Lists.newArrayList(); + LOG.debug("Listing history for {}:", noteId); + try { + Iterable<RevCommit> logs = git.log().addPath(noteId).call(); + for (RevCommit log: logs) { + history.add(new Revision(log.getName(), log.getShortMessage(), log.getCommitTime())); + LOG.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage()); + } + } catch (NoHeadException e) { + //when no initial commit exists + LOG.warn("No Head found for {}, {}", noteId, e.getMessage()); + } catch (GitAPIException e) { + LOG.error("Failed to get logs for {}", noteId, e); + } + return history; + } + + @Override + public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject) + throws IOException { + Note revisionNote = get(noteId, revId, subject); + if (revisionNote != null) { + save(revisionNote, subject); + } + return revisionNote; + } + + @Override + public void close() { + git.getRepository().close(); + } + + //DI replacements for Tests + protected Git getGit() { + return git; + } + + void setGit(Git git) { + this.git = git; + } + +} +
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..713407b --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.GitNotebookRepo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java new file mode 100644 index 0000000..58d7cc1 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.notebook.repo.NotebookRepoWithVersionControl.Revision; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.DiffEntry; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; + +public class GitNotebookRepoTest { + private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepoTest.class); + + private static final String TEST_NOTE_ID = "2A94M5J1Z"; + private static final String TEST_NOTE_ID2 = "2A94M5J2Z"; + + private File zeppelinDir; + private String notebooksDir; + private ZeppelinConfiguration conf; + private GitNotebookRepo notebookRepo; + + @Before + public void setUp() throws Exception { + String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + System.currentTimeMillis(); + zeppelinDir = new File(zpath); + zeppelinDir.mkdirs(); + new File(zeppelinDir, "conf").mkdirs(); + + notebooksDir = Joiner.on(File.separator).join(zpath, "notebook"); + File notebookDir = new File(notebooksDir); + notebookDir.mkdirs(); + + String testNoteDir = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID); + String testNoteDir2 = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID2); + FileUtils.copyDirectory( + new File( + GitNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID) + ).getFile() + ), + new File(testNoteDir)); + FileUtils.copyDirectory( + new File( + GitNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID2) + ).getFile() + ), + new File(testNoteDir2) + ); + + System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), zeppelinDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); + + conf = ZeppelinConfiguration.create(); + } + + @After + public void tearDown() throws Exception { + if (!FileUtils.deleteQuietly(zeppelinDir)) { + LOG.error("Failed to delete {} ", zeppelinDir.getName()); + } + } + + @Test + public void initNonemptyNotebookDir() throws IOException, GitAPIException { + //given - .git does not exit + File dotGit = new File(Joiner.on(File.separator).join(notebooksDir, ".git")); + assertThat(dotGit.exists()).isEqualTo(false); + + //when + notebookRepo = new GitNotebookRepo(conf); + + //then + Git git = notebookRepo.getGit(); + assertThat(git).isNotNull(); + + assertThat(dotGit.exists()).isEqualTo(true); + assertThat(notebookRepo.list(null)).isNotEmpty(); + + List<DiffEntry> diff = git.diff().call(); + // no commit, diff isn't empty + assertThat(diff).isNotEmpty(); + } + + @Test + public void showNotebookHistoryEmptyTest() throws GitAPIException, IOException { + //given + notebookRepo = new GitNotebookRepo(conf); + assertThat(notebookRepo.list(null)).isNotEmpty(); + + //when + List<Revision> testNotebookHistory = notebookRepo.revisionHistory(TEST_NOTE_ID, null); + + //then + //no initial commit, empty history + assertThat(testNotebookHistory).isEmpty(); + } + + @Test + public void showNotebookHistoryMultipleNotesTest() throws IOException { + //initial checks + notebookRepo = new GitNotebookRepo(conf); + assertThat(notebookRepo.list(null)).isNotEmpty(); + assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); + assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID2)).isTrue(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null)).isEmpty(); + + //add commit to both notes + notebookRepo.checkpoint(TEST_NOTE_ID, "first commit, note1", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); + notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); + + //modify, save and checkpoint first note + Note note = notebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map<String, Object> config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%md note1 test text"); + notebookRepo.save(note, null); + assertThat(notebookRepo.checkpoint(TEST_NOTE_ID, "second commit, note1", null)).isNotNull(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); + assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null)) + .isEqualTo(Revision.EMPTY); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); + + //modify, save and checkpoint second note + note = notebookRepo.get(TEST_NOTE_ID2, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", false); + p.setConfig(config); + p.setText("%md note2 test text"); + notebookRepo.save(note, null); + assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, "second commit, note2", null)).isNotNull(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(2); + } + + @Test + public void addCheckpointTest() throws IOException { + // initial checks + notebookRepo = new GitNotebookRepo(conf); + assertThat(notebookRepo.list(null)).isNotEmpty(); + assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + + notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null); + List<Revision> notebookHistoryBefore = notebookRepo.revisionHistory(TEST_NOTE_ID, null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isNotEmpty(); + int initialCount = notebookHistoryBefore.size(); + + // add changes to note + Note note = notebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map<String, Object> config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%md checkpoint test text"); + + // save and checkpoint note + notebookRepo.save(note, null); + notebookRepo.checkpoint(TEST_NOTE_ID, "second commit", null); + + // see if commit is added + List<Revision> notebookHistoryAfter = notebookRepo.revisionHistory(TEST_NOTE_ID, null); + assertThat(notebookHistoryAfter.size()).isEqualTo(initialCount + 1); + } + + private boolean containsNote(List<NoteInfo> notes, String noteId) { + for (NoteInfo note: notes) { + if (note.getId().equals(noteId)) { + return true; + } + } + return false; + } + + @Test + public void getRevisionTest() throws IOException { + // initial checks + notebookRepo = new GitNotebookRepo(conf); + assertThat(notebookRepo.list(null)).isNotEmpty(); + assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + + // add first checkpoint + Revision revision_1 = notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); + int paragraphCount_1 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size(); + + // add paragraph and save + Note note = notebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map<String, Object> config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); + p1.setText("checkpoint test text"); + notebookRepo.save(note, null); + + // second checkpoint + notebookRepo.checkpoint(TEST_NOTE_ID, "second commit", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); + int paragraphCount_2 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size(); + assertThat(paragraphCount_2).isEqualTo(paragraphCount_1 + 1); + + // get note from revision 1 + Note noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null); + assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1); + + // get current note + note = notebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_2); + + // add one more paragraph and save + Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config.put("enabled", false); + p2.setConfig(config); + p2.setText("get revision when modified note test text"); + notebookRepo.save(note, null); + note = notebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + int paragraphCount_3 = note.getParagraphs().size(); + assertThat(paragraphCount_3).isEqualTo(paragraphCount_2 + 1); + + // get revision 1 again + noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null); + assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1); + + // check that note is unchanged + note = notebookRepo.get(TEST_NOTE_ID, null); + assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_3); + } + + @Test + public void getRevisionFailTest() throws IOException { + // initial checks + notebookRepo = new GitNotebookRepo(conf); + assertThat(notebookRepo.list(null)).isNotEmpty(); + assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + + // add first checkpoint + Revision revision_1 = notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); + int paragraphCount_1 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size(); + + // get current note + Note note = notebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_1); + + // add one more paragraph and save + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map<String, Object> config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); + p1.setText("get revision when modified note test text"); + notebookRepo.save(note, null); + int paragraphCount_2 = note.getParagraphs().size(); + + // get note from revision 1 + Note noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null); + assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1); + + // get current note + note = notebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_2); + + // test for absent revision + Revision absentRevision = new Revision("absentId", StringUtils.EMPTY, 0); + note = notebookRepo.get(TEST_NOTE_ID, absentRevision.id, null); + assertThat(note).isNull(); + } + + @Test + public void setRevisionTest() throws IOException { + //create repo and check that note doesn't contain revisions + notebookRepo = new GitNotebookRepo(conf); + assertThat(notebookRepo.list(null)).isNotEmpty(); + assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + + // get current note + Note note = notebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + int paragraphCount_1 = note.getParagraphs().size(); + LOG.info("initial paragraph count: {}", paragraphCount_1); + + // checkpoint revision1 + Revision revision1 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: first commit", null); + //TODO(khalid): change to EMPTY after rebase + assertThat(revision1).isNotNull(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); + + // add one more paragraph and save + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map<String, Object> config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); + p1.setText("set revision sample text"); + notebookRepo.save(note, null); + int paragraphCount_2 = note.getParagraphs().size(); + assertThat(paragraphCount_2).isEqualTo(paragraphCount_1 + 1); + LOG.info("paragraph count after modification: {}", paragraphCount_2); + + // checkpoint revision2 + Revision revision2 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: second commit", null); + //TODO(khalid): change to EMPTY after rebase + assertThat(revision2).isNotNull(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); + + // set note to revision1 + Note returnedNote = notebookRepo.setNoteRevision(note.getId(), revision1.id, null); + assertThat(returnedNote).isNotNull(); + assertThat(returnedNote.getParagraphs().size()).isEqualTo(paragraphCount_1); + + // check note from repo + Note updatedNote = notebookRepo.get(note.getId(), null); + assertThat(updatedNote).isNotNull(); + assertThat(updatedNote.getParagraphs().size()).isEqualTo(paragraphCount_1); + + // set back to revision2 + returnedNote = notebookRepo.setNoteRevision(note.getId(), revision2.id, null); + assertThat(returnedNote).isNotNull(); + assertThat(returnedNote.getParagraphs().size()).isEqualTo(paragraphCount_2); + + // check note from repo + updatedNote = notebookRepo.get(note.getId(), null); + assertThat(updatedNote).isNotNull(); + assertThat(updatedNote.getParagraphs().size()).isEqualTo(paragraphCount_2); + + // try failure case - set to invalid revision + returnedNote = notebookRepo.setNoteRevision(note.getId(), "nonexistent_id", null); + assertThat(returnedNote).isNull(); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java new file mode 100644 index 0000000..738d708 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +//TODO(zjffdu) move it to zeppelin-zengine +public class NotebookRepoSyncInitializationTest { + private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncInitializationTest.class); + private String validFirstStorageClass = "org.apache.zeppelin.notebook.repo.VFSNotebookRepo"; + private String validSecondStorageClass = "org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"; + private String invalidStorageClass = "org.apache.zeppelin.notebook.repo.DummyNotebookRepo"; + private String validOneStorageConf = validFirstStorageClass; + private String validTwoStorageConf = validFirstStorageClass + "," + validSecondStorageClass; + private String invalidTwoStorageConf = validFirstStorageClass + "," + invalidStorageClass; + private String unsupportedStorageConf = validFirstStorageClass + "," + validSecondStorageClass + "," + validSecondStorageClass; + private String emptyStorageConf = ""; + + @Before + public void setUp(){ + System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); + //setup routine + } + + @After + public void tearDown() { + //tear-down routine + } + + @Test + public void validInitOneStorageTest() throws IOException { + // no need to initialize folder due to one storage + // set confs + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), validOneStorageConf); + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + // create repo + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + // check proper initialization of one storage + assertEquals(notebookRepoSync.getRepoCount(), 1); + assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo); + } + + @Test + public void validInitTwoStorageTest() throws IOException { + // initialize folders for each storage + String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); + File mainZepDir = new File(zpath); + mainZepDir.mkdirs(); + new File(mainZepDir, "conf").mkdirs(); + String mainNotePath = zpath+"/notebook"; + String secNotePath = mainNotePath + "_secondary"; + File mainNotebookDir = new File(mainNotePath); + File secNotebookDir = new File(secNotePath); + mainNotebookDir.mkdirs(); + secNotebookDir.mkdirs(); + + // set confs + System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), validTwoStorageConf); + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + // create repo + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + // check that both initialized + assertEquals(notebookRepoSync.getRepoCount(), 2); + assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo); + assertTrue(notebookRepoSync.getRepo(1) instanceof VFSNotebookRepoMock); + } + + @Test + public void invalidInitTwoStorageTest() throws IOException { + // set confs + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), invalidTwoStorageConf); + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + // create repo + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + // check that second didn't initialize + LOG.info(" " + notebookRepoSync.getRepoCount()); + assertEquals(notebookRepoSync.getRepoCount(), 1); + assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo); + } + + @Test + public void initUnsupportedNumberStoragesTest() throws IOException { + // initialize folders for each storage, currently for 2 only + String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); + File mainZepDir = new File(zpath); + mainZepDir.mkdirs(); + new File(mainZepDir, "conf").mkdirs(); + String mainNotePath = zpath+"/notebook"; + String secNotePath = mainNotePath + "_secondary"; + File mainNotebookDir = new File(mainNotePath); + File secNotebookDir = new File(secNotePath); + mainNotebookDir.mkdirs(); + secNotebookDir.mkdirs(); + + // set confs + System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), unsupportedStorageConf); + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + // create repo + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + // check that first two storages initialized instead of three + assertEquals(notebookRepoSync.getRepoCount(), 2); + assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo); + assertTrue(notebookRepoSync.getRepo(1) instanceof VFSNotebookRepoMock); + } + + @Test + public void initEmptyStorageTest() throws IOException { + // set confs + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), emptyStorageConf); + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + // create repo + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + // check initialization of one default storage + assertEquals(notebookRepoSync.getRepoCount(), 1); + assertTrue(notebookRepoSync.getRepo(0) instanceof NotebookRepoWithVersionControl); + } + + @Test + public void initOneDummyStorageTest() throws IOException { + // set confs + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), invalidStorageClass); + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + // create repo + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + // check initialization of one default storage instead of invalid one + assertEquals(notebookRepoSync.getRepoCount(), 1); + assertTrue(notebookRepoSync.getRepo(0) instanceof NotebookRepo); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java new file mode 100644 index 0000000..274c08a --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -0,0 +1,444 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.zeppelin.notebook.JobListenerFactory; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.notebook.NotebookAuthorization; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.notebook.ParagraphJobListener; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.search.SearchService; +import org.apache.zeppelin.storage.ConfigStorage; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.apache.zeppelin.user.Credentials; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.quartz.SchedulerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + + +//TODO(zjffdu) move it to zeppelin-zengine +public class NotebookRepoSyncTest implements JobListenerFactory { + + private File mainZepDir; + private ZeppelinConfiguration conf; + private SchedulerFactory schedulerFactory; + private File mainNotebookDir; + private File secNotebookDir; + private Notebook notebookSync; + private NotebookRepoSync notebookRepoSync; + private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; + private DependencyResolver depResolver; + private SearchService search; + private NotebookAuthorization notebookAuthorization; + private Credentials credentials; + private AuthenticationInfo anonymous; + private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class); + + @Before + public void setUp() throws Exception { + String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis(); + mainZepDir = new File(zpath); + mainZepDir.mkdirs(); + new File(mainZepDir, "conf").mkdirs(); + String mainNotePath = zpath+"/notebook"; + String secNotePath = mainNotePath + "_secondary"; + mainNotebookDir = new File(mainNotePath); + secNotebookDir = new File(secNotePath); + mainNotebookDir.mkdirs(); + secNotebookDir.mkdirs(); + + System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "false"); + System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), mainZepDir.getAbsolutePath() + "/conf"); + System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); + + LOG.info("main Note dir : " + mainNotePath); + LOG.info("secondary note dir : " + secNotePath); + conf = ZeppelinConfiguration.create(); + + ConfigStorage.reset(); + + this.schedulerFactory = SchedulerFactory.singleton(); + + depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); + interpreterSettingManager = new InterpreterSettingManager(conf, + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + factory = new InterpreterFactory(interpreterSettingManager); + + search = mock(SearchService.class); + notebookRepoSync = new NotebookRepoSync(conf); + notebookAuthorization = NotebookAuthorization.init(conf); + credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath(), null); + notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, + notebookAuthorization, credentials); + anonymous = new AuthenticationInfo("anonymous"); + } + + @After + public void tearDown() throws Exception { + delete(mainZepDir); + } + + @Test + public void testRepoCount() throws IOException { + assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount()); + } + + @Test + public void testSyncOnCreate() throws IOException { + /* check that both storage systems are empty */ + assertTrue(notebookRepoSync.getRepoCount() > 1); + assertEquals(0, notebookRepoSync.list(0, anonymous).size()); + assertEquals(0, notebookRepoSync.list(1, anonymous).size()); + + /* create note */ + Note note = notebookSync.createNote(anonymous); + + // check that automatically saved on both storages + assertEquals(1, notebookRepoSync.list(0, anonymous).size()); + assertEquals(1, notebookRepoSync.list(1, anonymous).size()); + assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId()); + + notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous); + } + + @Test + public void testSyncOnDelete() throws IOException { + /* create note */ + assertTrue(notebookRepoSync.getRepoCount() > 1); + assertEquals(0, notebookRepoSync.list(0, anonymous).size()); + assertEquals(0, notebookRepoSync.list(1, anonymous).size()); + + Note note = notebookSync.createNote(anonymous); + + /* check that created in both storage systems */ + assertEquals(1, notebookRepoSync.list(0, anonymous).size()); + assertEquals(1, notebookRepoSync.list(1, anonymous).size()); + assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId()); + + /* remove Note */ + notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous); + + /* check that deleted in both storages */ + assertEquals(0, notebookRepoSync.list(0, anonymous).size()); + assertEquals(0, notebookRepoSync.list(1, anonymous).size()); + + } + + @Test + public void testSyncUpdateMain() throws IOException { + + /* create note */ + Note note = notebookSync.createNote(anonymous); + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); + p1.setText("hello world"); + + /* new paragraph exists in note instance */ + assertEquals(1, note.getParagraphs().size()); + + /* new paragraph not yet saved into storages */ + assertEquals(0, notebookRepoSync.get(0, + notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + assertEquals(0, notebookRepoSync.get(1, + notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + + /* save to storage under index 0 (first storage) */ + notebookRepoSync.save(0, note, anonymous); + + /* check paragraph saved to first storage */ + assertEquals(1, notebookRepoSync.get(0, + notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + /* check paragraph isn't saved to second storage */ + assertEquals(0, notebookRepoSync.get(1, + notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + /* apply sync */ + notebookRepoSync.sync(null); + /* check whether added to second storage */ + assertEquals(1, notebookRepoSync.get(1, + notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + /* check whether same paragraph id */ + assertEquals(p1.getId(), notebookRepoSync.get(0, + notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getLastParagraph().getId()); + assertEquals(p1.getId(), notebookRepoSync.get(1, + notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getLastParagraph().getId()); + notebookRepoSync.remove(note.getId(), anonymous); + } + + @Test + public void testSyncOnReloadedList() throws IOException { + /* check that both storage repos are empty */ + assertTrue(notebookRepoSync.getRepoCount() > 1); + assertEquals(0, notebookRepoSync.list(0, anonymous).size()); + assertEquals(0, notebookRepoSync.list(1, anonymous).size()); + + File srcDir = new File("src/test/resources/2A94M5J1Z"); + File destDir = new File(secNotebookDir + "/2A94M5J1Z"); + + /* copy manually new notebook into secondary storage repo and check repos */ + try { + FileUtils.copyDirectory(srcDir, destDir); + } catch (IOException e) { + LOG.error(e.toString(), e); + } + assertEquals(0, notebookRepoSync.list(0, anonymous).size()); + assertEquals(1, notebookRepoSync.list(1, anonymous).size()); + + // After reloading notebooks repos should be synchronized + notebookSync.reloadAllNotes(anonymous); + assertEquals(1, notebookRepoSync.list(0, anonymous).size()); + assertEquals(1, notebookRepoSync.list(1, anonymous).size()); + } + + @Test + public void testOneWaySyncOnReloadedList() throws IOException, SchedulerException { + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "true"); + conf = ZeppelinConfiguration.create(); + notebookRepoSync = new NotebookRepoSync(conf); + notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, + notebookAuthorization, credentials); + + // check that both storage repos are empty + assertTrue(notebookRepoSync.getRepoCount() > 1); + assertEquals(0, notebookRepoSync.list(0, null).size()); + assertEquals(0, notebookRepoSync.list(1, null).size()); + + File srcDir = new File("src/test/resources/2A94M5J1Z"); + File destDir = new File(secNotebookDir + "/2A94M5J1Z"); + + // copy manually new notebook into secondary storage repo and check repos + try { + FileUtils.copyDirectory(srcDir, destDir); + } catch (IOException e) { + LOG.error(e.toString(), e); + } + assertEquals(0, notebookRepoSync.list(0, null).size()); + assertEquals(1, notebookRepoSync.list(1, null).size()); + + // after reloading the notebook should be wiped from secondary storage + notebookSync.reloadAllNotes(null); + assertEquals(0, notebookRepoSync.list(0, null).size()); + assertEquals(0, notebookRepoSync.list(1, null).size()); + + destDir = new File(mainNotebookDir + "/2A94M5J1Z"); + + // copy manually new notebook into primary storage repo and check repos + try { + FileUtils.copyDirectory(srcDir, destDir); + } catch (IOException e) { + LOG.error(e.toString(), e); + } + assertEquals(1, notebookRepoSync.list(0, null).size()); + assertEquals(0, notebookRepoSync.list(1, null).size()); + + // after reloading notebooks repos should be synchronized + notebookSync.reloadAllNotes(null); + assertEquals(1, notebookRepoSync.list(0, null).size()); + assertEquals(1, notebookRepoSync.list(1, null).size()); + } + + @Test + public void testCheckpointOneStorage() throws IOException, SchedulerException { + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); + ZeppelinConfiguration vConf = ZeppelinConfiguration.create(); + + NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf); + Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, + notebookAuthorization, credentials); + + // one git versioned storage initialized + assertThat(vRepoSync.getRepoCount()).isEqualTo(1); + assertThat(vRepoSync.getRepo(0)).isInstanceOf(GitNotebookRepo.class); + + GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0); + + // no notes + assertThat(vRepoSync.list(anonymous).size()).isEqualTo(0); + // create note + Note note = vNotebookSync.createNote(anonymous); + assertThat(vRepoSync.list(anonymous).size()).isEqualTo(1); + + String noteId = vRepoSync.list(anonymous).get(0).getId(); + // first checkpoint + vRepoSync.checkpoint(noteId, "checkpoint message", anonymous); + int vCount = gitRepo.revisionHistory(noteId, anonymous).size(); + assertThat(vCount).isEqualTo(1); + + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map<String, Object> config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%md checkpoint test"); + + // save and checkpoint again + vRepoSync.save(note, anonymous); + vRepoSync.checkpoint(noteId, "checkpoint message 2", anonymous); + assertThat(gitRepo.revisionHistory(noteId, anonymous).size()).isEqualTo(vCount + 1); + notebookRepoSync.remove(note.getId(), anonymous); + } + + @Test + public void testSyncWithAcl() throws IOException { + /* scenario 1 - note exists with acl on main storage */ + AuthenticationInfo user1 = new AuthenticationInfo("user1"); + Note note = notebookSync.createNote(user1); + assertEquals(0, note.getParagraphs().size()); + + // saved on both storages + assertEquals(1, notebookRepoSync.list(0, null).size()); + assertEquals(1, notebookRepoSync.list(1, null).size()); + + /* check that user1 is the only owner */ + NotebookAuthorization authInfo = NotebookAuthorization.getInstance(); + Set<String> entity = new HashSet<String>(); + entity.add(user1.getUser()); + assertEquals(true, authInfo.isOwner(note.getId(), entity)); + assertEquals(1, authInfo.getOwners(note.getId()).size()); + assertEquals(0, authInfo.getReaders(note.getId()).size()); + assertEquals(0, authInfo.getRunners(note.getId()).size()); + assertEquals(0, authInfo.getWriters(note.getId()).size()); + + /* update note and save on secondary storage */ + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + p1.setText("hello world"); + assertEquals(1, note.getParagraphs().size()); + notebookRepoSync.save(1, note, null); + + /* check paragraph isn't saved into first storage */ + assertEquals(0, notebookRepoSync.get(0, + notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size()); + /* check paragraph is saved into second storage */ + assertEquals(1, notebookRepoSync.get(1, + notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size()); + + /* now sync by user1 */ + notebookRepoSync.sync(user1); + + /* check that note updated and acl are same on main storage*/ + assertEquals(1, notebookRepoSync.get(0, + notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size()); + assertEquals(true, authInfo.isOwner(note.getId(), entity)); + assertEquals(1, authInfo.getOwners(note.getId()).size()); + assertEquals(0, authInfo.getReaders(note.getId()).size()); + assertEquals(0, authInfo.getRunners(note.getId()).size()); + assertEquals(0, authInfo.getWriters(note.getId()).size()); + + /* scenario 2 - note doesn't exist on main storage */ + /* remove from main storage */ + notebookRepoSync.remove(0, note.getId(), user1); + assertEquals(0, notebookRepoSync.list(0, null).size()); + assertEquals(1, notebookRepoSync.list(1, null).size()); + authInfo.removeNote(note.getId()); + assertEquals(0, authInfo.getOwners(note.getId()).size()); + assertEquals(0, authInfo.getReaders(note.getId()).size()); + assertEquals(0, authInfo.getRunners(note.getId()).size()); + assertEquals(0, authInfo.getWriters(note.getId()).size()); + + /* now sync - should bring note from secondary storage with added acl */ + notebookRepoSync.sync(user1); + assertEquals(1, notebookRepoSync.list(0, null).size()); + assertEquals(1, notebookRepoSync.list(1, null).size()); + assertEquals(1, authInfo.getOwners(note.getId()).size()); + assertEquals(1, authInfo.getReaders(note.getId()).size()); + assertEquals(1, authInfo.getRunners(note.getId()).size()); + assertEquals(1, authInfo.getWriters(note.getId()).size()); + assertEquals(true, authInfo.isOwner(note.getId(), entity)); + assertEquals(true, authInfo.isReader(note.getId(), entity)); + assertEquals(true, authInfo.isRunner(note.getId(), entity)); + assertEquals(true, authInfo.isWriter(note.getId(), entity)); + } + + static void delete(File file){ + if(file.isFile()) file.delete(); + else if(file.isDirectory()){ + File [] files = file.listFiles(); + if(files!=null && files.length>0){ + for(File f : files){ + delete(f); + } + } + file.delete(); + } + } + + @Override + public ParagraphJobListener getParagraphJobListener(Note note) { + return new ParagraphJobListener(){ + + @Override + public void onOutputAppend(Paragraph paragraph, int idx, String output) { + + } + + @Override + public void onOutputUpdate(Paragraph paragraph, int idx, InterpreterResultMessage msg) { + + } + + @Override + public void onOutputUpdateAll(Paragraph paragraph, List<InterpreterResultMessage> msgs) { + + } + + @Override + public void onProgressUpdate(Job job, int progress) { + } + + @Override + public void onStatusChange(Job job, Status before, Status after) { + + } + }; + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java new file mode 100644 index 0000000..7f450df --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.notebook.repo.mock; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; + +import java.io.IOException; + +public class VFSNotebookRepoMock extends VFSNotebookRepo { + + public VFSNotebookRepoMock() { + String secNotebookDir = ZeppelinConfiguration.create().getNotebookDir() + "_secondary"; + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), secNotebookDir); + } + + public void init(ZeppelinConfiguration conf) throws IOException { + super.init(conf); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J1Z/note.json ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J1Z/note.json b/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J1Z/note.json new file mode 100644 index 0000000..785ccea --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J1Z/note.json @@ -0,0 +1,341 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836981412_-1007008116", + "id": "20150213-231621_168813393", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:16:21 PM", + "dateStarted": "Apr 1, 2015 9:11:09 PM", + "dateFinished": "Apr 1, 2015 9:11:10 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "title": "Load data into table", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\";),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "title": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500779206_-1502780787", + "id": "20150210-015259_1403135953", + "result": { + "code": "SUCCESS", + "type": "TEXT", + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n" + }, + "dateCreated": "Feb 10, 2015 1:52:59 AM", + "dateStarted": "Jul 3, 2015 1:43:40 PM", + "dateFinished": "Jul 3, 2015 1:43:45 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500782552_-1439281894", + "id": "20150210-015302_1492795503", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n" + }, + "dateCreated": "Feb 10, 2015 1:53:02 AM", + "dateStarted": "Jul 3, 2015 1:43:17 PM", + "dateFinished": "Jul 3, 2015 1:43:23 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "maxAge": "35" + }, + "forms": { + "maxAge": { + "name": "maxAge", + "defaultValue": "30", + "hidden": false + } + } + }, + "jobName": "paragraph_1423720444030_-1424110477", + "id": "20150212-145404_867439529", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n" + }, + "dateCreated": "Feb 12, 2015 2:54:04 PM", + "dateStarted": "Jul 3, 2015 1:43:28 PM", + "dateFinished": "Jul 3, 2015 1:43:29 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "marital": "single" + }, + "forms": { + "marital": { + "name": "marital", + "defaultValue": "single", + "options": [ + { + "value": "single" + }, + { + "value": "divorced" + }, + { + "value": "married" + } + ], + "hidden": false + } + } + }, + "jobName": "paragraph_1423836262027_-210588283", + "id": "20150213-230422_1600658137", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n" + }, + "dateCreated": "Feb 13, 2015 11:04:22 PM", + "dateStarted": "Jul 3, 2015 1:43:33 PM", + "dateFinished": "Jul 3, 2015 1:43:34 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Apr 1, 2015 9:12:18 PM", + "dateFinished": "Apr 1, 2015 9:12:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```";, + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"; + }, + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "dateStarted": "Jul 3, 2015 1:44:56 PM", + "dateFinished": "Jul 3, 2015 1:44:56 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1435955447812_-158639899", + "id": "20150703-133047_853701097", + "dateCreated": "Jul 3, 2015 1:30:47 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Zeppelin Tutorial", + "id": "2A94M5J1Z", + "angularObjects": {}, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J2Z/note.json ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J2Z/note.json b/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J2Z/note.json new file mode 100644 index 0000000..79fe35c --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J2Z/note.json @@ -0,0 +1,87 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Apr 1, 2015 9:12:18 PM", + "dateFinished": "Apr 1, 2015 9:12:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```";, + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"; + }, + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "dateStarted": "Jul 3, 2015 1:44:56 PM", + "dateFinished": "Jul 3, 2015 1:44:56 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1435955447812_-158639899", + "id": "20150703-133047_853701097", + "dateCreated": "Jul 3, 2015 1:30:47 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Sample note - excerpt from Zeppelin Tutorial", + "id": "2A94M5J2Z", + "angularObjects": {}, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties b/zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties new file mode 100644 index 0000000..661632b --- /dev/null +++ b/zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n +#log4j.appender.stdout.layout.ConversionPattern= +#%5p [%t] (%F:%L) - %m%n +#%-4r [%t] %-5p %c %x - %m%n +# + +# Root logger option +log4j.rootLogger=INFO, stdout + +log4j.logger.org.apache.zeppelin.notebook.repo=DEBUG + +#mute some noisy guys +log4j.logger.org.apache.hadoop.mapred=WARN +log4j.logger.org.apache.hadoop.hive.ql=WARN +log4j.logger.org.apache.hadoop.hive.metastore=WARN +log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN + +log4j.logger.org.quartz=WARN +log4j.logger.DataNucleus=WARN +log4j.logger.DataNucleus.MetaData=ERROR +log4j.logger.DataNucleus.Datastore=ERROR + +# Log all JDBC parameters +log4j.logger.org.hibernate.type=ALL + +log4j.logger.org.apache.zeppelin.interpreter=DEBUG +log4j.logger.org.apache.zeppelin.scheduler=DEBUG +log4j.logger.org.apache.zeppelin.plugin=DEBUG + http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/pom.xml b/zeppelin-plugins/notebookrepo/github/pom.xml new file mode 100644 index 0000000..3673efb --- /dev/null +++ b/zeppelin-plugins/notebookrepo/github/pom.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zengine-plugins-parent</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>../../../zeppelin-plugins</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-github</artifactId> + <packaging>jar</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugin GitHubNotebookRepo</name> + <description>NotebookRepo implementation based on GitHub</description> + + <properties> + <plugin.name>NotebookRepo/GitHubNotebookRepo</plugin.name> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.zeppelin</groupId> + <artifactId>notebookrepo-git</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java new file mode 100644 index 0000000..6052e5f --- /dev/null +++ b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.PullCommand; +import org.eclipse.jgit.api.PushCommand; +import org.eclipse.jgit.api.RemoteAddCommand; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URISyntaxException; + +/** + * GitHub integration to store notebooks in a GitHub repository. + * It uses the same simple logic implemented in @see + * {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} + * + * The logic for updating the local repository from the remote repository is the following: + * - When the <code>GitHubNotebookRepo</code> is initialized + * - When pushing the changes to the remote repository + * + * The logic for updating the remote repository on GitHub from local repository is the following: + * - When commit the changes (saving the notebook) + */ +public class GitHubNotebookRepo extends GitNotebookRepo { + private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class); + private ZeppelinConfiguration zeppelinConfiguration; + private Git git; + + public GitHubNotebookRepo(ZeppelinConfiguration conf) throws IOException { + super(conf); + + this.git = super.getGit(); + this.zeppelinConfiguration = conf; + + configureRemoteStream(); + pullFromRemoteStream(); + } + + @Override + public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) { + Revision revision = super.checkpoint(pattern, commitMessage, subject); + + updateRemoteStream(); + + return revision; + } + + private void configureRemoteStream() { + try { + LOG.debug("Setting up remote stream"); + RemoteAddCommand remoteAddCommand = git.remoteAdd(); + remoteAddCommand.setName(zeppelinConfiguration.getZeppelinNotebookGitRemoteOrigin()); + remoteAddCommand.setUri(new URIish(zeppelinConfiguration.getZeppelinNotebookGitURL())); + remoteAddCommand.call(); + } catch (GitAPIException e) { + LOG.error("Error configuring GitHub", e); + } catch (URISyntaxException e) { + LOG.error("Error in GitHub URL provided", e); + } + } + + private void updateRemoteStream() { + LOG.debug("Updating remote stream"); + + pullFromRemoteStream(); + pushToRemoteSteam(); + } + + private void pullFromRemoteStream() { + try { + LOG.debug("Pull latest changed from remote stream"); + PullCommand pullCommand = git.pull(); + pullCommand.setCredentialsProvider( + new UsernamePasswordCredentialsProvider( + zeppelinConfiguration.getZeppelinNotebookGitUsername(), + zeppelinConfiguration.getZeppelinNotebookGitAccessToken() + ) + ); + + pullCommand.call(); + + } catch (GitAPIException e) { + LOG.error("Error when pulling latest changes from remote repository", e); + } + } + + private void pushToRemoteSteam() { + try { + LOG.debug("Push latest changed from remote stream"); + PushCommand pushCommand = git.push(); + pushCommand.setCredentialsProvider( + new UsernamePasswordCredentialsProvider( + zeppelinConfiguration.getZeppelinNotebookGitUsername(), + zeppelinConfiguration.getZeppelinNotebookGitAccessToken() + ) + ); + + pushCommand.call(); + } catch (GitAPIException e) { + LOG.error("Error when pushing latest changes from remote repository", e); + } + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..25d5145 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.GitHubNotebookRepo \ No newline at end of file
