Repository: incubator-zeppelin Updated Branches: refs/heads/master fd2a28853 -> ae08e713e
ZEPPELIN-367 Sync with secondary storage when listing notebooks This PR resolves https://issues.apache.org/jira/browse/ZEPPELIN-367 by synchronizing with secondary storage when reloading all notebooks into memory. Currently, existing `ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE` from https://github.com/apache/incubator-zeppelin/pull/331 is used as a flag since these functionalities are correlated, however it can be separated into distinct flag as well. Author: Khalid Huseynov <[email protected]> Closes #375 from khalidhuseynov/sync-on-list and squashes the following commits: e2cf5d9 [Khalid Huseynov] minor fix 878a23e [Khalid Huseynov] initial working commit Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/ae08e713 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/ae08e713 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/ae08e713 Branch: refs/heads/master Commit: ae08e713e43ce9ebfb11f0f9cf7a7ea42885d677 Parents: fd2a288 Author: Khalid Huseynov <[email protected]> Authored: Wed Oct 28 20:40:16 2015 +0900 Committer: Lee moon soo <[email protected]> Committed: Sun Nov 8 09:30:48 2015 +0900 ---------------------------------------------------------------------- .../notebook/repo/NotebookRepoSync.java | 6 ++++ .../notebook/repo/NotebookRepoSyncTest.java | 35 ++++++++++++++++++++ 2 files changed, 41 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/ae08e713/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java index 288cff9..49f40fc 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java @@ -42,12 +42,15 @@ public class NotebookRepoSync implements NotebookRepo{ private static final int maxRepoNum = 2; private static final String pushKey = "pushNoteIDs"; private static final String pullKey = "pullNoteIDs"; + private static ZeppelinConfiguration config; /** * @param (conf) * @throws - Exception */ public NotebookRepoSync(ZeppelinConfiguration conf) throws Exception { + + config = conf; String allStorageClassNames = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE).trim(); if (allStorageClassNames.isEmpty()) { @@ -72,6 +75,9 @@ public class NotebookRepoSync implements NotebookRepo{ /* by default lists from first repository */ public List<NoteInfo> list() throws IOException { + if (config.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE) && getRepoCount() > 1) { + sync(0, 1); + } return getRepo(0).list(); } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/ae08e713/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 981ed6e..bfc6561 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; +import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.interpreter.InterpreterFactory; @@ -180,6 +181,40 @@ public class NotebookRepoSyncTest implements JobListenerFactory{ notebookRepoSync.list(1).get(0).getId()).getLastParagraph().getId()); } + @Test + public void testSyncOnList() throws IOException { + + /* check that both storage repos are empty */ + assertTrue(notebookRepoSync.getRepoCount() > 1); + assertEquals(0, notebookRepoSync.list(0).size()); + assertEquals(0, notebookRepoSync.list(1).size()); + + File srcDir = new File("src/test/resources/2A94M5J1Z"); + File destDir = new File(secNotebookDir + "/2A94M5J1Z"); + + /* copy manually new notebook into secondary storage repo and check repos */ + try { + FileUtils.copyDirectory(srcDir, destDir); + } catch (IOException e) { + e.printStackTrace(); + } + assertEquals(0, notebookRepoSync.list(0).size()); + assertEquals(1, notebookRepoSync.list(1).size()); + + /* Although new notebook is added to secondary storage it's not displayed + * on list() with ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE set to false + */ + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE.getVarName(), "false"); + assertEquals(0, notebookRepoSync.list().size()); + + /* notebook is synced after ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE variable is set to true */ + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE.getVarName(), "true"); + assertEquals(1, notebookRepoSync.list().size()); + + assertEquals(1, notebookRepoSync.list(0).size()); + assertEquals(1, notebookRepoSync.list(1).size()); + } + private void delete(File file){ if(file.isFile()) file.delete(); else if(file.isDirectory()){
