Repository: incubator-zeppelin Updated Branches: refs/heads/master 6a011b993 -> b71f04eb9
Sync functionality with another storage system This PR is to handle the synchronization of notebooks between local Zeppelin fs and remote storage system. Corresponding issue: https://issues.apache.org/jira/browse/ZEPPELIN-133 TODO: - [x] sync non existing files at start - [x] sync modified files at start - [x] save to both storages on save() - [x] delete from both storages on remove() - [x] pull new/updated notes from secondary storage - [x] add comma separated storage class definitions - [x] corresponding tests - [x] handle failure cases of secondary storage - [x] handle updated notes on Paragraph level instead of Note class itself Author: Khalid Huseynov <[email protected]> Closes #123 from khalidhuseynov/sync-with-remote and squashes the following commits: 14c87ff [Khalid Huseynov] Merge branch 'master' into sync-with-remote Latest updates cbe167c [Khalid Huseynov] minor test change + log removal d33ffc7 [Khalid Huseynov] check latest modified from Paragraph fields c2827ff [Khalid Huseynov] delete modTime field from Note and NoteInfo 4293723 [Khalid Huseynov] add initial tests a441f9d [Khalid Huseynov] catch write to secondary storage failed exception, write to main storage 5a967f1 [Khalid Huseynov] logic ramification; add comma-separated storage class names; use list of repos 3314971 [Khalid Huseynov] remove more comments b6a3590 [Khalid Huseynov] minor refactoring of getting storage class 7c4366d [Khalid Huseynov] remove commented code section and redundant type conversion d001ebf [Khalid Huseynov] function name/description change 3899f8e [Khalid Huseynov] minor changes to log comments 359a41e [Khalid Huseynov] minor fix of default classname 51e6271 [Khalid Huseynov] change modTime of Note object on save() 5f7ff18 [Khalid Huseynov] use modTime field for uploading modified notes 5b8d994 [Khalid Huseynov] modify containsID() function to return object instead of boolean 032e59b [Khalid Huseynov] add modification time field for Note and NoteInfo e778c4e [Khalid Huseynov] initial sync implementation Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/b71f04eb Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/b71f04eb Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/b71f04eb Branch: refs/heads/master Commit: b71f04eb97d4c924f754b0ba7cc666dc09cefc55 Parents: 6a011b9 Author: Khalid Huseynov <[email protected]> Authored: Tue Jul 21 11:17:35 2015 +0900 Committer: Lee moon soo <[email protected]> Committed: Wed Jul 22 12:34:25 2015 +0900 ---------------------------------------------------------------------- .../apache/zeppelin/server/ZeppelinServer.java | 7 +- zeppelin-zengine/pom.xml | 19 +- .../org/apache/zeppelin/notebook/NoteInfo.java | 1 + .../notebook/repo/NotebookRepoSync.java | 297 +++++++++++++++++++ .../notebook/repo/NotebookRepoSyncTest.java | 214 +++++++++++++ .../notebook/repo/mock/VFSNotebookRepoMock.java | 26 ++ 6 files changed, 553 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 403d30a..2bd23bb 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -34,6 +34,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.repo.NotebookRepo; +import org.apache.zeppelin.notebook.repo.NotebookRepoSync; import org.apache.zeppelin.rest.InterpreterRestApi; import org.apache.zeppelin.rest.NotebookRestApi; import org.apache.zeppelin.rest.ZeppelinRestApi; @@ -307,11 +308,7 @@ public class ZeppelinServer extends Application { this.schedulerFactory = new SchedulerFactory(); this.replFactory = new InterpreterFactory(conf, notebookServer); - Class<?> notebookStorageClass = getClass().forName( - conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE)); - Constructor<?> constructor = notebookStorageClass.getConstructor( - ZeppelinConfiguration.class); - this.notebookRepo = (NotebookRepo) constructor.newInstance(conf); + this.notebookRepo = new NotebookRepoSync(conf); notebook = new Notebook(conf, notebookRepo, schedulerFactory, replFactory, notebookServer); } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index d4207a5..e6de0b8 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -89,7 +89,7 @@ <exclusions> <exclusion> <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> + <artifactId>commons-httpclient</artifactId> </exclusion> </exclusions> </dependency> @@ -105,17 +105,17 @@ <artifactId>quartz</artifactId> <version>2.2.1</version> </dependency> - + <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> </dependency> - + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> - + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> @@ -134,14 +134,21 @@ <exclusion> <groupId>xml-apis</groupId> <artifactId>xml-apis</artifactId> - </exclusion> - </exclusions> + </exclusion> + </exclusions> </dependency> <dependency> <!-- because there are two of them above --> <groupId>xml-apis</groupId> <artifactId>xml-apis</artifactId> <version>1.4.01</version> + </dependency> + + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>1.9.0</version> + <scope>test</scope> </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java index db07e50..f75a107 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java @@ -64,4 +64,5 @@ public class NoteInfo { public void setConfig(Map<String, Object> config) { this.config = config; } + } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java new file mode 100644 index 0000000..288cff9 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Paragraph; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Notebook repository sync with remote storage + */ +public class NotebookRepoSync implements NotebookRepo{ + private List<NotebookRepo> repos = new ArrayList<NotebookRepo>(); + private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSync.class); + private static final int maxRepoNum = 2; + private static final String pushKey = "pushNoteIDs"; + private static final String pullKey = "pullNoteIDs"; + + /** + * @param (conf) + * @throws - Exception + */ + public NotebookRepoSync(ZeppelinConfiguration conf) throws Exception { + + String allStorageClassNames = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE).trim(); + if (allStorageClassNames.isEmpty()) { + throw new IOException("Empty ZEPPELIN_NOTEBOOK_STORAGE conf parameter"); + } + String[] storageClassNames = allStorageClassNames.split(","); + if (storageClassNames.length > getMaxRepoNum()) { + throw new IOException("Unsupported number of storage classes (" + + storageClassNames.length + ") in ZEPPELIN_NOTEBOOK_STORAGE"); + } + + for (int i = 0; i < storageClassNames.length; i++) { + Class<?> notebookStorageClass = getClass().forName(storageClassNames[i].trim()); + Constructor<?> constructor = notebookStorageClass.getConstructor( + ZeppelinConfiguration.class); + repos.add((NotebookRepo) constructor.newInstance(conf)); + } + if (getRepoCount() > 1) { + sync(0, 1); + } + } + + /* by default lists from first repository */ + public List<NoteInfo> list() throws IOException { + return getRepo(0).list(); + } + + /* list from specific repo (for tests) */ + List<NoteInfo> list(int repoIndex) throws IOException { + return getRepo(repoIndex).list(); + } + + /* by default returns from first repository */ + public Note get(String noteId) throws IOException { + return getRepo(0).get(noteId); + } + + /* get note from specific repo (for tests) */ + Note get(int repoIndex, String noteId) throws IOException { + return getRepo(repoIndex).get(noteId); + } + + /* by default saves to all repos */ + public void save(Note note) throws IOException { + getRepo(0).save(note); + if (getRepoCount() > 1) { + try { + getRepo(1).save(note); + } + catch (IOException e) { + LOG.info(e.getMessage() + ": Failed to write to secondary storage"); + } + } + } + + /* save note to specific repo (for tests) */ + void save(int repoIndex, Note note) throws IOException { + getRepo(repoIndex).save(note); + } + + public void remove(String noteId) throws IOException { + for (NotebookRepo repo : repos) { + repo.remove(noteId); + } + /* TODO(khalid): handle case when removing from secondary storage fails */ + } + + /** + * copy new/updated notes from source to destination storage + * @throws IOException + */ + public void sync(int sourceRepoIndex, int destRepoIndex) throws IOException { + LOG.info("Sync started"); + NotebookRepo sourceRepo = getRepo(sourceRepoIndex); + NotebookRepo destRepo = getRepo(destRepoIndex); + List <NoteInfo> sourceNotes = sourceRepo.list(); + List <NoteInfo> destNotes = destRepo.list(); + + Map<String, List<String>> noteIDs = notesCheckDiff(sourceNotes, + sourceRepo, + destNotes, + destRepo); + List<String> pushNoteIDs = noteIDs.get(pushKey); + List<String> pullNoteIDs = noteIDs.get(pullKey); + if (!pushNoteIDs.isEmpty()) { + LOG.info("Notes with the following IDs will be pushed"); + for (String id : pushNoteIDs) { + LOG.info("ID : " + id); + } + pushNotes(pushNoteIDs, sourceRepo, destRepo); + } else { + LOG.info("Nothing to push"); + } + + if (!pullNoteIDs.isEmpty()) { + LOG.info("Notes with the following IDs will be pulled"); + for (String id : pullNoteIDs) { + LOG.info("ID : " + id); + } + pushNotes(pullNoteIDs, destRepo, sourceRepo); + } else { + LOG.info("Nothing to pull"); + } + + LOG.info("Sync ended"); + } + + public void sync() throws IOException { + sync(0, 1); + } + + private void pushNotes(List<String> ids, NotebookRepo localRepo, + NotebookRepo remoteRepo) throws IOException { + for (String id : ids) { + remoteRepo.save(localRepo.get(id)); + } + } + + int getRepoCount() { + return repos.size(); + } + + int getMaxRepoNum() { + return maxRepoNum; + } + + private NotebookRepo getRepo(int repoIndex) throws IOException { + if (repoIndex < 0 || repoIndex >= getRepoCount()) { + throw new IOException("Storage repo index is out of range"); + } + return repos.get(repoIndex); + } + + private Map<String, List<String>> notesCheckDiff(List <NoteInfo> sourceNotes, + NotebookRepo sourceRepo, + List <NoteInfo> destNotes, + NotebookRepo destRepo) throws IOException { + List <String> pushIDs = new ArrayList<String>(); + List <String> pullIDs = new ArrayList<String>(); + + NoteInfo dnote; + Date sdate, ddate; + for (NoteInfo snote : sourceNotes) { + dnote = containsID(destNotes, snote.getId()); + if (dnote != null) { + /* note exists in source and destination storage systems */ + sdate = lastModificationDate(sourceRepo.get(snote.getId())); + ddate = lastModificationDate(destRepo.get(dnote.getId())); + if (sdate.after(ddate)) { + /* source contains more up to date note - push */ + pushIDs.add(snote.getId()); + LOG.info("Modified note is added to push list : " + sdate); + } else if (sdate.compareTo(ddate) != 0) { + /* destination contains more up to date note - pull */ + LOG.info("Modified note is added to pull list : " + ddate); + pullIDs.add(snote.getId()); + } + } else { + /* note exists in source storage, and absent in destination + * view source as up to date - push + * (another scenario : note was deleted from destination - not considered)*/ + pushIDs.add(snote.getId()); + } + } + + for (NoteInfo note : destNotes) { + dnote = containsID(sourceNotes, note.getId()); + if (dnote == null) { + /* note exists in destination storage, and absent in source - pull*/ + pullIDs.add(note.getId()); + } + } + + Map<String, List<String>> map = new HashMap<String, List<String>>(); + map.put(pushKey, pushIDs); + map.put(pullKey, pullIDs); + return map; + } + + private NoteInfo containsID(List <NoteInfo> notes, String id) { + for (NoteInfo note : notes) { + if (note.getId().equals(id)) { + return note; + } + } + return null; + } + /** + * checks latest modification date based on Paragraph fields + * @return -Date + */ + private Date lastModificationDate(Note note) { + Date latest = new Date(0L); + Date tempCreated, tempStarted, tempFinished; + + for (Paragraph paragraph : note.getParagraphs()) { + tempCreated = paragraph.getDateCreated(); + tempStarted = paragraph.getDateStarted(); + tempFinished = paragraph.getDateFinished(); + + if (tempCreated != null && tempCreated.after(latest)) { + latest = tempCreated; + } + if (tempStarted != null && tempStarted.after(latest)) { + latest = tempStarted; + } + if (tempFinished != null && tempFinished.after(latest)) { + latest = tempFinished; + } + } + return latest; + } + + private void printParagraphs(Note note) { + LOG.info("Note name : " + note.getName()); + LOG.info("Note ID : " + note.id()); + for (Paragraph p : note.getParagraphs()) { + printParagraph(p); + } + } + + private void printParagraph(Paragraph paragraph) { + LOG.info("Date created : " + paragraph.getDateCreated()); + LOG.info("Date started : " + paragraph.getDateStarted()); + LOG.info("Date finished : " + paragraph.getDateFinished()); + LOG.info("Paragraph ID : " + paragraph.getId()); + LOG.info("Paragraph title : " + paragraph.getTitle()); + } + + private void printNoteInfos(List <NoteInfo> notes) { + LOG.info("The following is a list of note infos"); + for (NoteInfo note : notes) { + printNoteInfo(note); + } + } + + private void printNoteInfo(NoteInfo note) { + LOG.info("Note info of notebook with name : " + note.getName()); + LOG.info("ID : " + note.getId()); + Map<String, Object> configs = note.getConfig(); + for (Map.Entry<String, Object> entry : configs.entrySet()) { + LOG.info("Config Key = " + entry.getKey() + " , Value = " + + entry.getValue().toString() + "of class " + entry.getClass()); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java new file mode 100644 index 0000000..981ed6e --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.mock.MockInterpreter1; +import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.apache.zeppelin.notebook.JobListenerFactory; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.notebook.repo.NotebookRepoSync; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.scheduler.JobListener; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class NotebookRepoSyncTest implements JobListenerFactory{ + + private File mainZepDir; + private ZeppelinConfiguration conf; + private SchedulerFactory schedulerFactory; + private File mainNotebookDir; + private File secNotebookDir; + private Notebook notebookSync; + private NotebookRepoSync notebookRepoSync; + private InterpreterFactory factory; + private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class); + + @Before + public void setUp() throws Exception { + String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis(); + mainZepDir = new File(zpath); + mainZepDir.mkdirs(); + new File(mainZepDir, "conf").mkdirs(); + String mainNotePath = zpath+"/notebook"; + String secNotePath = mainNotePath + "_secondary"; + mainNotebookDir = new File(mainNotePath); + secNotebookDir = new File(secNotePath); + mainNotebookDir.mkdirs(); + secNotebookDir.mkdirs(); + + System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"); + LOG.info("main Note dir : " + mainNotePath); + LOG.info("secondary note dir : " + secNotePath); + conf = ZeppelinConfiguration.create(); + + this.schedulerFactory = new SchedulerFactory(); + + MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); + MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + + factory = new InterpreterFactory(conf, new InterpreterOption(false), null); + + notebookRepoSync = new NotebookRepoSync(conf); + notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this); + } + + @After + public void tearDown() throws Exception { + delete(mainZepDir); + } + + @Test + public void testRepoCount() throws IOException { + assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount()); + } + + @Test + public void testSyncOnCreate() throws IOException { + /* check that both storage systems are empty */ + assertTrue(notebookRepoSync.getRepoCount() > 1); + assertEquals(0, notebookRepoSync.list(0).size()); + assertEquals(0, notebookRepoSync.list(1).size()); + + /* create note */ + Note note = notebookSync.createNote(); + + // check that automatically saved on both storages + assertEquals(1, notebookRepoSync.list(0).size()); + assertEquals(1, notebookRepoSync.list(1).size()); + assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId()); + + } + + @Test + public void testSyncOnDelete() throws IOException { + /* create note */ + assertTrue(notebookRepoSync.getRepoCount() > 1); + assertEquals(0, notebookRepoSync.list(0).size()); + assertEquals(0, notebookRepoSync.list(1).size()); + + Note note = notebookSync.createNote(); + + /* check that created in both storage systems */ + assertEquals(1, notebookRepoSync.list(0).size()); + assertEquals(1, notebookRepoSync.list(1).size()); + assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId()); + + /* remove Note */ + notebookSync.removeNote(notebookRepoSync.list(0).get(0).getId()); + + /* check that deleted in both storages */ + assertEquals(0, notebookRepoSync.list(0).size()); + assertEquals(0, notebookRepoSync.list(1).size()); + + } + + @Test + public void testSyncUpdateMain() throws IOException { + + /* create note */ + Note note = notebookSync.createNote(); + Paragraph p1 = note.addParagraph(); + p1.setText("hello world"); + + /* new paragraph exists in note instance */ + assertEquals(1, note.getParagraphs().size()); + + /* new paragraph not yet saved into storages */ + assertEquals(0, notebookRepoSync.get(0, + notebookRepoSync.list(0).get(0).getId()).getParagraphs().size()); + assertEquals(0, notebookRepoSync.get(1, + notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); + + /* save to storage under index 0 (first storage) */ + notebookRepoSync.save(0, note); + + /* check paragraph saved to first storage */ + assertEquals(1, notebookRepoSync.get(0, + notebookRepoSync.list(0).get(0).getId()).getParagraphs().size()); + /* check paragraph isn't saved to second storage */ + assertEquals(0, notebookRepoSync.get(1, + notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); + /* apply sync */ + notebookRepoSync.sync(); + /* check whether added to second storage */ + assertEquals(1, notebookRepoSync.get(1, + notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); + /* check whether same paragraph id */ + assertEquals(p1.getId(), notebookRepoSync.get(0, + notebookRepoSync.list(0).get(0).getId()).getLastParagraph().getId()); + assertEquals(p1.getId(), notebookRepoSync.get(1, + notebookRepoSync.list(1).get(0).getId()).getLastParagraph().getId()); + } + + private void delete(File file){ + if(file.isFile()) file.delete(); + else if(file.isDirectory()){ + File [] files = file.listFiles(); + if(files!=null && files.length>0){ + for(File f : files){ + delete(f); + } + } + file.delete(); + } + + } + @Override + public JobListener getParagraphJobListener(Note note) { + return new JobListener(){ + + @Override + public void onProgressUpdate(Job job, int progress) { + } + + @Override + public void beforeStatusChange(Job job, Status before, Status after) { + } + + @Override + public void afterStatusChange(Job job, Status before, Status after) { + } + }; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java new file mode 100644 index 0000000..dd4e513 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java @@ -0,0 +1,26 @@ +package org.apache.zeppelin.notebook.repo.mock; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import org.apache.commons.vfs2.VFS; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; + +public class VFSNotebookRepoMock extends VFSNotebookRepo { + + private static ZeppelinConfiguration modifyNotebookDir(ZeppelinConfiguration conf) { + String secNotebookDir = conf.getNotebookDir() + "_secondary"; + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), secNotebookDir); + ZeppelinConfiguration secConf = ZeppelinConfiguration.create(); + return secConf; + } + + public VFSNotebookRepoMock(ZeppelinConfiguration conf) throws IOException { + super(modifyNotebookDir(conf)); + } + +}
