Repository: zeppelin Updated Branches: refs/heads/master 69d58f373 -> d6203c51e
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 634ac30..f4b8c32 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -17,31 +17,28 @@ package org.apache.zeppelin.notebook; -import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; - -import com.google.common.collect.Maps; -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - import com.google.common.collect.Sets; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.AbstractInterpreterTest; +import org.apache.zeppelin.interpreter.ClassloaderInterpreter; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; import org.apache.zeppelin.resource.LocalResourcePool; -import org.apache.zeppelin.resource.ResourcePoolUtils; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.SchedulerFactory; @@ -56,18 +53,35 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonatype.aether.RepositoryException; -public class NotebookTest implements JobListenerFactory{ +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +public class NotebookTest extends AbstractInterpreterTest implements JobListenerFactory { private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class); - private File tmpDir; - private ZeppelinConfiguration conf; private SchedulerFactory schedulerFactory; - private File notebookDir; private Notebook notebook; private NotebookRepo notebookRepo; - private InterpreterFactory factory; - private InterpreterSettingManager interpreterSettingManager; - private DependencyResolver depResolver; private NotebookAuthorization notebookAuthorization; private Credentials credentials; private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS; @@ -75,57 +89,30 @@ public class NotebookTest implements JobListenerFactory{ @Before public void setUp() throws Exception { + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true"); + System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2"); + super.setUp(); - tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); - tmpDir.mkdirs(); - new File(tmpDir, "conf").mkdirs(); - notebookDir = new File(tmpDir + "/notebook"); - notebookDir.mkdirs(); - - System.setProperty(ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.toString() + "/conf"); - System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); - - conf = ZeppelinConfiguration.create(); - - this.schedulerFactory = new SchedulerFactory(); - - depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); - - ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>(); - interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>())); - interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(), - Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null); - interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>()); - - ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>(); - interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>())); - interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(), - Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null); - interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>()); - + schedulerFactory = SchedulerFactory.singleton(); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); notebookAuthorization = NotebookAuthorization.init(conf); credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath()); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search, + notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search, notebookAuthorization, credentials); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true"); } @After public void tearDown() throws Exception { - delete(tmpDir); + super.tearDown(); } @Test public void testSelectingReplImplementation() throws IOException { Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); // run with default repl Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -259,7 +246,7 @@ public class NotebookTest implements JobListenerFactory{ Notebook notebook2 = new Notebook( conf, notebookRepo, schedulerFactory, - new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager), + new InterpreterFactory(interpreterSettingManager), interpreterSettingManager, null, null, null, null); assertEquals(1, notebook2.getAllNotes().size()); @@ -316,7 +303,7 @@ public class NotebookTest implements JobListenerFactory{ @Test public void testRunAll() throws IOException { Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds()); // p1 Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -355,7 +342,7 @@ public class NotebookTest implements JobListenerFactory{ public void testSchedule() throws InterruptedException, IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds()); Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); Map config = new HashMap<>(); @@ -428,8 +415,8 @@ public class NotebookTest implements JobListenerFactory{ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{ // create a note and a paragraph Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); - + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); Map config = new HashMap<>(); p.setConfig(config); @@ -448,21 +435,17 @@ public class NotebookTest implements JobListenerFactory{ notebook.refreshCron(note.getId()); - MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter()) - .getInnerInterpreter())); + RemoteInterpreter mock1 = (RemoteInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock1"); - MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter()) - .getInnerInterpreter())); + RemoteInterpreter mock2 = (RemoteInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock2"); // wait until interpreters are started - while (!mock1.isOpen() || !mock2.isOpen()) { + while (!mock1.isOpened() || !mock2.isOpened()) { Thread.yield(); } // wait until interpreters are closed - while (mock1.isOpen() || mock2.isOpen()) { + while (mock1.isOpened() || mock2.isOpened()) { Thread.yield(); } @@ -481,7 +464,7 @@ public class NotebookTest implements JobListenerFactory{ public void testExportAndImportNote() throws IOException, CloneNotSupportedException, InterruptedException, InterpreterException, SchedulerException, RepositoryException { Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds()); final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); String simpleText = "hello world"; @@ -520,7 +503,7 @@ public class NotebookTest implements JobListenerFactory{ public void testCloneNote() throws IOException, CloneNotSupportedException, InterruptedException, InterpreterException, SchedulerException, RepositoryException { Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds()); final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); p.setText("hello world"); @@ -554,7 +537,7 @@ public class NotebookTest implements JobListenerFactory{ public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException, InterruptedException { Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous); assertEquals(cloneNote.getName(), "Note " + cloneNote.getId()); @@ -566,7 +549,7 @@ public class NotebookTest implements JobListenerFactory{ public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException, InterruptedException { Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); p.setText("hello world"); @@ -591,28 +574,28 @@ public class NotebookTest implements JobListenerFactory{ @Test public void testResourceRemovealOnParagraphNoteRemove() throws IOException { Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); - for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) { - intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId())); - } + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); p1.setText("hello"); Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); p2.setText("%mock2 world"); - + for (InterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) { + intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId())); + } note.runAll(); while (p1.isTerminated() == false || p1.getResult() == null) Thread.yield(); while (p2.isTerminated() == false || p2.getResult() == null) Thread.yield(); - assertEquals(2, ResourcePoolUtils.getAllResources().size()); + assertEquals(2, interpreterSettingManager.getAllResources().size()); // remove a paragraph note.removeParagraph(anonymous.getUser(), p1.getId()); - assertEquals(1, ResourcePoolUtils.getAllResources().size()); + assertEquals(1, interpreterSettingManager.getAllResources().size()); // remove note notebook.removeNote(note.getId(), anonymous); - assertEquals(0, ResourcePoolUtils.getAllResources().size()); + assertEquals(0, interpreterSettingManager.getAllResources().size()); } @Test @@ -620,10 +603,10 @@ public class NotebookTest implements JobListenerFactory{ IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); AngularObjectRegistry registry = interpreterSettingManager - .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") + .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -653,10 +636,10 @@ public class NotebookTest implements JobListenerFactory{ IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); AngularObjectRegistry registry = interpreterSettingManager - .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") + .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -687,10 +670,10 @@ public class NotebookTest implements JobListenerFactory{ IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); AngularObjectRegistry registry = interpreterSettingManager - .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") + .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); // add local scope object @@ -700,14 +683,13 @@ public class NotebookTest implements JobListenerFactory{ // restart interpreter interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId()); - registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") - .getAngularObjectRegistry(); - - // local and global scope object should be removed - // But InterpreterGroup does not implement angularObjectRegistry per session (scoped, isolated) - // So for now, does not have good way to remove all objects in particular session on restart. - assertNotNull(registry.get("o1", note.getId(), null)); - assertNotNull(registry.get("o2", null, null)); + registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0) + .getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess") + .getAngularObjectRegistry(); + + // New InterpreterGroup will be created and its AngularObjectRegistry will be created + assertNull(registry.get("o1", note.getId(), null)); + assertNull(registry.get("o2", null, null)); notebook.removeNote(note.getId(), anonymous); } @@ -815,7 +797,7 @@ public class NotebookTest implements JobListenerFactory{ public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException, IOException { Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); // create three paragraphs Paragraph p1 = note.addNewParagraph(anonymous); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java index 7bd6819..8a4bea0 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java @@ -21,6 +21,7 @@ package org.apache.zeppelin.notebook; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -39,7 +40,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectBuilder; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.Input; -import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -47,10 +48,7 @@ import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.apache.zeppelin.interpreter.InterpreterResultMessage; -import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.InterpreterSetting.Status; -import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; @@ -173,7 +171,7 @@ public class ParagraphTest { Interpreter mockInterpreter = mock(Interpreter.class); doReturn(mockInterpreter).when(spyParagraph).getRepl(anyString()); - InterpreterGroup mockInterpreterGroup = mock(InterpreterGroup.class); + ManagedInterpreterGroup mockInterpreterGroup = mock(ManagedInterpreterGroup.class); when(mockInterpreter.getInterpreterGroup()).thenReturn(mockInterpreterGroup); when(mockInterpreterGroup.getId()).thenReturn("mock_id_1"); when(mockInterpreterGroup.getAngularObjectRegistry()).thenReturn(mock(AngularObjectRegistry.class)); @@ -186,7 +184,7 @@ public class ParagraphTest { when(mockInterpreterOption.permissionIsSet()).thenReturn(false); when(mockInterpreterSetting.getStatus()).thenReturn(Status.READY); when(mockInterpreterSetting.getId()).thenReturn("mock_id_1"); - when(mockInterpreterSetting.getInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup); + when(mockInterpreterSetting.getOrCreateInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup); spyInterpreterSettingList.add(mockInterpreterSetting); when(mockNote.getId()).thenReturn("any_id"); when(mockInterpreterSettingManager.getInterpreterSettings(anyString())).thenReturn(spyInterpreterSettingList); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index a6c9393..8a0b5a1 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -33,10 +33,12 @@ import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterFactory; -import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.InterpreterSettingManager; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.apache.zeppelin.notebook.*; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; @@ -69,7 +71,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory { private Credentials credentials; private AuthenticationInfo anonymous; private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class); - + @Before public void setUp() throws Exception { String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis(); @@ -91,12 +93,13 @@ public class NotebookRepoSyncTest implements JobListenerFactory { LOG.info("secondary note dir : " + secNotePath); conf = ZeppelinConfiguration.create(); - this.schedulerFactory = new SchedulerFactory(); + this.schedulerFactory = SchedulerFactory.singleton(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); - + interpreterSettingManager = new InterpreterSettingManager(conf, + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + factory = new InterpreterFactory(interpreterSettingManager); + search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); notebookAuthorization = NotebookAuthorization.init(conf); @@ -110,19 +113,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory { public void tearDown() throws Exception { delete(mainZepDir); } - + @Test public void testRepoCount() throws IOException { assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount()); } - + @Test public void testSyncOnCreate() throws IOException { /* check that both storage systems are empty */ assertTrue(notebookRepoSync.getRepoCount() > 1); assertEquals(0, notebookRepoSync.list(0, anonymous).size()); assertEquals(0, notebookRepoSync.list(1, anonymous).size()); - + /* create note */ Note note = notebookSync.createNote(anonymous); @@ -130,7 +133,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory { assertEquals(1, notebookRepoSync.list(0, anonymous).size()); assertEquals(1, notebookRepoSync.list(1, anonymous).size()); assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId()); - + notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous); } @@ -140,26 +143,26 @@ public class NotebookRepoSyncTest implements JobListenerFactory { assertTrue(notebookRepoSync.getRepoCount() > 1); assertEquals(0, notebookRepoSync.list(0, anonymous).size()); assertEquals(0, notebookRepoSync.list(1, anonymous).size()); - + Note note = notebookSync.createNote(anonymous); /* check that created in both storage systems */ assertEquals(1, notebookRepoSync.list(0, anonymous).size()); assertEquals(1, notebookRepoSync.list(1, anonymous).size()); assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId()); - + /* remove Note */ notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous); - + /* check that deleted in both storages */ assertEquals(0, notebookRepoSync.list(0, anonymous).size()); assertEquals(0, notebookRepoSync.list(1, anonymous).size()); - + } - + @Test public void testSyncUpdateMain() throws IOException { - + /* create note */ Note note = notebookSync.createNote(anonymous); Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -167,19 +170,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory { config.put("enabled", true); p1.setConfig(config); p1.setText("hello world"); - + /* new paragraph exists in note instance */ assertEquals(1, note.getParagraphs().size()); - + /* new paragraph not yet saved into storages */ assertEquals(0, notebookRepoSync.get(0, notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size()); assertEquals(0, notebookRepoSync.get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); - - /* save to storage under index 0 (first storage) */ + + /* save to storage under index 0 (first storage) */ notebookRepoSync.save(0, note, anonymous); - + /* check paragraph saved to first storage */ assertEquals(1, notebookRepoSync.get(0, notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size()); @@ -284,45 +287,45 @@ public class NotebookRepoSyncTest implements JobListenerFactory { // one git versioned storage initialized assertThat(vRepoSync.getRepoCount()).isEqualTo(1); assertThat(vRepoSync.getRepo(0)).isInstanceOf(GitNotebookRepo.class); - + GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0); - + // no notes assertThat(vRepoSync.list(anonymous).size()).isEqualTo(0); // create note Note note = vNotebookSync.createNote(anonymous); assertThat(vRepoSync.list(anonymous).size()).isEqualTo(1); - + String noteId = vRepoSync.list(anonymous).get(0).getId(); // first checkpoint vRepoSync.checkpoint(noteId, "checkpoint message", anonymous); int vCount = gitRepo.revisionHistory(noteId, anonymous).size(); assertThat(vCount).isEqualTo(1); - + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); Map<String, Object> config = p.getConfig(); config.put("enabled", true); p.setConfig(config); p.setText("%md checkpoint test"); - + // save and checkpoint again vRepoSync.save(note, anonymous); vRepoSync.checkpoint(noteId, "checkpoint message 2", anonymous); assertThat(gitRepo.revisionHistory(noteId, anonymous).size()).isEqualTo(vCount + 1); notebookRepoSync.remove(note.getId(), anonymous); } - + @Test public void testSyncWithAcl() throws IOException { /* scenario 1 - note exists with acl on main storage */ AuthenticationInfo user1 = new AuthenticationInfo("user1"); Note note = notebookSync.createNote(user1); assertEquals(0, note.getParagraphs().size()); - + // saved on both storages assertEquals(1, notebookRepoSync.list(0, null).size()); assertEquals(1, notebookRepoSync.list(1, null).size()); - + /* check that user1 is the only owner */ NotebookAuthorization authInfo = NotebookAuthorization.getInstance(); Set<String> entity = new HashSet<String>(); @@ -332,23 +335,23 @@ public class NotebookRepoSyncTest implements JobListenerFactory { assertEquals(0, authInfo.getReaders(note.getId()).size()); assertEquals(0, authInfo.getRunners(note.getId()).size()); assertEquals(0, authInfo.getWriters(note.getId()).size()); - + /* update note and save on secondary storage */ Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); p1.setText("hello world"); assertEquals(1, note.getParagraphs().size()); notebookRepoSync.save(1, note, null); - + /* check paragraph isn't saved into first storage */ assertEquals(0, notebookRepoSync.get(0, notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size()); /* check paragraph is saved into second storage */ assertEquals(1, notebookRepoSync.get(1, notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size()); - + /* now sync by user1 */ notebookRepoSync.sync(user1); - + /* check that note updated and acl are same on main storage*/ assertEquals(1, notebookRepoSync.get(0, notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size()); @@ -357,7 +360,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory { assertEquals(0, authInfo.getReaders(note.getId()).size()); assertEquals(0, authInfo.getRunners(note.getId()).size()); assertEquals(0, authInfo.getWriters(note.getId()).size()); - + /* scenario 2 - note doesn't exist on main storage */ /* remove from main storage */ notebookRepoSync.remove(0, note.getId(), user1); @@ -368,7 +371,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory { assertEquals(0, authInfo.getReaders(note.getId()).size()); assertEquals(0, authInfo.getRunners(note.getId()).size()); assertEquals(0, authInfo.getWriters(note.getId()).size()); - + /* now sync - should bring note from secondary storage with added acl */ notebookRepoSync.sync(user1); assertEquals(1, notebookRepoSync.list(0, null).size()); @@ -428,5 +431,5 @@ public class NotebookRepoSyncTest implements JobListenerFactory { } }; } - + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 6f85bf6..98a08d9 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -22,25 +22,14 @@ import static org.mockito.Mockito.mock; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Properties; -import com.google.common.collect.Maps; import org.apache.commons.io.FileUtils; -import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.dep.DependencyResolver; -import org.apache.zeppelin.interpreter.InterpreterFactory; -import org.apache.zeppelin.interpreter.InterpreterInfo; -import org.apache.zeppelin.interpreter.InterpreterOption; -import org.apache.zeppelin.interpreter.DefaultInterpreterProperty; -import org.apache.zeppelin.interpreter.InterpreterProperty; -import org.apache.zeppelin.interpreter.InterpreterSettingManager; -import org.apache.zeppelin.interpreter.mock.MockInterpreter1; + +import org.apache.zeppelin.interpreter.AbstractInterpreterTest; + import org.apache.zeppelin.notebook.JobListenerFactory; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Notebook; @@ -58,59 +47,34 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableMap; -public class VFSNotebookRepoTest implements JobListenerFactory { +public class VFSNotebookRepoTest extends AbstractInterpreterTest implements JobListenerFactory { + private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepoTest.class); - private ZeppelinConfiguration conf; + private SchedulerFactory schedulerFactory; private Notebook notebook; private NotebookRepo notebookRepo; - private InterpreterSettingManager interpreterSettingManager; - private InterpreterFactory factory; - private DependencyResolver depResolver; private NotebookAuthorization notebookAuthorization; - private File mainZepDir; - private File mainNotebookDir; - @Before public void setUp() throws Exception { - String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); - mainZepDir = new File(zpath); - mainZepDir.mkdirs(); - new File(mainZepDir, "conf").mkdirs(); - String mainNotePath = zpath + "/notebook"; - mainNotebookDir = new File(mainNotePath); - mainNotebookDir.mkdirs(); - - System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2"); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo"); - conf = ZeppelinConfiguration.create(); - - this.schedulerFactory = new SchedulerFactory(); - - this.schedulerFactory = new SchedulerFactory(); - depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); - ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>(); - interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>())); - interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(), - Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null); - interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>()); + super.setUp(); + this.schedulerFactory = SchedulerFactory.singleton(); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); notebookAuthorization = NotebookAuthorization.init(conf); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search, + notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search, notebookAuthorization, null); } @After public void tearDown() throws Exception { - if (!FileUtils.deleteQuietly(mainZepDir)) { - LOG.error("Failed to delete {} ", mainZepDir.getName()); + if (!FileUtils.deleteQuietly(testRootDir)) { + LOG.error("Failed to delete {} ", testRootDir.getName()); } } @@ -120,7 +84,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory { int numNotes = notebookRepo.list(null).size(); // when create invalid json file - File testNoteDir = new File(mainNotebookDir, "test"); + File testNoteDir = new File(notebookDir, "interpreter/test"); testNoteDir.mkdir(); FileUtils.writeStringToFile(new File(testNoteDir, "note.json"), ""); @@ -132,7 +96,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory { public void testSaveNotebook() throws IOException, InterruptedException { AuthenticationInfo anonymous = new AuthenticationInfo("anonymous"); Note note = notebook.createNote(anonymous); - interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds()); Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); Map<String, Object> config = p1.getConfig(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java index 46134e5..469d5b5 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java @@ -17,35 +17,27 @@ package org.apache.zeppelin.resource; import com.google.gson.Gson; -import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller; -import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool; +import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.File; import java.util.HashMap; import java.util.LinkedList; -import java.util.Properties; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; /** * Unittest for DistributedResourcePool */ -public class DistributedResourcePoolTest { - private static final String INTERPRETER_SCRIPT = - System.getProperty("os.name").startsWith("Windows") ? - "../bin/interpreter.cmd" : - "../bin/interpreter.sh"; - private InterpreterGroup intpGroup1; - private InterpreterGroup intpGroup2; - private HashMap<String, String> env; +public class DistributedResourcePoolTest extends AbstractInterpreterTest { + private RemoteInterpreter intp1; private RemoteInterpreter intp2; private InterpreterContext context; @@ -55,50 +47,10 @@ public class DistributedResourcePoolTest { @Before public void setUp() throws Exception { - env = new HashMap<>(); - env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); - - Properties p = new Properties(); - - intp1 = new RemoteInterpreter( - p, - "note", - MockInterpreterResourcePool.class.getName(), - new File(INTERPRETER_SCRIPT).getAbsolutePath(), - "fake", - "fakeRepo", - env, - 10 * 1000, - null, - null, - "anonymous", - false - ); - - intpGroup1 = new InterpreterGroup("intpGroup1"); - intpGroup1.put("note", new LinkedList<Interpreter>()); - intpGroup1.get("note").add(intp1); - intp1.setInterpreterGroup(intpGroup1); - - intp2 = new RemoteInterpreter( - p, - "note", - MockInterpreterResourcePool.class.getName(), - new File(INTERPRETER_SCRIPT).getAbsolutePath(), - "fake", - "fakeRepo", - env, - 10 * 1000, - null, - null, - "anonymous", - false - ); - - intpGroup2 = new InterpreterGroup("intpGroup2"); - intpGroup2.put("note", new LinkedList<Interpreter>()); - intpGroup2.get("note").add(intp2); - intp2.setInterpreterGroup(intpGroup2); + super.setUp(); + InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("mock_resource_pool"); + intp1 = (RemoteInterpreter) interpreterSetting.getInterpreter("user1", "note1", "mock_resource_pool"); + intp2 = (RemoteInterpreter) interpreterSetting.getInterpreter("user2", "note1", "mock_resource_pool"); context = new InterpreterContext( "note", @@ -117,26 +69,13 @@ public class DistributedResourcePoolTest { intp1.open(); intp2.open(); - eventPoller1 = new RemoteInterpreterEventPoller(null, null); - eventPoller1.setInterpreterGroup(intpGroup1); - eventPoller1.setInterpreterProcess(intpGroup1.getRemoteInterpreterProcess()); - - eventPoller2 = new RemoteInterpreterEventPoller(null, null); - eventPoller2.setInterpreterGroup(intpGroup2); - eventPoller2.setInterpreterProcess(intpGroup2.getRemoteInterpreterProcess()); - - eventPoller1.start(); - eventPoller2.start(); + eventPoller1 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller(); + eventPoller2 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller(); } @After public void tearDown() throws Exception { - eventPoller1.shutdown(); - intp1.close(); - intpGroup1.close(); - eventPoller2.shutdown(); - intp2.close(); - intpGroup2.close(); + interpreterSettingManager.close(); } @Test @@ -235,13 +174,13 @@ public class DistributedResourcePoolTest { // then get all resources. - assertEquals(4, ResourcePoolUtils.getAllResources().size()); + assertEquals(4, interpreterSettingManager.getAllResources().size()); // when remove all resources from note1 - ResourcePoolUtils.removeResourcesBelongsToNote("note1"); + interpreterSettingManager.removeResourcesBelongsToNote("note1"); // then resources should be removed. - assertEquals(2, ResourcePoolUtils.getAllResources().size()); + assertEquals(2, interpreterSettingManager.getAllResources().size()); assertEquals("", gson.fromJson( intp1.interpret("get note1:paragraph1:key1", context).message().get(0).getData(), String.class)); @@ -251,10 +190,10 @@ public class DistributedResourcePoolTest { // when remove all resources from note2:paragraph1 - ResourcePoolUtils.removeResourcesBelongsToParagraph("note2", "paragraph1"); + interpreterSettingManager.removeResourcesBelongsToParagraph("note2", "paragraph1"); // then 1 - assertEquals(1, ResourcePoolUtils.getAllResources().size()); + assertEquals(1, interpreterSettingManager.getAllResources().size()); assertEquals("value2", gson.fromJson( intp1.interpret("get note2:paragraph2:key2", context).message().get(0).getData(), String.class)); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index ebb5100..a758591 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -17,83 +17,84 @@ package org.apache.zeppelin.scheduler; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.Properties; - -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterInfo; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterRunner; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA; import org.apache.zeppelin.resource.LocalResourcePool; import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { private static final String INTERPRETER_SCRIPT = - System.getProperty("os.name").startsWith("Windows") ? - "../bin/interpreter.cmd" : - "../bin/interpreter.sh"; + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; + + private InterpreterSetting interpreterSetting; private SchedulerFactory schedulerSvc; private static final int TICK_WAIT = 100; private static final int MAX_WAIT_CYCLES = 100; @Before - public void setUp() throws Exception{ + public void setUp() throws Exception { schedulerSvc = new SchedulerFactory(); + + InterpreterOption interpreterOption = new InterpreterOption(); + interpreterOption.setRemote(true); + InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterA.class.getName(), "mock", true, new HashMap<String, Object>()); + List<InterpreterInfo> interpreterInfos = new ArrayList<>(); + interpreterInfos.add(interpreterInfo1); + InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT); + interpreterSetting = new InterpreterSetting.Builder() + .setId("test") + .setName("test") + .setGroup("test") + .setInterpreterInfos(interpreterInfos) + .setOption(interpreterOption) + .setRunner(runner) + .setInterpreterDir("../interpeters/test") + .create(); } @After - public void tearDown(){ - + public void tearDown() { + interpreterSetting.close(); } @Test public void test() throws Exception { - Properties p = new Properties(); - final InterpreterGroup intpGroup = new InterpreterGroup(); - Map<String, String> env = new HashMap<>(); - env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); - - final RemoteInterpreter intpA = new RemoteInterpreter( - p, - "note", - MockInterpreterA.class.getName(), - new File(INTERPRETER_SCRIPT).getAbsolutePath(), - "fake", - "fakeRepo", - env, - 10 * 1000, - this, - null, - "anonymous", - false); - - intpGroup.put("note", new LinkedList<Interpreter>()); - intpGroup.get("note").add(intpA); - intpA.setInterpreterGroup(intpGroup); + final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1"); intpA.open(); - Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note", - intpA.getInterpreterProcess(), - 10); + Scheduler scheduler = intpA.getScheduler(); Job job = new Job("jobId", "jobName", null, 200) { Object results; + @Override public Object getReturn() { return results; @@ -120,7 +121,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null), + null, new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null)); return "1000"; @@ -145,7 +146,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { } assertTrue(job.isRunning()); - Thread.sleep(5*TICK_WAIT); + Thread.sleep(5 * TICK_WAIT); assertEquals(0, scheduler.getJobsWaiting().size()); assertEquals(1, scheduler.getJobsRunning().size()); @@ -165,34 +166,10 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { @Test public void testAbortOnPending() throws Exception { - Properties p = new Properties(); - final InterpreterGroup intpGroup = new InterpreterGroup(); - Map<String, String> env = new HashMap<>(); - env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); - - final RemoteInterpreter intpA = new RemoteInterpreter( - p, - "note", - MockInterpreterA.class.getName(), - new File(INTERPRETER_SCRIPT).getAbsolutePath(), - "fake", - "fakeRepo", - env, - 10 * 1000, - this, - null, - "anonymous", - false); - - intpGroup.put("note", new LinkedList<Interpreter>()); - intpGroup.get("note").add(intpA); - intpA.setInterpreterGroup(intpGroup); - + final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1"); intpA.open(); - Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note", - intpA.getInterpreterProcess(), - 10); + Scheduler scheduler = intpA.getScheduler(); Job job1 = new Job("jobId1", "jobName1", null, 200) { Object results; @@ -205,7 +182,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null), + null, new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null); @@ -255,7 +232,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { new AuthenticationInfo(), new HashMap<String, Object>(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null), + null, new LocalResourcePool("pool1"), new LinkedList<InterpreterContextRunner>(), null); @@ -358,7 +335,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { } @Override - public void onParaInfosReceived(String noteId, String paragraphId, - String interpreterSettingId, Map<String, String> metaInfos) { + public void onParaInfosReceived(String noteId, String paragraphId, + String interpreterSettingId, Map<String, String> metaInfos) { } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/conf/interpreter.json ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/resources/conf/interpreter.json b/zeppelin-zengine/src/test/resources/conf/interpreter.json new file mode 100644 index 0000000..45e1d60 --- /dev/null +++ b/zeppelin-zengine/src/test/resources/conf/interpreter.json @@ -0,0 +1,115 @@ +{ + "interpreterSettings": { + "2C3RWCVAG": { + "id": "2C3RWCVAG", + "name": "test", + "group": "test", + "properties": { + "property_1": "value_1", + "property_2": "new_value_2", + "property_3": "value_3" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "echo", + "class": "org.apache.zeppelin.interpreter.EchoInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "java", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + }, + + "2CKWE7B19": { + "id": "2CKWE7B19", + "name": "test2", + "group": "test", + "properties": { + "property_1": "value_1", + "property_2": "new_value_2", + "property_3": "value_3" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "echo", + "class": "org.apache.zeppelin.interpreter.EchoInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "java", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + } + }, + "interpreterBindings": { + "2C6793KRV": [ + "2C48Y7FSJ", + "2C63XW4XE", + "2C66GE1VB", + "2C5VH924X", + "2C4BJDRRZ", + "2C3SQSB7V", + "2C4HKDCQW", + "2C3DR183X", + "2C66Z9XPQ", + "2C3PTPMUH", + "2C69WE69N", + "2C5SRRXHM", + "2C4ZD49PF", + "2C6V3D44K", + "2C4UB1UZA", + "2C5S1R21W", + "2C5DCRVGM", + "2C686X8ZH", + "2C3RWCVAG", + "2C3JKFMJU", + "2C3VECEG2" + ] + }, + "interpreterRepositories": [ + { + "id": "central", + "type": "default", + "url": "http://repo1.maven.org/maven2/";, + "releasePolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "snapshotPolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "mirroredRepositories": [], + "repositoryManager": false + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json deleted file mode 100644 index 65568ef..0000000 --- a/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "group": "mock11", - "name": "mock11", - "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter11", - "properties": { - }, - "editor": { - "language": "java" - } - } -] http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json new file mode 100644 index 0000000..2f628da --- /dev/null +++ b/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json @@ -0,0 +1,19 @@ +[ + { + "group": "mock1", + "name": "mock1", + "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter1", + "properties": { + }, + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + } +] http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json new file mode 100644 index 0000000..fae03f6 --- /dev/null +++ b/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json @@ -0,0 +1,19 @@ +[ + { + "group": "mock2", + "name": "mock2", + "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter2", + "properties": { + }, + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "isolated", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + } +] http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json new file mode 100644 index 0000000..4dfe0a7 --- /dev/null +++ b/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json @@ -0,0 +1,19 @@ +[ + { + "group": "mock_resource_pool", + "name": "mock_resource_pool", + "className": "org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool", + "properties": { + }, + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + } +] http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json new file mode 100644 index 0000000..596fc91 --- /dev/null +++ b/zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json @@ -0,0 +1,53 @@ +[ + { + "group": "test", + "name": "double_echo", + "className": "org.apache.zeppelin.interpreter.DoubleEchoInterpreter", + "properties": { + "property_1": { + "envName": "PROPERTY_1", + "propertyName": "property_1", + "defaultValue": "value_1", + "description": "desc_1" + }, + "property_2": { + "envName": "PROPERTY_2", + "propertyName": "property_2", + "defaultValue": "value_2", + "description": "desc_2" + } + }, + "editor": { + "language": "java", + "editOnDblClick": false + } + }, + + { + "group": "test", + "name": "echo", + "defaultInterpreter": true, + "className": "org.apache.zeppelin.interpreter.EchoInterpreter", + "properties": { + "property_1": { + "envName": "PROPERTY_1", + "propertyName": "property_1", + "defaultValue": "value_1", + "description": "desc_1" + }, + "property_2": { + "envName": "PROPERTY_2", + "propertyName": "property_2", + "defaultValue": "value_2", + "description": "desc_2" + } + }, + "runner": { + "linux": "linux_runner" + }, + "editor": { + "language": "java", + "editOnDblClick": false + } + } +] http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/resources/log4j.properties b/zeppelin-zengine/src/test/resources/log4j.properties index 001a222..74f619b 100644 --- a/zeppelin-zengine/src/test/resources/log4j.properties +++ b/zeppelin-zengine/src/test/resources/log4j.properties @@ -35,7 +35,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN log4j.logger.org.apache.hadoop.hive.ql=WARN log4j.logger.org.apache.hadoop.hive.metastore=WARN log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN -log4j.logger.org.apache.zeppelin.scheduler=WARN log4j.logger.org.quartz=WARN log4j.logger.DataNucleus=WARN @@ -45,4 +44,6 @@ log4j.logger.DataNucleus.Datastore=ERROR # Log all JDBC parameters log4j.logger.org.hibernate.type=ALL +log4j.logger.org.apache.zeppelin.interpreter=DEBUG +log4j.logger.org.apache.zeppelin.scheduler=DEBUG
