http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java new file mode 100644 index 0000000..9dd9fbf --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.notebook.repo.zeppelinhub; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.repo.NotebookRepoWithVersionControl; +import org.apache.zeppelin.notebook.repo.NotebookRepoSettingsInfo; +import org.apache.zeppelin.notebook.repo.zeppelinhub.model.Instance; +import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserTokenContainer; +import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserSessionContainer; +import org.apache.zeppelin.notebook.repo.zeppelinhub.rest.ZeppelinhubRestApiHandler; +import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.Client; +import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils.ZeppelinhubUtils; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +/** + * ZeppelinHub repo class. + */ +public class ZeppelinHubRepo implements NotebookRepoWithVersionControl { + private static final Logger LOG = LoggerFactory.getLogger(ZeppelinHubRepo.class); + private static final String DEFAULT_SERVER = "https://www.zeppelinhub.com";; + static final String ZEPPELIN_CONF_PROP_NAME_SERVER = "zeppelinhub.api.address"; + static final String ZEPPELIN_CONF_PROP_NAME_TOKEN = "zeppelinhub.api.token"; + + private static final Gson GSON = new Gson(); + private static final Note EMPTY_NOTE = new Note(); + private Client websocketClient; + private UserTokenContainer tokenManager; + + private String token; + private ZeppelinhubRestApiHandler restApiClient; + + private ZeppelinConfiguration conf; + + public ZeppelinHubRepo() { + + } + + public ZeppelinHubRepo(ZeppelinConfiguration conf) { + this(); + init(conf); + } + + public void init(ZeppelinConfiguration conf) { + this.conf = conf; + String zeppelinHubUrl = getZeppelinHubUrl(conf); + LOG.info("Initializing ZeppelinHub integration module"); + + token = conf.getString("ZEPPELINHUB_API_TOKEN", ZEPPELIN_CONF_PROP_NAME_TOKEN, ""); + restApiClient = ZeppelinhubRestApiHandler.newInstance(zeppelinHubUrl); + //TODO(khalid): check which realm for authentication, pass to token manager + tokenManager = UserTokenContainer.init(restApiClient, token); + + websocketClient = Client.initialize(getZeppelinWebsocketUri(conf), + getZeppelinhubWebsocketUri(conf), token, conf); + websocketClient.start(); + } + + private String getZeppelinHubWsUri(URI api) throws URISyntaxException { + URI apiRoot = api; + String scheme = apiRoot.getScheme(); + int port = apiRoot.getPort(); + if (port <= 0) { + port = (scheme != null && scheme.equals("https")) ? 443 : 80; + } + + if (scheme == null) { + LOG.info("{} is not a valid zeppelinhub server address. proceed with default address {}", + apiRoot, DEFAULT_SERVER); + apiRoot = new URI(DEFAULT_SERVER); + scheme = apiRoot.getScheme(); + port = apiRoot.getPort(); + if (port <= 0) { + port = (scheme != null && scheme.equals("https")) ? 443 : 80; + } + } + String ws = scheme.equals("https") ? "wss://" : "ws://"; + return ws + apiRoot.getHost() + ":" + port + "/async"; + } + + String getZeppelinhubWebsocketUri(ZeppelinConfiguration conf) { + String zeppelinHubUri = StringUtils.EMPTY; + try { + zeppelinHubUri = getZeppelinHubWsUri(new URI(conf.getString("ZEPPELINHUB_API_ADDRESS", + ZEPPELIN_CONF_PROP_NAME_SERVER, DEFAULT_SERVER))); + } catch (URISyntaxException e) { + LOG.error("Cannot get ZeppelinHub URI", e); + } + return zeppelinHubUri; + } + + private String getZeppelinWebsocketUri(ZeppelinConfiguration conf) { + int port = conf.getServerPort(); + if (port <= 0) { + port = 80; + } + String ws = conf.useSsl() ? "wss" : "ws"; + return ws + "://localhost:" + port + "/ws"; + } + + // Used in tests + void setZeppelinhubRestApiHandler(ZeppelinhubRestApiHandler zeppelinhub) { + restApiClient = zeppelinhub; + } + + String getZeppelinHubUrl(ZeppelinConfiguration conf) { + if (conf == null) { + LOG.error("Invalid configuration, cannot be null. Using default address {}", DEFAULT_SERVER); + return DEFAULT_SERVER; + } + URI apiRoot; + String zeppelinhubUrl; + try { + String url = conf.getString("ZEPPELINHUB_API_ADDRESS", + ZEPPELIN_CONF_PROP_NAME_SERVER, + DEFAULT_SERVER); + apiRoot = new URI(url); + } catch (URISyntaxException e) { + LOG.error("Invalid zeppelinhub url, using default address {}", DEFAULT_SERVER, e); + return DEFAULT_SERVER; + } + + String scheme = apiRoot.getScheme(); + if (scheme == null) { + LOG.info("{} is not a valid zeppelinhub server address. proceed with default address {}", + apiRoot, DEFAULT_SERVER); + zeppelinhubUrl = DEFAULT_SERVER; + } else { + zeppelinhubUrl = scheme + "://" + apiRoot.getHost(); + if (apiRoot.getPort() > 0) { + zeppelinhubUrl += ":" + apiRoot.getPort(); + } + } + return zeppelinhubUrl; + } + + private boolean isSubjectValid(AuthenticationInfo subject) { + if (subject == null) { + return false; + } + return (subject.isAnonymous() && !conf.isAnonymousAllowed()) ? false : true; + } + + @Override + public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + if (!isSubjectValid(subject)) { + return Collections.emptyList(); + } + String token = getUserToken(subject.getUser()); + String response = restApiClient.get(token, StringUtils.EMPTY); + List<NoteInfo> notes = GSON.fromJson(response, new TypeToken<List<NoteInfo>>() {}.getType()); + if (notes == null) { + return Collections.emptyList(); + } + LOG.info("ZeppelinHub REST API listing notes "); + return notes; + } + + @Override + public Note get(String noteId, AuthenticationInfo subject) throws IOException { + if (StringUtils.isBlank(noteId) || !isSubjectValid(subject)) { + return EMPTY_NOTE; + } + String token = getUserToken(subject.getUser()); + String response = restApiClient.get(token, noteId); + Note note = Note.fromJson(response); + if (note == null) { + return EMPTY_NOTE; + } + LOG.info("ZeppelinHub REST API get note {} ", noteId); + return note; + } + + @Override + public void save(Note note, AuthenticationInfo subject) throws IOException { + if (note == null || !isSubjectValid(subject)) { + throw new IOException("Zeppelinhub failed to save note"); + } + String jsonNote = note.toJson(); + String token = getUserToken(subject.getUser()); + LOG.info("ZeppelinHub REST API saving note {} ", note.getId()); + restApiClient.put(token, jsonNote); + } + + @Override + public void remove(String noteId, AuthenticationInfo subject) throws IOException { + if (StringUtils.isBlank(noteId) || !isSubjectValid(subject)) { + throw new IOException("Zeppelinhub failed to remove note"); + } + String token = getUserToken(subject.getUser()); + LOG.info("ZeppelinHub REST API removing note {} ", noteId); + restApiClient.del(token, noteId); + } + + @Override + public void close() { + websocketClient.stop(); + restApiClient.close(); + } + + @Override + public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject) + throws IOException { + if (StringUtils.isBlank(noteId) || !isSubjectValid(subject)) { + return Revision.EMPTY; + } + String endpoint = Joiner.on("/").join(noteId, "checkpoint"); + String content = GSON.toJson(ImmutableMap.of("message", checkpointMsg)); + + String token = getUserToken(subject.getUser()); + String response = restApiClient.putWithResponseBody(token, endpoint, content); + + return GSON.fromJson(response, Revision.class); + } + + @Override + public Note get(String noteId, String revId, AuthenticationInfo subject) throws IOException { + if (StringUtils.isBlank(noteId) || StringUtils.isBlank(revId) || !isSubjectValid(subject)) { + return EMPTY_NOTE; + } + String endpoint = Joiner.on("/").join(noteId, "checkpoint", revId); + String token = getUserToken(subject.getUser()); + String response = restApiClient.get(token, endpoint); + + Note note = Note.fromJson(response); + if (note == null) { + return EMPTY_NOTE; + } + LOG.info("ZeppelinHub REST API get note {} revision {}", noteId, revId); + return note; + } + + @Override + public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) { + if (StringUtils.isBlank(noteId) || !isSubjectValid(subject)) { + return Collections.emptyList(); + } + String endpoint = Joiner.on("/").join(noteId, "checkpoint"); + List<Revision> history = Collections.emptyList(); + try { + String token = getUserToken(subject.getUser()); + String response = restApiClient.get(token, endpoint); + history = GSON.fromJson(response, new TypeToken<List<Revision>>(){}.getType()); + } catch (IOException e) { + LOG.error("Cannot get note history", e); + } + return history; + } + + private String getUserToken(String user) { + return tokenManager.getUserToken(user); + } + + @Override + public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { + if (!isSubjectValid(subject)) { + return Collections.emptyList(); + } + + List<NotebookRepoSettingsInfo> settings = Lists.newArrayList(); + String user = subject.getUser(); + String zeppelinHubUserSession = UserSessionContainer.instance.getSession(user); + String userToken = getUserToken(user); + List<Instance> instances; + List<Map<String, String>> values = Lists.newLinkedList(); + + try { + instances = tokenManager.getUserInstances(zeppelinHubUserSession); + } catch (IOException e) { + LOG.warn("Couldnt find instances for the session {}, returning empty collection", + zeppelinHubUserSession); + // user not logged + //TODO(xxx): handle this case. + instances = Collections.emptyList(); + } + + NotebookRepoSettingsInfo repoSetting = NotebookRepoSettingsInfo.newInstance(); + repoSetting.type = NotebookRepoSettingsInfo.Type.DROPDOWN; + for (Instance instance : instances) { + if (instance.token.equals(userToken)) { + repoSetting.selected = Integer.toString(instance.id); + } + values.add(ImmutableMap.of("name", instance.name, "value", Integer.toString(instance.id))); + } + + repoSetting.value = values; + repoSetting.name = "Instance"; + settings.add(repoSetting); + return settings; + } + + private void changeToken(int instanceId, String user) { + if (instanceId <= 0) { + LOG.error("User {} tried to switch to a non valid instance {}", user, instanceId); + return; + } + + LOG.info("User {} will switch instance", user); + String ticket = UserSessionContainer.instance.getSession(user); + List<Instance> instances; + String currentToken = StringUtils.EMPTY, targetToken = StringUtils.EMPTY; + try { + instances = tokenManager.getUserInstances(ticket); + if (instances.isEmpty()) { + return; + } + currentToken = tokenManager.getExistingUserToken(user); + for (Instance instance : instances) { + if (instance.id == instanceId) { + LOG.info("User {} switched to instance {}", user, instance.name); + tokenManager.setUserToken(user, instance.token); + targetToken = instance.token; + break; + } + } + if (!StringUtils.isBlank(currentToken) && !StringUtils.isBlank(targetToken)) { + ZeppelinhubUtils.userSwitchTokenRoutine(user, currentToken, targetToken); + } + } catch (IOException e) { + LOG.error("Cannot switch instance for user {}", user, e); + } + } + + @Override + public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { + if (!isSubjectValid(subject)) { + LOG.error("Invalid subject, cannot update Zeppelinhub settings"); + return; + } + if (settings == null || settings.isEmpty()) { + LOG.error("Cannot update ZeppelinHub repo settings because of invalid settings"); + return; + } + + int instanceId = 0; + if (settings.containsKey("Instance")) { + try { + instanceId = Integer.parseInt(settings.get("Instance")); + } catch (NumberFormatException e) { + LOG.error("ZeppelinHub Instance Id in not a valid integer", e); + } + } + changeToken(instanceId, subject.getUser()); + } + + @Override + public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject) + throws IOException { + // Auto-generated method stub + return null; + } + +}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo new file mode 100644 index 0000000..395d1cd --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepoTest.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepoTest.java new file mode 100644 index 0000000..2fa6d41 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepoTest.java @@ -0,0 +1,155 @@ +package org.apache.zeppelin.notebook.repo.zeppelinhub; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.commons.httpclient.HttpException; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.repo.zeppelinhub.rest.ZeppelinhubRestApiHandler; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.io.Files; + + +public class ZeppelinHubRepoTest { + final String token = "AAA-BBB-CCC-00"; + final String testAddr = "http://zeppelinhub.ltd";; + final AuthenticationInfo auth = new AuthenticationInfo("anthony"); + + private ZeppelinHubRepo repo; + private File pathOfNotebooks = new File(System.getProperty("user.dir") + "/src/test/resources/list_of_notes"); + private File pathOfNotebook = new File(System.getProperty("user.dir") + "/src/test/resources/note"); + + @Before + public void setUp() throws Exception { + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, testAddr); + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_TOKEN, token); + + ZeppelinConfiguration conf = new ZeppelinConfiguration(); + repo = new ZeppelinHubRepo(); + repo.init(conf); + repo.setZeppelinhubRestApiHandler(getMockedZeppelinHandler()); + } + + private ZeppelinhubRestApiHandler getMockedZeppelinHandler() throws HttpException, IOException { + ZeppelinhubRestApiHandler mockedZeppelinhubHandler = mock(ZeppelinhubRestApiHandler.class); + + byte[] listOfNotesResponse = Files.toByteArray(pathOfNotebooks); + when(mockedZeppelinhubHandler.get("AAA-BBB-CCC-00", "")) + .thenReturn(new String(listOfNotesResponse)); + + byte[] noteResponse = Files.toByteArray(pathOfNotebook); + when(mockedZeppelinhubHandler.get("AAA-BBB-CCC-00", "AAAAA")) + .thenReturn(new String(noteResponse)); + + return mockedZeppelinhubHandler; + } + + @Test + public void testGetZeppelinhubUrl() { + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, testAddr); + + ZeppelinConfiguration config = new ZeppelinConfiguration(); + ZeppelinHubRepo repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinHubUrl(config)).isEqualTo("http://zeppelinhub.ltd";); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "yolow"); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinHubUrl(config)).isEqualTo("https://www.zeppelinhub.com";); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "http://zeppelinhub.ltd:4242";); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinHubUrl(config)).isEqualTo("http://zeppelinhub.ltd:4242";); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "http://zeppelinhub.ltd:0";); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinHubUrl(config)).isEqualTo("http://zeppelinhub.ltd";); + } + + @Test + public void testGetZeppelinHubWsEndpoint() { + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, testAddr); + + ZeppelinConfiguration config = new ZeppelinConfiguration(); + ZeppelinHubRepo repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("ws://zeppelinhub.ltd:80/async"); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "https://zeppelinhub.ltd";); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("wss://zeppelinhub.ltd:443/async"); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "yolow"); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("wss://www.zeppelinhub.com:443/async"); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "http://zeppelinhub.ltd:4242";); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("ws://zeppelinhub.ltd:4242/async"); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "https://www.zeppelinhub.com";); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("wss://www.zeppelinhub.com:443/async"); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "http://www.zeppelinhub.com";); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("ws://www.zeppelinhub.com:80/async"); + + System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "https://www.zeppelinhub.com:4242";); + + config = new ZeppelinConfiguration(); + repository = new ZeppelinHubRepo(config); + assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("wss://www.zeppelinhub.com:4242/async"); + } + + @Test + public void testGetAllNotes() throws IOException { + List<NoteInfo> notebooks = repo.list(auth); + assertThat(notebooks).isNotEmpty(); + assertThat(notebooks.size()).isEqualTo(3); + } + + @Test + public void testGetNote() throws IOException { + Note notebook = repo.get("AAAAA", auth); + assertThat(notebook).isNotNull(); + assertThat(notebook.getId()).isEqualTo("2A94M5J1Z"); + } + + @Test + public void testRemoveNote() throws IOException { + // not suppose to throw + repo.remove("AAAAA", auth); + } + + @Test + public void testRemoveNoteError() throws IOException { + // not suppose to throw + repo.remove("BBBBB", auth); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinClientTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinClientTest.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinClientTest.java new file mode 100644 index 0000000..b8e52e4 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinClientTest.java @@ -0,0 +1,127 @@ +package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock.MockEchoWebsocketServer; +import org.apache.zeppelin.notebook.socket.Message; +import org.apache.zeppelin.notebook.socket.Message.OP; +import org.eclipse.jetty.websocket.api.Session; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Maps; + +public class ZeppelinClientTest { + private Logger LOG = LoggerFactory.getLogger(ZeppelinClientTest.class); + private final int zeppelinPort = 8080; + private final String validWebsocketUrl = "ws://localhost:" + zeppelinPort + "/ws"; + private ExecutorService executor; + private MockEchoWebsocketServer echoServer; + + @Before + public void setUp() throws Exception { + startWebsocketServer(); + } + + @After + public void tearDown() throws Exception { + //tear down routine + echoServer.stop(); + executor.shutdown(); + } + + private void startWebsocketServer() throws InterruptedException { + // mock zeppelin websocket server setup + executor = Executors.newFixedThreadPool(1); + echoServer = new MockEchoWebsocketServer(zeppelinPort); + executor.submit(echoServer); + } + + @Test + public void zeppelinConnectionTest() { + try { + // Wait for websocket server to start + Thread.sleep(2000); + } catch (InterruptedException e) { + LOG.warn("Cannot wait for websocket server to start, returning"); + return; + } + // Initialize and start Zeppelin client + ZeppelinClient client = ZeppelinClient.initialize(validWebsocketUrl, "dummy token", null); + client.start(); + LOG.info("Zeppelin websocket client started"); + + // Connection to note AAAA + Session connectionA = client.getZeppelinConnection("AAAA", "anonymous", "anonymous"); + assertNotNull(connectionA); + assertTrue(connectionA.isOpen()); + + assertEquals(client.countConnectedNotes(), 1); + assertEquals(connectionA, client.getZeppelinConnection("AAAA", "anonymous", "anonymous")); + + // Connection to note BBBB + Session connectionB = client.getZeppelinConnection("BBBB", "anonymous", "anonymous"); + assertNotNull(connectionB); + assertTrue(connectionB.isOpen()); + + assertEquals(client.countConnectedNotes(), 2); + assertEquals(connectionB, client.getZeppelinConnection("BBBB", "anonymous", "anonymous")); + + // Remove connection to note AAAA + client.removeNoteConnection("AAAA"); + assertEquals(client.countConnectedNotes(), 1); + assertNotEquals(connectionA, client.getZeppelinConnection("AAAA", "anonymous", "anonymous")); + assertEquals(client.countConnectedNotes(), 2); + client.stop(); + } + + @Test + public void zeppelinClientSingletonTest() { + ZeppelinClient client1 = ZeppelinClient.getInstance(); + if (client1 == null) { + client1 = ZeppelinClient.initialize(validWebsocketUrl, "TOKEN", null); + } + assertNotNull(client1); + ZeppelinClient client2 = ZeppelinClient.getInstance(); + assertNotNull(client2); + assertEquals(client1, client2); + } + + @Test + public void zeppelinMessageSerializationTest() { + Message msg = new Message(OP.LIST_NOTES); + msg.data = Maps.newHashMap(); + msg.data.put("key", "value"); + ZeppelinClient client = ZeppelinClient.initialize(validWebsocketUrl, "TOKEN", null); + String serializedMsg = client.serialize(msg); + Message deserializedMsg = client.deserialize(serializedMsg); + assertEquals(msg.op, deserializedMsg.op); + assertEquals(msg.data.get("key"), deserializedMsg.data.get("key")); + + String invalidMsg = "random text"; + deserializedMsg =client.deserialize(invalidMsg); + assertNull(deserializedMsg); + } + + @Test + public void sendToZeppelinTest() { + ZeppelinClient client = ZeppelinClient.initialize(validWebsocketUrl, "TOKEN", null); + client.start(); + Message msg = new Message(OP.LIST_NOTES); + msg.data = Maps.newHashMap(); + msg.data.put("key", "value"); + client.send(msg, "DDDD"); + client.removeNoteConnection("DDDD"); + client.stop(); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinhubClientTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinhubClientTest.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinhubClientTest.java new file mode 100644 index 0000000..384cfe4 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinhubClientTest.java @@ -0,0 +1,72 @@ +package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket; + +import static org.junit.Assert.*; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinhubClient; +import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock.MockEchoWebsocketServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZeppelinhubClientTest { + private Logger LOG = LoggerFactory.getLogger(ZeppelinClientTest.class); + private final int zeppelinPort = 8090; + private final String validWebsocketUrl = "ws://localhost:" + zeppelinPort + "/ws"; + private ExecutorService executor; + private MockEchoWebsocketServer echoServer; + private final String runNotebookMsg = + "{\"op\":\"RUN_NOTEBOOK\"," + + "\"data\":[{\"id\":\"20150112-172845_1301897143\",\"title\":null,\"config\":{},\"params\":{},\"data\":null}," + + "{\"id\":\"20150112-172845_1301897143\",\"title\":null,\"config\":{},\"params\":{},\"data\":null}]," + + "\"meta\":{\"owner\":\"author\",\"instance\":\"my-zepp\",\"noteId\":\"2AB7SY361\"}}"; + private final String invalidRunNotebookMsg = "some random string"; + + @Before + public void setUp() throws Exception { + startWebsocketServer(); + } + + @After + public void tearDown() throws Exception { + //tear down routine + echoServer.stop(); + executor.shutdown(); + } + + private void startWebsocketServer() throws InterruptedException { + // mock zeppelin websocket server setup + executor = Executors.newFixedThreadPool(1); + echoServer = new MockEchoWebsocketServer(zeppelinPort); + executor.submit(echoServer); + } + + @Test + public void zeppelinhubClientSingletonTest() { + ZeppelinhubClient client1 = ZeppelinhubClient.getInstance(); + if (client1 == null) { + client1 = ZeppelinhubClient.initialize(validWebsocketUrl, "TOKEN"); + } + assertNotNull(client1); + ZeppelinhubClient client2 = ZeppelinhubClient.getInstance(); + assertNotNull(client2); + assertEquals(client1, client2); + } + + @Test + public void runAllParagraphTest() throws Exception { + Client.initialize(validWebsocketUrl, validWebsocketUrl, "TOKEN", null); + Client.getInstance().start(); + ZeppelinhubClient zeppelinhubClient = ZeppelinhubClient.getInstance(); + boolean runStatus = zeppelinhubClient.runAllParagraph("2AB7SY361", runNotebookMsg); + assertTrue(runStatus); + runStatus = zeppelinhubClient.runAllParagraph("2AB7SY361", invalidRunNotebookMsg); + assertFalse(runStatus); + Client.getInstance().stop(); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEchoWebsocketServer.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEchoWebsocketServer.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEchoWebsocketServer.java new file mode 100644 index 0000000..e9959e9 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEchoWebsocketServer.java @@ -0,0 +1,46 @@ +package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.slf4j.LoggerFactory; + +public class MockEchoWebsocketServer implements Runnable { + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MockEchoWebsocketServer.class); + private Server server; + + public MockEchoWebsocketServer(int port) { + server = new Server(); + ServerConnector connector = new ServerConnector(server); + connector.setPort(port); + server.addConnector(connector); + + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + context.setContextPath("/"); + server.setHandler(context); + + //ServletHolder holderEvents = new ServletHolder("ws-events", MockEventServlet.class); + context.addServlet(MockEventServlet.class, "/ws/*"); + } + + public void start() throws Exception { + LOG.info("Starting mock echo websocket server"); + server.start(); + server.join(); + } + + public void stop() throws Exception { + LOG.info("Stopping mock echo websocket server"); + server.stop(); + } + + @Override + public void run() { + try { + this.start(); + } catch (Exception e) { + LOG.error("Couldn't start mock echo websocket server", e); + } + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventServlet.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventServlet.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventServlet.java new file mode 100644 index 0000000..c84f2c3 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventServlet.java @@ -0,0 +1,14 @@ +package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock; + +import org.eclipse.jetty.websocket.servlet.WebSocketServlet; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; + +@SuppressWarnings("serial") +public class MockEventServlet extends WebSocketServlet +{ + @Override + public void configure(WebSocketServletFactory factory) + { + factory.register(MockEventSocket.class); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventSocket.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventSocket.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventSocket.java new file mode 100644 index 0000000..0f39b01 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventSocket.java @@ -0,0 +1,38 @@ +package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock; + +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MockEventSocket extends WebSocketAdapter { + private static final Logger LOG = LoggerFactory.getLogger(MockEventServlet.class); + private Session session; + + @Override + public void onWebSocketConnect(Session session) { + super.onWebSocketConnect(session); + this.session = session; + LOG.info("Socket Connected: " + session); + } + + @Override + public void onWebSocketText(String message) { + super.onWebSocketText(message); + session.getRemote().sendStringByFuture(message); + LOG.info("Received TEXT message: {}", message); + + } + + @Override + public void onWebSocketClose(int statusCode, String reason) { + super.onWebSocketClose(statusCode, reason); + LOG.info("Socket Closed: [{}] {}", statusCode, reason); + } + + @Override + public void onWebSocketError(Throwable cause) { + super.onWebSocketError(cause); + LOG.error("Websocket error: {}", cause); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/protocol/ZeppelinhubMessageTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/protocol/ZeppelinhubMessageTest.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/protocol/ZeppelinhubMessageTest.java new file mode 100644 index 0000000..1f07f4f --- /dev/null +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/protocol/ZeppelinhubMessageTest.java @@ -0,0 +1,43 @@ +package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol; + +import static org.junit.Assert.assertEquals; + +import java.util.Map; + +import org.apache.zeppelin.notebook.socket.Message.OP; +import org.junit.Test; + +import com.google.common.collect.Maps; + +public class ZeppelinhubMessageTest { + + private String msg = "{\"op\":\"LIST_NOTES\",\"data\":\"my data\",\"meta\":{\"key1\":\"val1\"}}"; + + @Test + public void testThatCanSerializeZeppelinHubMessage() { + Map<String,String> meta = Maps.newHashMap(); + meta.put("key1", "val1"); + String zeppelinHubMsg = ZeppelinhubMessage.newMessage(OP.LIST_NOTES, "my data", meta).toJson(); + + assertEquals(msg, zeppelinHubMsg); + } + + @Test + public void testThastCanDeserialiseZeppelinhubMessage() { + Map<String,String> meta = Maps.newHashMap(); + meta.put("key1", "val1"); + ZeppelinhubMessage expected = ZeppelinhubMessage.newMessage(OP.LIST_NOTES.toString(), "my data", meta); + ZeppelinhubMessage zeppelinHubMsg = ZeppelinhubMessage.fromJson(msg); + + assertEquals(expected.op, zeppelinHubMsg.op); + assertEquals(expected.data, zeppelinHubMsg.data); + assertEquals(expected.meta, zeppelinHubMsg.meta); + } + + @Test + public void testThatInvalidStringReturnEmptyZeppelinhubMessage() { + assertEquals(ZeppelinhubMessage.EMPTY, ZeppelinhubMessage.fromJson("")); + assertEquals(ZeppelinhubMessage.EMPTY, ZeppelinhubMessage.fromJson("dwfewewrewr")); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/pom.xml b/zeppelin-plugins/pom.xml new file mode 100644 index 0000000..f436411 --- /dev/null +++ b/zeppelin-plugins/pom.xml @@ -0,0 +1,124 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>zeppelin</artifactId> + <groupId>org.apache.zeppelin</groupId> + <version>0.9.0-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <groupId>org.apache.zeppelin</groupId> + <artifactId>zengine-plugins-parent</artifactId> + <packaging>pom</packaging> + <version>0.9.0-SNAPSHOT</version> + <name>Zeppelin: Plugins Parent</name> + <description>Zeppelin Plugins Parent</description> + + <modules> + <module>notebookrepo/s3</module> + <module>notebookrepo/vfs</module> + <module>notebookrepo/git</module> + <module>notebookrepo/github</module> + <module>notebookrepo/azure</module> + <module>notebookrepo/gcs</module> + <module>notebookrepo/mongodb</module> + <module>notebookrepo/zeppelin-hub</module> + <module>notebookrepo/filesystem</module> + </modules> + + <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>zeppelin-zengine</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- Test libraries --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-plugin-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/../../../../plugins/${plugin.name}</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + <includeScope>runtime</includeScope> + </configuration> + </execution> + <execution> + <id>copy-plugin-artifact</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/../../../../plugins/${plugin.name}</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + <artifactItems> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>${project.artifactId}</artifactId> + <version>${project.version}</version> + <type>${project.packaging}</type> + </artifactItem> + </artifactItems> + </configuration> + </execution> + </executions> + </plugin> + + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-server/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 3ee83bd..fc6d0dd 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -420,42 +420,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>start-zeppelin</id> - <phase>pre-integration-test</phase> - <configuration> - <target unless="skipTests"> - <exec executable="./zeppelin-daemon.sh" dir="${zeppelin.daemon.package.base}" spawn="true"> - <arg value="start" /> - </exec> - </target> - </configuration> - <goals> - <goal>run</goal> - </goals> - </execution> - - <execution> - <id>stop-zeppelin</id> - <phase>post-integration-test</phase> - <configuration> - <target unless="skipTests"> - <exec executable="./zeppelin-daemon.sh" dir="${zeppelin.daemon.package.base}" spawn="false"> - <arg value="stop" /> - </exec> - </target> - </configuration> - <goals> - <goal>run</goal> - </goals> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> </plugin> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-server/src/main/java/org/apache/zeppelin/realm/PamRealm.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/PamRealm.java b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/PamRealm.java index 5b3ae10..d7f62ec 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/PamRealm.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/PamRealm.java @@ -43,7 +43,7 @@ import java.util.Set; */ public class PamRealm extends AuthorizingRealm { - private static final Logger LOG = LoggerFactory.getLogger(ZeppelinHubRealm.class); + private static final Logger LOG = LoggerFactory.getLogger(PamRealm.class); private String service; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java index 113469f..1d6cdf9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java @@ -88,23 +88,23 @@ public class ZeppelinHubRealm extends AuthorizingRealm { LOG.debug("{} successfully login via ZeppelinHub", user.login); return new SimpleAuthenticationInfo(user.login, token.getPassword(), name); } - + @Override protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) { // TODO(xxx): future work will be done here. return null; } - + protected void onInit() { super.onInit(); } - + /** * Setter of ZeppelinHub URL, this will be called by Shiro based on zeppelinhubUrl property * in shiro.ini file.</p> - * It will also perform a check of ZeppelinHub url {@link #isZeppelinHubUrlValid}, + * It will also perform a check of ZeppelinHub url {@link #isZeppelinHubUrlValid}, * if the url is not valid, the default zeppelinhub url will be used. - * + * * @param url */ public void setZeppelinhubUrl(String url) { @@ -118,9 +118,9 @@ public class ZeppelinHubRealm extends AuthorizingRealm { } /** - * Send to ZeppelinHub a login request based on the request body which is a JSON that contains 2 + * Send to ZeppelinHub a login request based on the request body which is a JSON that contains 2 * fields "login" and "password". - * + * * @param requestBody JSON string of ZeppelinHub payload. * @return Account object with login, name (if set in ZeppelinHub), and mail. * @throws AuthenticationException if fail to login. @@ -141,12 +141,12 @@ public class ZeppelinHubRealm extends AuthorizingRealm { responseBody = put.getResponseBodyAsString(); userSession = put.getResponseHeader(USER_SESSION_HEADER).getValue(); put.releaseConnection(); - + } catch (IOException e) { LOG.error("Cannot login user", e); throw new AuthenticationException(e.getMessage()); } - + User account = null; try { account = User.fromJson(responseBody); @@ -156,7 +156,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm { } onLoginSuccess(account.login, userSession); - + return account; } @@ -182,7 +182,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm { * Perform a Simple URL check by using <code>URI(url).toURL()</code>. * If the url is not valid, the try-catch condition will catch the exceptions and return false, * otherwise true will be returned. - * + * * @param url * @return */ @@ -215,7 +215,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm { return gson.fromJson(json, User.class); } } - + public void onLoginSuccess(String username, String session) { UserSessionContainer.instance.setSession(username, session); @@ -227,7 +227,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm { ZeppelinhubUtils.userLoginRoutine(username); } - + @Override public void onLogout(PrincipalCollection principals) { ZeppelinhubUtils.userLogoutRoutine((String) principals.getPrimaryPrincipal()); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index b8f9be9..85fc631 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -30,7 +30,6 @@ import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang.StringUtils; -import org.apache.commons.vfs2.FileSystemException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.*; @@ -1052,7 +1051,7 @@ public class NotebookServer extends WebSocketServlet note.persist(subject); addConnectionToNote(note.getId(), (NotebookSocket) conn); conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note))); - } catch (FileSystemException e) { + } catch (IOException e) { LOG.error("Exception from createNote", e); conn.send(serializeMessage(new Message(OP.ERROR_INFO).put("info", "Oops! There is something wrong with the notebook file system. " @@ -1840,7 +1839,7 @@ public class NotebookServer extends WebSocketServlet try { note.persist(p.getAuthenticationInfo()); return true; - } catch (FileSystemException ex) { + } catch (IOException ex) { LOG.error("Exception from run", ex); conn.send(serializeMessage(new Message(OP.ERROR_INFO).put("info", "Oops! There is something wrong with the notebook file system. " http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-server/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/resources/log4j.properties b/zeppelin-server/src/test/resources/log4j.properties index aff3b79..ce9bc50 100644 --- a/zeppelin-server/src/test/resources/log4j.properties +++ b/zeppelin-server/src/test/resources/log4j.properties @@ -41,5 +41,4 @@ log4j.logger.DataNucleus.Datastore=ERROR # Log all JDBC parameters log4j.logger.org.hibernate.type=ALL - -log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.hadoop=WARN \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-zengine/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index a7d9d5f..d37f6ea 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -38,18 +38,13 @@ <!--library versions--> <hadoop.version>2.7.3</hadoop.version> <commons.lang3.version>3.4</commons.lang3.version> - <commons.vfs2.version>2.0</commons.vfs2.version> - <gcs.storage.version>1.14.0</gcs.storage.version> - <aws.sdk.s3.version>1.10.62</aws.sdk.s3.version> - <adl.sdk.version>2.1.4</adl.sdk.version> <jackrabbit.webdav.version>1.5.2</jackrabbit.webdav.version> <quartz.scheduler.version>2.2.1</quartz.scheduler.version> <lucene.version>5.3.1</lucene.version> <org.reflections.version>0.9.8</org.reflections.version> <xml.apis.version>1.4.01</xml.apis.version> - <eclipse.jgit.version>4.5.4.201711221230-r</eclipse.jgit.version> <frontend.maven.plugin.version>1.3</frontend.maven.plugin.version> - + <aws.sdk.s3.version>1.10.62</aws.sdk.s3.version> <!--test library versions--> <google.truth.version>0.27</google.truth.version> <google.testing.nio.version>0.32.0-alpha</google.testing.nio.version> @@ -83,15 +78,8 @@ </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-vfs2</artifactId> - <version>${commons.vfs2.version}</version> - <exclusions> - <exclusion> - <groupId>org.codehaus.plexus</groupId> - <artifactId>plexus-utils</artifactId> - </exclusion> - </exclusions> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> </dependency> <dependency> @@ -117,86 +105,6 @@ </dependency> <dependency> - <groupId>com.google.cloud</groupId> - <artifactId>google-cloud-storage</artifactId> - <version>${gcs.storage.version}</version> - <exclusions> - <exclusion> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.api</groupId> - <artifactId>api-common</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.http-client</groupId> - <artifactId>google-http-client-jackson2</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.http-client</groupId> - <artifactId>google-http-client</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-core-asl</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>com.google.api</groupId> - <artifactId>api-common</artifactId> - <version>1.2.0</version> - <exclusions> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>com.google.http-client</groupId> - <artifactId>google-http-client-jackson2</artifactId> - <version>1.23.0</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-s3</artifactId> - <version>${aws.sdk.s3.version}</version> - <exclusions> - <exclusion> - <groupId>joda-time</groupId> - <artifactId>joda-time</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-client</artifactId> <version>${jetty.version}</version> @@ -250,6 +158,12 @@ </dependency> <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-s3</artifactId> + <version>${aws.sdk.s3.version}</version> + </dependency> + + <dependency> <groupId>org.reflections</groupId> <artifactId>reflections</artifactId> <version>${org.reflections.version}</version> @@ -266,12 +180,6 @@ </dependency> <dependency> - <groupId>org.eclipse.jgit</groupId> - <artifactId>org.eclipse.jgit</artifactId> - <version>${eclipse.jgit.version}</version> - </dependency> - - <dependency> <groupId>com.github.eirslett</groupId> <artifactId>frontend-maven-plugin</artifactId> <version>${frontend.maven.plugin.version}</version> @@ -304,23 +212,6 @@ </dependency> <dependency> - <groupId>com.google.cloud</groupId> - <artifactId>google-cloud-nio</artifactId> - <version>${google.testing.nio.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> <groupId>com.google.truth</groupId> <artifactId>truth</artifactId> <version>${google.truth.version}</version> @@ -366,12 +257,6 @@ <version>1.5</version> </dependency> - <dependency> - <groupId>org.mongodb</groupId> - <artifactId>mongo-java-driver</artifactId> - <version>3.4.1</version> - </dependency> - </dependencies> <build> @@ -745,73 +630,6 @@ </exclusion> </exclusions> </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-azure</artifactId> - <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </exclusion> - <exclusion> - <groupId>com.jcraf</groupId> - <artifactId>jsch</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.microsoft.azure</groupId> - <artifactId>azure-data-lake-store-sdk</artifactId> - <version>${adl.sdk.version}</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-aws</artifactId> - <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>joda-time</groupId> - <artifactId>joda-time</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> </profile> @@ -837,104 +655,6 @@ <version>${hadoop.version}</version> <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-azure</artifactId> - <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>com.jcraft</groupId> - <artifactId>jsch</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - </exclusion> - <exclusion> - <groupId>com.nimbusds</groupId> - <artifactId>nimbus-jose-jwt</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </exclusion> - <exclusion> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - </exclusion> - <exclusion> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlet</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-core-asl</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - </exclusion> - <exclusion> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-azure-datalake</artifactId> - <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-aws</artifactId> - <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>joda-time</groupId> - <artifactId>joda-time</artifactId> - </exclusion> - </exclusions> - </dependency> </dependencies> </profile> </profiles> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java index 9b1b6cb..4fc860d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java @@ -18,13 +18,7 @@ package org.apache.zeppelin.interpreter.recovery; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.InterpreterSettingManager; @@ -34,15 +28,10 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess; import org.apache.zeppelin.notebook.FileSystemStorage; -import org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 85a5f1c..3728cd3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -167,6 +167,11 @@ public class Note implements ParagraphJobListener, JsonSerializable { return id; } + @VisibleForTesting + public void setId(String id) { + this.id = id; + } + public String getName() { if (isNameEmpty()) { name = getId(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index ba83449..e9903c7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -233,7 +233,7 @@ public class Notebook implements NoteEventListener { * @throws IOException, CloneNotSupportedException, IllegalArgumentException */ public Note cloneNote(String sourceNoteId, String newNoteName, AuthenticationInfo subject) - throws IOException, CloneNotSupportedException, IllegalArgumentException { + throws IOException, IllegalArgumentException { Note sourceNote = getNote(sourceNoteId); if (sourceNote == null) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java deleted file mode 100644 index 3b01088..0000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.notebook.repo; - -import com.microsoft.azure.storage.CloudStorageAccount; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.file.CloudFile; -import com.microsoft.azure.storage.file.CloudFileClient; -import com.microsoft.azure.storage.file.CloudFileDirectory; -import com.microsoft.azure.storage.file.CloudFileShare; -import com.microsoft.azure.storage.file.ListFileItem; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.notebook.Note; -import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Azure storage backend for notebooks - */ -public class AzureNotebookRepo implements NotebookRepo { - private static final Logger LOG = LoggerFactory.getLogger(S3NotebookRepo.class); - - private final ZeppelinConfiguration conf; - private final String user; - private final String shareName; - private final CloudFileDirectory rootDir; - - public AzureNotebookRepo(ZeppelinConfiguration conf) - throws URISyntaxException, InvalidKeyException, StorageException { - this.conf = conf; - user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER); - shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE); - - CloudStorageAccount account = CloudStorageAccount.parse( - conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING)); - CloudFileClient client = account.createCloudFileClient(); - CloudFileShare share = client.getShareReference(shareName); - share.createIfNotExists(); - - CloudFileDirectory userDir = StringUtils.isBlank(user) ? - share.getRootDirectoryReference() : - share.getRootDirectoryReference().getDirectoryReference(user); - userDir.createIfNotExists(); - - rootDir = userDir.getDirectoryReference("notebook"); - rootDir.createIfNotExists(); - } - - @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { - List<NoteInfo> infos = new LinkedList<>(); - NoteInfo info = null; - - for (ListFileItem item : rootDir.listFilesAndDirectories()) { - if (item.getClass() == CloudFileDirectory.class) { - CloudFileDirectory dir = (CloudFileDirectory) item; - - try { - if (dir.getFileReference("note.json").exists()) { - info = new NoteInfo(getNote(dir.getName())); - - if (info != null) { - infos.add(info); - } - } - } catch (StorageException | URISyntaxException e) { - String msg = "Error enumerating notebooks from Azure storage"; - LOG.error(msg, e); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - } - } - - return infos; - } - - private Note getNote(String noteId) throws IOException { - InputStream ins = null; - - try { - CloudFileDirectory dir = rootDir.getDirectoryReference(noteId); - CloudFile file = dir.getFileReference("note.json"); - - ins = file.openRead(); - } catch (URISyntaxException | StorageException e) { - String msg = String.format("Error reading notebook %s from Azure storage", noteId); - - LOG.error(msg, e); - - throw new IOException(msg, e); - } - - String json = IOUtils.toString(ins, - conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING)); - ins.close(); - return Note.fromJson(json); - } - - @Override - public Note get(String noteId, AuthenticationInfo subject) throws IOException { - return getNote(noteId); - } - - @Override - public void save(Note note, AuthenticationInfo subject) throws IOException { - String json = note.toJson(); - - ByteArrayOutputStream output = new ByteArrayOutputStream(); - Writer writer = new OutputStreamWriter(output); - writer.write(json); - writer.close(); - output.close(); - - byte[] buffer = output.toByteArray(); - - try { - CloudFileDirectory dir = rootDir.getDirectoryReference(note.getId()); - dir.createIfNotExists(); - - CloudFile cloudFile = dir.getFileReference("note.json"); - cloudFile.uploadFromByteArray(buffer, 0, buffer.length); - } catch (URISyntaxException | StorageException e) { - String msg = String.format("Error saving notebook %s to Azure storage", note.getId()); - - LOG.error(msg, e); - - throw new IOException(msg, e); - } - } - - // unfortunately, we need to use a recursive delete here - private void delete(ListFileItem item) throws StorageException { - if (item.getClass() == CloudFileDirectory.class) { - CloudFileDirectory dir = (CloudFileDirectory) item; - - for (ListFileItem subItem : dir.listFilesAndDirectories()) { - delete(subItem); - } - - dir.deleteIfExists(); - } else if (item.getClass() == CloudFile.class) { - CloudFile file = (CloudFile) item; - - file.deleteIfExists(); - } - } - - @Override - public void remove(String noteId, AuthenticationInfo subject) throws IOException { - try { - CloudFileDirectory dir = rootDir.getDirectoryReference(noteId); - - delete(dir); - } catch (URISyntaxException | StorageException e) { - String msg = String.format("Error deleting notebook %s from Azure storage", noteId); - - LOG.error(msg, e); - - throw new IOException(msg, e); - } - } - - @Override - public void close() { - } - - @Override - public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { - LOG.warn("Method not implemented"); - return Collections.emptyList(); - } - - @Override - public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { - LOG.warn("Method not implemented"); - } - -} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java deleted file mode 100644 index bfc1178..0000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.apache.zeppelin.notebook.repo; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.notebook.FileSystemStorage; -import org.apache.zeppelin.notebook.Note; -import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * NotebookRepos for hdfs. - * - * Assume the notebook directory structure is as following - * - notebookdir - * - noteId/note.json - * - noteId/note.json - * - noteId/note.json - */ -public class FileSystemNotebookRepo implements NotebookRepo { - private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemNotebookRepo.class); - - private FileSystemStorage fs; - private Path notebookDir; - - public FileSystemNotebookRepo(ZeppelinConfiguration zConf) throws IOException { - this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir()); - LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() + - " for Zeppelin Notebook."); - this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir())); - LOGGER.info("Using folder {} to store notebook", notebookDir); - this.fs.tryMkDir(notebookDir); - } - - @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { - List<Path> notePaths = fs.list(new Path(notebookDir, "*/note.json")); - List<NoteInfo> noteInfos = new ArrayList<>(); - for (Path path : notePaths) { - NoteInfo noteInfo = new NoteInfo(path.getParent().getName(), "", null); - noteInfos.add(noteInfo); - } - return noteInfos; - } - - @Override - public Note get(final String noteId, AuthenticationInfo subject) throws IOException { - String content = this.fs.readFile( - new Path(notebookDir.toString() + "/" + noteId + "/note.json")); - return Note.fromJson(content); - } - - @Override - public void save(final Note note, AuthenticationInfo subject) throws IOException { - this.fs.writeFile(note.toJson(), - new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"), - true); - } - - @Override - public void remove(final String noteId, AuthenticationInfo subject) throws IOException { - this.fs.delete(new Path(notebookDir.toString() + "/" + noteId)); - } - - @Override - public void close() { - LOGGER.warn("close is not implemented for HdfsNotebookRepo"); - } - - @Override - public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { - LOGGER.warn("getSettings is not implemented for HdfsNotebookRepo"); - return null; - } - - @Override - public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { - LOGGER.warn("updateSettings is not implemented for HdfsNotebookRepo"); - } - -}
