Repository: incubator-zeppelin Updated Branches: refs/heads/master e87f5b1ab -> 218a3b5bc
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java new file mode 100644 index 0000000..8886add --- /dev/null +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.httpclient.methods.DeleteMethod; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.httpclient.methods.PutMethod; +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.server.ZeppelinServer; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import static org.junit.Assert.*; + +/** + * Zeppelin interpreter rest api tests + */ +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class InterpreterRestApiTest extends AbstractTestRestApi { + Gson gson = new Gson(); + + @BeforeClass + public static void init() throws Exception { + AbstractTestRestApi.startUp(); + } + + @AfterClass + public static void destroy() throws Exception { + AbstractTestRestApi.shutDown(); + } + + @Test + public void getAvailableInterpreters() throws IOException { + // when + GetMethod get = httpGet("/interpreter"); + + // then + assertThat(get, isAllowed()); + Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() { + }.getType()); + Map<String, Object> body = (Map<String, Object>) resp.get("body"); + assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size()); + get.releaseConnection(); + } + + @Test + public void getSettings() throws IOException { + // when + GetMethod get = httpGet("/interpreter/setting"); + + // then + Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() { + }.getType()); + assertThat(get, isAllowed()); + get.releaseConnection(); + } + + @Test + public void testSettingsCRUD() throws IOException { + // Call Create Setting REST API + String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," + + "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," + + "\"dependencies\":[]}"; + PostMethod post = httpPost("/interpreter/setting/", jsonRequest); + LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString()); + assertThat("test create method:", post, isCreated()); + + Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() { + }.getType()); + Map<String, Object> body = (Map<String, Object>) resp.get("body"); + //extract id from body string {id=2AWMQDNX7, name=md2, group=md, + String newSettingId = body.toString().split(",")[0].split("=")[1]; + post.releaseConnection(); + + // Call Update Setting REST API + jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"}," + + "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," + + "\"dependencies\":[]}"; + PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest); + LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString()); + assertThat("test update method:", put, isAllowed()); + put.releaseConnection(); + + // Call Delete Setting REST API + DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId); + LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString()); + assertThat("Test delete method:", delete, isAllowed()); + delete.releaseConnection(); + } + + @Test + public void testInterpreterAutoBinding() throws IOException { + // create note + Note note = ZeppelinServer.notebook.createNote(); + + // check interpreter is binded + GetMethod get = httpGet("/notebook/interpreter/bind/" + note.id()); + assertThat(get, isAllowed()); + get.addRequestHeader("Origin", "http://localhost";); + Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() { + }.getType()); + List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body"); + assertTrue(0 < body.size()); + + get.releaseConnection(); + //cleanup + ZeppelinServer.notebook.removeNote(note.getId()); + } + + @Test + public void testInterpreterRestart() throws IOException, InterruptedException { + // create new note + Note note = ZeppelinServer.notebook.createNote(); + note.addParagraph(); + Paragraph p = note.getLastParagraph(); + Map config = p.getConfig(); + config.put("enabled", true); + + // run markdown paragraph + p.setConfig(config); + p.setText("%md markdown"); + note.run(p.getId()); + while (p.getStatus() != Status.FINISHED) { + Thread.sleep(100); + } + assertEquals("<p>markdown</p>\n", p.getResult().message()); + + + // restart interpreter + for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) { + if (setting.getName().equals("md")) { + // Call Restart Interpreter REST API + PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), ""); + assertThat("test interpreter restart:", put, isAllowed()); + put.releaseConnection(); + break; + } + } + + // run markdown paragraph, again + p = note.addParagraph(); + p.setConfig(config); + p.setText("%md markdown restarted"); + note.run(p.getId()); + while (p.getStatus() != Status.FINISHED) { + Thread.sleep(100); + } + assertEquals("<p>markdown restarted</p>\n", p.getResult().message()); + //cleanup + ZeppelinServer.notebook.removeNote(note.getId()); + } + + @Test + public void testListRepository() throws IOException { + GetMethod get = httpGet("/interpreter/repository"); + assertThat(get, isAllowed()); + get.releaseConnection(); + } + + @Test + public void testAddDeleteRepository() throws IOException { + // Call create repository REST API + String repoId = "securecentral"; + String jsonRequest = "{\"id\":\"" + repoId + + "\",\"url\":\"https://repo1.maven.org/maven2\",\"snapshot\":\"false\"}";; + + PostMethod post = httpPost("/interpreter/repository/", jsonRequest); + assertThat("Test create method:", post, isCreated()); + post.releaseConnection(); + + // Call delete repository REST API + DeleteMethod delete = httpDelete("/interpreter/repository/" + repoId); + assertThat("Test delete method:", delete, isAllowed()); + delete.releaseConnection(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index d8049cc..3c7c7d0 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -72,124 +72,6 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi { httpGetRoot.releaseConnection(); } - - @Test - public void getAvailableInterpreters() throws IOException { - // when - GetMethod get = httpGet("/interpreter"); - - // then - assertThat(get, isAllowed()); - Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() { - }.getType()); - Map<String, Object> body = (Map<String, Object>) resp.get("body"); - assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size()); - get.releaseConnection(); - } - - @Test - public void getSettings() throws IOException { - // when - GetMethod get = httpGet("/interpreter/setting"); - - // then - Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() { - }.getType()); - assertThat(get, isAllowed()); - get.releaseConnection(); - } - - @Test - public void testSettingsCRUD() throws IOException { - // Call Create Setting REST API - String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"},\"" + - "interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]}"; - PostMethod post = httpPost("/interpreter/setting/", jsonRequest); - LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString()); - assertThat("test create method:", post, isCreated()); - - Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() { - }.getType()); - Map<String, Object> body = (Map<String, Object>) resp.get("body"); - //extract id from body string {id=2AWMQDNX7, name=md2, group=md, - String newSettingId = body.toString().split(",")[0].split("=")[1]; - post.releaseConnection(); - - // Call Update Setting REST API - jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"},\"" + - "interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]}"; - PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest); - LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString()); - assertThat("test update method:", put, isAllowed()); - put.releaseConnection(); - - // Call Delete Setting REST API - DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId); - LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString()); - assertThat("Test delete method:", delete, isAllowed()); - delete.releaseConnection(); - } - @Test - public void testInterpreterAutoBinding() throws IOException { - // create note - Note note = ZeppelinServer.notebook.createNote(); - - // check interpreter is binded - GetMethod get = httpGet("/notebook/interpreter/bind/"+note.id()); - assertThat(get, isAllowed()); - get.addRequestHeader("Origin", "http://localhost";); - Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>(){}.getType()); - List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body"); - assertTrue(0 < body.size()); - - get.releaseConnection(); - //cleanup - ZeppelinServer.notebook.removeNote(note.getId()); - } - - @Test - public void testInterpreterRestart() throws IOException, InterruptedException { - // create new note - Note note = ZeppelinServer.notebook.createNote(); - note.addParagraph(); - Paragraph p = note.getLastParagraph(); - Map config = p.getConfig(); - config.put("enabled", true); - - // run markdown paragraph - p.setConfig(config); - p.setText("%md markdown"); - note.run(p.getId()); - while (p.getStatus() != Status.FINISHED) { - Thread.sleep(100); - } - assertEquals("<p>markdown</p>\n", p.getResult().message()); - - - // restart interpreter - for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) { - if (setting.getName().equals("md")) { - // Call Restart Interpreter REST API - PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), ""); - assertThat("test interpreter restart:", put, isAllowed()); - put.releaseConnection(); - break; - } - } - - // run markdown paragraph, again - p = note.addParagraph(); - p.setConfig(config); - p.setText("%md markdown restarted"); - note.run(p.getId()); - while (p.getStatus() != Status.FINISHED) { - Thread.sleep(100); - } - assertEquals("<p>markdown restarted</p>\n", p.getResult().message()); - //cleanup - ZeppelinServer.notebook.removeNote(note.getId()); - } - @Test public void testGetNotebookInfo() throws IOException { LOG.info("testGetNotebookInfo"); @@ -303,7 +185,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi { } @Test - public void testDeleteNote() throws IOException { + public void testDeleteNote() throws IOException { LOG.info("testDeleteNote"); //Create note and get ID Note note = ZeppelinServer.notebook.createNote(); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/app.js ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index d13faf0..33a9daf 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -81,7 +81,8 @@ ngToastProvider.configure({ dismissButton: true, dismissOnClick: false, - timeout: 6000 + timeout: 6000, + verticalPosition: 'bottom' }); }); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/home/home.css ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/home/home.css b/zeppelin-web/src/app/home/home.css index 0202142..dada898 100644 --- a/zeppelin-web/src/app/home/home.css +++ b/zeppelin-web/src/app/home/home.css @@ -276,31 +276,6 @@ kbd { } /* - ngToast Style -*/ - -.ng-toast .alert { - color: white !important; - border: none !important; -} - -.ng-toast .alert-danger { - background: #A94442 !important; -} - -.ng-toast .alert-warning { - background: #CE9532 !important; -} - -.ng-toast .alert-info { - background: #589EC1 !important; -} - -.ng-toast .alert-success { - background: #428443 !important; -} - -/* temporary fix for bootstrap issue (https://github.com/twbs/bootstrap/issues/5865) This part should be removed when new version of bootstrap handles this issue. */ http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html index 38ae7cc..6f46c4e 100644 --- a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html +++ b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html @@ -12,10 +12,10 @@ See the License for the specific language governing permissions and limitations under the License. --> <div> - <div class="row"> - <div class="col-md-12"> - <div class="interpreterSettingAdd" ng-show="showAddNewSetting"> - <hr /> + <div class="row interpreter"> + <div class="col-md-12" ng-show="showAddNewSetting"> + <hr /> + <div class="interpreterSettingAdd"> <h4>Create new interpreter</h4> <div class="form-group" style="width:200px"> @@ -67,6 +67,50 @@ limitations under the License. </tr> </table> + <b>Dependencies</b> + <table class="table table-striped properties"> + <tr> + <th>artifact</th> + <th>exclude</th> + <th>action</th> + </tr> + + <tr ng-repeat="dep in newInterpreterSetting.dependencies"> + <td> + <input ng-model="dep.groupArtifactVersion" style="width:100%" /> + </td> + <td> + <textarea msd-elastic ng-model="dep.exclusions" + ng-list + placeholder="(Optional) comma separated groupId:artifactId list"> + </textarea> + </td> + <td> + <div class="btn btn-default btn-sm fa fa-remove" + ng-click="removeInterpreterDependency(dep.groupArtifactVersion)"> + </div> + </td> + </tr> + + <tr> + <td> + <input ng-model="newInterpreterSetting.depArtifact" + placeholder="groupId:artifactId:version or local file path" + style="width: 100%" /> + </td> + <td> + <textarea msd-elastic ng-model="newInterpreterSetting.depExclude" + ng-list + placeholder="(Optional) comma separated groupId:artifactId list"> + </textarea> + </td> + <td> + <div class="btn btn-default btn-sm fa fa-plus" ng-click="addNewInterpreterDependency()"> + </div> + </td> + </tr> + </table> + <span class="btn btn-primary" ng-click="addNewInterpreterSetting()"> Save </span> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/interpreter/interpreter.controller.js ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 396a552..1e3800a 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -15,37 +15,42 @@ 'use strict'; angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, $route, $routeParams, $location, $rootScope, - $http, baseUrlSrv) { + $http, baseUrlSrv, ngToast) { var interpreterSettingsTmp = []; $scope.interpreterSettings = []; $scope.availableInterpreters = {}; $scope.showAddNewSetting = false; + $scope.showRepositoryInfo = false; $scope._ = _; var getInterpreterSettings = function() { $http.get(baseUrlSrv.getRestApiBase()+'/interpreter/setting'). - success(function(data, status, headers, config) { - $scope.interpreterSettings = data.body; - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); + success(function(data, status, headers, config) { + $scope.interpreterSettings = data.body; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); }; var getAvailableInterpreters = function() { $http.get(baseUrlSrv.getRestApiBase()+'/interpreter'). - success(function(data, status, headers, config) { - $scope.availableInterpreters = data.body; - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); + success(function(data, status, headers, config) { + $scope.availableInterpreters = data.body; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); }; var emptyNewProperty = function(object) { angular.extend(object, {propertyValue: '', propertyKey: ''}); }; + var emptyNewDependency = function(object) { + angular.extend(object, {depArtifact: '', depExclude: ''}); + }; + var removeTMPSettings = function(index) { interpreterSettingsTmp.splice(index, 1); }; @@ -55,29 +60,34 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, interpreterSettingsTmp[index] = angular.copy($scope.interpreterSettings[index]); }; - $scope.updateInterpreterSetting = function(settingId) { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Do you want to update this interpreter and restart with new settings?', - callback: function(result) { - if (result) { - var index = _.findIndex($scope.interpreterSettings, {'id': settingId}); - var request = { - properties: angular.copy($scope.interpreterSettings[index].properties), - }; - - $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request). - success(function (data, status, headers, config) { - $scope.interpreterSettings[index] = data.body; - removeTMPSettings(index); - }). - error(function (data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); - } + $scope.updateInterpreterSetting = function(form, settingId) { + var result = confirm('Do you want to update this interpreter and restart with new settings?'); + if (result) { + var index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + var setting = $scope.interpreterSettings[index]; + if (setting.propertyKey !== '' || setting.propertyKey) { + $scope.addNewInterpreterProperty(settingId); } - }); + if (setting.depArtifact !== '' || setting.depArtifact) { + $scope.addNewInterpreterDependency(settingId); + } + + var request = { + properties: angular.copy(setting.properties), + dependencies: angular.copy(setting.dependencies) + }; + + $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request). + success(function (data, status, headers, config) { + $scope.interpreterSettings[index] = data.body; + removeTMPSettings(index); + }). + error(function (data, status, headers, config) { + console.log('Error %o %o', status, data.message); + ngToast.danger(data.message); + form.$show(); + }); + } }; $scope.resetInterpreterSetting = function(settingId){ @@ -117,8 +127,8 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, var intpInfo = el[i]; for (var key in intpInfo) { properties[key] = { - value : intpInfo[key].defaultValue, - description : intpInfo[key].description + value: intpInfo[key].defaultValue, + description: intpInfo[key].description }; } } @@ -165,32 +175,46 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, return; } - var newSetting = angular.copy($scope.newInterpreterSetting); + var newSetting = $scope.newInterpreterSetting; + if (newSetting.propertyKey !== '' || newSetting.propertyKey) { + $scope.addNewInterpreterProperty(); + } + if (newSetting.depArtifact !== '' || newSetting.depArtifact) { + $scope.addNewInterpreterDependency(); + } + + var request = angular.copy($scope.newInterpreterSetting); - for (var p in $scope.newInterpreterSetting.properties) { - newSetting.properties[p] = $scope.newInterpreterSetting.properties[p].value; + // Change properties to proper request format + var newProperties = {}; + for (var p in newSetting.properties) { + newProperties[p] = newSetting.properties[p].value; } + request.properties = newProperties; - $http.post(baseUrlSrv.getRestApiBase()+'/interpreter/setting', newSetting). - success(function(data, status, headers, config) { - $scope.resetNewInterpreterSetting(); - getInterpreterSettings(); - $scope.showAddNewSetting = false; - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); + $http.post(baseUrlSrv.getRestApiBase() + '/interpreter/setting', request). + success(function(data, status, headers, config) { + $scope.resetNewInterpreterSetting(); + getInterpreterSettings(); + $scope.showAddNewSetting = false; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + ngToast.danger(data.message); + }); }; $scope.cancelInterpreterSetting = function() { $scope.showAddNewSetting = false; + $scope.resetNewInterpreterSetting(); }; $scope.resetNewInterpreterSetting = function() { $scope.newInterpreterSetting = { - name : undefined, - group : undefined, - properties : {} + name: undefined, + group: undefined, + properties: {}, + dependencies: [] }; emptyNewProperty($scope.newInterpreterSetting); }; @@ -205,6 +229,21 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, } }; + $scope.removeInterpreterDependency = function(artifact, settingId) { + if (settingId === undefined) { + $scope.newInterpreterSetting.dependencies = _.reject($scope.newInterpreterSetting.dependencies, + function(el) { + return el.groupArtifactVersion === artifact; + }); + } else { + var index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + $scope.interpreterSettings[index].dependencies = _.reject($scope.interpreterSettings[index].dependencies, + function(el) { + return el.groupArtifactVersion === artifact; + }); + } + }; + $scope.addNewInterpreterProperty = function(settingId) { if(settingId === undefined) { // Add new property from create form @@ -230,10 +269,126 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, } }; + $scope.addNewInterpreterDependency = function(settingId) { + if(settingId === undefined) { + // Add new dependency from create form + if (!$scope.newInterpreterSetting.depArtifact || $scope.newInterpreterSetting.depArtifact === '') { + return; + } + + // overwrite if artifact already exists + var newSetting = $scope.newInterpreterSetting; + for(var d in newSetting.dependencies) { + if (newSetting.dependencies[d].groupArtifactVersion === newSetting.depArtifact) { + newSetting.dependencies[d] = { + 'groupArtifactVersion': newSetting.depArtifact, + 'exclusions': newSetting.depExclude + }; + newSetting.dependencies.splice(d, 1); + } + } + + newSetting.dependencies.push({ + 'groupArtifactVersion': newSetting.depArtifact, + 'exclusions': (newSetting.depExclude === '')? []: newSetting.depExclude + }); + emptyNewDependency(newSetting); + } + else { + // Add new dependency from edit form + var index = _.findIndex($scope.interpreterSettings, { 'id': settingId }); + var setting = $scope.interpreterSettings[index]; + if (!setting.depArtifact || setting.depArtifact === '') { + return; + } + + // overwrite if artifact already exists + for(var dep in setting.dependencies) { + if (setting.dependencies[dep].groupArtifactVersion === setting.depArtifact) { + setting.dependencies[dep] = { + 'groupArtifactVersion': setting.depArtifact, + 'exclusions': setting.depExclude + }; + setting.dependencies.splice(dep, 1); + } + } + + setting.dependencies.push({ + 'groupArtifactVersion': setting.depArtifact, + 'exclusions': (setting.depExclude === '')? []: setting.depExclude + }); + emptyNewDependency(setting); + } + }; + + $scope.resetNewRepositorySetting = function() { + $scope.newRepoSetting = { + id: undefined, + url: undefined, + snapshot: false, + username: undefined, + password: undefined + }; + }; + + var getRepositories = function() { + $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/repository'). + success(function(data, status, headers, config) { + $scope.repositories = data.body; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); + }; + + $scope.addNewRepository = function() { + var request = angular.copy($scope.newRepoSetting); + + $http.post(baseUrlSrv.getRestApiBase() + '/interpreter/repository', request). + success(function(data, status, headers, config) { + getRepositories(); + $scope.resetNewRepositorySetting(); + angular.element('#repoModal').modal('hide'); + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', headers, config); + }); + }; + + $scope.removeRepository = function(repoId) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Do you want to delete this repository?', + callback: function(result) { + if (result) { + $http.delete(baseUrlSrv.getRestApiBase()+'/interpreter/repository/' + repoId). + success(function(data, status, headers, config) { + var index = _.findIndex($scope.repositories, { 'id': repoId }); + $scope.repositories.splice(index, 1); + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); + } + } + }); + }; + + $scope.isDefaultRepository = function(repoId) { + if (repoId === 'central' || repoId === 'local') { + return true; + } else { + return false; + } + }; + var init = function() { $scope.resetNewInterpreterSetting(); + $scope.resetNewRepositorySetting(); getInterpreterSettings(); getAvailableInterpreters(); + getRepositories(); }; init(); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/interpreter/interpreter.css ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/interpreter/interpreter.css b/zeppelin-web/src/app/interpreter/interpreter.css index b6fe133..8695f94 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.css +++ b/zeppelin-web/src/app/interpreter/interpreter.css @@ -34,6 +34,13 @@ font-size: 12px; } +.interpreter input { + width: 100%; + display: block; + height: 23px; + border: 1px solid #CCCCCC; +} + .interpreter .interpreter-title { font-size: 20px; font-weight: bold; @@ -81,6 +88,11 @@ float: left; } -.empty-properties-message { +.gray40-message { color: #666; } + +.blackOpc:hover { + color: #000; + opacity: .5; +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/interpreter/interpreter.html ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/interpreter/interpreter.html b/zeppelin-web/src/app/interpreter/interpreter.html index 57984da..f1f9a02 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.html +++ b/zeppelin-web/src/app/interpreter/interpreter.html @@ -18,11 +18,18 @@ limitations under the License. <h3 class="new_h3"> Interpreters </h3> - <span class="btn btn-default fa fa-plus" - ng-click="showAddNewSetting = !showAddNewSetting" - style="float:right;margin-top:10px;"> - Create - </span> + <div class="pull-right" style="margin-top:10px;"> + <span class="btn btn-default fa fa-plus" + ng-click="showAddNewSetting = !showAddNewSetting" + style="margin-right:6px;"> + Create + </span> + <span style="cursor:pointer;margin-right:4px;" + ng-click="showRepositoryInfo = !showRepositoryInfo" + tooltip-placement="bottom" tooltip="Repository information"> + <i class="fa fa-cog" ng-style="{color: showRepositoryInfo ? '#3071A9' : 'black' }"></i> + </span> + </div> </div> </div> <div class="row"> @@ -31,6 +38,42 @@ limitations under the License. </div> </div> </div> + + <div class="row" ng-if="showRepositoryInfo"> + <div class="col-md-12"> + <hr /> + <h4>Repositories</h4> + <p>Available repository lists. These repositories are used to resolve external dependencies of interpreter.</p> + <ul class="noDot"> + <li class="liVertical" ng-repeat="repo in repositories"> + <a tabindex="0" class="btn btn-info" role="button" + popover-trigger="focus" + popover-placement="right" + popover-html-unsafe=" + <label>URL: </label> + {{repo.url}}<br> + <label>Snapshot: </label> + {{repo.snapshotPolicy.enabled}}<br> + <label>Username: </label> + {{repo.authentication.username}}"> + <span class="fa fa-database"></span> + {{repo.id}} + <span ng-if="!isDefaultRepository(repo.id)" + class="fa fa-close blackOpc" ng-click="removeRepository(repo.id)"></span> + </a> + </li> + <li class="liVertical"> + <div ng-include src="'components/repository-create/repository-dialog.html'"></div> + <div class="btn btn-default" + data-toggle="modal" + data-target="#repoModal"> + <span class="fa fa-plus"></span> + </div> + </li> + </ul> + </div> + </div> + <div ng-include src="'app/interpreter/interpreter-create/interpreter-create.html'"></div> </div> @@ -66,8 +109,8 @@ limitations under the License. </div> </div> <div class="row interpreter"> - <div ng-show="_.isEmpty(setting.properties) || valueform.$hidden" class="col-md-12 empty-properties-message"> - <em>Currently there are no properties set for this interpreter</em> + <div ng-show="_.isEmpty(setting.properties) && _.isEmpty(setting.dependencies) || valueform.$hidden" class="col-md-12 gray40-message"> + <em>Currently there are no properties and dependencies set for this interpreter</em> </div> <div class="col-md-12" ng-show="!_.isEmpty(setting.properties) || valueform.$visible"> <h5>Properties</h5> @@ -96,8 +139,7 @@ limitations under the License. <td> <input ng-model="setting.propertyKey" pu-elastic-input - pu-elastic-input-minwidth="180px"> - </input> + pu-elastic-input-minwidth="180px" /> </td> <td> <textarea msd-elastic ng-model="setting.propertyValue"></textarea> @@ -109,9 +151,66 @@ limitations under the License. </td> </tr> </table> - <form editable-form name="valueform" onaftersave="updateInterpreterSetting(setting.id)" ng-show="valueform.$visible"> - <button type="submit" class="btn btn-primary" - ng-disabled="valueform.$waiting"> + </div> + + <div class="col-md-12" ng-show="!_.isEmpty(setting.dependencies) || valueform.$visible"> + <h5>Dependencies</h5> + <p class="gray40-message" style="font-size:12px" ng-show="valueform.$visible"> + These dependencies will be added to classpath when interpreter process starts.</p> + <table class="table table-striped"> + <thead> + <tr> + <th style="width:40%">artifact</th> + <th>exclude</th> + <th ng-if="valueform.$visible">action</th> + </tr> + </thead> + <tr ng-repeat="dep in setting.dependencies"> + <td> + <span editable-text="dep.groupArtifactVersion" e-placeholder="groupId:artifactId:version or local file path" + e-form="valueform" e-msd-elastic e-style="width:100%"> + {{dep.groupArtifactVersion | breakFilter}} + </span> + </td> + <td> + <textarea ng-if="valueform.$visible" ng-model="dep.exclusions" + placeholder="(Optional) comma separated groupId:artifactId list" + form="valueform" + e-msd-elastic + ng-list=""> + </textarea> + <div ng-if="!valueform.$visible">{{dep.exclusions.join()}}</div> + </td> + <td ng-if="valueform.$visible"> + <div class="btn btn-default btn-sm fa fa-remove" + ng-click="removeInterpreterDependency(dep.groupArtifactVersion, setting.id)"> + </div> + </td> + </tr> + <tr ng-if="valueform.$visible"> + <td> + <input ng-model="setting.depArtifact" + placeholder="groupId:artifactId:version or local file path" + style="width: 100%" /> + </td> + <td> + <textarea ng-model="setting.depExclude" + placeholder="(Optional) comma separated groupId:artifactId list" + msd-elastic + ng-list=""> + </textarea> + </td> + <td> + <div class="btn btn-default btn-sm fa fa-plus" + ng-click="addNewInterpreterDependency(setting.id)"> + </div> + </td> + </tr> + </table> + <form editable-form name="valueform" + onaftersave="updateInterpreterSetting(valueform, setting.id)" + ng-show="valueform.$visible"> + <button type="submit" class="btn btn-primary"> Save </button> <button type="button" class="btn btn-default" http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/components/repository-create/repository-dialog.html ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/components/repository-create/repository-dialog.html b/zeppelin-web/src/components/repository-create/repository-dialog.html new file mode 100644 index 0000000..98b1469 --- /dev/null +++ b/zeppelin-web/src/components/repository-create/repository-dialog.html @@ -0,0 +1,76 @@ +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<div id="repoModal" class="modal fade" role="dialog" + tabindex='-1'> + <div class="modal-dialog"> + + <!-- Modal content--> + <div class="modal-content"> + <div class="modal-header"> + <button type="button" class="close" data-dismiss="modal">×</button> + <h4 class="modal-title">Add New Repository</h4> + </div> + + <form class="form-horizontal" ng-submit="addNewRepository()"> + <div class="modal-body"> + <div class="form-group"> + <label class="control-label col-sm-2" for="repoId">ID</label> + <div class="col-sm-10"> + <input type="text" class="form-control" id="repoId" ng-model="newRepoSetting.id" + placeholder="Repository id" required> + </div> + </div> + <div class="form-group"> + <label class="control-label col-sm-2" for="repoUrl">URL</label> + <div class="col-sm-10"> + <input type="text" pattern="(http|https|file)://.*" title="Please enter URL starts with http://, https:// or file://" + class="form-control" id="repoUrl" ng-model="newRepoSetting.url" + placeholder="Repository url" required> + </div> + </div> + <div class="form-group"> + <label class="control-label col-sm-2" for="repoSnapshot">Snapshot</label> + <div class="col-sm-10"> + <select class="form-control" id="repoSnapshot" ng-model="newRepoSetting.snapshot"> + <option ng-selected="true">false</option> + <option>true</option> + </select> + </div> + </div> + <div class="form-group"> + <label class="control-label col-sm-2" for="repoUsername">Username</label> + <div class="col-sm-10"> + <input type="text" class="form-control" id="repoUsername" ng-model="newRepoSetting.username"> + </div> + </div> + <div class="form-group"> + <label class="control-label col-sm-2" for="repoPassword">Password</label> + <div class="col-sm-10"> + <input type="password" class="form-control" id="repoPassword" ng-model="newRepoSetting.password"> + </div> + </div> + </div> + <div class="modal-footer"> + <button type="submit" + class="btn btn-default">Add + </button> + <button type="button" data-dismiss="modal" + class="btn btn-default" ng-click="resetNewRepositorySetting()">Cancel + </button> + </div> + </form> + </div> + </div> +</div> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/index.html ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index a8e082c..1f1166b 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -63,8 +63,8 @@ limitations under the License. </head> <body ng-class="{'bodyAsIframe': asIframe}" > <!--[if lt IE 7]> -<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/";>upgrade your browser</a> to improve your experience.</p> -<![endif]--> + <p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/";>upgrade your browser</a> to improve your experience.</p> + <![endif]--> <toast></toast> <div ng-include src="'components/navbar/navbar.html'"></div> <!-- Add your site or application content here --> @@ -81,9 +81,9 @@ limitations under the License. </div> <!-- build:js(.) scripts/oldieshim.js --> <!--[if lt IE 9]> -<script src="bower_components/es5-shim/es5-shim.js"></script> -<script src="bower_components/json3/lib/json3.min.js"></script> -<![endif]--> + <script src="bower_components/es5-shim/es5-shim.js"></script> + <script src="bower_components/json3/lib/json3.min.js"></script> + <![endif]--> <!-- endbuild --> <!-- build:js(.) scripts/vendor.js --> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 9e606ee..421b3d4 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -346,6 +346,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER); } + public String getInterpreterLocalRepoPath() { + return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO); + } + public String getRelativeDir(ConfVars c) { return getRelativeDir(getString(c)); } @@ -456,6 +460,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { + "org.apache.zeppelin.scalding.ScaldingInterpreter," + "org.apache.zeppelin.jdbc.JDBCInterpreter"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), + ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"), ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000), ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10), ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"), http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 3cd1257..3761709 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -19,10 +19,13 @@ package org.apache.zeppelin.interpreter; import com.google.gson.Gson; import com.google.gson.GsonBuilder; + +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.NullArgumentException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.Dependency; import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; @@ -34,6 +37,9 @@ import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; +import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.RemoteRepository; import java.io.*; import java.lang.reflect.Constructor; @@ -60,6 +66,7 @@ public class InterpreterFactory { new HashMap<String, InterpreterSetting>(); private Map<String, List<String>> interpreterBindings = new HashMap<String, List<String>>(); + private List<RemoteRepository> interpreterRepositories; private Gson gson; @@ -68,13 +75,13 @@ public class InterpreterFactory { AngularObjectRegistryListener angularObjectRegistryListener; private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; - DependencyResolver depResolver; + private DependencyResolver depResolver; public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, DependencyResolver depResolver) - throws InterpreterException, IOException { + throws InterpreterException, IOException, RepositoryException { this(conf, new InterpreterOption(true), angularObjectRegistryListener, remoteInterpreterProcessListener, depResolver); } @@ -84,11 +91,12 @@ public class InterpreterFactory { AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, DependencyResolver depResolver) - throws InterpreterException, IOException { + throws InterpreterException, IOException, RepositoryException { this.conf = conf; this.defaultOption = defaultOption; this.angularObjectRegistryListener = angularObjectRegistryListener; this.depResolver = depResolver; + this.interpreterRepositories = depResolver.getRepos(); this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS); interpreterClassList = replsConf.split(","); @@ -101,7 +109,7 @@ public class InterpreterFactory { init(); } - private void init() throws InterpreterException, IOException { + private void init() throws InterpreterException, IOException, RepositoryException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); // Load classes @@ -172,7 +180,11 @@ public class InterpreterFactory { if (found) { // add all interpreters in group - add(groupName, groupName, defaultOption, p); + add(groupName, + groupName, + new LinkedList<Dependency>(), + defaultOption, + p); groupClassNameMap.remove(groupName); break; } @@ -186,7 +198,7 @@ public class InterpreterFactory { logger.info("Interpreter setting group {} : id={}, name={}", setting.getGroup(), settingId, setting.getName()); for (Interpreter interpreter : setting.getInterpreterGroup()) { - logger.info(" className = {}", interpreter.getClassName()); + logger.info(" className = {}", interpreter.getClassName()); } } } @@ -225,12 +237,11 @@ public class InterpreterFactory { // previously created setting should turn this feature on here. setting.getOption().setRemote(true); - - InterpreterSetting intpSetting = new InterpreterSetting( setting.id(), setting.getName(), setting.getGroup(), + setting.getDependencies(), setting.getOption()); InterpreterGroup interpreterGroup = createInterpreterGroup( @@ -244,8 +255,41 @@ public class InterpreterFactory { } this.interpreterBindings = info.interpreterBindings; + + if (info.interpreterRepositories != null) { + for (RemoteRepository repo : info.interpreterRepositories) { + if (!depResolver.getRepos().contains(repo)) { + this.interpreterRepositories.add(repo); + } + } + } } + private void loadInterpreterDependencies(InterpreterSetting intSetting) + throws IOException, RepositoryException { + // dependencies to prevent library conflict + File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + intSetting.id()); + if (localRepoDir.exists()) { + FileUtils.cleanDirectory(localRepoDir); + } + + // load dependencies + List<Dependency> deps = intSetting.getDependencies(); + if (deps != null) { + for (Dependency d: deps) { + if (d.getExclusions() != null) { + depResolver.load( + d.getGroupArtifactVersion(), + d.getExclusions(), + conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + } else { + depResolver.load( + d.getGroupArtifactVersion(), + conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + } + } + } + } private void saveToFile() throws IOException { String jsonString; @@ -254,6 +298,7 @@ public class InterpreterFactory { InterpreterInfoSaving info = new InterpreterInfoSaving(); info.interpreterBindings = interpreterBindings; info.interpreterSettings = interpreterSettings; + info.interpreterRepositories = interpreterRepositories; jsonString = gson.toJson(info); } @@ -330,15 +375,21 @@ public class InterpreterFactory { * @throws IOException */ public InterpreterGroup add(String name, String groupName, + List<Dependency> dependencies, InterpreterOption option, Properties properties) - throws InterpreterException, IOException { + throws InterpreterException, IOException, RepositoryException { synchronized (interpreterSettings) { InterpreterSetting intpSetting = new InterpreterSetting( name, groupName, + dependencies, option); + if (dependencies.size() > 0) { + loadInterpreterDependencies(intpSetting); + } + InterpreterGroup interpreterGroup = createInterpreterGroup( intpSetting.id(), groupName, option, properties); @@ -354,13 +405,13 @@ public class InterpreterFactory { String groupName, InterpreterOption option, Properties properties) - throws InterpreterException , NullArgumentException { + throws InterpreterException, NullArgumentException { //When called from REST API without option we receive NPE - if (option == null ) + if (option == null) throw new NullArgumentException("option"); //When called from REST API without option we receive NPE - if (properties == null ) + if (properties == null) throw new NullArgumentException("properties"); AngularObjectRegistry angularObjectRegistry; @@ -393,7 +444,8 @@ public class InterpreterFactory { if (option.isRemote()) { intp = createRemoteRepl(info.getPath(), info.getClassName(), - properties); + properties, + interpreterGroup.id); } else { intp = createRepl(info.getPath(), info.getClassName(), @@ -428,6 +480,9 @@ public class InterpreterFactory { saveToFile(); } } + + File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id); + FileUtils.deleteDirectory(localRepoDir); } /** @@ -511,8 +566,10 @@ public class InterpreterFactory { * @param properties * @throws IOException */ - public void setPropertyAndRestart(String id, InterpreterOption option, - Properties properties) throws IOException { + public void setPropertyAndRestart(String id, + InterpreterOption option, + Properties properties, + List<Dependency> dependencies) throws IOException, RepositoryException { synchronized (interpreterSettings) { InterpreterSetting intpsetting = interpreterSettings.get(id); if (intpsetting != null) { @@ -523,11 +580,14 @@ public class InterpreterFactory { intpsetting.getInterpreterGroup().destroy(); intpsetting.setOption(option); + intpsetting.setDependencies(dependencies); InterpreterGroup interpreterGroup = createInterpreterGroup( intpsetting.id(), intpsetting.getGroup(), option, properties); intpsetting.setInterpreterGroup(interpreterGroup); + + loadInterpreterDependencies(intpsetting); saveToFile(); } else { throw new InterpreterException("Interpreter setting id " + id @@ -661,13 +721,14 @@ public class InterpreterFactory { private Interpreter createRemoteRepl(String interpreterPath, String className, - Properties property) { - + Properties property, String interpreterId) { int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); + String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterId; int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE); LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter( property, className, conf.getInterpreterRemoteRunnerPath(), - interpreterPath, connectTimeout, maxPoolSize, remoteInterpreterProcessListener)); + interpreterPath, localRepoPath, connectTimeout, + maxPoolSize, remoteInterpreterProcessListener)); return intp; } @@ -690,4 +751,19 @@ public class InterpreterFactory { return new URL[] {path.toURI().toURL()}; } } + + public List<RemoteRepository> getRepositories() { + return this.interpreterRepositories; + } + + public void addRepository(String id, String url, boolean snapshot, Authentication auth) + throws IOException { + depResolver.addRepo(id, url, snapshot, auth); + saveToFile(); + } + + public void removeRepository(String id) throws IOException { + depResolver.delRepo(id); + saveToFile(); + } } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java index ae507d4..786a723 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java @@ -17,6 +17,8 @@ package org.apache.zeppelin.interpreter; +import org.sonatype.aether.repository.RemoteRepository; + import java.util.List; import java.util.Map; @@ -26,4 +28,5 @@ import java.util.Map; public class InterpreterInfoSaving { public Map<String, InterpreterSetting> interpreterSettings; public Map<String, List<String>> interpreterBindings; + public List<RemoteRepository> interpreterRepositories; } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 301ed23..e8080a2 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -17,9 +17,12 @@ package org.apache.zeppelin.interpreter; +import java.util.LinkedList; +import java.util.List; import java.util.Properties; import java.util.Random; +import org.apache.zeppelin.dep.Dependency; import org.apache.zeppelin.notebook.utility.IdHashes; /** @@ -32,21 +35,26 @@ public class InterpreterSetting { private String description; private Properties properties; private InterpreterGroup interpreterGroup; + private List<Dependency> dependencies; private InterpreterOption option; - public InterpreterSetting(String id, String name, + public InterpreterSetting(String id, + String name, String group, + List<Dependency> dependencies, InterpreterOption option) { this.id = id; this.name = name; this.group = group; + this.dependencies = dependencies; this.option = option; } public InterpreterSetting(String name, String group, + List<Dependency> dependencies, InterpreterOption option) { - this(generateId(), name, group, option); + this(generateId(), name, group, dependencies, option); } public String id() { @@ -90,6 +98,17 @@ public class InterpreterSetting { return properties; } + public List<Dependency> getDependencies() { + if (dependencies == null) { + return new LinkedList<Dependency>(); + } + return dependencies; + } + + public void setDependencies(List<Dependency> dependencies) { + this.dependencies = dependencies; + } + public InterpreterOption getOption() { if (option == null) { option = new InterpreterOption(); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 17d91cc..8fea693 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -24,17 +24,21 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; +import java.util.LinkedList; import java.util.List; import java.util.Properties; import org.apache.commons.lang.NullArgumentException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter2; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.sonatype.aether.RepositoryException; public class InterpreterFactoryTest { @@ -42,6 +46,7 @@ public class InterpreterFactoryTest { private File tmpDir; private ZeppelinConfiguration conf; private InterpreterContext context; + private DependencyResolver depResolver; @Before public void setUp() throws Exception { @@ -55,7 +60,8 @@ public class InterpreterFactoryTest { System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); conf = new ZeppelinConfiguration(); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); context = new InterpreterContext("note", "id", "title", "text", null, null, null, null, null); } @@ -98,14 +104,14 @@ public class InterpreterFactoryTest { } @Test - public void testFactoryDefaultList() throws IOException { + public void testFactoryDefaultList() throws IOException, RepositoryException { // get default list from default setting List<String> all = factory.getDefaultInterpreterSettingList(); assertEquals(2, all.size()); assertEquals(factory.get(all.get(0)).getInterpreterGroup().getFirst().getClassName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); // add setting - factory.add("a mock", "mock2", new InterpreterOption(false), new Properties()); + factory.add("a mock", "mock2", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties()); all = factory.getDefaultInterpreterSettingList(); assertEquals(2, all.size()); assertEquals("mock1", factory.get(all.get(0)).getName()); @@ -113,16 +119,16 @@ public class InterpreterFactoryTest { } @Test - public void testExceptions() throws IOException { + public void testExceptions() throws InterpreterException, IOException, RepositoryException { List<String> all = factory.getDefaultInterpreterSettingList(); // add setting with null option & properties expected nullArgumentException.class try { - factory.add("a mock", "mock2", null, new Properties()); + factory.add("a mock", "mock2", new LinkedList<Dependency>(), null, new Properties()); } catch(NullArgumentException e) { assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage()); } try { - factory.add("a mock" , "mock2" , new InterpreterOption(false),null); + factory.add("a mock", "mock2", new LinkedList<Dependency>(), new InterpreterOption(false), null); } catch (NullArgumentException e){ assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage()); } @@ -130,17 +136,17 @@ public class InterpreterFactoryTest { @Test - public void testSaveLoad() throws InterpreterException, IOException { + public void testSaveLoad() throws IOException, RepositoryException { // interpreter settings assertEquals(2, factory.get().size()); // check if file saved assertTrue(new File(conf.getInterpreterSettingPath()).exists()); - factory.add("newsetting", "mock1", new InterpreterOption(false), new Properties()); + factory.add("newsetting", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties()); assertEquals(3, factory.get().size()); - InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null); + InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver); assertEquals(3, factory2.get().size()); } } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 4fa8ef6..04a6ceb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -25,6 +25,7 @@ import java.util.HashMap; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; @@ -40,6 +41,7 @@ public class NoteInterpreterLoaderTest { private File tmpDir; private ZeppelinConfiguration conf; private InterpreterFactory factory; + private DependencyResolver depResolver; @Before public void setUp() throws Exception { @@ -58,7 +60,8 @@ public class NoteInterpreterLoaderTest { MockInterpreter11.register("mock11", "group1", "org.apache.zeppelin.interpreter.mock.MockInterpreter11"); MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); } @After http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 55cd6ba..1988f1c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -35,6 +35,7 @@ import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; @@ -55,6 +56,7 @@ import org.junit.Test; import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; public class NotebookTest implements JobListenerFactory{ private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class); @@ -66,6 +68,7 @@ public class NotebookTest implements JobListenerFactory{ private Notebook notebook; private NotebookRepo notebookRepo; private InterpreterFactory factory; + private DependencyResolver depResolver; @Before public void setUp() throws Exception { @@ -86,7 +89,8 @@ public class NotebookTest implements JobListenerFactory{ MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); @@ -161,7 +165,7 @@ public class NotebookTest implements JobListenerFactory{ } @Test - public void testPersist() throws IOException, SchedulerException{ + public void testPersist() throws IOException, SchedulerException, RepositoryException { Note note = notebook.createNote(); // run with default repl @@ -173,8 +177,8 @@ public class NotebookTest implements JobListenerFactory{ note.persist(); Notebook notebook2 = new Notebook( - conf, notebookRepo, schedulerFactory, new InterpreterFactory(conf, null, null, null), this, - null); + conf, notebookRepo, schedulerFactory, + new InterpreterFactory(conf, null, null, null, depResolver), this, null); assertEquals(1, notebook2.getAllNotes().size()); } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 31970af..753fab2 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterOutput; @@ -57,6 +58,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory { private Notebook notebookSync; private NotebookRepoSync notebookRepoSync; private InterpreterFactory factory; + private DependencyResolver depResolver; private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class); @Before @@ -85,7 +87,8 @@ public class NotebookRepoSyncTest implements JobListenerFactory { MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 2e2801c..0d4ff86 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; @@ -48,6 +49,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory { private Notebook notebook; private NotebookRepo notebookRepo; private InterpreterFactory factory; + private DependencyResolver depResolver; private File mainZepDir; private File mainNotebookDir; @@ -73,7 +75,8 @@ public class VFSNotebookRepoTest implements JobListenerFactory { MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); this.schedulerFactory = new SchedulerFactory(); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf);
