http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java index d76bb15..7d1df9b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java @@ -17,13 +17,14 @@ package org.apache.zeppelin.interpreter; -import java.io.IOException; -import java.io.StringReader; -import java.util.Properties; import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.StringReader; +import java.util.Properties; + /** * Special Interpreter for Interpreter Configuration customization. It is attached to each * InterpreterGroup implicitly by Zeppelin. @@ -36,11 +37,11 @@ public class ConfInterpreter extends Interpreter { protected String interpreterGroupId; protected InterpreterSetting interpreterSetting; - public ConfInterpreter( - Properties properties, - String sessionId, - String interpreterGroupId, - InterpreterSetting interpreterSetting) { + + public ConfInterpreter(Properties properties, + String sessionId, + String interpreterGroupId, + InterpreterSetting interpreterSetting) { super(properties); this.sessionId = sessionId; this.interpreterGroupId = interpreterGroupId; @@ -48,10 +49,14 @@ public class ConfInterpreter extends Interpreter { } @Override - public void open() throws InterpreterException {} + public void open() throws InterpreterException { + + } @Override - public void close() throws InterpreterException {} + public void close() throws InterpreterException { + + } @Override public InterpreterResult interpret(String st, InterpreterContext context) @@ -63,8 +68,8 @@ public class ConfInterpreter extends Interpreter { Properties newProperties = new Properties(); newProperties.load(new StringReader(st)); finalProperties.putAll(newProperties); - LOGGER.debug( - "Properties for InterpreterGroup: " + interpreterGroupId + " is " + finalProperties); + LOGGER.debug("Properties for InterpreterGroup: " + interpreterGroupId + " is " + + finalProperties); interpreterSetting.setInterpreterGroupProperties(interpreterGroupId, finalProperties); return new InterpreterResult(InterpreterResult.Code.SUCCESS); } catch (IOException e) { @@ -74,7 +79,9 @@ public class ConfInterpreter extends Interpreter { } @Override - public void cancel(InterpreterContext context) throws InterpreterException {} + public void cancel(InterpreterContext context) throws InterpreterException { + + } @Override public FormType getFormType() throws InterpreterException {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index be8e961..e045a59 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -17,14 +17,18 @@ package org.apache.zeppelin.interpreter; +import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + /** * //TODO(zjffdu) considering to move to InterpreterSettingManager * - * <p>Factory class for creating interpreters. + * Factory class for creating interpreters. + * */ public class InterpreterFactory { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterFactory.class); @@ -35,8 +39,10 @@ public class InterpreterFactory { this.interpreterSettingManager = interpreterSettingManager; } - public Interpreter getInterpreter( - String user, String noteId, String replName, String defaultInterpreterSetting) + public Interpreter getInterpreter(String user, + String noteId, + String replName, + String defaultInterpreterSetting) throws InterpreterNotFoundException { if (StringUtils.isBlank(replName)) { @@ -60,9 +66,10 @@ public class InterpreterFactory { } throw new InterpreterNotFoundException("No interpreter setting named: " + group); - } else if (replNameSplits.length == 1) { + } else if (replNameSplits.length == 1){ // first assume group is omitted - InterpreterSetting setting = interpreterSettingManager.getByName(defaultInterpreterSetting); + InterpreterSetting setting = + interpreterSettingManager.getByName(defaultInterpreterSetting); if (setting != null) { Interpreter interpreter = setting.getInterpreter(user, noteId, replName); if (null != interpreter) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java index a721ea0..fd632ce 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java @@ -18,23 +18,21 @@ package org.apache.zeppelin.interpreter; import com.google.gson.annotations.SerializedName; + import java.util.Map; /** - * Information of interpreters in this interpreter setting. this will be serialized for - * conf/interpreter.json and REST api response. + * Information of interpreters in this interpreter setting. + * this will be serialized for conf/interpreter.json and REST api response. */ public class InterpreterInfo { private String name; - - @SerializedName("class") - private String className; - + @SerializedName("class") private String className; private boolean defaultInterpreter = false; private Map<String, Object> editor; - public InterpreterInfo( - String className, String name, boolean defaultInterpreter, Map<String, Object> editor) { + public InterpreterInfo(String className, String name, boolean defaultInterpreter, + Map<String, Object> editor) { this.className = className; this.name = name; this.defaultInterpreter = defaultInterpreter; @@ -74,10 +72,9 @@ public class InterpreterInfo { boolean sameName = null == getName() ? null == other.getName() : getName().equals(other.getName()); - boolean sameClassName = - null == getClassName() - ? null == other.getClassName() - : getClassName().equals(other.getClassName()); + boolean sameClassName = null == getClassName() ? + null == other.getClassName() : + getClassName().equals(other.getClassName()); boolean sameIsDefaultInterpreter = defaultInterpreter == other.isDefaultInterpreter(); return sameName && sameClassName && sameIsDefaultInterpreter; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java index 22eb66c..8f89448 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java @@ -17,32 +17,37 @@ package org.apache.zeppelin.interpreter; -import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; -import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; - import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.gson.internal.StringMap; +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.common.JsonSerializable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.repository.RemoteRepository; + import java.io.BufferedReader; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; import java.util.*; -import org.apache.commons.io.IOUtils; -import org.apache.zeppelin.common.JsonSerializable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.repository.RemoteRepository; -/** */ +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; + +/** + * + */ public class InterpreterInfoSaving implements JsonSerializable { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterInfoSaving.class); - private static final Gson gson = new GsonBuilder().setPrettyPrinting().create(); + private static final Gson gson = new GsonBuilder().setPrettyPrinting().create(); public Map<String, InterpreterSetting> interpreterSettings = new HashMap<>(); public List<RemoteRepository> interpreterRepositories = new ArrayList<>(); @@ -58,8 +63,7 @@ public class InterpreterInfoSaving implements JsonSerializable { if (infoSaving != null && infoSaving.interpreterSettings != null) { for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) { interpreterSetting.convertPermissionsFromUsersToOwners( - jsonObject - .getAsJsonObject("interpreterSettings") + jsonObject.getAsJsonObject("interpreterSettings") .getAsJsonObject(interpreterSetting.getId())); } } @@ -76,8 +80,7 @@ public class InterpreterInfoSaving implements JsonSerializable { } catch (UnsupportedOperationException e) { // File system does not support Posix file permissions (likely windows) - continue anyway. LOGGER.warn("unable to setPosixFilePermissions on '{}'.", file); - } - ; + }; } LOGGER.info("Save Interpreter Settings to " + file); IOUtils.write(this.toJson(), new FileOutputStream(file.toFile())); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java index 15b5fdb..192e822 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java @@ -1,9 +1,12 @@ package org.apache.zeppelin.interpreter; -/** Exception for no interpreter is found */ +/** + * Exception for no interpreter is found + */ public class InterpreterNotFoundException extends InterpreterException { - public InterpreterNotFoundException() {} + public InterpreterNotFoundException() { + } public InterpreterNotFoundException(String message) { super(message); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 335503c..36fc1f1 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -17,10 +17,6 @@ package org.apache.zeppelin.interpreter; -import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE; -import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; -import static org.apache.zeppelin.util.IdHashes.generateId; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -29,19 +25,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.annotations.SerializedName; import com.google.gson.internal.StringMap; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.dep.Dependency; @@ -62,29 +45,51 @@ import org.apache.zeppelin.plugin.PluginManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Represent one InterpreterSetting in the interpreter setting page */ +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE; +import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; +import static org.apache.zeppelin.util.IdHashes.generateId; + +/** + * Represent one InterpreterSetting in the interpreter setting page + */ public class InterpreterSetting { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSetting.class); private static final String SHARED_PROCESS = "shared_process"; private static final String SHARED_SESSION = "shared_session"; - private static final Map<String, Object> DEFAULT_EDITOR = - ImmutableMap.of("language", (Object) "text", "editOnDblClick", false); + private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of( + "language", (Object) "text", + "editOnDblClick", false); private String id; private String name; // the original interpreter setting template name where it is created from private String group; - // TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json + //TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json /** - * properties can be either Properties or Map<String, InterpreterProperty> properties should be: - - * Properties when Interpreter instances are saved to `conf/interpreter.json` file - Map<String, - * InterpreterProperty> when Interpreters are registered : this is needed after - * https://github.com/apache/zeppelin/pull/1145 which changed the way of getting default - * interpreter setting AKA interpreterSettingsRef Note(mina): In order to simplify the - * implementation, I chose to change properties from Properties to Object instead of creating new - * classes. + * properties can be either Properties or Map<String, InterpreterProperty> + * properties should be: + * - Properties when Interpreter instances are saved to `conf/interpreter.json` file + * - Map<String, InterpreterProperty> when Interpreters are registered + * : this is needed after https://github.com/apache/zeppelin/pull/1145 + * which changed the way of getting default interpreter setting AKA interpreterSettingsRef + * Note(mina): In order to simplify the implementation, I chose to change properties + * from Properties to Object instead of creating new classes. */ private Object properties = new Properties(); @@ -125,7 +130,9 @@ public class InterpreterSetting { private transient RemoteInterpreterEventServer interpreterEventServer; /////////////////////////////////////////////////////////////////////////////////////////// - /** Builder class for InterpreterSetting */ + /** + * Builder class for InterpreterSetting + */ public static class Builder { private InterpreterSetting interpreterSetting; @@ -199,14 +206,13 @@ public class InterpreterSetting { return this; } - public Builder setRemoteInterpreterEventServer( - RemoteInterpreterEventServer interpreterEventServer) { + public Builder setRemoteInterpreterEventServer(RemoteInterpreterEventServer interpreterEventServer) { interpreterSetting.interpreterEventServer = interpreterEventServer; return this; } - public Builder setRemoteInterpreterProcessListener( - RemoteInterpreterProcessListener remoteInterpreterProcessListener) { + public Builder setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener + remoteInterpreterProcessListener) { interpreterSetting.remoteInterpreterProcessListener = remoteInterpreterProcessListener; return this; } @@ -271,8 +277,8 @@ public class InterpreterSetting { this.id = o.name; this.name = o.name; this.group = o.group; - this.properties = - convertInterpreterProperties((Map<String, DefaultInterpreterProperty>) o.getProperties()); + this.properties = convertInterpreterProperties( + (Map<String, DefaultInterpreterProperty>) o.getProperties()); this.interpreterInfos = new ArrayList<>(o.getInterpreterInfos()); this.option = InterpreterOption.fromInterpreterOption(o.getOption()); this.dependencies = new ArrayList<>(o.getDependencies()); @@ -282,8 +288,8 @@ public class InterpreterSetting { } private void createLauncher() throws IOException { - this.launcher = - PluginManager.get().loadInterpreterLauncher(getLauncherPlugin(), recoveryStorage); + this.launcher = PluginManager.get().loadInterpreterLauncher( + getLauncherPlugin(), recoveryStorage); } public AngularObjectRegistryListener getAngularObjectRegistryListener() { @@ -306,8 +312,8 @@ public class InterpreterSetting { return interpreterSettingManager; } - public InterpreterSetting setAngularObjectRegistryListener( - AngularObjectRegistryListener angularObjectRegistryListener) { + public InterpreterSetting setAngularObjectRegistryListener(AngularObjectRegistryListener + angularObjectRegistryListener) { this.angularObjectRegistryListener = angularObjectRegistryListener; return this; } @@ -317,8 +323,8 @@ public class InterpreterSetting { return this; } - public InterpreterSetting setRemoteInterpreterProcessListener( - RemoteInterpreterProcessListener remoteInterpreterProcessListener) { + public InterpreterSetting setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener + remoteInterpreterProcessListener) { this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; return this; } @@ -380,7 +386,7 @@ public class InterpreterSetting { key = SHARED_PROCESS; } - // TODO(zjffdu) we encode interpreter setting id into groupId, this is not a good design + //TODO(zjffdu) we encode interpreter setting id into groupId, this is not a good design return id + ":" + key; } @@ -406,18 +412,14 @@ public class InterpreterSetting { try { interpreterGroupWriteLock.lock(); if (!interpreterGroups.containsKey(groupId)) { - LOGGER.info( - "Create InterpreterGroup with groupId: {} for user: {} and note: {}", - groupId, - user, - noteId); + LOGGER.info("Create InterpreterGroup with groupId: {} for user: {} and note: {}", + groupId, user, noteId); ManagedInterpreterGroup intpGroup = createInterpreterGroup(groupId); interpreterGroups.put(groupId, intpGroup); } return interpreterGroups.get(groupId); } finally { - interpreterGroupWriteLock.unlock(); - ; + interpreterGroupWriteLock.unlock();; } } @@ -436,8 +438,7 @@ public class InterpreterSetting { interpreterGroupReadLock.lock(); return interpreterGroups.get(groupId); } finally { - interpreterGroupReadLock.unlock(); - ; + interpreterGroupReadLock.unlock();; } } @@ -495,6 +496,7 @@ public class InterpreterSetting { } } + public Object getProperties() { return properties; } @@ -511,25 +513,25 @@ public class InterpreterSetting { Map<String, InterpreterProperty> iProperties = (Map<String, InterpreterProperty>) properties; for (Map.Entry<String, InterpreterProperty> entry : iProperties.entrySet()) { if (entry.getValue().getValue() != null) { - jProperties.setProperty( - entry.getKey().trim(), entry.getValue().getValue().toString().trim()); + jProperties.setProperty(entry.getKey().trim(), + entry.getValue().getValue().toString().trim()); } } if (!jProperties.containsKey("zeppelin.interpreter.output.limit")) { - jProperties.setProperty( - "zeppelin.interpreter.output.limit", conf.getInt(ZEPPELIN_INTERPRETER_OUTPUT_LIMIT) + ""); + jProperties.setProperty("zeppelin.interpreter.output.limit", + conf.getInt(ZEPPELIN_INTERPRETER_OUTPUT_LIMIT) + ""); } if (!jProperties.containsKey("zeppelin.interpreter.max.poolsize")) { - jProperties.setProperty( - "zeppelin.interpreter.max.poolsize", + jProperties.setProperty("zeppelin.interpreter.max.poolsize", conf.getInt(ZEPPELIN_INTERPRETER_MAX_POOL_SIZE) + ""); } String interpreterLocalRepoPath = conf.getInterpreterLocalRepoPath(); - // TODO(zjffdu) change it to interpreterDir/{interpreter_name} - jProperties.setProperty("zeppelin.interpreter.localRepo", interpreterLocalRepoPath + "/" + id); + //TODO(zjffdu) change it to interpreterDir/{interpreter_name} + jProperties.setProperty("zeppelin.interpreter.localRepo", + interpreterLocalRepoPath + "/" + id); return jProperties; } @@ -596,7 +598,9 @@ public class InterpreterSetting { this.name = name; } - /** * Interpreter status */ + /*** + * Interpreter status + */ public enum Status { DOWNLOADING_DEPENDENCIES, ERROR, @@ -658,19 +662,15 @@ public class InterpreterSetting { List<InterpreterInfo> interpreterInfos = getInterpreterInfos(); Properties intpProperties = getJavaProperties(); for (InterpreterInfo info : interpreterInfos) { - Interpreter interpreter = - new RemoteInterpreter( - intpProperties, sessionId, info.getClassName(), user, lifecycleManager); + Interpreter interpreter = new RemoteInterpreter(intpProperties, sessionId, + info.getClassName(), user, lifecycleManager); if (info.isDefaultInterpreter()) { interpreters.add(0, interpreter); } else { interpreters.add(interpreter); } - LOGGER.info( - "Interpreter {} created for user: {}, sessionId: {}", - interpreter.getClassName(), - user, - sessionId); + LOGGER.info("Interpreter {} created for user: {}, sessionId: {}", + interpreter.getClassName(), user, sessionId); } // TODO(zjffdu) this kind of hardcode is ugly. For now SessionConfInterpreter is used @@ -685,23 +685,16 @@ public class InterpreterSetting { return interpreters; } - synchronized RemoteInterpreterProcess createInterpreterProcess( - String interpreterGroupId, String userName, Properties properties) throws IOException { + synchronized RemoteInterpreterProcess createInterpreterProcess(String interpreterGroupId, + String userName, + Properties properties) + throws IOException { if (launcher == null) { createLauncher(); } - InterpreterLaunchContext launchContext = - new InterpreterLaunchContext( - properties, - option, - interpreterRunner, - userName, - interpreterGroupId, - id, - group, - name, - interpreterEventServer.getPort(), - interpreterEventServer.getHost()); + InterpreterLaunchContext launchContext = new + InterpreterLaunchContext(properties, option, interpreterRunner, userName, + interpreterGroupId, id, group, name, interpreterEventServer.getPort(), interpreterEventServer.getHost()); RemoteInterpreterProcess process = (RemoteInterpreterProcess) launcher.launch(launchContext); recoveryStorage.onInterpreterClientStart(process); return process; @@ -709,8 +702,8 @@ public class InterpreterSetting { List<Interpreter> getOrCreateSession(String user, String noteId) { ManagedInterpreterGroup interpreterGroup = getOrCreateInterpreterGroup(user, noteId); - Preconditions.checkNotNull( - interpreterGroup, "No InterpreterGroup existed for user {}, " + "noteId {}", user, noteId); + Preconditions.checkNotNull(interpreterGroup, "No InterpreterGroup existed for user {}, " + + "noteId {}", user, noteId); String sessionId = getInterpreterSessionId(user, noteId); return interpreterGroup.getOrCreateSession(user, sessionId); } @@ -745,7 +738,7 @@ public class InterpreterSetting { return info.getClassName(); } } - // TODO(zjffdu) It requires user can not create interpreter with name `conf`, + //TODO(zjffdu) It requires user can not create interpreter with name `conf`, // conf is a reserved word of interpreter name if (replName.equals("conf")) { if (group.equals("livy")) { @@ -778,12 +771,11 @@ public class InterpreterSetting { ManagedInterpreterGroup interpreterGroup = this.interpreterGroups.get(interpreterGroupId); for (List<Interpreter> session : interpreterGroup.sessions.values()) { for (Interpreter intp : session) { - if (!intp.getProperties().equals(properties) - && interpreterGroup.getRemoteInterpreterProcess() != null - && interpreterGroup.getRemoteInterpreterProcess().isRunning()) { - throw new IOException( - "Can not change interpreter properties when interpreter process " - + "has already been launched"); + if (!intp.getProperties().equals(properties) && + interpreterGroup.getRemoteInterpreterProcess() != null && + interpreterGroup.getRemoteInterpreterProcess().isRunning()) { + throw new IOException("Can not change interpreter properties when interpreter process " + + "has already been launched"); } intp.setProperties(properties); } @@ -793,58 +785,53 @@ public class InterpreterSetting { private void loadInterpreterDependencies() { setStatus(Status.DOWNLOADING_DEPENDENCIES); setErrorReason(null); - Thread t = - new Thread() { - public void run() { + Thread t = new Thread() { + public void run() { + try { + // dependencies to prevent library conflict + File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id); + if (localRepoDir.exists()) { try { - // dependencies to prevent library conflict - File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id); - if (localRepoDir.exists()) { - try { - FileUtils.forceDelete(localRepoDir); - } catch (FileNotFoundException e) { - LOGGER.info("A file that does not exist cannot be deleted, nothing to worry", e); - } - } + FileUtils.forceDelete(localRepoDir); + } catch (FileNotFoundException e) { + LOGGER.info("A file that does not exist cannot be deleted, nothing to worry", e); + } + } - // load dependencies - List<Dependency> deps = getDependencies(); - if (deps != null) { - for (Dependency d : deps) { - File destDir = - new File( - conf.getRelativeDir( - ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO)); - - if (d.getExclusions() != null) { - dependencyResolver.load( - d.getGroupArtifactVersion(), d.getExclusions(), new File(destDir, id)); - } else { - dependencyResolver.load(d.getGroupArtifactVersion(), new File(destDir, id)); - } - } + // load dependencies + List<Dependency> deps = getDependencies(); + if (deps != null) { + for (Dependency d : deps) { + File destDir = new File( + conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO)); + + if (d.getExclusions() != null) { + dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(), + new File(destDir, id)); + } else { + dependencyResolver + .load(d.getGroupArtifactVersion(), new File(destDir, id)); } - - setStatus(Status.READY); - setErrorReason(null); - } catch (Exception e) { - LOGGER.error( - String.format( - "Error while downloading repos for interpreter group : %s," - + " go to interpreter setting page click on edit and save it again to make " - + "this interpreter work properly. : %s", - getGroup(), e.getLocalizedMessage()), - e); - setErrorReason(e.getLocalizedMessage()); - setStatus(Status.ERROR); } } - }; + + setStatus(Status.READY); + setErrorReason(null); + } catch (Exception e) { + LOGGER.error(String.format("Error while downloading repos for interpreter group : %s," + + " go to interpreter setting page click on edit and save it again to make " + + "this interpreter work properly. : %s", + getGroup(), e.getLocalizedMessage()), e); + setErrorReason(e.getLocalizedMessage()); + setStatus(Status.ERROR); + } + } + }; t.start(); } - // TODO(zjffdu) ugly code, should not use JsonObject as parameter. not readable + //TODO(zjffdu) ugly code, should not use JsonObject as parameter. not readable public void convertPermissionsFromUsersToOwners(JsonObject jsonObject) { if (jsonObject != null) { JsonObject option = jsonObject.getAsJsonObject("option"); @@ -870,11 +857,10 @@ public class InterpreterSetting { for (Object o : p.entrySet()) { Map.Entry entry = (Map.Entry) o; if (!(entry.getValue() instanceof StringMap)) { - InterpreterProperty newProperty = - new InterpreterProperty( - entry.getKey().toString(), - entry.getValue(), - InterpreterPropertyType.STRING.getValue()); + InterpreterProperty newProperty = new InterpreterProperty( + entry.getKey().toString(), + entry.getValue(), + InterpreterPropertyType.STRING.getValue()); newProperties.put(entry.getKey().toString(), newProperty); } else { // already converted @@ -884,7 +870,8 @@ public class InterpreterSetting { return newProperties; } else if (properties instanceof Map) { - Map<String, Object> dProperties = (Map<String, Object>) properties; + Map<String, Object> dProperties = + (Map<String, Object>) properties; Map<String, InterpreterProperty> newProperties = new HashMap<>(); for (String key : dProperties.keySet()) { Object value = dProperties.get(key); @@ -892,29 +879,31 @@ public class InterpreterSetting { return (Map<String, InterpreterProperty>) properties; } else if (value instanceof StringMap) { StringMap stringMap = (StringMap) value; - InterpreterProperty newProperty = - new InterpreterProperty( - key, - stringMap.get("value"), - stringMap.containsKey("type") ? stringMap.get("type").toString() : "string"); + InterpreterProperty newProperty = new InterpreterProperty( + key, + stringMap.get("value"), + stringMap.containsKey("type") ? stringMap.get("type").toString() : "string"); newProperties.put(newProperty.getName(), newProperty); - } else if (value instanceof DefaultInterpreterProperty) { + } else if (value instanceof DefaultInterpreterProperty){ DefaultInterpreterProperty dProperty = (DefaultInterpreterProperty) value; - InterpreterProperty property = - new InterpreterProperty( - key, - dProperty.getValue(), - dProperty.getType() != null ? dProperty.getType() : "string" - // in case user forget to specify type in interpreter-setting.json - ); + InterpreterProperty property = new InterpreterProperty( + key, + dProperty.getValue(), + dProperty.getType() != null ? dProperty.getType() : "string" + // in case user forget to specify type in interpreter-setting.json + ); newProperties.put(key, property); } else if (value instanceof String) { - InterpreterProperty newProperty = new InterpreterProperty(key, value, "string"); + InterpreterProperty newProperty = new InterpreterProperty( + key, + value, + "string"); newProperties.put(newProperty.getName(), newProperty); } else { - throw new RuntimeException("Can not convert this type of property: " + value.getClass()); + throw new RuntimeException("Can not convert this type of property: " + + value.getClass()); } } return newProperties; @@ -923,9 +912,8 @@ public class InterpreterSetting { } public void waitForReady() throws InterruptedException { - while (getStatus() - .equals( - org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) { + while (getStatus().equals( + org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) { Thread.sleep(200); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index d96c59f..d730db4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -25,26 +25,6 @@ import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.DirectoryStream.Filter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; @@ -63,38 +43,65 @@ import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; -import org.apache.zeppelin.storage.ConfigStorage; import org.apache.zeppelin.util.ReflectionUtils; +import org.apache.zeppelin.storage.ConfigStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonatype.aether.repository.Authentication; import org.sonatype.aether.repository.Proxy; import org.sonatype.aether.repository.RemoteRepository; +import org.sonatype.aether.repository.Authentication; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + /** * InterpreterSettingManager is the component which manage all the interpreter settings. - * (load/create/update/remove/get) TODO(zjffdu) We could move it into another separated component. + * (load/create/update/remove/get) + * TODO(zjffdu) We could move it into another separated component. */ public class InterpreterSettingManager implements InterpreterSettingManagerMBean { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSettingManager.class); - private static final Map<String, Object> DEFAULT_EDITOR = - ImmutableMap.of("language", (Object) "text", "editOnDblClick", false); + private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of( + "language", (Object) "text", + "editOnDblClick", false); private final ZeppelinConfiguration conf; private final Path interpreterDirPath; /** - * This is only InterpreterSetting templates with default name and properties name --> - * InterpreterSetting + * This is only InterpreterSetting templates with default name and properties + * name --> InterpreterSetting */ private final Map<String, InterpreterSetting> interpreterSettingTemplates = Maps.newConcurrentMap(); /** - * This is used by creating and running Interpreters id --> InterpreterSetting TODO(zjffdu) change - * it to name --> InterpreterSetting + * This is used by creating and running Interpreters + * id --> InterpreterSetting + * TODO(zjffdu) change it to name --> InterpreterSetting */ - private final Map<String, InterpreterSetting> interpreterSettings = Maps.newConcurrentMap(); + private final Map<String, InterpreterSetting> interpreterSettings = + Maps.newConcurrentMap(); private final List<RemoteRepository> interpreterRepositories; private InterpreterOption defaultOption; @@ -110,23 +117,20 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean private ConfigStorage configStorage; private RemoteInterpreterEventServer interpreterEventServer; - public InterpreterSettingManager( - ZeppelinConfiguration zeppelinConfiguration, - AngularObjectRegistryListener angularObjectRegistryListener, - RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appEventListener) + public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration, + AngularObjectRegistryListener angularObjectRegistryListener, + RemoteInterpreterProcessListener + remoteInterpreterProcessListener, + ApplicationEventListener appEventListener) throws IOException { - this( - zeppelinConfiguration, - new InterpreterOption(), + this(zeppelinConfiguration, new InterpreterOption(), angularObjectRegistryListener, remoteInterpreterProcessListener, appEventListener, ConfigStorage.getInstance(zeppelinConfiguration)); } - public InterpreterSettingManager( - ZeppelinConfiguration conf, + public InterpreterSettingManager(ZeppelinConfiguration conf, InterpreterOption defaultOption, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, @@ -174,7 +178,7 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean loadInterpreterSettingFromDefaultDir(false); Set<String> newlyAddedInterpreters = Sets.newHashSet(interpreterSettingTemplates.keySet()); newlyAddedInterpreters.removeAll(installedInterpreters); - if (!newlyAddedInterpreters.isEmpty()) { + if(!newlyAddedInterpreters.isEmpty()) { saveToFile(); } } catch (IOException e) { @@ -182,9 +186,9 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean } } + private void initInterpreterSetting(InterpreterSetting interpreterSetting) { - interpreterSetting - .setConf(conf) + interpreterSetting.setConf(conf) .setInterpreterSettingManager(this) .setAngularObjectRegistryListener(angularObjectRegistryListener) .setRemoteInterpreterProcessListener(remoteInterpreterProcessListener) @@ -196,9 +200,12 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean .postProcessing(); } - /** Load interpreter setting from interpreter.json */ + /** + * Load interpreter setting from interpreter.json + */ private void loadFromFile() throws IOException { - InterpreterInfoSaving infoSaving = configStorage.loadInterpreterSettings(); + InterpreterInfoSaving infoSaving = + configStorage.loadInterpreterSettings(); if (infoSaving == null) { // it is fresh zeppelin instance if there's no interpreter.json, just create interpreter // setting from interpreterSettingTemplates @@ -210,10 +217,11 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return; } - // TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving) + //TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving) for (InterpreterSetting savedInterpreterSetting : infoSaving.interpreterSettings.values()) { - savedInterpreterSetting.setProperties( - InterpreterSetting.convertInterpreterProperties(savedInterpreterSetting.getProperties())); + savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties( + savedInterpreterSetting.getProperties() + )); initInterpreterSetting(savedInterpreterSetting); InterpreterSetting interpreterSettingTemplate = @@ -225,16 +233,14 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean savedInterpreterSetting.setInterpreterDir(interpreterSettingTemplate.getInterpreterDir()); // merge properties from interpreter-setting.json and interpreter.json Map<String, InterpreterProperty> mergedProperties = - new HashMap<>( - InterpreterSetting.convertInterpreterProperties( - interpreterSettingTemplate.getProperties())); - Map<String, InterpreterProperty> savedProperties = - InterpreterSetting.convertInterpreterProperties( - savedInterpreterSetting.getProperties()); + new HashMap<>(InterpreterSetting.convertInterpreterProperties( + interpreterSettingTemplate.getProperties())); + Map<String, InterpreterProperty> savedProperties = InterpreterSetting + .convertInterpreterProperties(savedInterpreterSetting.getProperties()); for (Map.Entry<String, InterpreterProperty> entry : savedProperties.entrySet()) { // only merge properties whose value is not empty - if (entry.getValue().getValue() != null - && !StringUtils.isBlank(entry.getValue().toString())) { + if (entry.getValue().getValue() != null && ! + StringUtils.isBlank(entry.getValue().toString())) { mergedProperties.put(entry.getKey(), entry.getValue()); } } @@ -245,11 +251,9 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean savedInterpreterSetting.setInterpreterRunner( interpreterSettingTemplate.getInterpreterRunner()); } else { - LOGGER.warn( - "No InterpreterSetting Template found for InterpreterSetting: " - + savedInterpreterSetting.getGroup() - + ", but it is found in interpreter.json, " - + "it would be skipped."); + LOGGER.warn("No InterpreterSetting Template found for InterpreterSetting: " + + savedInterpreterSetting.getGroup() + ", but it is found in interpreter.json, " + + "it would be skipped."); continue; } @@ -261,8 +265,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean } } savedInterpreterSetting.postProcessing(); - LOGGER.info( - "Create Interpreter Setting {} from interpreter.json", savedInterpreterSetting.getName()); + LOGGER.info("Create Interpreter Setting {} from interpreter.json", + savedInterpreterSetting.getName()); interpreterSettings.put(savedInterpreterSetting.getId(), savedInterpreterSetting); } @@ -301,24 +305,24 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean String interpreterJson = conf.getInterpreterJson(); ClassLoader cl = Thread.currentThread().getContextClassLoader(); if (Files.exists(interpreterDirPath)) { - for (Path interpreterDir : - Files.newDirectoryStream( - interpreterDirPath, - new Filter<Path>() { - @Override - public boolean accept(Path entry) throws IOException { - return Files.exists(entry) && Files.isDirectory(entry); - } - })) { + for (Path interpreterDir : Files + .newDirectoryStream(interpreterDirPath, new Filter<Path>() { + @Override + public boolean accept(Path entry) throws IOException { + return Files.exists(entry) && Files.isDirectory(entry); + } + })) { String interpreterDirString = interpreterDir.toString(); /** - * Register interpreter by the following ordering 1. Register it from path - * {ZEPPELIN_HOME}/interpreter/{interpreter_name}/ interpreter-setting.json 2. Register it - * from interpreter-setting.json in classpath {ZEPPELIN_HOME}/interpreter/{interpreter_name} + * Register interpreter by the following ordering + * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/ + * interpreter-setting.json + * 2. Register it from interpreter-setting.json in classpath + * {ZEPPELIN_HOME}/interpreter/{interpreter_name} */ if (!registerInterpreterFromPath(interpreterDirString, interpreterJson, override)) { - if (!registerInterpreterFromResource( - cl, interpreterDirString, interpreterJson, override)) { + if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson, + override)) { LOGGER.warn("No interpreter-setting.json found in " + interpreterDirString); } } @@ -336,9 +340,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return appEventListener; } - private boolean registerInterpreterFromResource( - ClassLoader cl, String interpreterDir, String interpreterJson, boolean override) - throws IOException { + private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir, + String interpreterJson, boolean override) throws IOException { URL[] urls = recursiveBuildLibList(new File(interpreterDir)); ClassLoader tempClassLoader = new URLClassLoader(urls, null); @@ -354,8 +357,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return true; } - private boolean registerInterpreterFromPath( - String interpreterDir, String interpreterJson, boolean override) throws IOException { + private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson, + boolean override) throws IOException { Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson); if (Files.exists(interpreterJsonPath)) { @@ -369,12 +372,13 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean } private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) { - Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {}.getType(); + Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() { + }.getType(); return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType); } - private void registerInterpreterSetting( - List<RegisteredInterpreter> registeredInterpreters, String interpreterDir, boolean override) { + private void registerInterpreterSetting(List<RegisteredInterpreter> registeredInterpreters, + String interpreterDir, boolean override) { Map<String, DefaultInterpreterProperty> properties = new HashMap<>(); List<InterpreterInfo> interpreterInfos = new ArrayList<>(); @@ -382,13 +386,10 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean String group = null; InterpreterRunner runner = null; for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) { - // TODO(zjffdu) merge RegisteredInterpreter & InterpreterInfo + //TODO(zjffdu) merge RegisteredInterpreter & InterpreterInfo InterpreterInfo interpreterInfo = - new InterpreterInfo( - registeredInterpreter.getClassName(), - registeredInterpreter.getName(), - registeredInterpreter.isDefaultInterpreter(), - registeredInterpreter.getEditor()); + new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(), + registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor()); group = registeredInterpreter.getGroup(); runner = registeredInterpreter.getRunner(); // use defaultOption if it is not specified in interpreter-setting.json @@ -399,23 +400,22 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean interpreterInfos.add(interpreterInfo); } - InterpreterSetting interpreterSettingTemplate = - new InterpreterSetting.Builder() - .setGroup(group) - .setName(group) - .setInterpreterInfos(interpreterInfos) - .setProperties(properties) - .setDependencies(new ArrayList<Dependency>()) - .setOption(option) - .setRunner(runner) - .setInterpreterDir(interpreterDir) - .setRunner(runner) - .setConf(conf) - .setIntepreterSettingManager(this) - .create(); + InterpreterSetting interpreterSettingTemplate = new InterpreterSetting.Builder() + .setGroup(group) + .setName(group) + .setInterpreterInfos(interpreterInfos) + .setProperties(properties) + .setDependencies(new ArrayList<Dependency>()) + .setOption(option) + .setRunner(runner) + .setInterpreterDir(interpreterDir) + .setRunner(runner) + .setConf(conf) + .setIntepreterSettingManager(this) + .create(); String key = interpreterSettingTemplate.getName(); - if (override || !interpreterSettingTemplates.containsKey(key)) { + if(override || !interpreterSettingTemplates.containsKey(key)) { LOGGER.info("Register InterpreterSettingTemplate: {}", key); interpreterSettingTemplates.put(key, interpreterSettingTemplate); } @@ -453,9 +453,9 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return null; } - // TODO(zjffdu) logic here is a little ugly - public Map<String, Object> getEditorSetting( - Interpreter interpreter, String user, String noteId, String replName) { + //TODO(zjffdu) logic here is a little ugly + public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId, + String replName) { Map<String, Object> editor = DEFAULT_EDITOR; String group = StringUtils.EMPTY; try { @@ -491,7 +491,7 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return interpreterGroups; } - // TODO(zjffdu) move Resource related api to ResourceManager + //TODO(zjffdu) move Resource related api to ResourceManager public ResourceSet getAllResources() { return getAllResourcesExcept(null); } @@ -499,8 +499,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) { ResourceSet resourceSet = new ResourceSet(); for (ManagedInterpreterGroup intpGroup : getAllInterpreterGroup()) { - if (interpreterGroupExcludsion != null - && intpGroup.getId().equals(interpreterGroupExcludsion)) { + if (interpreterGroupExcludsion != null && + intpGroup.getId().equals(interpreterGroupExcludsion)) { continue; } @@ -511,15 +511,13 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean resourceSet.addAll(localPool.getAll()); } } else if (remoteInterpreterProcess.isRunning()) { - List<String> resourceList = - remoteInterpreterProcess.callRemoteFunction( - new RemoteInterpreterProcess.RemoteFunction<List<String>>() { - @Override - public List<String> call(RemoteInterpreterService.Client client) - throws Exception { - return client.resourcePoolGetAll(); - } - }); + List<String> resourceList = remoteInterpreterProcess.callRemoteFunction( + new RemoteInterpreterProcess.RemoteFunction<List<String>>() { + @Override + public List<String> call(RemoteInterpreterService.Client client) throws Exception { + return client.resourcePoolGetAll(); + } + }); if (resourceList != null) { for (String res : resourceList) { resourceSet.add(Resource.fromJson(res)); @@ -557,15 +555,13 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean r.getResourceId().getName()); } } else if (remoteInterpreterProcess.isRunning()) { - List<String> resourceList = - remoteInterpreterProcess.callRemoteFunction( - new RemoteInterpreterProcess.RemoteFunction<List<String>>() { - @Override - public List<String> call(RemoteInterpreterService.Client client) - throws Exception { - return client.resourcePoolGetAll(); - } - }); + List<String> resourceList = remoteInterpreterProcess.callRemoteFunction( + new RemoteInterpreterProcess.RemoteFunction<List<String>>() { + @Override + public List<String> call(RemoteInterpreterService.Client client) throws Exception { + return client.resourcePoolGetAll(); + } + }); for (String res : resourceList) { resourceSet.add(Resource.fromJson(res)); } @@ -605,45 +601,42 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean */ private void copyDependenciesFromLocalPath(final InterpreterSetting setting) { setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); - final Thread t = - new Thread() { - public void run() { - try { - List<Dependency> deps = setting.getDependencies(); - if (deps != null) { - for (Dependency d : deps) { - File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); - - int numSplits = d.getGroupArtifactVersion().split(":").length; - if (!(numSplits >= 3 && numSplits <= 6)) { - dependencyResolver.copyLocalDependency( - d.getGroupArtifactVersion(), new File(destDir, setting.getId())); - } + final Thread t = new Thread() { + public void run() { + try { + List<Dependency> deps = setting.getDependencies(); + if (deps != null) { + for (Dependency d : deps) { + File destDir = new File( + conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + + int numSplits = d.getGroupArtifactVersion().split(":").length; + if (!(numSplits >= 3 && numSplits <= 6)) { + dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(), + new File(destDir, setting.getId())); } } - setting.setStatus(InterpreterSetting.Status.READY); - } catch (Exception e) { - LOGGER.error( - String.format( - "Error while copying deps for interpreter group : %s," - + " go to interpreter setting page click on edit and save it again to make " - + "this interpreter work properly.", - setting.getGroup()), - e); - setting.setErrorReason(e.getLocalizedMessage()); - setting.setStatus(InterpreterSetting.Status.ERROR); - } finally { - } + setting.setStatus(InterpreterSetting.Status.READY); + } catch (Exception e) { + LOGGER.error(String.format("Error while copying deps for interpreter group : %s," + + " go to interpreter setting page click on edit and save it again to make " + + "this interpreter work properly.", + setting.getGroup()), e); + setting.setErrorReason(e.getLocalizedMessage()); + setting.setStatus(InterpreterSetting.Status.ERROR); + } finally { + } - }; - t.start(); + } + }; + t.start(); } /** - * Return ordered interpreter setting list. The list does not contain more than one setting from - * the same interpreter class. Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), - * Interpreter setting name + * Return ordered interpreter setting list. + * The list does not contain more than one setting from the same interpreter class. + * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name */ public List<String> getInterpreterSettingIds() { List<String> settingIdList = new ArrayList<>(); @@ -653,12 +646,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return settingIdList; } - public InterpreterSetting createNewSetting( - String name, - String group, - List<Dependency> dependencies, - InterpreterOption option, - Map<String, InterpreterProperty> p) + public InterpreterSetting createNewSetting(String name, String group, + List<Dependency> dependencies, InterpreterOption option, Map<String, InterpreterProperty> p) throws IOException { if (name.indexOf(".") >= 0) { @@ -673,7 +662,7 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean InterpreterSetting setting = new InterpreterSetting(interpreterSettingTemplates.get(group)); setting.setName(name); setting.setGroup(group); - // TODO(zjffdu) Should use setDependencies + //TODO(zjffdu) Should use setDependencies setting.appendDependencies(dependencies); setting.setInterpreterOption(option); setting.setProperties(p); @@ -683,6 +672,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return setting; } + + @VisibleForTesting public void closeNote(String user, String noteId) { // close interpreters in this note session @@ -712,7 +703,7 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean } return urls; } else { - return new URL[] {path.toURI().toURL()}; + return new URL[]{path.toURI().toURL()}; } } @@ -720,9 +711,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return this.interpreterRepositories; } - public void addRepository( - String id, String url, boolean snapshot, Authentication auth, Proxy proxy) - throws IOException { + public void addRepository(String id, String url, boolean snapshot, Authentication auth, + Proxy proxy) throws IOException { dependencyResolver.addRepo(id, url, snapshot, auth, proxy); saveToFile(); } @@ -807,23 +797,23 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean FileUtils.deleteDirectory(localRepoDir); } - /** Get interpreter settings */ + /** + * Get interpreter settings + */ public List<InterpreterSetting> get() { List<InterpreterSetting> orderedSettings = new ArrayList<>(interpreterSettings.values()); - Collections.sort( - orderedSettings, - new Comparator<InterpreterSetting>() { - @Override - public int compare(InterpreterSetting o1, InterpreterSetting o2) { - if (o1.getName().equals(defaultInterpreterGroup)) { - return -1; - } else if (o2.getName().equals(defaultInterpreterGroup)) { - return 1; - } else { - return o1.getName().compareTo(o2.getName()); - } - } - }); + Collections.sort(orderedSettings, new Comparator<InterpreterSetting>() { + @Override + public int compare(InterpreterSetting o1, InterpreterSetting o2) { + if (o1.getName().equals(defaultInterpreterGroup)) { + return -1; + } else if (o2.getName().equals(defaultInterpreterGroup)) { + return 1; + } else { + return o1.getName().compareTo(o2.getName()); + } + } + }); return orderedSettings; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java index adcdcd4..f36cb0d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java @@ -15,12 +15,18 @@ * limitations under the License. */ + package org.apache.zeppelin.interpreter; -/** Interface for managing the lifecycle of interpreters */ + +/** + * Interface for managing the lifecycle of interpreters + */ public interface LifecycleManager { void onInterpreterProcessStarted(ManagedInterpreterGroup interpreterGroup); - void onInterpreterUse(ManagedInterpreterGroup interpreterGroup, String sessionId); + void onInterpreterUse(ManagedInterpreterGroup interpreterGroup, + String sessionId); + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java index c6d5216..e1470df 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java @@ -15,12 +15,10 @@ * limitations under the License. */ + package org.apache.zeppelin.interpreter; -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Properties; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Scheduler; @@ -28,7 +26,14 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** ManagedInterpreterGroup runs under zeppelin server */ +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Properties; + +/** + * ManagedInterpreterGroup runs under zeppelin server + */ public class ManagedInterpreterGroup extends InterpreterGroup { private static final Logger LOGGER = LoggerFactory.getLogger(ManagedInterpreterGroup.class); @@ -38,7 +43,6 @@ public class ManagedInterpreterGroup extends InterpreterGroup { /** * Create InterpreterGroup with given id and interpreterSetting, used in ZeppelinServer - * * @param id * @param interpreterSetting */ @@ -51,16 +55,16 @@ public class ManagedInterpreterGroup extends InterpreterGroup { return interpreterSetting; } - public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess( - String userName, Properties properties) throws IOException { + public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess(String userName, + Properties properties) + throws IOException { if (remoteInterpreterProcess == null) { LOGGER.info("Create InterpreterProcess for InterpreterGroup: " + getId()); - remoteInterpreterProcess = - interpreterSetting.createInterpreterProcess(id, userName, properties); + remoteInterpreterProcess = interpreterSetting.createInterpreterProcess(id, userName, + properties); remoteInterpreterProcess.start(userName); interpreterSetting.getLifecycleManager().onInterpreterProcessStarted(this); - getInterpreterSetting() - .getRecoveryStorage() + getInterpreterSetting().getRecoveryStorage() .onInterpreterClientStart(remoteInterpreterProcess); } return remoteInterpreterProcess; @@ -74,7 +78,10 @@ public class ManagedInterpreterGroup extends InterpreterGroup { return remoteInterpreterProcess; } - /** Close all interpreter instances in this group */ + + /** + * Close all interpreter instances in this group + */ public synchronized void close() { LOGGER.info("Close InterpreterGroup: " + id); for (String sessionId : sessions.keySet()) { @@ -84,17 +91,13 @@ public class ManagedInterpreterGroup extends InterpreterGroup { /** * Close all interpreter instances in this session - * * @param sessionId */ public synchronized void close(String sessionId) { - LOGGER.info( - "Close Session: " - + sessionId - + " for interpreter setting: " - + interpreterSetting.getName()); + LOGGER.info("Close Session: " + sessionId + " for interpreter setting: " + + interpreterSetting.getName()); close(sessions.remove(sessionId)); - // TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server + //TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server if (sessions.isEmpty() && interpreterSetting != null) { LOGGER.info("Remove this InterpreterGroup: {} as all the sessions are closed", id); interpreterSetting.removeInterpreterGroup(id); @@ -134,7 +137,7 @@ public class ManagedInterpreterGroup extends InterpreterGroup { } catch (InterpreterException e) { LOGGER.warn("Fail to close interpreter " + interpreter.getClassName(), e); } - // TODO(zjffdu) move the close of schedule to Interpreter + //TODO(zjffdu) move the close of schedule to Interpreter if (null != scheduler) { SchedulerFactory.singleton().removeScheduler(scheduler.getName()); } @@ -154,4 +157,5 @@ public class ManagedInterpreterGroup extends InterpreterGroup { return interpreters; } } + }
