Repository: incubator-zeppelin Updated Branches: refs/heads/branch-0.5 a4fc928d7 -> d7774c69a
Configurable interpreter process connection waiting timeout and larger default value. https://issues.apache.org/jira/browse/ZEPPELIN-124 Increase connection timeout and make it configurable Author: Lee moon soo <[email protected]> Closes #116 from Leemoonsoo/ZEPPELIN-124 and squashes the following commits: 7e70289 [Lee moon soo] Increase interpreter connection timeout and make it configurable (cherry picked from commit 8045d70829c1f9e5c0a151f620edda19425d727f) Signed-off-by: Lee moon soo <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/d7774c69 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/d7774c69 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/d7774c69 Branch: refs/heads/branch-0.5 Commit: d7774c69a14c8afe46bc6980e19ae1ea77de844d Parents: a4fc928 Author: Lee moon soo <[email protected]> Authored: Mon Jun 22 16:38:17 2015 -0700 Committer: Lee moon soo <[email protected]> Committed: Wed Jun 24 13:33:29 2015 -0700 ---------------------------------------------------------------------- conf/zeppelin-site.xml.template | 7 +++++ .../interpreter/remote/RemoteInterpreter.java | 11 +++++-- .../remote/RemoteInterpreterProcess.java | 13 +++++--- .../remote/RemoteAngularObjectTest.java | 3 +- .../remote/RemoteInterpreterProcessTest.java | 4 +-- .../remote/RemoteInterpreterTest.java | 33 +++++++++++++------- .../zeppelin/scheduler/RemoteSchedulerTest.java | 3 +- .../zeppelin/conf/ZeppelinConfiguration.java | 1 + .../interpreter/InterpreterFactory.java | 4 ++- 9 files changed, 55 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/conf/zeppelin-site.xml.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index cd72f12..c5dc5a1 100644 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -71,6 +71,13 @@ </property> <property> + <name>zeppelin.interpreter.connect.timeout</name> + <value>30000</value> + <description>Interpreter process connect timeout in msec.</description> +</property> + + +<property> <name>zeppelin.ssl</name> <value>false</value> <description>Should SSL be used by the servers?</description> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 1637e9c..22818fc 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -58,11 +58,13 @@ public class RemoteInterpreter extends Interpreter { = new HashMap<String, RemoteInterpreterProcess>(); private InterpreterContextRunnerPool interpreterContextRunnerPool; + private int connectTimeout; public RemoteInterpreter(Properties property, String className, String interpreterRunner, - String interpreterPath) { + String interpreterPath, + int connectTimeout) { super(property); this.className = className; @@ -71,18 +73,21 @@ public class RemoteInterpreter extends Interpreter { this.interpreterPath = interpreterPath; env = new HashMap<String, String>(); interpreterContextRunnerPool = new InterpreterContextRunnerPool(); + this.connectTimeout = connectTimeout; } public RemoteInterpreter(Properties property, String className, String interpreterRunner, String interpreterPath, - Map<String, String> env) { + Map<String, String> env, + int connectTimeout) { super(property); this.className = className; this.interpreterRunner = interpreterRunner; this.interpreterPath = interpreterPath; this.env = env; + this.connectTimeout = connectTimeout; } @Override @@ -333,7 +338,7 @@ public class RemoteInterpreter extends Interpreter { || (!intpProcess.isRunning() && intpProcess.getPort() == -1)) { interpreterGroupReference.put(getInterpreterGroupKey(interpreterGroup), new RemoteInterpreterProcess(interpreterRunner, - interpreterPath, env, interpreterContextRunnerPool)); + interpreterPath, env, interpreterContextRunnerPool, connectTimeout)); logger.info("setInterpreterGroup = " + getInterpreterGroupKey(interpreterGroup) + " class=" + className http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 61fcb70..5dd2a65 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -50,26 +50,29 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler { private Map<String, String> env; private final RemoteInterpreterEventPoller remoteInterpreterEventPoller; private final InterpreterContextRunnerPool interpreterContextRunnerPool; + private int connectTimeout; public RemoteInterpreterProcess(String intpRunner, String intpDir, Map<String, String> env, - InterpreterContextRunnerPool interpreterContextRunnerPool) { + InterpreterContextRunnerPool interpreterContextRunnerPool, int connectTimeout) { this(intpRunner, intpDir, env, interpreterContextRunnerPool, - new RemoteInterpreterEventPoller()); + new RemoteInterpreterEventPoller(), connectTimeout); } RemoteInterpreterProcess(String intpRunner, String intpDir, Map<String, String> env, InterpreterContextRunnerPool interpreterContextRunnerPool, - RemoteInterpreterEventPoller remoteInterpreterEventPoller) { + RemoteInterpreterEventPoller remoteInterpreterEventPoller, + int connectTimeout) { this.interpreterRunner = intpRunner; this.interpreterDir = intpDir; this.env = env; this.interpreterContextRunnerPool = interpreterContextRunnerPool; referenceCount = new AtomicInteger(0); this.remoteInterpreterEventPoller = remoteInterpreterEventPoller; + this.connectTimeout = connectTimeout; } @@ -113,7 +116,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler { long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < 5 * 1000) { + while (System.currentTimeMillis() - startTime < connectTimeout) { if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) { break; } else { @@ -123,7 +126,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler { } } } - + clientPool = new GenericObjectPool<Client>(new ClientFactory("localhost", port)); remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index d4909e3..e6da1ec 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -68,7 +68,8 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener { MockInterpreterAngular.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intp); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index 4ea9a30..0043272 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -33,7 +33,7 @@ public class RemoteInterpreterProcessTest { public void testStartStop() { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap<String, String>(), - new InterpreterContextRunnerPool()); + new InterpreterContextRunnerPool(), 10 * 1000); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); assertEquals(1, rip.reference(intpGroup)); @@ -49,7 +49,7 @@ public class RemoteInterpreterProcessTest { public void testClientFactory() throws Exception { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap<String, String>(), - new InterpreterContextRunnerPool(), mock(RemoteInterpreterEventPoller.class)); + new InterpreterContextRunnerPool(), mock(RemoteInterpreterEventPoller.class), 10 * 1000); rip.reference(intpGroup); assertEquals(0, rip.getNumActiveClient()); assertEquals(0, rip.getNumIdleClient()); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 4d5636d..b49f86d 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -74,7 +74,8 @@ public class RemoteInterpreterTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpA); @@ -85,7 +86,8 @@ public class RemoteInterpreterTest { MockInterpreterB.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpB); @@ -135,7 +137,8 @@ public class RemoteInterpreterTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpA); @@ -146,7 +149,8 @@ public class RemoteInterpreterTest { MockInterpreterB.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpB); @@ -197,7 +201,8 @@ public class RemoteInterpreterTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpA); @@ -208,7 +213,8 @@ public class RemoteInterpreterTest { MockInterpreterB.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpB); @@ -311,7 +317,8 @@ public class RemoteInterpreterTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpA); @@ -390,7 +397,8 @@ public class RemoteInterpreterTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpA); @@ -468,7 +476,8 @@ public class RemoteInterpreterTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpA.setInterpreterGroup(intpGroup); @@ -489,7 +498,8 @@ public class RemoteInterpreterTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpA.setInterpreterGroup(intpGroup); @@ -513,7 +523,8 @@ public class RemoteInterpreterTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpA.setInterpreterGroup(intpGroup); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index bb0fb80..2a1075a 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -62,7 +62,8 @@ public class RemoteSchedulerTest { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", - env + env, + 10 * 1000 ); intpGroup.add(intpA); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 95d77b7..aa15e51 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -399,6 +399,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { + "org.apache.zeppelin.tajo.TajoInterpreter," + "org.apache.zeppelin.flink.FlinkInterpreter"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), + ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000), ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"), ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"), ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", VFSNotebookRepo.class.getName()), http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index c8fc485..77df7c5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -621,8 +621,10 @@ public class InterpreterFactory { private Interpreter createRemoteRepl(String interpreterPath, String className, Properties property) { + int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter( - property, className, conf.getInterpreterRemoteRunnerPath(), interpreterPath)); + property, className, conf.getInterpreterRemoteRunnerPath(), + interpreterPath, connectTimeout)); return intp; }
