zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m...@apache.org
Subject incubator-zeppelin git commit: Configurable interpreter process connection waiting timeout and larger default value.
Date Wed, 24 Jun 2015 20:33:32 GMT
Repository: incubator-zeppelin
Updated Branches:
  refs/heads/branch-0.5 a4fc928d7 -> d7774c69a


Configurable interpreter process connection waiting timeout and larger default value.

https://issues.apache.org/jira/browse/ZEPPELIN-124
Increase connection timeout and make it configurable

Author: Lee moon soo <moon@apache.org>

Closes #116 from Leemoonsoo/ZEPPELIN-124 and squashes the following commits:

7e70289 [Lee moon soo] Increase interpreter connection timeout and make it configurable

(cherry picked from commit 8045d70829c1f9e5c0a151f620edda19425d727f)
Signed-off-by: Lee moon soo <moon@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/d7774c69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/d7774c69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/d7774c69

Branch: refs/heads/branch-0.5
Commit: d7774c69a14c8afe46bc6980e19ae1ea77de844d
Parents: a4fc928
Author: Lee moon soo <moon@apache.org>
Authored: Mon Jun 22 16:38:17 2015 -0700
Committer: Lee moon soo <moon@apache.org>
Committed: Wed Jun 24 13:33:29 2015 -0700

----------------------------------------------------------------------
 conf/zeppelin-site.xml.template                 |  7 +++++
 .../interpreter/remote/RemoteInterpreter.java   | 11 +++++--
 .../remote/RemoteInterpreterProcess.java        | 13 +++++---
 .../remote/RemoteAngularObjectTest.java         |  3 +-
 .../remote/RemoteInterpreterProcessTest.java    |  4 +--
 .../remote/RemoteInterpreterTest.java           | 33 +++++++++++++-------
 .../zeppelin/scheduler/RemoteSchedulerTest.java |  3 +-
 .../zeppelin/conf/ZeppelinConfiguration.java    |  1 +
 .../interpreter/InterpreterFactory.java         |  4 ++-
 9 files changed, 55 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index cd72f12..c5dc5a1 100644
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -71,6 +71,13 @@
 </property>
 
 <property>
+  <name>zeppelin.interpreter.connect.timeout</name>
+  <value>30000</value>
+  <description>Interpreter process connect timeout in msec.</description>
+</property>
+
+
+<property>
   <name>zeppelin.ssl</name>
   <value>false</value>
   <description>Should SSL be used by the servers?</description>

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
index 1637e9c..22818fc 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
@@ -58,11 +58,13 @@ public class RemoteInterpreter extends Interpreter {
     = new HashMap<String, RemoteInterpreterProcess>();
 
   private InterpreterContextRunnerPool interpreterContextRunnerPool;
+  private int connectTimeout;
 
   public RemoteInterpreter(Properties property,
       String className,
       String interpreterRunner,
-      String interpreterPath) {
+      String interpreterPath,
+      int connectTimeout) {
     super(property);
 
     this.className = className;
@@ -71,18 +73,21 @@ public class RemoteInterpreter extends Interpreter {
     this.interpreterPath = interpreterPath;
     env = new HashMap<String, String>();
     interpreterContextRunnerPool = new InterpreterContextRunnerPool();
+    this.connectTimeout = connectTimeout;
   }
 
   public RemoteInterpreter(Properties property,
       String className,
       String interpreterRunner,
       String interpreterPath,
-      Map<String, String> env) {
+      Map<String, String> env,
+      int connectTimeout) {
     super(property);
     this.className = className;
     this.interpreterRunner = interpreterRunner;
     this.interpreterPath = interpreterPath;
     this.env = env;
+    this.connectTimeout = connectTimeout;  
   }
 
   @Override
@@ -333,7 +338,7 @@ public class RemoteInterpreter extends Interpreter {
           || (!intpProcess.isRunning() && intpProcess.getPort() == -1)) {
         interpreterGroupReference.put(getInterpreterGroupKey(interpreterGroup),
             new RemoteInterpreterProcess(interpreterRunner,
-                interpreterPath, env, interpreterContextRunnerPool));
+                interpreterPath, env, interpreterContextRunnerPool, connectTimeout));
 
         logger.info("setInterpreterGroup = "
             + getInterpreterGroupKey(interpreterGroup) + " class=" + className

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
index 61fcb70..5dd2a65 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
@@ -50,26 +50,29 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler
{
   private Map<String, String> env;
   private final RemoteInterpreterEventPoller remoteInterpreterEventPoller;
   private final InterpreterContextRunnerPool interpreterContextRunnerPool;
+  private int connectTimeout;
 
   public RemoteInterpreterProcess(String intpRunner,
       String intpDir,
       Map<String, String> env,
-      InterpreterContextRunnerPool interpreterContextRunnerPool) {
+      InterpreterContextRunnerPool interpreterContextRunnerPool, int connectTimeout) {
     this(intpRunner, intpDir, env, interpreterContextRunnerPool, 
-        new RemoteInterpreterEventPoller());
+        new RemoteInterpreterEventPoller(), connectTimeout);
   }
 
   RemoteInterpreterProcess(String intpRunner,
       String intpDir,
       Map<String, String> env,
       InterpreterContextRunnerPool interpreterContextRunnerPool,
-      RemoteInterpreterEventPoller remoteInterpreterEventPoller) {
+      RemoteInterpreterEventPoller remoteInterpreterEventPoller,
+      int connectTimeout) {
     this.interpreterRunner = intpRunner;
     this.interpreterDir = intpDir;
     this.env = env;
     this.interpreterContextRunnerPool = interpreterContextRunnerPool;
     referenceCount = new AtomicInteger(0);
     this.remoteInterpreterEventPoller = remoteInterpreterEventPoller;
+    this.connectTimeout = connectTimeout;
   }
 
 
@@ -113,7 +116,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler
{
 
 
         long startTime = System.currentTimeMillis();
-        while (System.currentTimeMillis() - startTime < 5 * 1000) {
+        while (System.currentTimeMillis() - startTime < connectTimeout) {
           if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port))
{
             break;
           } else {
@@ -123,7 +126,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler
{
             }
           }
         }
-
+        
         clientPool = new GenericObjectPool<Client>(new ClientFactory("localhost", port));
 
         remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup);

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
index d4909e3..e6da1ec 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
@@ -68,7 +68,8 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener
{
         MockInterpreterAngular.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intp);

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
index 4ea9a30..0043272 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
@@ -33,7 +33,7 @@ public class RemoteInterpreterProcessTest {
   public void testStartStop() {
     InterpreterGroup intpGroup = new InterpreterGroup();
     RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh",
"nonexists", new HashMap<String, String>(),
-        new InterpreterContextRunnerPool());
+        new InterpreterContextRunnerPool(), 10 * 1000);
     assertFalse(rip.isRunning());
     assertEquals(0, rip.referenceCount());
     assertEquals(1, rip.reference(intpGroup));
@@ -49,7 +49,7 @@ public class RemoteInterpreterProcessTest {
   public void testClientFactory() throws Exception {
     InterpreterGroup intpGroup = new InterpreterGroup();
     RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh",
"nonexists", new HashMap<String, String>(),
-        new InterpreterContextRunnerPool(), mock(RemoteInterpreterEventPoller.class));
+        new InterpreterContextRunnerPool(), mock(RemoteInterpreterEventPoller.class), 10
* 1000);
     rip.reference(intpGroup);
     assertEquals(0, rip.getNumActiveClient());
     assertEquals(0, rip.getNumIdleClient());

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
index 4d5636d..b49f86d 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
@@ -74,7 +74,8 @@ public class RemoteInterpreterTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpA);
@@ -85,7 +86,8 @@ public class RemoteInterpreterTest {
         MockInterpreterB.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpB);
@@ -135,7 +137,8 @@ public class RemoteInterpreterTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpA);
@@ -146,7 +149,8 @@ public class RemoteInterpreterTest {
         MockInterpreterB.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpB);
@@ -197,7 +201,8 @@ public class RemoteInterpreterTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpA);
@@ -208,7 +213,8 @@ public class RemoteInterpreterTest {
         MockInterpreterB.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpB);
@@ -311,7 +317,8 @@ public class RemoteInterpreterTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpA);
@@ -390,7 +397,8 @@ public class RemoteInterpreterTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpA);
@@ -468,7 +476,8 @@ public class RemoteInterpreterTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpA.setInterpreterGroup(intpGroup);
@@ -489,7 +498,8 @@ public class RemoteInterpreterTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpA.setInterpreterGroup(intpGroup);
@@ -513,7 +523,8 @@ public class RemoteInterpreterTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpA.setInterpreterGroup(intpGroup);

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
index bb0fb80..2a1075a 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
@@ -62,7 +62,8 @@ public class RemoteSchedulerTest {
         MockInterpreterA.class.getName(),
         new File("../bin/interpreter.sh").getAbsolutePath(),
         "fake",
-        env
+        env,
+        10 * 1000
         );
 
     intpGroup.add(intpA);

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 95d77b7..aa15e51 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -399,6 +399,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
         + "org.apache.zeppelin.tajo.TajoInterpreter,"
         + "org.apache.zeppelin.flink.FlinkInterpreter"),
     ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
+    ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
     ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
     ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
     ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", VFSNotebookRepo.class.getName()),

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/d7774c69/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index c8fc485..77df7c5 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -621,8 +621,10 @@ public class InterpreterFactory {
   private Interpreter createRemoteRepl(String interpreterPath, String className,
       Properties property) {
 
+    int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
     LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter(
-        property, className, conf.getInterpreterRemoteRunnerPath(), interpreterPath));
+        property, className, conf.getInterpreterRemoteRunnerPath(),
+        interpreterPath, connectTimeout));
     return intp;
   }
 


Mime
View raw message