zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject zeppelin git commit: ZEPPELIN-3352. Improve RemoteInterpreterProcess creation timeout mechanism
Date Sun, 17 Jun 2018 03:04:57 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 5c7d65b18 -> 191324a67


ZEPPELIN-3352. Improve RemoteInterpreterProcess creation timeout mechanism

### What is this PR for?
This PR allow interpreter override `zeppelin.interpreter.connect.timeout` in zeppelin-site.xml.
So that each interpreter can specify its own timeout threshold without restarting the zeppelin
server in case he hit the timeout issue. Besides that I change the default timeout to be 60
seconds. This is what zeppelin did before. Although the default timeout is 30 seconds before,
but RemoteInterpreterManagedProcess will still wait for 2 times of timeout threshold. So I
directly change it to 60 secodns in this PR.

### What type of PR is it?
[ Improvement ]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3352

### How should this be tested?
* CI pass

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versons? No
* Does this needs documentation? No

Author: Jeff Zhang <zjffdu@apache.org>

Closes #3021 from zjffdu/ZEPPELIN-3352 and squashes the following commits:

9426d8e41 [Jeff Zhang] ZEPPELIN-3352. Improve RemoteInterpreterProcess creation timeout mechanism


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/191324a6
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/191324a6
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/191324a6

Branch: refs/heads/master
Commit: 191324a67aa26601731bbb801fdb790282466bd3
Parents: 5c7d65b
Author: Jeff Zhang <zjffdu@apache.org>
Authored: Fri Jun 15 11:32:19 2018 +0800
Committer: Jeff Zhang <zjffdu@apache.org>
Committed: Sun Jun 17 11:04:53 2018 +0800

----------------------------------------------------------------------
 .../zeppelin/conf/ZeppelinConfiguration.java    |  2 +-
 .../interpreter/launcher/InterpreterClient.java |  4 +++-
 .../launcher/InterpreterLauncher.java           | 13 ++++++++++-
 .../launcher/ShellScriptLauncher.java           |  3 +--
 .../remote/RemoteInterpreterManagedProcess.java | 10 +++++---
 .../launcher/ShellScriptLauncherTest.java       | 24 ++++++++++++++++++++
 .../launcher/SparkInterpreterLauncherTest.java  | 22 ++++++++++++++++++
 7 files changed, 70 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index dc46dd0..635adbf 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -728,7 +728,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
     ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo",
         "http://repo1.maven.org/maven2/"),
-    ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
+    ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 60000),
     ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
     ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
         + "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
index 26da270..bfd3e44 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
@@ -17,6 +17,8 @@
 
 package org.apache.zeppelin.interpreter.launcher;
 
+import java.io.IOException;
+
 /**
  * Interface to InterpreterClient which is created by InterpreterLauncher. This is the component
  * that is used to for the communication from zeppelin-server process to zeppelin interpreter
@@ -26,7 +28,7 @@ public interface InterpreterClient {
 
   String getInterpreterSettingName();
 
-  void start(String userName);
+  void start(String userName) throws IOException;
 
   void stop();
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
index 1cee20e..30cf995 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
@@ -37,5 +37,16 @@ public abstract class InterpreterLauncher {
     this.recoveryStorage = recoveryStorage;
   }
 
-  public abstract  InterpreterClient launch(InterpreterLaunchContext context) throws IOException;
+  protected int getConnectTimeout() {
+    int connectTimeout =
+        zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
+    if (properties.containsKey(
+        ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName()))
{
+      connectTimeout = Integer.parseInt(properties.getProperty(
+          ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName()));
+    }
+    return connectTimeout;
+  }
+
+  public abstract InterpreterClient launch(InterpreterLaunchContext context) throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
index b271d9b..18a6dde 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
@@ -51,8 +51,7 @@ public class ShellScriptLauncher extends InterpreterLauncher {
     InterpreterRunner runner = context.getRunner();
     String groupName = context.getInterpreterSettingGroup();
     String name = context.getInterpreterSettingName();
-    int connectTimeout =
-        zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
+    int connectTimeout = getConnectTimeout();
 
     if (option.isExistingProcess()) {
       return new RemoteInterpreterRunningProcess(

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
index 5160522..db6d263 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
@@ -97,7 +97,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
   }
 
   @Override
-  public void start(String userName) {
+  public void start(String userName) throws IOException {
     // start server process
     CommandLine cmdLine = CommandLine.parse(interpreterRunner);
     cmdLine.addArgument("-d", false);
@@ -143,11 +143,15 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
     try {
       synchronized (running) {
         if (!running.get()) {
-          running.wait(getConnectTimeout() * 2);
+          running.wait(getConnectTimeout());
         }
       }
       if (!running.get()) {
-        throw new RuntimeException(new String(cmdOut.toByteArray()));
+        throw new IOException(new String(
+            String.format("Interpreter Process creation is time out in %d seconds",
+                getConnectTimeout()/1000) + "\n" + "You can increase timeout threshold via
" +
+                "setting zeppelin.interpreter.connect.timeout of this interpreter.\n" +
+                cmdOut.toString()));
       }
     } catch (InterruptedException e) {
       logger.error("Remote interpreter is not accessible");

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
index 6301d0d..ace3f31 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
@@ -53,10 +53,34 @@ public class ShellScriptLauncherTest {
     assertEquals("name", interpreterProcess.getInterpreterSettingName());
     assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
     assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
+    assertEquals(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getIntValue(),
+        interpreterProcess.getConnectTimeout());
     assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
     assertEquals(1, interpreterProcess.getEnv().size());
     assertEquals("VALUE_1", interpreterProcess.getEnv().get("ENV_1"));
     assertEquals(true, interpreterProcess.isUserImpersonated());
   }
 
+  @Test
+  public void testConnectTimeOut() throws IOException {
+    ZeppelinConfiguration zConf = new ZeppelinConfiguration();
+    ShellScriptLauncher launcher = new ShellScriptLauncher(zConf, null);
+    Properties properties = new Properties();
+    properties.setProperty(
+        ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(),
"10000");
+    InterpreterOption option = new InterpreterOption();
+    option.setUserImpersonate(true);
+    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null,
"user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
+    InterpreterClient client = launcher.launch(context);
+    assertTrue( client instanceof RemoteInterpreterManagedProcess);
+    RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess)
client;
+    assertEquals("name", interpreterProcess.getInterpreterSettingName());
+    assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
+    assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
+    assertEquals(10000, interpreterProcess.getConnectTimeout());
+    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
+    assertEquals(0, interpreterProcess.getEnv().size());
+    assertEquals(true, interpreterProcess.isUserImpersonated());
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
index 2e4b165..7aab026 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
@@ -38,6 +38,28 @@ public class SparkInterpreterLauncherTest {
   }
 
   @Test
+  public void testConnectTimeOut() throws IOException {
+    ZeppelinConfiguration zConf = new ZeppelinConfiguration();
+    SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
+    Properties properties = new Properties();
+    properties.setProperty(
+        ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(),
"10000");
+    InterpreterOption option = new InterpreterOption();
+    option.setUserImpersonate(true);
+    InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null,
"user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
+    InterpreterClient client = launcher.launch(context);
+    assertTrue( client instanceof RemoteInterpreterManagedProcess);
+    RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess)
client;
+    assertEquals("name", interpreterProcess.getInterpreterSettingName());
+    assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
+    assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
+    assertEquals(10000, interpreterProcess.getConnectTimeout());
+    assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
+    assertTrue(interpreterProcess.getEnv().size() >= 2);
+    assertEquals(true, interpreterProcess.isUserImpersonated());
+  }
+
+  @Test
   public void testLocalMode() throws IOException {
     ZeppelinConfiguration zConf = new ZeppelinConfiguration();
     SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);


Mime
View raw message