zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From felixche...@apache.org
Subject zeppelin git commit: ZEPPELIN-1852. Use multiple InterpreterResult for displaying appInfo
Date Wed, 18 Jan 2017 07:18:05 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 8daf32563 -> 69b866adc


ZEPPELIN-1852. Use multiple InterpreterResult for displaying appInfo

### What is this PR for?
Refactor the livy interpreter to use multiple `InterpreterResult` for displaying appInfo.

### What type of PR is it?
[Refactoring]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1852

### How should this be tested?
Add integration test and also test it manually

### Screenshots (if appropriate)
![image](https://cloud.githubusercontent.com/assets/164491/21446774/691b84fe-c905-11e6-91a1-9dcc57c75f3c.png)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Jeff Zhang <zjffdu@apache.org>

Closes #1796 from zjffdu/ZEPPELIN-1852 and squashes the following commits:

724f22e [Jeff Zhang] update doc
e1c2eb9 [Jeff Zhang] ZEPPELIN-1852. Use multiple InterpreterResult for displaying appInfo


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/69b866ad
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/69b866ad
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/69b866ad

Branch: refs/heads/master
Commit: 69b866adca5ecac00b26dfd3573013eaae133102
Parents: 8daf325
Author: Jeff Zhang <zjffdu@apache.org>
Authored: Wed Dec 28 19:08:22 2016 +0800
Committer: Felix Cheung <felixcheung@apache.org>
Committed: Tue Jan 17 23:18:01 2017 -0800

----------------------------------------------------------------------
 docs/interpreter/livy.md                        |  5 +++
 .../zeppelin/livy/BaseLivyInterprereter.java    | 34 +++++++++-----------
 .../src/main/resources/interpreter-setting.json |  8 ++---
 .../apache/zeppelin/livy/LivyInterpreterIT.java | 34 +++++++++++++++++++-
 4 files changed, 58 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/69b866ad/docs/interpreter/livy.md
----------------------------------------------------------------------
diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md
index 47ebc46..6f04244 100644
--- a/docs/interpreter/livy.md
+++ b/docs/interpreter/livy.md
@@ -61,6 +61,11 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory`
     <td>Max number of Spark SQL result to display.</td>
   </tr>
   <tr>
+    <td>zeppelin.livy.session.create_timeout</td>
+    <td>120</td>
+    <td>Timeout in seconds for session creation</td>
+  </tr>
+  <tr>
     <td>zeppelin.livy.displayAppInfo</td>
     <td>false</td>
     <td>Whether to display app info</td>

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/69b866ad/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
index 3d84363..f0591fd 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
@@ -62,8 +62,10 @@ public abstract class BaseLivyInterprereter extends Interpreter {
   public BaseLivyInterprereter(Properties property) {
     super(property);
     this.livyURL = property.getProperty("zeppelin.livy.url");
+    this.displayAppInfo = Boolean.parseBoolean(
+        property.getProperty("zeppelin.livy.displayAppInfo", "false"));
     this.sessionCreationTimeout = Integer.parseInt(
-        property.getProperty("zeppelin.livy.create.session.timeout", 120 + ""));
+        property.getProperty("zeppelin.livy.session.create_timeout", 120 + ""));
     this.pullStatusInterval = Integer.parseInt(
         property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + ""));
   }
@@ -77,7 +79,6 @@ public abstract class BaseLivyInterprereter extends Interpreter {
     } catch (LivyException e) {
       String msg = "Fail to create session, please check livy interpreter log and " +
           "livy server log";
-      LOGGER.error(msg);
       throw new RuntimeException(msg, e);
     }
   }
@@ -102,10 +103,11 @@ public abstract class BaseLivyInterprereter extends Interpreter {
                 .get(0).getData());
       }
 
-      interpret(
-          "val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get",
-          null, false, false);
-      if (StringUtils.isEmpty(sessionInfo.appInfo.get("sparkUiUrl"))) {
+      if (sessionInfo.appInfo == null ||
+          StringUtils.isEmpty(sessionInfo.appInfo.get("sparkUiUrl"))) {
+        interpret(
+            "val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get",
+            null, false, false);
         sessionInfo.webUIAddress = extractStatementResult(
             interpret(
                 "webui.getClass.getMethod(\"appUIAddress\").invoke(webui)", null, false,
false)
@@ -215,14 +217,12 @@ public abstract class BaseLivyInterprereter extends Interpreter {
         if (sessionInfo.isFinished()) {
           String msg = "Session " + sessionInfo.id + " is finished, appId: " + sessionInfo.appId
               + ", log: " + sessionInfo.log;
-          LOGGER.error(msg);
           throw new LivyException(msg);
         }
         if ((System.currentTimeMillis() - start) / 1000 > sessionCreationTimeout) {
           String msg = "The creation of session " + sessionInfo.id + " is timeout within
"
               + sessionCreationTimeout + " seconds, appId: " + sessionInfo.appId
               + ", log: " + sessionInfo.log;
-          LOGGER.error(msg);
           throw new LivyException(msg);
         }
         Thread.sleep(pullStatusInterval);
@@ -361,16 +361,14 @@ public abstract class BaseLivyInterprereter extends Interpreter {
 
       if (displayAppInfo) {
         //TODO(zjffdu), use multiple InterpreterResult to display appInfo
-        StringBuilder outputBuilder = new StringBuilder();
-        outputBuilder.append("%angular ");
-        outputBuilder.append("<pre><code>");
-        outputBuilder.append(result);
-        outputBuilder.append("</code></pre>");
-        outputBuilder.append("<hr/>");
-        outputBuilder.append("Spark Application Id:" + sessionInfo.appId + "<br/>");
-        outputBuilder.append("Spark WebUI: <a href=" + sessionInfo.webUIAddress + ">"
-            + sessionInfo.webUIAddress + "</a>");
-        return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputBuilder.toString());
+        InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
+        interpreterResult.add(InterpreterResult.Type.TEXT, result);
+        String appInfoHtml = "<hr/>Spark Application Id: " + sessionInfo.appId + "<br/>"
+            + "Spark WebUI: <a href=\"" + sessionInfo.webUIAddress + "\">"
+            + sessionInfo.webUIAddress + "</a>";
+        LOGGER.info("appInfoHtml:" + appInfoHtml);
+        interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml);
+        return interpreterResult;
       } else {
         return new InterpreterResult(InterpreterResult.Code.SUCCESS, result);
       }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/69b866ad/livy/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json
index a2b9758..42f64cf 100644
--- a/livy/src/main/resources/interpreter-setting.json
+++ b/livy/src/main/resources/interpreter-setting.json
@@ -11,9 +11,9 @@
         "defaultValue": "http://localhost:8998",
         "description": "The URL for Livy Server."
       },
-      "zeppelin.livy.create.session.retries": {
-        "envName": "ZEPPELIN_LIVY_CREATE_SESSION_RETRIES",
-        "propertyName": "zeppelin.livy.create.session.timeout",
+      "zeppelin.livy.session.create_timeout": {
+        "envName": "ZEPPELIN_LIVY_SESSION_CREATE_TIMEOUT",
+        "propertyName": "zeppelin.livy.session.create_timeout",
         "defaultValue": "120",
         "description": "Livy Server create session timeout (seconds)."
       },
@@ -87,7 +87,7 @@
         "defaultValue": "",
         "description": "Adding extra libraries to livy interpreter"
       },
-      "livy.spark.displayAppInfo": {
+      "zeppelin.livy.displayAppInfo": {
         "propertyName": "zeppelin.livy.displayAppInfo",
         "defaultValue": "false",
         "description": "Whether display app info"

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/69b866ad/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
----------------------------------------------------------------------
diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
index 8ca8842..ada91ed 100644
--- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
+++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
@@ -51,7 +51,7 @@ public class LivyInterpreterIT {
     LOGGER.info("Starting livy at {}", cluster.livyEndpoint());
     properties = new Properties();
     properties.setProperty("zeppelin.livy.url", cluster.livyEndpoint());
-    properties.setProperty("zeppelin.livy.create.session.timeout", "120");
+    properties.setProperty("zeppelin.livy.session.create_timeout", "120");
     properties.setProperty("zeppelin.livy.spark.sql.maxResult", "100");
   }
 
@@ -314,6 +314,38 @@ public class LivyInterpreterIT {
   }
 
   @Test
+  public void testSparkInterpreterWithDisplayAppInfo() {
+    if (!checkPreCondition()) {
+      return;
+    }
+    InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
+    interpreterGroup.put("session_1", new ArrayList<Interpreter>());
+    Properties properties2 = new Properties(properties);
+    properties2.put("zeppelin.livy.displayAppInfo", "true");
+    // enable spark ui because it is disabled by livy integration test
+    properties2.put("livy.spark.ui.enabled", "true");
+    LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties2);
+    sparkInterpreter.setInterpreterGroup(interpreterGroup);
+    interpreterGroup.get("session_1").add(sparkInterpreter);
+    AuthenticationInfo authInfo = new AuthenticationInfo("user1");
+    MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
+    InterpreterOutput output = new InterpreterOutput(outputListener);
+    InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
+        "title", "text", authInfo, null, null, null, null, null, output);
+    sparkInterpreter.open();
+
+    try {
+      InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      assertEquals(2, result.message().size());
+      assertTrue(result.message().get(0).getData().contains("1.5.2"));
+      assertTrue(result.message().get(1).getData().contains("Spark Application Id"));
+    } finally {
+      sparkInterpreter.close();
+    }
+  }
+
+  @Test
   public void testSparkRInterpreter() {
     if (!checkPreCondition()) {
       return;


Mime
View raw message