zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From felixche...@apache.org
Subject zeppelin git commit: [ZEPPELIN-2714] Soft-code Spark UI button visualization
Date Sun, 09 Jul 2017 07:43:59 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 86bc9332e -> 96d78ee57


[ZEPPELIN-2714] Soft-code Spark UI button visualization

### What is this PR for?
When "spark.ui.enabled" property is set to "false" we should not show the Spark UI button.
We keep the same behaviour when this property does not exist or when it exists and it's set
to true.

### What type of PR is it?
[ Improvement]

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-2714

### How should this be tested?
1. Open Zeppelin, go to Interpreter > Spark
1. Click spark ui > msg: "No spark application running"
1. Go to a paragraph, run "sc.parallelize(1 to 100).count()", check "SPARK JOB" button
1. Go to Interpreter > Spark > Set "spark.ui.enabled" to "false"
1. Rerun paragraph > "SPARK JOB" button not visible
1. Go to Interpreter > Spark > "spark ui" button not visible
Also, test with "spark.ui.enabled"="true" and other workflow combinations

### Questions:
* Does the licenses files need update? N
* Is there breaking changes for older versions? N
* Does this needs documentation? N

Author: Nelson Costa <nelson.costa85@gmail.com>

Closes #2456 from necosta/zeppelin2714 and squashes the following commits:

278a23e [Nelson Costa] [ZEPPELIN-2714] Fixed broken unit-test
edf29cc [Nelson Costa] [ZEPPELIN-2714] Minor final fix
294dea8 [Nelson Costa] [ZEPPELIN-2714] Improvements to change request
a85864c [Nelson Costa] [ZEPPELIN-2714] Soft-code Spark UI button visualization


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/96d78ee5
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/96d78ee5
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/96d78ee5

Branch: refs/heads/master
Commit: 96d78ee57b5340b2441a1b8fd866d327d998913b
Parents: 86bc933
Author: Nelson Costa <nelson.costa85@gmail.com>
Authored: Thu Jul 6 15:04:04 2017 +0100
Committer: Felix Cheung <felixcheung@apache.org>
Committed: Sun Jul 9 00:43:54 2017 -0700

----------------------------------------------------------------------
 docs/usage/rest_api/interpreter.md              | 23 ++++++++++++
 .../apache/zeppelin/spark/SparkInterpreter.java | 37 ++++++++++++-------
 .../zeppelin/rest/InterpreterRestApi.java       | 23 ++++--------
 .../zeppelin/rest/InterpreterRestApiTest.java   | 39 +++++++++++++++++---
 .../app/interpreter/interpreter.controller.js   | 13 +++++--
 5 files changed, 96 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/docs/usage/rest_api/interpreter.md
----------------------------------------------------------------------
diff --git a/docs/usage/rest_api/interpreter.md b/docs/usage/rest_api/interpreter.md
index 4869866..23a7c66 100644
--- a/docs/usage/rest_api/interpreter.md
+++ b/docs/usage/rest_api/interpreter.md
@@ -604,3 +604,26 @@ The role of registered interpreters, settings and interpreters group
are describ
     </td>        
   </table>  
   
+<br/>
+### Get interpreter settings metadata info
+
+  <table class="table-configuration">
+    <col width="200">
+    <tr>
+      <td>Description</td>
+      <td>This ```GET``` method returns interpreter settings metadata info. </td>
+    </tr>
+    <tr>
+      <td>URL</td>
+      <td>```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/metadata/[setting
ID]```</td>
+    </tr>
+    <tr>
+      <td>Success code</td>
+      <td>200</td>
+    </tr>
+    <tr>
+      <td>Fail code</td>
+      <td> 500 </td>
+    </tr>
+  </table>
+  
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index c170e4e..dd1fa11 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -184,11 +184,14 @@ public class SparkInterpreter extends Interpreter {
         super.onJobStart(jobStart);
         int jobId = jobStart.jobId();
         String jobGroupId = jobStart.properties().getProperty("spark.jobGroup.id");
+        String uiEnabled = jobStart.properties().getProperty("spark.ui.enabled");
         String jobUrl = getJobUrl(jobId);
         String noteId = Utils.getNoteId(jobGroupId);
         String paragraphId = Utils.getParagraphId(jobGroupId);
-
-        if (jobUrl != null && noteId != null && paragraphId != null) {
+        // Button visible if Spark UI property not set, set as invalid boolean or true
+        java.lang.Boolean showSparkUI =
+                uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false");
+        if (showSparkUI && jobUrl != null) {
           RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient();
           Map<String, String> infos = new java.util.HashMap<>();
           infos.put("jobUrl", jobUrl);
@@ -1040,21 +1043,29 @@ public class SparkInterpreter extends Interpreter {
   }
 
   public void populateSparkWebUrl(InterpreterContext ctx) {
-    if (sparkUrl == null) {
-      sparkUrl = getSparkUIUrl();
-      Map<String, String> infos = new java.util.HashMap<>();
-      if (sparkUrl != null) {
-        infos.put("url", sparkUrl);
-        if (ctx != null && ctx.getClient() != null) {
-          logger.info("Sending metainfos to Zeppelin server: {}", infos.toString());
-          getZeppelinContext().setEventClient(ctx.getClient());
-          ctx.getClient().onMetaInfosReceived(infos);
-        }
+    sparkUrl = getSparkUIUrl();
+    Map<String, String> infos = new java.util.HashMap<>();
+    infos.put("url", sparkUrl);
+    String uiEnabledProp = property.getProperty("spark.ui.enabled", "true");
+    java.lang.Boolean uiEnabled = java.lang.Boolean.parseBoolean(
+            uiEnabledProp.trim());
+    if (!uiEnabled) {
+      infos.put("message", "Spark UI disabled");
+    } else {
+      if (StringUtils.isNotBlank(sparkUrl)) {
+        infos.put("message", "Spark UI enabled");
+      } else {
+        infos.put("message", "No spark url defined");
       }
     }
+    if (ctx != null && ctx.getClient() != null) {
+      logger.info("Sending metadata to Zeppelin server: {}", infos.toString());
+      getZeppelinContext().setEventClient(ctx.getClient());
+      ctx.getClient().onMetaInfosReceived(infos);
+    }
   }
 
-  public List<File> currentClassPath() {
+  private List<File> currentClassPath() {
     List<File> paths = classPath(Thread.currentThread().getContextClassLoader());
     String[] cps = System.getProperty("java.class.path").split(File.pathSeparator);
     if (cps != null) {

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
index 17c4027..cd0210e 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
@@ -246,28 +246,19 @@ public class InterpreterRestApi {
   }
 
   /**
-   * get the metainfo property value
+   * get metadata values
    */
   @GET
-  @Path("getmetainfos/{settingId}")
+  @Path("metadata/{settingId}")
+  @ZeppelinApi
   public Response getMetaInfo(@Context HttpServletRequest req,
       @PathParam("settingId") String settingId) {
-    String propName = req.getParameter("propName");
-    if (propName == null) {
-      return new JsonResponse<>(Status.BAD_REQUEST).build();
-    }
-    String propValue = null;
     InterpreterSetting interpreterSetting = interpreterSettingManager.get(settingId);
-    Map<String, String> infos = interpreterSetting.getInfos();
-    if (infos != null) {
-      propValue = infos.get(propName);
+    if (interpreterSetting == null) {
+      return new JsonResponse<>(Status.NOT_FOUND).build();
     }
-    Map<String, String> respMap = new HashMap<>();
-    respMap.put(propName, propValue);
-    logger.debug("Get meta info");
-    logger.debug("Interpretersetting Id: {}, property Name:{}, property value: {}", settingId,
-        propName, propValue);
-    return new JsonResponse<>(Status.OK, respMap).build();
+    Map<String, String> infos = interpreterSetting.getInfos();
+    return new JsonResponse<>(Status.OK, "metadata", infos).build();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
index 5333385..28541bd 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
@@ -54,8 +54,8 @@ import static org.hamcrest.MatcherAssert.assertThat;
  */
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public class InterpreterRestApiTest extends AbstractTestRestApi {
-  Gson gson = new Gson();
-  AuthenticationInfo anonymous;
+  private Gson gson = new Gson();
+  private AuthenticationInfo anonymous;
 
   @BeforeClass
   public static void init() throws Exception {
@@ -365,21 +365,48 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
     delete.releaseConnection();
   }
 
-  public JsonObject getBodyFieldFromResponse(String rawResponse) {
+  @Test
+  public void testGetMetadataInfo() throws IOException {
+    String jsonRequest = "{\"name\":\"spark\",\"group\":\"spark\"," +
+            "\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\",
\"type\": \"textarea\"}}," +
+            "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}],"
+
+            "\"dependencies\":[]," +
+            "\"option\": { \"remote\": true, \"session\": false }}";
+    PostMethod post = httpPost("/interpreter/setting/", jsonRequest);
+    InterpreterSetting created = convertResponseToInterpreterSetting(post.getResponseBodyAsString());
+    String settingId = created.getId();
+    Map<String, String> infos = new java.util.HashMap<>();
+    infos.put("key1", "value1");
+    infos.put("key2", "value2");
+    ZeppelinServer.notebook.getInterpreterSettingManager().get(settingId).setInfos(infos);
+    GetMethod get = httpGet("/interpreter/metadata/" + settingId);
+    assertThat(get, isAllowed());
+    JsonObject body = getBodyFieldFromResponse(get.getResponseBodyAsString());
+    assertEquals(body.entrySet().size(), infos.size());
+    java.util.Map.Entry<String, JsonElement> item = body.entrySet().iterator().next();
+    if (item.getKey().equals("key1")) {
+      assertEquals(item.getValue().getAsString(), "value1");
+    } else {
+      assertEquals(item.getValue().getAsString(), "value2");
+    }
+    get.releaseConnection();
+  }
+
+  private JsonObject getBodyFieldFromResponse(String rawResponse) {
     JsonObject response = gson.fromJson(rawResponse, JsonElement.class).getAsJsonObject();
     return response.getAsJsonObject("body");
   }
 
-  public JsonArray getArrayBodyFieldFromResponse(String rawResponse) {
+  private JsonArray getArrayBodyFieldFromResponse(String rawResponse) {
     JsonObject response = gson.fromJson(rawResponse, JsonElement.class).getAsJsonObject();
     return response.getAsJsonArray("body");
   }
 
-  public InterpreterSetting convertResponseToInterpreterSetting(String rawResponse) {
+  private InterpreterSetting convertResponseToInterpreterSetting(String rawResponse) {
     return gson.fromJson(getBodyFieldFromResponse(rawResponse), InterpreterSetting.class);
   }
 
-  public static String getSimulatedMarkdownResult(String markdown) {
+  private static String getSimulatedMarkdownResult(String markdown) {
     return String.format("<div class=\"markdown-body\">\n<p>%s</p>\n</div>",
markdown);
   }
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/zeppelin-web/src/app/interpreter/interpreter.controller.js
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js
index e75740f..dc3619e 100644
--- a/zeppelin-web/src/app/interpreter/interpreter.controller.js
+++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js
@@ -754,16 +754,21 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast,
$timeou
   }
 
   $scope.showSparkUI = function (settingId) {
-    $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/getmetainfos/' + settingId + '?propName=url')
+    $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/metadata/' + settingId)
       .success(function (data, status, headers, config) {
-        let url = data.body.url
-        if (!url) {
+        if (data.body === undefined) {
           BootstrapDialog.alert({
             message: 'No spark application running'
           })
           return
         }
-        window.open(url, '_blank')
+        if (data.body.url) {
+          window.open(data.body.url, '_blank')
+        } else {
+          BootstrapDialog.alert({
+            message: data.body.message
+          })
+        }
       }).error(function (data, status, headers, config) {
         console.log('Error %o %o', status, data.message)
       })


Mime
View raw message