zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject zeppelin git commit: ZEPPELIN-3668. Can't hide Spark Jobs (Spark UI) button
Date Wed, 01 Aug 2018 00:33:41 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master e71fac375 -> 343fd178e


ZEPPELIN-3668. Can't hide Spark Jobs (Spark UI) button

### What is this PR for?

This is to fix the bug of unable to hide spark jobs.

### What type of PR is it?
[Bug Fix]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3668

### How should this be tested?
* Unit test added
* Verify it manually as well.

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Jeff Zhang <zjffdu@apache.org>

Closes #3104 from zjffdu/ZEPPELIN-3668 and squashes the following commits:

c7ddecc6e [Jeff Zhang] ZEPPELIN-3668. Can't hide Spark Jobs (Spark UI) button


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/343fd178
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/343fd178
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/343fd178

Branch: refs/heads/master
Commit: 343fd178edf85bb7880ebd4fcecf0b11a7f38561
Parents: e71fac3
Author: Jeff Zhang <zjffdu@apache.org>
Authored: Tue Jul 31 09:04:57 2018 +0800
Committer: Jeff Zhang <zjffdu@apache.org>
Committed: Wed Aug 1 08:33:36 2018 +0800

----------------------------------------------------------------------
 .../zeppelin/spark/NewSparkInterpreterTest.java | 34 +++++++++++++++++---
 .../apache/zeppelin/spark/SparkShimsTest.java   |  5 ++-
 .../org/apache/zeppelin/spark/SparkShims.java   | 22 +++++--------
 .../org/apache/zeppelin/spark/Spark1Shims.java  |  4 ++-
 .../org/apache/zeppelin/spark/Spark2Shims.java  |  4 ++-
 5 files changed, 45 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
----------------------------------------------------------------------
diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
index 48be45b..59b39dd 100644
--- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
+++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
@@ -44,7 +44,6 @@ import java.net.URL;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -53,6 +52,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 
 
@@ -163,7 +163,7 @@ public class NewSparkInterpreterTest {
 
     result = interpreter.interpret(
         "case class Bank(age:Integer, job:String, marital : String, education : String, balance
: Integer)\n" +
-        "val bank = bankText.map(s=>s.split(\";\")).filter(s => s(0)!=\"\\\"age\\\"\").map(\n"
+
+            "val bank = bankText.map(s=>s.split(\";\")).filter(s => s(0)!=\"\\\"age\\\"\").map(\n"
+
             "    s => Bank(s(0).toInt, \n" +
             "            s(1).replaceAll(\"\\\"\", \"\"),\n" +
             "            s(2).replaceAll(\"\\\"\", \"\"),\n" +
@@ -188,7 +188,7 @@ public class NewSparkInterpreterTest {
               "df.show()", getInterpreterContext());
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertTrue(output.contains(
-              "+---+----+\n" +
+          "+---+----+\n" +
               "| _1|  _2|\n" +
               "+---+----+\n" +
               "|  1|   a|\n" +
@@ -203,7 +203,7 @@ public class NewSparkInterpreterTest {
               "df.show()", getInterpreterContext());
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertTrue(output.contains(
-              "+---+----+\n" +
+          "+---+----+\n" +
               "| _1|  _2|\n" +
               "+---+----+\n" +
               "|  1|   a|\n" +
@@ -318,7 +318,7 @@ public class NewSparkInterpreterTest {
     interpretThread.start();
     boolean nonZeroProgress = false;
     int progress = 0;
-    while(interpretThread.isAlive()) {
+    while (interpretThread.isAlive()) {
       progress = interpreter.getProgress(context2);
       assertTrue(progress >= 0);
       if (progress != 0 && progress != 100) {
@@ -463,6 +463,30 @@ public class NewSparkInterpreterTest {
     assertEquals(null, interpreter.getSparkContext().getLocalProperty("spark.scheduler.pool"));
   }
 
+  @Test
+  public void testDisableSparkUI() throws InterpreterException {
+    Properties properties = new Properties();
+    properties.setProperty("spark.master", "local");
+    properties.setProperty("spark.app.name", "test");
+    properties.setProperty("zeppelin.spark.maxResult", "100");
+    properties.setProperty("zeppelin.spark.test", "true");
+    properties.setProperty("zeppelin.spark.useNew", "true");
+    properties.setProperty("spark.ui.enabled", "false");
+
+    interpreter = new SparkInterpreter(properties);
+    assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
+    interpreter.setInterpreterGroup(mock(InterpreterGroup.class));
+    InterpreterContext.set(getInterpreterContext());
+    interpreter.open();
+
+    InterpreterContext context = getInterpreterContext();
+    InterpreterResult result = interpreter.interpret("sc.range(1, 10).sum", context);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+
+    // spark job url is not sent
+    verify(mockRemoteEventClient, never()).onParaInfosReceived(any(Map.class));
+  }
+
   @After
   public void tearDown() throws InterpreterException {
     if (this.interpreter != null) {

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
----------------------------------------------------------------------
diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
index ccebac3..fd47ce2 100644
--- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
+++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
@@ -108,7 +108,6 @@ public class SparkShimsTest {
   @PrepareForTest({BaseZeppelinContext.class, VersionInfo.class})
   @PowerMockIgnore({"javax.net.*", "javax.security.*"})
   public static class SingleTests {
-    @Mock Properties mockProperties;
     @Captor ArgumentCaptor<Map<String, String>> argumentCaptor;
 
     SparkShims sparkShims;
@@ -130,7 +129,7 @@ public class SparkShimsTest {
 
     @Test
     public void runUnderLocalTest() {
-      sparkShims.buildSparkJobUrl("local", "http://sparkurl", 0, mockProperties, mockContext);
+      sparkShims.buildSparkJobUrl("local", "http://sparkurl", 0, mockContext);
 
       Map<String, String> mapValue = argumentCaptor.getValue();
       assertTrue(mapValue.keySet().contains("jobUrl"));
@@ -140,7 +139,7 @@ public class SparkShimsTest {
     @Test
     public void runUnderYarnTest() {
 
-      sparkShims.buildSparkJobUrl("yarn", "http://sparkurl", 0, mockProperties, mockContext);
+      sparkShims.buildSparkJobUrl("yarn", "http://sparkurl", 0, mockContext);
 
       Map<String, String> mapValue = argumentCaptor.getValue();
       assertTrue(mapValue.keySet().contains("jobUrl"));

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java
----------------------------------------------------------------------
diff --git a/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java b/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java
index d308762..1287c57 100644
--- a/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java
+++ b/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java
@@ -104,26 +104,20 @@ public abstract class SparkShims {
   protected void buildSparkJobUrl(String master,
                                   String sparkWebUrl,
                                   int jobId,
-                                  Properties jobProperties,
                                   InterpreterContext context) {
-    String uiEnabled = jobProperties.getProperty("spark.ui.enabled");
     String jobUrl = sparkWebUrl + "/jobs/job?id=" + jobId;
-    // Button visible if Spark UI property not set, set as invalid boolean or true
-    boolean showSparkUI =
-        uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false");
     String version = VersionInfo.getVersion();
     if (master.toLowerCase().contains("yarn") && !supportYarn6615(version)) {
       jobUrl = sparkWebUrl + "/jobs";
     }
-    if (showSparkUI && jobUrl != null) {
-      Map<String, String> infos = new java.util.HashMap<String, String>();
-      infos.put("jobUrl", jobUrl);
-      infos.put("label", "SPARK JOB");
-      infos.put("tooltip", "View in Spark web UI");
-      infos.put("noteId", context.getNoteId());
-      infos.put("paraId", context.getParagraphId());
-      context.getIntpEventClient().onParaInfosReceived(infos);
-    }
+
+    Map<String, String> infos = new java.util.HashMap<String, String>();
+    infos.put("jobUrl", jobUrl);
+    infos.put("label", "SPARK JOB");
+    infos.put("tooltip", "View in Spark web UI");
+    infos.put("noteId", context.getNoteId());
+    infos.put("paraId", context.getParagraphId());
+    context.getIntpEventClient().onParaInfosReceived(infos);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
----------------------------------------------------------------------
diff --git a/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
b/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
index db0727c..3981204 100644
--- a/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
+++ b/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
@@ -38,7 +38,9 @@ public class Spark1Shims extends SparkShims {
     sc.addSparkListener(new JobProgressListener(sc.getConf()) {
       @Override
       public void onJobStart(SparkListenerJobStart jobStart) {
-        buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context);
+        if (sc.getConf().getBoolean("spark.ui.enabled", true)) {
+          buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), context);
+        }
       }
     });
   }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
----------------------------------------------------------------------
diff --git a/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
b/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
index 177b0ac..5f0cf87 100644
--- a/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
+++ b/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
@@ -39,7 +39,9 @@ public class Spark2Shims extends SparkShims {
     sc.addSparkListener(new SparkListener() {
       @Override
       public void onJobStart(SparkListenerJobStart jobStart) {
-        buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context);
+        if (sc.getConf().getBoolean("spark.ui.enabled", true)) {
+          buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), context);
+        }
       }
     });
   }


Mime
View raw message