zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject zeppelin git commit: ZEPPELIN-3519. zeppelin.spark.printREPLOutput doesn't work for new SparkInterpreter
Date Fri, 15 Jun 2018 09:16:16 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 1caa6aa2f -> 6ed3d88a2


ZEPPELIN-3519. zeppelin.spark.printREPLOutput doesn't work for new SparkInterpreter

### What is this PR for?
Straightforward fix to make `zeppelin.spark.printREPLOutput` work in `NewSparkInterpreter`

### What type of PR is it?
[Bug Fix]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3519

### How should this be tested?
* CI pass

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Jeff Zhang <zjffdu@apache.org>

Closes #3022 from zjffdu/ZEPPELIN-3519 and squashes the following commits:

9d3bbe578 [Jeff Zhang] ZEPPELIN-3519. zeppelin.spark.printREPLOutput doesn't work for new
SparkInterpreter


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/6ed3d88a
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/6ed3d88a
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/6ed3d88a

Branch: refs/heads/master
Commit: 6ed3d88a2a43e33c6bbd0931c92c13304d7ab0de
Parents: 1caa6aa
Author: Jeff Zhang <zjffdu@apache.org>
Authored: Fri Jun 1 12:54:16 2018 +0800
Committer: Jeff Zhang <zjffdu@apache.org>
Committed: Fri Jun 15 17:16:10 2018 +0800

----------------------------------------------------------------------
 .../zeppelin/spark/NewSparkInterpreter.java     |  7 +++---
 .../zeppelin/spark/NewSparkInterpreterTest.java | 26 ++++++++++++++++++++
 .../spark/SparkScala210Interpreter.scala        |  9 ++++---
 .../spark/SparkScala211Interpreter.scala        | 11 ++++++---
 .../spark/BaseSparkScalaInterpreter.scala       |  3 ++-
 5 files changed, 46 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6ed3d88a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
index b60e38b..c532074 100644
--- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
+++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
@@ -95,9 +95,10 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter {
 
       String innerIntpClassName = innerInterpreterClassMap.get(scalaVersion);
       Class clazz = Class.forName(innerIntpClassName);
-      this.innerInterpreter =
-          (BaseSparkScalaInterpreter) clazz.getConstructor(SparkConf.class, List.class)
-              .newInstance(conf, getDependencyFiles());
+      this.innerInterpreter = (BaseSparkScalaInterpreter)
+          clazz.getConstructor(SparkConf.class, List.class, Boolean.class)
+              .newInstance(conf, getDependencyFiles(),
+                  Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput", "true")));
       this.innerInterpreter.open();
 
       sc = this.innerInterpreter.sc();

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6ed3d88a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
----------------------------------------------------------------------
diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
index 84bdc43..a694f08 100644
--- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
+++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
@@ -390,6 +390,32 @@ public class NewSparkInterpreterTest {
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
   }
 
+  @Test
+  public void testDisableReplOutput() throws InterpreterException {
+    Properties properties = new Properties();
+    properties.setProperty("spark.master", "local");
+    properties.setProperty("spark.app.name", "test");
+    properties.setProperty("zeppelin.spark.maxResult", "100");
+    properties.setProperty("zeppelin.spark.test", "true");
+    properties.setProperty("zeppelin.spark.useNew", "true");
+    properties.setProperty("zeppelin.spark.printREPLOutput", "false");
+
+    interpreter = new SparkInterpreter(properties);
+    assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
+    interpreter.setInterpreterGroup(mock(InterpreterGroup.class));
+    interpreter.open();
+
+    InterpreterResult result = interpreter.interpret("val a=\"hello world\"", getInterpreterContext());
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    // no output for define new variable
+    assertEquals("", output);
+
+    result = interpreter.interpret("print(a)", getInterpreterContext());
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    // output from print statement will still be displayed
+    assertEquals("hello world", output);
+  }
+
   @After
   public void tearDown() throws InterpreterException {
     if (this.interpreter != null) {

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6ed3d88a/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala
----------------------------------------------------------------------
diff --git a/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala
b/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala
index 9d371ba..a9943fb 100644
--- a/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala
+++ b/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala
@@ -35,8 +35,9 @@ import scala.tools.nsc.interpreter._
   * SparkInterpreter for scala-2.10
   */
 class SparkScala210Interpreter(override val conf: SparkConf,
-                               override val depFiles: java.util.List[String])
-  extends BaseSparkScalaInterpreter(conf, depFiles) {
+                               override val depFiles: java.util.List[String],
+                               override val printReplOutput: java.lang.Boolean)
+  extends BaseSparkScalaInterpreter(conf, depFiles, printReplOutput) {
 
   lazy override val LOGGER: Logger = LoggerFactory.getLogger(getClass)
 
@@ -66,7 +67,9 @@ class SparkScala210Interpreter(override val conf: SparkConf,
     settings.embeddedDefaults(Thread.currentThread().getContextClassLoader())
     settings.usejavacp.value = true
     settings.classpath.value = getUserJars.mkString(File.pathSeparator)
-    Console.setOut(interpreterOutput)
+    if (printReplOutput) {
+      Console.setOut(interpreterOutput)
+    }
     sparkILoop = new SparkILoop()
 
     setDeclaredField(sparkILoop, "settings", settings)

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6ed3d88a/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala
----------------------------------------------------------------------
diff --git a/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala
b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala
index e145260..7ddb3fb 100644
--- a/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala
+++ b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala
@@ -35,8 +35,9 @@ import scala.tools.nsc.interpreter._
   * SparkInterpreter for scala-2.11
   */
 class SparkScala211Interpreter(override val conf: SparkConf,
-                               override val depFiles: java.util.List[String])
-  extends BaseSparkScalaInterpreter(conf, depFiles) {
+                               override val depFiles: java.util.List[String],
+                               override val printReplOutput: java.lang.Boolean)
+  extends BaseSparkScalaInterpreter(conf, depFiles, printReplOutput) {
 
   lazy override val LOGGER: Logger = LoggerFactory.getLogger(getClass)
 
@@ -66,7 +67,11 @@ class SparkScala211Interpreter(override val conf: SparkConf,
     settings.usejavacp.value = true
     settings.classpath.value = getUserJars.mkString(File.pathSeparator)
 
-    val replOut = new JPrintWriter(interpreterOutput, true)
+    val replOut = if (printReplOutput) {
+      new JPrintWriter(interpreterOutput, true)
+    } else {
+      new JPrintWriter(Console.out, true)
+    }
     sparkILoop = new ILoop(None, replOut)
     sparkILoop.settings = settings
     sparkILoop.createInterpreter()

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6ed3d88a/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala
----------------------------------------------------------------------
diff --git a/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala
b/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala
index 883beb0..7f9674c 100644
--- a/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala
+++ b/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala
@@ -38,7 +38,8 @@ import scala.util.control.NonFatal
   * @param depFiles
   */
 abstract class BaseSparkScalaInterpreter(val conf: SparkConf,
-                                         val depFiles: java.util.List[String]) {
+                                         val depFiles: java.util.List[String],
+                                         val printReplOutput: java.lang.Boolean) {
 
   protected lazy val LOGGER: Logger = LoggerFactory.getLogger(getClass)
 


Mime
View raw message