zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m...@apache.org
Subject zeppelin git commit: [ZEPPELIN-2067] SparkInterpreter prints unnecessary newline
Date Wed, 08 Feb 2017 08:26:04 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 5900f0d5f -> 0ab026e07


[ZEPPELIN-2067] SparkInterpreter prints unnecessary newline

### What is this PR for?
Spark interpreter prints unnecessary new line before the evaluation output is printed.
See https://github.com/apache/zeppelin/pull/1975#issuecomment-277581660.

This PR make SparkInterpreter ignores unnecessary preceding newline from ReplReporter.error()

### What type of PR is it?
Bug Fix

### Todos
* [x] - Ignore unnecessary preceding newline
* [x] - unittest

### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-2067

### How should this be tested?
run
```
%spark
import java.util.Date
import java.net.URL
```

and see if result looks like (no new line in front of each lines)

```
import java.util.Date
import java.net.URL
```

### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no

Author: Lee moon soo <moon@apache.org>

Closes #1981 from Leemoonsoo/ZEPPELIN-2067 and squashes the following commits:

1a31a66 [Lee moon soo] add issue id in the comment
23d5d55 [Lee moon soo] Add unittest
6908bdf [Lee moon soo] Ignore preceding newline from scala RepleReporter.error


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/0ab026e0
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/0ab026e0
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/0ab026e0

Branch: refs/heads/master
Commit: 0ab026e07b7c852454dd2b9a281f81249cf3d52f
Parents: 5900f0d
Author: Lee moon soo <moon@apache.org>
Authored: Tue Feb 7 13:17:55 2017 +0900
Committer: Lee moon soo <moon@apache.org>
Committed: Wed Feb 8 17:25:52 2017 +0900

----------------------------------------------------------------------
 .../apache/zeppelin/spark/SparkInterpreter.java |  2 +-
 .../util/InterpreterOutputStream.java           | 30 ++++++++++++++------
 .../zeppelin/rest/ZeppelinSparkClusterTest.java | 24 ++++++++++++++++
 3 files changed, 46 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0ab026e0/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 8ce2ba3..30ff4bc 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -993,6 +993,7 @@ public class SparkInterpreter extends Interpreter {
   }
 
   private Results.Result interpret(String line) {
+    out.ignoreLeadingNewLinesFromScalaReporter();
     return (Results.Result) Utils.invokeMethod(
         intp,
         "interpret",
@@ -1261,7 +1262,6 @@ public class SparkInterpreter extends Interpreter {
     if (varName == null || varName.isEmpty()) {
       return;
     }
-
     Object lastObj = null;
     try {
       if (Utils.isScala2_10()) {

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0ab026e0/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
index b6f01b1..6bdc2db 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
@@ -30,6 +30,7 @@ import java.io.IOException;
 public class InterpreterOutputStream extends LogOutputStream {
   public static Logger logger;
   InterpreterOutput interpreterOutput;
+  boolean ignoreLeadingNewLinesFromScalaReporter = false;
 
   public InterpreterOutputStream(Logger logger) {
     this.logger = logger;
@@ -45,6 +46,18 @@ public class InterpreterOutputStream extends LogOutputStream {
 
   @Override
   public void write(int b) throws IOException {
+    if (ignoreLeadingNewLinesFromScalaReporter && b == '\n') {
+      StackTraceElement[] stacks = Thread.currentThread().getStackTrace();
+      for (StackTraceElement stack : stacks) {
+        if (stack.getClassName().equals("scala.tools.nsc.interpreter.ReplReporter") &&
+            stack.getMethodName().equals("error")) {
+          // ignore. Please see ZEPPELIN-2067
+          return;
+        }
+      }
+    } else {
+      ignoreLeadingNewLinesFromScalaReporter = false;
+    }
     super.write(b);
     if (interpreterOutput != null) {
       interpreterOutput.write(b);
@@ -53,17 +66,13 @@ public class InterpreterOutputStream extends LogOutputStream {
 
   @Override
   public void write(byte [] b) throws IOException {
-    super.write(b);
-    if (interpreterOutput != null) {
-      interpreterOutput.write(b);
-    }
+    write(b, 0, b.length);
   }
 
   @Override
-  public void write(byte [] b, int offset, int len) throws IOException {
-    super.write(b, offset, len);
-    if (interpreterOutput != null) {
-      interpreterOutput.write(b, offset, len);
+  public void write(byte [] b, int off, int len) throws IOException {
+    for (int i = off; i < len; i++) {
+      write(b[i]);
     }
   }
 
@@ -80,7 +89,6 @@ public class InterpreterOutputStream extends LogOutputStream {
     }
   }
 
-
   @Override
   public void flush() throws IOException {
     super.flush();
@@ -88,4 +96,8 @@ public class InterpreterOutputStream extends LogOutputStream {
       interpreterOutput.flush();
     }
   }
+
+  public void ignoreLeadingNewLinesFromScalaReporter() {
+    ignoreLeadingNewLinesFromScalaReporter = true;
+  }
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0ab026e0/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
index 4816e4f..f0cfab8 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
@@ -76,6 +76,30 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
     }
 
     @Test
+    public void scalaOutputTest() throws IOException {
+        // create new note
+        Note note = ZeppelinServer.notebook.createNote(anonymous);
+        Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
+        Map config = p.getConfig();
+        config.put("enabled", true);
+        p.setConfig(config);
+        p.setText("%spark import java.util.Date\n" +
+            "import java.net.URL\n" +
+            "println(\"hello\")\n"
+        );
+        p.setAuthenticationInfo(anonymous);
+        note.run(p.getId());
+        waitForFinish(p);
+        assertEquals(Status.FINISHED, p.getStatus());
+        assertEquals("import java.util.Date\n" +
+            "import java.net.URL\n" +
+            "hello\n", p.getResult().message().get(0).getData());
+        ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
+    }
+
+
+
+    @Test
     public void basicRDDTransformationAndActionTest() throws IOException {
         // create new note
         Note note = ZeppelinServer.notebook.createNote(anonymous);


Mime
View raw message