zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject zeppelin git commit: ZEPPELIN-3105 Notebook not running via REST API after Zeppelin Restart
Date Fri, 12 Jan 2018 01:17:17 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 922a228bc -> f7df879cc


ZEPPELIN-3105 Notebook not running via REST API after Zeppelin Restart

### What is this PR for?
Fixes regression introduced with [ZEPPELIN-3013](https://issues.apache.org/jira/browse/ZEPPELIN-3013).
Executing Notebook and
Paragraph via Rest API was not possible anymore, because some parsing
was introduced paragraph.setText(...). The setter was not invoked after
deserialization, thus transient variables paragraph.intpText and paragraph.scriptText were
not set.

This PR fixes that by invoking paragraph.parseText() after deserialization.

Also a test was added, and AbstractTestRestApi.shutdown() method was
overloaded to support the use case of not deleting the interpreter.json
during server restart within one test case.

### What type of PR is it?
[Bug Fix]

### Todos
-

### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-3105

### How should this be tested?
Refer to the test case added,
org.apache.zeppelin.rest.NotebookRestApiTest.testRunWithServerRestart()

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? - no
* Is there breaking changes for older versions? - no
* Does this needs documentation? - no

Author: Andreas Weise <a.weise@avm.de>

Closes #2706 from weand/ZEPPELIN-3105 and squashes the following commits:

50865b1 [Andreas Weise] Merge remote-tracking branch 'origin/master' into ZEPPELIN-3105
d46182f [Andreas Weise] Merge branch 'master' of https://github.com/apache/zeppelin.git into
ZEPPELIN-3105
02e1f08 [Andreas Weise] Merge branch 'master' of https://github.com/apache/zeppelin.git into
ZEPPELIN-3105
ff43983 [Andreas Weise] Merge branch 'master' of https://github.com/apache/zeppelin.git into
ZEPPELIN-3105
ea8f3ea [Andreas Weise] ZEPPELIN-3105 Notebook not running via REST API after Zeppelin Restart
591ec8d [Andreas Weise] ZEPPELIN-3105 Notebook not running via REST API after Zeppelin Restart


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/f7df879c
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/f7df879c
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/f7df879c

Branch: refs/heads/master
Commit: f7df879cc4d5a147030c0e7880bb1cba4f359979
Parents: 922a228
Author: Andreas Weise <a.weise@avm.de>
Authored: Fri Jan 12 01:19:11 2018 +0100
Committer: Jeff Zhang <zjffdu@apache.org>
Committed: Fri Jan 12 09:17:12 2018 +0800

----------------------------------------------------------------------
 .../zeppelin/rest/AbstractTestRestApi.java      |  6 ++-
 .../zeppelin/rest/NotebookRestApiTest.java      | 52 +++++++++++++++++++-
 .../java/org/apache/zeppelin/notebook/Note.java | 29 ++++++++---
 .../org/apache/zeppelin/notebook/Paragraph.java |  8 ++-
 4 files changed, 84 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f7df879c/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index ad0e1fd..7d4c21c 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -370,6 +370,10 @@ public abstract class AbstractTestRestApi {
   }
 
   protected static void shutDown() throws Exception {
+    shutDown(true);
+  }
+
+  protected static void shutDown(final boolean deleteConfDir) throws Exception {
     if (!wasRunning) {
       // restart interpreter to stop all interpreter processes
       List<InterpreterSetting> settingList = ZeppelinServer.notebook.getInterpreterSettingManager().get();
@@ -407,7 +411,7 @@ public abstract class AbstractTestRestApi {
             .clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED.getVarName());
       }
 
-      if (!ZeppelinServer.notebook.getConf().isRecoveryEnabled()) {
+      if (deleteConfDir && !ZeppelinServer.notebook.getConf().isRecoveryEnabled())
{
         // don't delete interpreter.json when recovery is enabled. otherwise the interpreter
setting
         // id will change after zeppelin restart, then we can not recover interpreter process
         // properly

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f7df879c/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
index 8f249c3..d5c29c9 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
@@ -19,6 +19,7 @@ package org.apache.zeppelin.rest;
 
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
+
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.methods.PostMethod;
 import org.apache.commons.httpclient.methods.PutMethod;
@@ -199,7 +200,7 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
     }.getType());
     Map<String, Object> resp2Body = (Map<String, Object>) resp2.get("body");
 
-    assertEquals((String)resp2Body.get("name"), "Note " + clonedNoteId);
+    assertEquals(resp2Body.get("name"), "Note " + clonedNoteId);
     get.releaseConnection();
 
     //cleanup
@@ -269,4 +270,53 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
     //cleanup
     ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
   }
+
+  @Test
+  public void testRunWithServerRestart() throws Exception {
+    Note note1 = ZeppelinServer.notebook.createNote(anonymous);
+    // 2 paragraphs
+    // P1:
+    //    %python
+    //    import time
+    //    time.sleep(1)
+    //    from __future__ import print_function
+    //    print(user)
+    // P2:
+    //    %python
+    //    user='abc'
+    //
+    Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+    Paragraph p2 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+    p1.setText("%python import time\ntime.sleep(1)\nuser='abc'");
+    p2.setText("%python from __future__ import print_function\nprint(user)");
+
+    PostMethod post1 = httpPost("/notebook/job/" + note1.getId(), "");
+    assertThat(post1, isAllowed());
+    post1.releaseConnection();
+    PutMethod put = httpPut("/notebook/" + note1.getId() + "/clear", "");
+    LOG.info("test clear paragraph output response\n" + put.getResponseBodyAsString());
+    assertThat(put, isAllowed());
+    put.releaseConnection();
+
+    // restart server (while keeping interpreter configuration)
+    AbstractTestRestApi.shutDown(false);
+    startUp(NotebookRestApiTest.class.getSimpleName());
+
+    note1 = ZeppelinServer.notebook.getNote(note1.getId());
+    p1 = note1.getParagraph(p1.getId());
+    p2 = note1.getParagraph(p2.getId());
+
+    PostMethod post2 = httpPost("/notebook/job/" + note1.getId(), "");
+    assertThat(post2, isAllowed());
+    Map<String, Object> resp = gson.fromJson(post2.getResponseBodyAsString(),
+        new TypeToken<Map<String, Object>>() {}.getType());
+    assertEquals(resp.get("status"), "OK");
+    post2.releaseConnection();
+
+    assertEquals(Job.Status.FINISHED, p1.getStatus());
+    assertEquals(Job.Status.FINISHED, p2.getStatus());
+    assertNotNull(p2.getResult());
+    assertEquals("abc\n", p2.getResult().message().get(0).getData());
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f7df879c/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 19f396e..281c4de 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -20,13 +20,17 @@ package org.apache.zeppelin.notebook;
 import static java.lang.String.format;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.gson.GsonBuilder;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zeppelin.common.JsonSerializable;
 import org.apache.zeppelin.completer.CompletionType;
@@ -34,7 +38,13 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.Input;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterInfo;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
 import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.notebook.repo.NotebookRepo;
@@ -47,8 +57,10 @@ import org.apache.zeppelin.user.Credentials;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 
 /**
  * Binded interpreters for a note
@@ -60,7 +72,8 @@ public class Note implements ParagraphJobListener, JsonSerializable {
       .setPrettyPrinting()
       .setDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
       .registerTypeAdapter(Date.class, new NotebookImportDeserializer())
-      .registerTypeAdapterFactory(Input.TypeAdapterFactory).create();
+      .registerTypeAdapterFactory(Input.TypeAdapterFactory)
+      .create();
 
   // threadpool for delayed persist of note
   private static final ScheduledThreadPoolExecutor delayedPersistThreadPool =
@@ -934,6 +947,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
     return !interpreterSettingManager.getInterpreterSettings(getId()).isEmpty();
   }
 
+  @Override
   public String toJson() {
     return gson.toJson(this);
   }
@@ -941,13 +955,14 @@ public class Note implements ParagraphJobListener, JsonSerializable
{
   public static Note fromJson(String json) {
     Note note = gson.fromJson(json, Note.class);
     convertOldInput(note);
-    note.resetRuntimeInfos();
+    note.postProcessParagraphs();
     return note;
   }
 
-  public void resetRuntimeInfos() {
+  public void postProcessParagraphs() {
     for (Paragraph p : paragraphs) {
       p.clearRuntimeInfos();
+      p.parseText();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f7df879c/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
index 32b9b73..20f9ff8 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
@@ -18,6 +18,7 @@
 package org.apache.zeppelin.notebook;
 
 import java.io.IOException;
+import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
@@ -26,12 +27,10 @@ import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.security.SecureRandom;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.lang.StringUtils;
 import org.apache.zeppelin.common.JsonSerializable;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
@@ -182,6 +181,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable
{
     // strip white space from the beginning
     this.text = newText;
     this.dateUpdated = new Date();
+    parseText();
+  }
+
+  public void parseText() {
     // parse text to get interpreter component
     if (this.text != null) {
       Matcher matcher = REPL_PATTERN.matcher(this.text);
@@ -822,6 +825,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable
{
     return result1;
   }
 
+  @Override
   public String toJson() {
     return Note.getGson().toJson(this);
   }


Mime
View raw message