zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject [zeppelin] branch master updated: [ZEPPELIN-4497] Change zeppelin-jupyter-adapter to zeppelin-jupyter-interpreter
Date Mon, 23 Dec 2019 07:19:16 GMT
This is an automated email from the ASF dual-hosted git repository.

zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git


The following commit(s) were added to refs/heads/master by this push:
     new 3fdc590  [ZEPPELIN-4497] Change zeppelin-jupyter-adapter to zeppelin-jupyter-interpreter
3fdc590 is described below

commit 3fdc590cdcf7388997e17f6a0b2283f5b54c1bf1
Author: Jeff Zhang <zjffdu@apache.org>
AuthorDate: Fri Dec 20 22:52:21 2019 +0800

    [ZEPPELIN-4497] Change zeppelin-jupyter-adapter to zeppelin-jupyter-interpreter
    
    ### What is this PR for?
    In ZEPPELIN-4480, we introduce zeppelin-jupyter-adapter which build a bridge between zeppelin
and jupyter, zeppelin can extends this class to leverage any kernel of jupyter.
    After a second thought, I think it is not enough. We can allow user to use jupyter kernel
directly instead of implement one by one. So the purpose is to allow user use jupyter kernel
like this
    
    ```
    %jupyter(kernel=ipython)
    
    python_code
    ...
    ```
    
    Of course user can enhance the jupyter kernel in Zeppelin, e.g. IPythonInterpreter of
Zeppelin can enhance the ipython kernel by adding support of ZeppelinContext
    
    ### What type of PR is it?
    [Refactoring]
    
    ### Todos
    * [ ] - Task
    
    ### What is the Jira issue?
    * https://jira.apache.org/jira/browse/ZEPPELIN-4497#
    
    ### How should this be tested?
    * CI pass
    
    ### Screenshots (if appropriate)
    
    ### Questions:
    * Does the licenses files need update? No
    * Is there breaking changes for older versions? No
    * Does this needs documentation? No
    
    ![image](https://user-images.githubusercontent.com/164491/71322469-c8f6c800-2502-11ea-9ad1-15c8dc78771a.png)
    
    ![image](https://user-images.githubusercontent.com/164491/71322797-833bfe80-2506-11ea-84e3-64d037cc739e.png)
    
    ![image](https://user-images.githubusercontent.com/164491/71322800-8fc05700-2506-11ea-92fb-be802b3a8b07.png)
    
    Author: Jeff Zhang <zjffdu@apache.org>
    
    Closes #3558 from zjffdu/ZEPPELIN-4497 and squashes the following commits:
    
    2efdc1e0e [Jeff Zhang] [ZEPPELIN-4497] Change zeppelin-jupyter-adapter to zeppelin-jupyter-interpreter
---
 pom.xml                                            |   2 +-
 python/pom.xml                                     |   2 +-
 .../apache/zeppelin/python/IPythonInterpreter.java |   6 +-
 .../pom.xml                                        |  16 ++-
 .../zeppelin/jupyter/JupyterInterpreter.java       | 112 +++++++++++++++++++++
 .../zeppelin/jupyter}/JupyterKernelClient.java     |   2 +-
 .../jupyter}/JupyterKernelInterpreter.java         |  29 ++++--
 .../zeppelin/jupyter/JupyterZeppelinContext.java   |  46 +++++++++
 .../src/main/proto/kernel.proto                    |   0
 .../src/main/resources/grpc/generate_rpc.sh        |   0
 .../main/resources/grpc/jupyter/kernel_client.py   |   0
 .../src/main/resources/grpc/jupyter/kernel_pb2.py  |   0
 .../main/resources/grpc/jupyter/kernel_pb2_grpc.py |   0
 .../main/resources/grpc/jupyter/kernel_server.py   |   0
 .../src/main/resources/interpreter-setting.json    |  14 +++
 15 files changed, 214 insertions(+), 15 deletions(-)

diff --git a/pom.xml b/pom.xml
index 130b01a..96a2912 100644
--- a/pom.xml
+++ b/pom.xml
@@ -58,7 +58,7 @@
     <module>zeppelin-zengine</module>
     <module>zeppelin-display</module>
     <module>rlang</module>
-    <module>zeppelin-jupyter-adapter</module>
+    <module>zeppelin-jupyter-interpreter</module>
     <module>kotlin</module>
     <module>groovy</module>
     <module>spark</module>
diff --git a/python/pom.xml b/python/pom.xml
index a167069..76fc0d5 100644
--- a/python/pom.xml
+++ b/python/pom.xml
@@ -43,7 +43,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.zeppelin</groupId>
-      <artifactId>zeppelin-jupyter-adapter</artifactId>
+      <artifactId>zeppelin-jupyter-interpreter</artifactId>
       <version>${project.version}</version>
     </dependency>
 
diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
index d6dcec0..3a42624 100644
--- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
+++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
@@ -22,11 +22,11 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zeppelin.interpreter.BaseZeppelinContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.JupyterKernelInterpreter;
 import org.apache.zeppelin.interpreter.jupyter.proto.ExecuteRequest;
 import org.apache.zeppelin.interpreter.jupyter.proto.ExecuteResponse;
 import org.apache.zeppelin.interpreter.jupyter.proto.ExecuteStatus;
 import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
+import org.apache.zeppelin.jupyter.JupyterKernelInterpreter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import py4j.GatewayServer;
@@ -39,7 +39,9 @@ import java.util.Map;
 import java.util.Properties;
 
 /**
- * IPython Interpreter for Zeppelin
+ * IPython Interpreter for Zeppelin. It enhances the JupyterKernelInterpreter by setting
up
+ * communication between JVM and Python process via py4j. So that in IPythonInterpreter
+ * you can use ZeppelinContext.
  */
 public class IPythonInterpreter extends JupyterKernelInterpreter {
 
diff --git a/zeppelin-jupyter-adapter/pom.xml b/zeppelin-jupyter-interpreter/pom.xml
similarity index 92%
rename from zeppelin-jupyter-adapter/pom.xml
rename to zeppelin-jupyter-interpreter/pom.xml
index 2b180d7..ffc8daa 100644
--- a/zeppelin-jupyter-adapter/pom.xml
+++ b/zeppelin-jupyter-interpreter/pom.xml
@@ -21,20 +21,20 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>zeppelin-interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.9.0-SNAPSHOT</version>
-    <relativePath>../pom.xml</relativePath>
+    <relativePath>../zeppelin-interpreter-parent/pom.xml</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
-  <artifactId>zeppelin-jupyter-adapter</artifactId>
+  <artifactId>zeppelin-jupyter-interpreter</artifactId>
   <packaging>jar</packaging>
   <version>0.9.0-SNAPSHOT</version>
   <name>Zeppelin: Jupyter Adapter</name>
 
   <properties>
-    <interpreter.name>python</interpreter.name>
+    <interpreter.name>jupyter</interpreter.name>
     <python.py4j.version>0.10.7</python.py4j.version>
     <grpc.version>1.15.0</grpc.version>
   </properties>
@@ -179,6 +179,14 @@
       </plugin>
 
       <plugin>
+        <artifactId>maven-dependency-plugin</artifactId>
+      </plugin>
+
+      <plugin>
+        <artifactId>maven-shade-plugin</artifactId>
+      </plugin>
+
+      <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-checkstyle-plugin</artifactId>
         <configuration>
diff --git a/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterInterpreter.java
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterInterpreter.java
new file mode 100644
index 0000000..b359468
--- /dev/null
+++ b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterInterpreter.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.jupyter;
+
+import org.apache.zeppelin.interpreter.AbstractInterpreter;
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Interpreter for Jupyter kernel. It can work for any Jupyter kernel as long as the kernel
+ * is installed. Specify the kernel name in paragraph properties.
+ * Run it via `%jupyter(kernel=ipython)`
+ *
+ */
+public class JupyterInterpreter extends AbstractInterpreter {
+
+  private Map<String, JupyterKernelInterpreter> kernelInterpreterMap = new HashMap<>();
+
+  public JupyterInterpreter(Properties properties) {
+    super(properties);
+  }
+
+  @Override
+  public BaseZeppelinContext getZeppelinContext() {
+    return new JupyterZeppelinContext(getInterpreterGroup().getInterpreterHookRegistry(),
1000);
+  }
+
+  @Override
+  protected InterpreterResult internalInterpret(
+          String st, InterpreterContext context) throws InterpreterException {
+    String kernel = context.getLocalProperties().get("kernel");
+    if (kernel == null) {
+      return new InterpreterResult(InterpreterResult.Code.ERROR, "No kernel is specified");
+    }
+    JupyterKernelInterpreter kernelInterpreter = null;
+    synchronized (kernelInterpreterMap) {
+      if (kernelInterpreterMap.containsKey(kernel)) {
+        kernelInterpreter = kernelInterpreterMap.get(kernel);
+      } else {
+        kernelInterpreter = new JupyterKernelInterpreter(kernel, properties);
+        kernelInterpreter.open();
+        kernelInterpreterMap.put(kernel, kernelInterpreter);
+      }
+    }
+    return kernelInterpreter.interpret(st, context);
+  }
+
+  @Override
+  public void open() throws InterpreterException {
+    // do nothing
+  }
+
+  @Override
+  public void close() throws InterpreterException {
+    for (JupyterKernelInterpreter kernelInterpreter : kernelInterpreterMap.values()) {
+      kernelInterpreter.close();
+    }
+  }
+
+  @Override
+  public void cancel(InterpreterContext context) throws InterpreterException {
+    String kernel = context.getLocalProperties().get("kernel");
+    if (kernel == null) {
+      throw new InterpreterException("No kernel is specified");
+    }
+    JupyterKernelInterpreter kernelInterpreter = kernelInterpreterMap.get(kernel);
+    if (kernelInterpreter != null) {
+      throw new InterpreterException("No such interpreter: " + kernel);
+    }
+    kernelInterpreter.cancel(context);
+  }
+
+  @Override
+  public FormType getFormType() throws InterpreterException {
+    return FormType.NATIVE;
+  }
+
+  @Override
+  public int getProgress(InterpreterContext context) throws InterpreterException {
+    String kernel = context.getLocalProperties().get("kernel");
+    if (kernel == null) {
+      throw new InterpreterException("No kernel is specified");
+    }
+    JupyterKernelInterpreter kernelInterpreter = kernelInterpreterMap.get(kernel);
+    if (kernelInterpreter == null) {
+      throw new InterpreterException("No such interpreter: " + kernel);
+    }
+    return  kernelInterpreter.getProgress(context);
+  }
+}
diff --git a/zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelClient.java
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelClient.java
similarity index 99%
rename from zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelClient.java
rename to zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelClient.java
index f8ba79e..2a65b0c 100644
--- a/zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelClient.java
+++ b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelClient.java
@@ -15,7 +15,7 @@
 * limitations under the License.
 */
 
-package org.apache.zeppelin.interpreter;
+package org.apache.zeppelin.jupyter;
 
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
diff --git a/zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelInterpreter.java
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
similarity index 93%
rename from zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelInterpreter.java
rename to zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
index 0aedbbf..c680f61 100644
--- a/zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelInterpreter.java
+++ b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.zeppelin.interpreter;
+package org.apache.zeppelin.jupyter;
 
 import com.google.common.annotations.VisibleForTesting;
 import io.grpc.ManagedChannelBuilder;
@@ -25,6 +25,11 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.jupyter.proto.CancelRequest;
 import org.apache.zeppelin.interpreter.jupyter.proto.CompletionRequest;
 import org.apache.zeppelin.interpreter.jupyter.proto.CompletionResponse;
@@ -52,10 +57,11 @@ import java.util.Map;
 import java.util.Properties;
 
 /**
- * Jupyter Kernel adapter for Zeppelin. All the jupyter kernel could be used by Zeppelin
- * by extending this class.
+ * Jupyter Kernel Interpreter for Zeppelin. One instance of this class represents one
+ * Jupyter Kernel. You can enhance the jupyter kernel by extending this class.
+ * e.g. IPythonInterpreter.
  */
-public abstract class JupyterKernelInterpreter extends Interpreter {
+public class JupyterKernelInterpreter extends Interpreter {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(JupyterKernelInterpreter.class);
 
@@ -63,6 +69,8 @@ public abstract class JupyterKernelInterpreter extends Interpreter {
   protected JupyterKernelClient jupyterKernelClient;
 
   protected BaseZeppelinContext zeppelinContext;
+
+  private String kernel;
   // working directory of jupyter kernel
   protected File kernelWorkDir;
   // python executable file for launching the jupyter kernel
@@ -71,11 +79,18 @@ public abstract class JupyterKernelInterpreter extends Interpreter {
 
   private InterpreterOutputStream interpreterOutput = new InterpreterOutputStream(LOGGER);
 
+  public JupyterKernelInterpreter(String kernel, Properties properties) {
+    this(properties);
+    this.kernel = kernel;
+  }
+
   public JupyterKernelInterpreter(Properties properties) {
     super(properties);
   }
 
-  public abstract String getKernelName();
+  public String getKernelName() {
+    return this.kernel;
+  }
 
   public List<String> getRequiredPackages() {
     List<String> requiredPackages = new ArrayList<>();
@@ -85,7 +100,9 @@ public abstract class JupyterKernelInterpreter extends Interpreter {
     return requiredPackages;
   }
 
-  public abstract BaseZeppelinContext buildZeppelinContext();
+  protected BaseZeppelinContext buildZeppelinContext() {
+    return new JupyterZeppelinContext(null, 1000);
+  }
 
   @Override
   public void open() throws InterpreterException {
diff --git a/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterZeppelinContext.java
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterZeppelinContext.java
new file mode 100644
index 0000000..22b28d4
--- /dev/null
+++ b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterZeppelinContext.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.jupyter;
+
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
+
+import java.util.List;
+import java.util.Map;
+
+public class JupyterZeppelinContext extends BaseZeppelinContext {
+  public JupyterZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
+    super(hooks, maxResult);
+  }
+
+  @Override
+  public Map<String, String> getInterpreterClassMap() {
+    return null;
+  }
+
+  @Override
+  public List<Class> getSupportedClasses() {
+    return null;
+  }
+
+  @Override
+  public String showData(Object obj, int maxResult) {
+    return null;
+  }
+}
diff --git a/zeppelin-jupyter-adapter/src/main/proto/kernel.proto b/zeppelin-jupyter-interpreter/src/main/proto/kernel.proto
similarity index 100%
rename from zeppelin-jupyter-adapter/src/main/proto/kernel.proto
rename to zeppelin-jupyter-interpreter/src/main/proto/kernel.proto
diff --git a/zeppelin-jupyter-adapter/src/main/resources/grpc/generate_rpc.sh b/zeppelin-jupyter-interpreter/src/main/resources/grpc/generate_rpc.sh
similarity index 100%
rename from zeppelin-jupyter-adapter/src/main/resources/grpc/generate_rpc.sh
rename to zeppelin-jupyter-interpreter/src/main/resources/grpc/generate_rpc.sh
diff --git a/zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_client.py b/zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_client.py
similarity index 100%
rename from zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_client.py
rename to zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_client.py
diff --git a/zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_pb2.py b/zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_pb2.py
similarity index 100%
rename from zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_pb2.py
rename to zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_pb2.py
diff --git a/zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_pb2_grpc.py b/zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_pb2_grpc.py
similarity index 100%
rename from zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_pb2_grpc.py
rename to zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_pb2_grpc.py
diff --git a/zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_server.py b/zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_server.py
similarity index 100%
rename from zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_server.py
rename to zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_server.py
diff --git a/zeppelin-jupyter-interpreter/src/main/resources/interpreter-setting.json b/zeppelin-jupyter-interpreter/src/main/resources/interpreter-setting.json
new file mode 100644
index 0000000..19181f7
--- /dev/null
+++ b/zeppelin-jupyter-interpreter/src/main/resources/interpreter-setting.json
@@ -0,0 +1,14 @@
+[
+  {
+    "group": "jupyter",
+    "name": "jupyter",
+    "className": "org.apache.zeppelin.jupyter.JupyterInterpreter",
+    "properties": {
+    },
+    "editor": {
+      "language": "text",
+      "editOnDblClick": false,
+      "completionSupport": true
+    }
+  }
+]


Mime
View raw message