zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject [zeppelin] branch master updated: ZEPPELIN-4227. Move StandardInterpreterLauncher and SparkInterpreterLauncher into zeppelin-zengine
Date Tue, 10 Sep 2019 07:35:11 GMT
This is an automated email from the ASF dual-hosted git repository.

zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git


The following commit(s) were added to refs/heads/master by this push:
     new b9006a2  ZEPPELIN-4227. Move StandardInterpreterLauncher and SparkInterpreterLauncher
into zeppelin-zengine
b9006a2 is described below

commit b9006a247043aa3151a798d10c85b7edf6ea9fe3
Author: Jeff Zhang <zjffdu@apache.org>
AuthorDate: Sun Jul 7 22:25:17 2019 +0800

    ZEPPELIN-4227. Move StandardInterpreterLauncher and SparkInterpreterLauncher into zeppelin-zengine
    
    ### What is this PR for?
    We should include these default interpreter launcher into zeppelin-zengine, so that user
can use zeppelin even without building zeppelin-plugins module.
    
    ### What type of PR is it?
    [Improvement]
    
    ### Todos
    * [ ] - Task
    
    ### What is the Jira issue?
    * https://issues.apache.org/jira/browse/ZEPPELIN-4227
    
    ### How should this be tested?
    * CI pass
    
    ### Screenshots (if appropriate)
    
    ### Questions:
    * Does the licenses files need update? No
    * Is there breaking changes for older versions? No
    * Does this needs documentation? No
    
    Author: Jeff Zhang <zjffdu@apache.org>
    
    Closes #3421 from zjffdu/ZEPPELIN-4227 and squashes the following commits:
    
    ee1c7bcc1 [Jeff Zhang] ZEPPELIN-4227. Move StandardInterpreterLauncher and SparkInterpreterLauncher
into zeppelin-zengine
---
 .travis.yml                                        |  4 +-
 zeppelin-plugins/launcher/cluster/pom.xml          |  7 +--
 zeppelin-plugins/launcher/docker/pom.xml           |  5 --
 zeppelin-plugins/launcher/spark/pom.xml            | 67 ----------------------
 zeppelin-plugins/launcher/standard/pom.xml         | 49 ----------------
 zeppelin-plugins/pom.xml                           |  2 -
 .../apache/zeppelin/rest/NotebookRestApiTest.java  |  4 +-
 .../launcher/SparkInterpreterLauncher.java         |  0
 .../launcher/StandardInterpreterLauncher.java      |  0
 .../org/apache/zeppelin/plugin/PluginManager.java  | 13 +++++
 .../launcher/SparkInterpreterLauncherTest.java     |  9 +++
 .../launcher/StandardInterpreterLauncherTest.java  |  0
 12 files changed, 27 insertions(+), 133 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ef2193a..2838e1b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,7 +86,7 @@ matrix:
     - sudo: required
       jdk: "openjdk8"
       dist: xenial
-      env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Phelium-dev -Pexamples -Pspark-scala-2.11"
BUILD_FLAG="clean install -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr
-DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/JdbcIntegrationTest.java,**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java
-DfailIfNoTests=false"
+      env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Phelium-dev -Pexamples -Pspark-scala-2.11"
BUILD_FLAG="install -Pbuild-distr -DskipRat -DskipTests" TEST_FLAG="verify -Pusing-packaged-distr
-DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/JdbcIntegrationTest.java,**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java
-DfailIfNoTests=false"
 
     # Test selenium with spark module for spark 2.3
     - jdk: "openjdk8"
@@ -98,7 +98,7 @@ matrix:
     # Test interpreter modules
     - jdk: "openjdk8"
       dist: xenial
-      env: PYTHON="3" SPARKR="true" SCALA_VER="2.10" TENSORFLOW="1.0.0" PROFILE="-Pscala-2.10"
BUILD_FLAG="clean install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat" MODULES="-pl
$(echo .,zeppelin-interpreter,zeppelin-interpreter-api,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
+      env: PYTHON="3" SPARKR="true" SCALA_VER="2.10" TENSORFLOW="1.0.0" PROFILE="-Pscala-2.10"
BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo
.,zeppelin-interpreter,zeppelin-interpreter-api,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
 
     # Run Spark integration test and unit test separately for each spark version
 
diff --git a/zeppelin-plugins/launcher/cluster/pom.xml b/zeppelin-plugins/launcher/cluster/pom.xml
index bf24546..08b03be 100644
--- a/zeppelin-plugins/launcher/cluster/pom.xml
+++ b/zeppelin-plugins/launcher/cluster/pom.xml
@@ -39,15 +39,10 @@
   <properties>
     <plugin.name>Launcher/ClusterInterpreterLauncher</plugin.name>
   </properties>
-
+  
   <dependencies>
     <dependency>
       <groupId>org.apache.zeppelin</groupId>
-      <artifactId>launcher-standard</artifactId>
-      <version>0.9.0-SNAPSHOT</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.zeppelin</groupId>
       <artifactId>launcher-docker</artifactId>
       <version>0.9.0-SNAPSHOT</version>
     </dependency>
diff --git a/zeppelin-plugins/launcher/docker/pom.xml b/zeppelin-plugins/launcher/docker/pom.xml
index 6d029fe..b9a47cb 100644
--- a/zeppelin-plugins/launcher/docker/pom.xml
+++ b/zeppelin-plugins/launcher/docker/pom.xml
@@ -42,11 +42,6 @@
 
   <dependencies>
     <dependency>
-      <groupId>org.apache.zeppelin</groupId>
-      <artifactId>launcher-spark</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
       <groupId>com.hubspot.jinjava</groupId>
       <artifactId>jinjava</artifactId>
       <version>2.4.12</version>
diff --git a/zeppelin-plugins/launcher/spark/pom.xml b/zeppelin-plugins/launcher/spark/pom.xml
deleted file mode 100644
index 189c27f..0000000
--- a/zeppelin-plugins/launcher/spark/pom.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <artifactId>zengine-plugins-parent</artifactId>
-        <groupId>org.apache.zeppelin</groupId>
-        <version>0.9.0-SNAPSHOT</version>
-        <relativePath>../../../zeppelin-plugins</relativePath>
-    </parent>
-
-    <groupId>org.apache.zeppelin</groupId>
-    <artifactId>launcher-spark</artifactId>
-    <packaging>jar</packaging>
-    <version>0.9.0-SNAPSHOT</version>
-    <name>Zeppelin: Plugin SparkInterpreterLauncher</name>
-    <description>Spark Launcher implementation based on shell script interpreter.sh</description>
-
-    <properties>
-        <plugin.name>Launcher/SparkInterpreterLauncher</plugin.name>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.zeppelin</groupId>
-            <artifactId>launcher-standard</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-dependency-plugin</artifactId>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>${plugin.surefire.version}</version>
-                <configuration>
-                    <environmentVariables>
-                        <ZEPPELIN_HOME>${project.basedir}/../../..</ZEPPELIN_HOME>
-                    </environmentVariables>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-</project>
diff --git a/zeppelin-plugins/launcher/standard/pom.xml b/zeppelin-plugins/launcher/standard/pom.xml
deleted file mode 100644
index c8c68ce..0000000
--- a/zeppelin-plugins/launcher/standard/pom.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <artifactId>zengine-plugins-parent</artifactId>
-        <groupId>org.apache.zeppelin</groupId>
-        <version>0.9.0-SNAPSHOT</version>
-        <relativePath>../../../zeppelin-plugins</relativePath>
-    </parent>
-
-    <groupId>org.apache.zeppelin</groupId>
-    <artifactId>launcher-standard</artifactId>
-    <packaging>jar</packaging>
-    <version>0.9.0-SNAPSHOT</version>
-    <name>Zeppelin: Plugin StandardLauncher</name>
-    <description>Launcher implementation based on shell script interpreter.sh</description>
-
-    <properties>
-        <plugin.name>Launcher/StandardInterpreterLauncher</plugin.name>
-    </properties>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-dependency-plugin</artifactId>
-            </plugin>
-        </plugins>
-    </build>
-</project>
diff --git a/zeppelin-plugins/pom.xml b/zeppelin-plugins/pom.xml
index dea3a47..bca6421 100644
--- a/zeppelin-plugins/pom.xml
+++ b/zeppelin-plugins/pom.xml
@@ -43,9 +43,7 @@
         <module>notebookrepo/zeppelin-hub</module>
         <module>notebookrepo/filesystem</module>
 
-        <module>launcher/standard</module>
         <module>launcher/k8s-standard</module>
-        <module>launcher/spark</module>
         <module>launcher/cluster</module>
         <module>launcher/docker</module>
     </modules>
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
index 1835b51..079fe5f 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
@@ -422,8 +422,8 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
       assertEquals(resp.get("status"), "OK");
       post2.releaseConnection();
 
-      assertEquals(Job.Status.FINISHED, p1.getStatus());
-      assertEquals(Job.Status.FINISHED, p2.getStatus());
+      assertEquals(p1.getReturn().toString(), Job.Status.FINISHED, p1.getStatus());
+      assertEquals(p2.getReturn().toString(), Job.Status.FINISHED, p2.getStatus());
       assertNotNull(p2.getReturn());
       assertEquals("abc\n", p2.getReturn().message().get(0).getData());
     } finally {
diff --git a/zeppelin-plugins/launcher/spark/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
similarity index 100%
rename from zeppelin-plugins/launcher/spark/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
diff --git a/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java
similarity index 100%
rename from zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java
rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
index c3a2b9b..cdcfed0 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
@@ -140,6 +140,19 @@ public class PluginManager {
       return cachedLaunchers.get(launcherPlugin);
     }
     LOGGER.info("Loading Interpreter Launcher Plugin: " + launcherPlugin);
+    // load plugin from classpath directly first for these builtin InterpreterLauncher.
+    // If fails, then try to load it from plugin folder.
+    try {
+      InterpreterLauncher launcher = (InterpreterLauncher)
+              (Class.forName("org.apache.zeppelin.interpreter.launcher." + launcherPlugin))
+                      .getConstructor(ZeppelinConfiguration.class, RecoveryStorage.class)
+                      .newInstance(zConf, recoveryStorage);
+      return launcher;
+    } catch (InstantiationException | IllegalAccessException | ClassNotFoundException
+            | NoSuchMethodException | InvocationTargetException e) {
+      LOGGER.warn("Fail to instantiate InterpreterLauncher from classpath directly:" + launcherPlugin,
e);
+    }
+
     URLClassLoader pluginClassLoader = getPluginClassLoader(pluginsDir, "Launcher", launcherPlugin);
     String pluginClass = "org.apache.zeppelin.interpreter.launcher." + launcherPlugin;
     InterpreterLauncher launcher = null;
diff --git a/zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
similarity index 97%
rename from zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
rename to zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
index 118e7d4..8500de6 100644
--- a/zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
@@ -25,7 +25,10 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess;
 import org.apache.zeppelin.util.Util;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -37,6 +40,8 @@ import static org.junit.Assert.assertTrue;
 
 public class SparkInterpreterLauncherTest {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterLauncher.class);
+
   private String sparkHome;
   private String zeppelinHome;
 
@@ -47,7 +52,11 @@ public class SparkInterpreterLauncherTest {
     }
 
     sparkHome = DownloadUtils.downloadSpark("2.3.2");
+    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(),
+            new File("..").getAbsolutePath());
+
     zeppelinHome = ZeppelinConfiguration.create().getZeppelinHome();
+    LOGGER.info("ZEPPELIN_HOME: " + zeppelinHome);
   }
 
   @Test
diff --git a/zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java
similarity index 100%
rename from zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java
rename to zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java


Mime
View raw message