zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jongy...@apache.org
Subject zeppelin git commit: [ZEPPELIN-1520] Change test environment to use the local mode of Apache Spark
Date Tue, 11 Oct 2016 08:13:29 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 52202afd3 -> d2f63f6f9


[ZEPPELIN-1520] Change test environment to use the local mode of Apache Spark

### What is this PR for?
Simplify travis test to reduce usage of resources

### What type of PR is it?
[Improvement]

### Todos
* [x] - Remove start-up/stop SparkCluster

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1520

### How should this be tested?
Travis will pass without any error

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no

Author: Jongyoul Lee <jongyoul@gmail.com>

Closes #1487 from jongyoul/ZEPPELIN-1520 and squashes the following commits:

3bccf66 [Jongyoul Lee] Removed some unused scripts anymore
15a3711 [Jongyoul Lee] Cleaned up commented lines
1237658 [Jongyoul Lee] Removed checking mechanism
f37dacf [Jongyoul Lee] Changed master to local[2]
2aac444 [Jongyoul Lee] Remove scripts of start/stop SparkCluster


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d2f63f6f
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d2f63f6f
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d2f63f6f

Branch: refs/heads/master
Commit: d2f63f6f96f1921ebc901915abc606f431461cb8
Parents: 52202af
Author: Jongyoul Lee <jongyoul@gmail.com>
Authored: Mon Oct 10 00:02:25 2016 +0900
Committer: Jongyoul Lee <jongyoul@apache.org>
Committed: Tue Oct 11 17:13:20 2016 +0900

----------------------------------------------------------------------
 .travis.yml                                     |  4 --
 testing/startSparkCluster.sh                    | 67 --------------------
 testing/stopSparkCluster.sh                     | 42 ------------
 .../zeppelin/rest/AbstractTestRestApi.java      | 12 +---
 4 files changed, 3 insertions(+), 122 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index e2a115c..d314b2f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -89,7 +89,6 @@ install:
 
 before_script:
   - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
-  - ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
   - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
   - tail conf/zeppelin-env.sh
 
@@ -109,6 +108,3 @@ after_failure:
   - cat zeppelin-web/npm-debug.log
   - cat spark-*/logs/*
 
-after_script:
-  - ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER
-

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/testing/startSparkCluster.sh
----------------------------------------------------------------------
diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh
deleted file mode 100755
index dc7613d..0000000
--- a/testing/startSparkCluster.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-if [[ "$#" -ne 2 ]]; then
-    echo "usage) $0 [spark version] [hadoop version]"
-    echo "   eg) $0 1.3.1 2.6"
-    exit 1
-fi
-
-SPARK_VERSION="${1}"
-HADOOP_VERSION="${2}"
-
-echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
-if [[ "$?" -eq 0 ]]; then
-  echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
-  if [[ "$?" -eq 0 ]]; then
-    SPARK_VER_RANGE="<=1.2"
-  else
-    SPARK_VER_RANGE="<=1.3"
-  fi
-else
-  SPARK_VER_RANGE=">1.3"
-fi
-
-set -xe
-
-FWDIR="$(dirname "${BASH_SOURCE-$0}")"
-ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
-
-SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
-export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
-echo "SPARK_HOME is ${SPARK_HOME}"
-
-# create PID dir. test case detect pid file so they can select active spark home dir for
test
-export SPARK_PID_DIR="${SPARK_HOME}/run"
-mkdir -p "${SPARK_PID_DIR}"
-
-# start
-export SPARK_MASTER_PORT=7071
-export SPARK_MASTER_WEBUI_PORT=7072
-export SPARK_WORKER_WEBUI_PORT=8082
-${SPARK_HOME}/sbin/start-master.sh
-
-if [ "${SPARK_VER_RANGE}" == "<=1.3" ]||[ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
-    # spark 1.3 or prior
-    ${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT}
-else
-    ${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071
-fi
-
-set +xe

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/testing/stopSparkCluster.sh
----------------------------------------------------------------------
diff --git a/testing/stopSparkCluster.sh b/testing/stopSparkCluster.sh
deleted file mode 100755
index e049ec4..0000000
--- a/testing/stopSparkCluster.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-if [[ "$#" -ne 2 ]]; then
-    echo "usage) $0 [spark version] [hadoop version]"
-    echo "   eg) $0 1.3.1 2.6"
-    exit 1
-fi
-
-SPARK_VERSION="${1}"
-HADOOP_VERSION="${2}"
-
-set -xe
-
-FWDIR="$(dirname "${BASH_SOURCE-$0}")"
-ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
-
-SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
-export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
-echo "SPARK_HOME is ${SPARK_HOME}"
-
-# set create PID dir
-export SPARK_PID_DIR="${SPARK_HOME}/run"
-
-${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
-${SPARK_HOME}/sbin/stop-master.sh
-set +xe

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index eb080fe..de2607f 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -152,7 +152,7 @@ public abstract class AbstractTestRestApi {
         }
 
         // set spark master and other properties
-        sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname()
+ ":7071");
+        sparkIntpSetting.getProperties().setProperty("master", "local[2]");
         sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
         sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false");
         // set spark home for pyspark
@@ -175,7 +175,7 @@ public abstract class AbstractTestRestApi {
             sparkIntpSetting.getProperties().setProperty("master", System.getenv("SPARK_MASTER"));
           } else {
             sparkIntpSetting.getProperties()
-                    .setProperty("master", "spark://" + getHostname() + ":7071");
+                    .setProperty("master", "local[2]");
           }
           sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
           // set spark home for pyspark
@@ -238,13 +238,7 @@ public abstract class AbstractTestRestApi {
   }
 
   private static boolean isActiveSparkHome(File dir) {
-    if (dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+")) {
-      File pidDir = new File(dir, "run");
-      if (pidDir.isDirectory() && pidDir.listFiles().length > 0) {
-        return true;
-      }
-    }
-    return false;
+    return dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+");
   }
 
   protected static void shutDown() throws Exception {


Mime
View raw message