distributedlog-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject incubator-distributedlog git commit: DL-141: Fix broken scripts in tutorials and add integration test to travis ci
Date Fri, 16 Dec 2016 19:15:26 GMT
Repository: incubator-distributedlog
Updated Branches:
  refs/heads/master 927f0ca01 -> dc1bb30ba


DL-141: Fix broken scripts in tutorials and add integration test to travis ci

- fix the broken scripts used in tutorials
- fix the write_proxy configuration (a change was merged to validate configuration which it
brokes the configuration too)
- add integration test to validate the basic end-to-end functionalities

Author: Sijie Guo <sijieg@twitter.com>

Reviewers: Leigh Stewart <lstewart@apache.org>

Closes #76 from sijie/sijie/script_for_ci


Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/dc1bb30b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/dc1bb30b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/dc1bb30b

Branch: refs/heads/master
Commit: dc1bb30ba3ad2ab0898cf7f074261f1de0bd2d79
Parents: 927f0ca
Author: Sijie Guo <sijieg@twitter.com>
Authored: Fri Dec 16 11:15:21 2016 -0800
Committer: Sijie Guo <sijieg@twitter.com>
Committed: Fri Dec 16 11:15:21 2016 -0800

----------------------------------------------------------------------
 .travis.yml                                     |   4 +
 distributedlog-core/bin/dlog                    |   3 -
 distributedlog-core/conf/write_proxy.conf       |   2 +-
 .../distributedlog-kafka/bin/runner             |   1 -
 scripts/common.sh                               |   6 +-
 scripts/integration/smoketest.sh                | 113 +++++++++++++++++++
 scripts/runner                                  |  11 +-
 7 files changed, 124 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 50343d3..a064f24 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,6 +40,10 @@ script:
   - travis_retry mvn --batch-mode clean apache-rat:check
   - travis_wait 60 mvn --batch-mode clean package findbugs:check
 
+# run integration test after success
+after_success:
+  - ./scripts/integration/smoketest.sh
+
 cache:
   directories:
     - $HOME/.m2

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/distributedlog-core/bin/dlog
----------------------------------------------------------------------
diff --git a/distributedlog-core/bin/dlog b/distributedlog-core/bin/dlog
index 522e47c..6632fd0 100755
--- a/distributedlog-core/bin/dlog
+++ b/distributedlog-core/bin/dlog
@@ -54,9 +54,6 @@ source "${DLOG_ROOT}"/scripts/common.sh
 COMMAND=$1
 shift
 
-#Change to DLOG_HOME to support relative paths
-cd "$DLOG_HOME"
-
 case "${COMMAND}" in
   local)
     exec java $OPTS $JMX_ARGS com.twitter.distributedlog.LocalDLMEmulator $@

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/distributedlog-core/conf/write_proxy.conf
----------------------------------------------------------------------
diff --git a/distributedlog-core/conf/write_proxy.conf b/distributedlog-core/conf/write_proxy.conf
index adad68a..271f177 100644
--- a/distributedlog-core/conf/write_proxy.conf
+++ b/distributedlog-core/conf/write_proxy.conf
@@ -39,7 +39,7 @@ bkcZKRetryMaxBackoffMillis=200
 
 # bookkeeper client timeouts
 bkcWriteTimeoutSeconds=2
-bkcReadTimeoutSeconds=1
+bkcReadTimeoutSeconds=2
 bkcNumWorkerThreads=32
 bkc.numChannelsPerBookie=1
 bkc.enableTaskExecutionStats=true

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/distributedlog-tutorials/distributedlog-kafka/bin/runner
----------------------------------------------------------------------
diff --git a/distributedlog-tutorials/distributedlog-kafka/bin/runner b/distributedlog-tutorials/distributedlog-kafka/bin/runner
deleted file mode 120000
index a0ce2ef..0000000
--- a/distributedlog-tutorials/distributedlog-kafka/bin/runner
+++ /dev/null
@@ -1 +0,0 @@
-../../../scripts/runner
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/scripts/common.sh
----------------------------------------------------------------------
diff --git a/scripts/common.sh b/scripts/common.sh
index f37ed1b..8df5248 100755
--- a/scripts/common.sh
+++ b/scripts/common.sh
@@ -54,13 +54,13 @@ DEFAULT_LOG_CONF="${DLOG_HOME}/conf/log4j.properties"
 [ -f "${DLOG_HOME}/conf/dlogenv.sh" ] && source "${DLOG_HOME}/conf/dlogenv.sh"
 
 # exclude tests jar
-RELEASE_JAR=$(ls "${DLOG_HOME}/distributedlog-*.jar" 2> /dev/null | egrep -v 'tests|javadoc|sources'
| tail -1)
+RELEASE_JAR=$(ls "${DLOG_HOME}/distributedlog-*.jar" 2> /dev/null | grep -v 'tests|javadoc|sources'
| tail -1)
 if [ $? == 0 ]; then
   DLOG_JAR="${RELEASE_JAR}"
 fi
 
 # exclude tests jar
-BUILT_JAR=$(ls "${DLOG_HOME}"/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources'
| tail -1)
+BUILT_JAR=$(ls "${DLOG_HOME}"/target/distributedlog-*.jar 2> /dev/null | grep -v 'tests|javadoc|sources'
| tail -1)
 
 if [ -e "${BUILD_JAR}" ] && [ -e "${DLOG_JAR}" ]; then
   echo "\nCouldn't find dlog jar.";
@@ -79,7 +79,7 @@ add_maven_deps_to_classpath() {
   # Need to generate classpath from maven pom. This is costly so generate it
   # and cache it. Save the file into our target dir so a mvn clean will get
   # clean it up and force us create a new one.
-  f="${DLOG_HOME}/target/cached_classpath.txt"
+  f="${PWD}/${DLOG_HOME}/target/cached_classpath.txt"
   if [ ! -f "${f}" ]; then
     "${MVN}" -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}"
&> /dev/null
   fi

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/scripts/integration/smoketest.sh
----------------------------------------------------------------------
diff --git a/scripts/integration/smoketest.sh b/scripts/integration/smoketest.sh
new file mode 100755
index 0000000..5141756
--- /dev/null
+++ b/scripts/integration/smoketest.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+set -e
+
+BASEDIR=$(dirname "$0")
+DLOG_ROOT="${BASEDIR}/../.."
+
+LOG_DIR="${DLOG_ROOT}/logs"
+mkdir -p ${LOG_DIR}
+
+# DL ZK & Namespace
+NAMESPACE="${USER}-${RANDOM}"
+ZK_PORT=7000
+
+# write proxies
+export WP_SHARD_ID=1
+export WP_SERVICE_PORT=8000
+export WP_STATS_PORT=8001
+export WP_NAMESPACE="distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE}"
+
+# streams
+SMOKESTREAM_PREFIX="smoketest-stream-"
+
+# start the sandbox
+nohup ${DLOG_ROOT}/distributedlog-service/bin/dlog local ${ZK_PORT} > ${LOG_DIR}/sandbox.out
2>&1&
+echo $! > ${LOG_DIR}/sandbox.pid
+
+# create namespace
+${DLOG_ROOT}/distributedlog-core/bin/dlog admin bind -l /ledgers -s 127.0.0.1:${ZK_PORT}
-c distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE}
+
+if [ $? -ne 0 ]; then
+  echo "Failed to create namespace '${NAMESPACE}'."
+  exit 1
+fi
+
+# create streams
+${DLOG_ROOT}/distributedlog-service/bin/dlog tool create -u distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE}
-r ${SMOKESTREAM_PREFIX} -e 1-5 -f
+
+if [ $? -ne 0 ]; then
+  echo "Failed to create streams prefixed with '${SMOKESTREAM_PREFIX}'."
+  exit 1
+fi
+
+# list streams
+STREAM_OUTPUT=$(${DLOG_ROOT}/distributedlog-service/bin/dlog tool list -u distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE}
| grep "${SMOKESTREAM_PREFIX}")
+
+echo "Create streams : ${STREAM_OUTPUT}."
+
+for i in {1..5}; do
+  if [[ ${STREAM_OUTPUT} != *"${SMOKESTREAM_PREFIX}${i}"* ]]; then
+    echo "Stream '${SMOKESTREAM_PREFIX}${i}' is not created."
+    exit 1
+  fi
+done
+
+# start a write proxy
+${DLOG_ROOT}/distributedlog-service/bin/dlog-daemon.sh start writeproxy
+
+# tail the the streams
+nohup ${DLOG_ROOT}/distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.MultiReader
distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE} ${SMOKESTREAM_PREFIX}1,${SMOKESTREAM_PREFIX}2,${SMOKESTREAM_PREFIX}3,${SMOKESTREAM_PREFIX}4,${SMOKESTREAM_PREFIX}5
> ${LOG_DIR}/reader.out 2>&1&
+echo $! > ${LOG_DIR}/reader.pid
+
+# generate the records
+nohup ${DLOG_ROOT}/distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.RecordGenerator
"inet!127.0.0.1:${WP_SERVICE_PORT}" ${SMOKESTREAM_PREFIX}1 1 > ${LOG_DIR}/writer.out 2>&1&
+echo $! > ${LOG_DIR}/writer.pid
+
+# wait for 20 seconds
+sleep 20
+
+# kill the writer
+kill `cat ${LOG_DIR}/writer.pid`
+
+# stop the reader
+kill `cat ${LOG_DIR}/reader.pid`
+
+# check the number of records received
+NUM_RECORDS=`cat ${LOG_DIR}/reader.out | grep "record-" | wc -l`
+
+echo "Received ${NUM_RECORDS} records : "
+cat ${LOG_DIR}/reader.out
+
+if [ $NUM_RECORDS -lt 18 ]; then
+  echo "[FAILURE] Received less than 18 records."
+  exit 1
+fi
+
+# stop a write proxy
+${DLOG_ROOT}/distributedlog-service/bin/dlog-daemon.sh stop writeproxy
+
+# stop the sandbox
+kill `cat ${LOG_DIR}/sandbox.pid`
+
+exit 0

http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/scripts/runner
----------------------------------------------------------------------
diff --git a/scripts/runner b/scripts/runner
index 937829a..78ce126 100755
--- a/scripts/runner
+++ b/scripts/runner
@@ -23,9 +23,8 @@
 set -e
 
 BASEDIR=$(dirname "$0")
-DLOG_ROOT="${BASEDIR}/.."
-
-DLOG_HOME="${DLOG_ROOT}/distributedlog-service"
+DLOG_ROOT="${BASEDIR}/../../.."
+DLOG_HOME="${BASEDIR}/.."
 
 usage() {
   cat <<EOF
@@ -45,16 +44,12 @@ These variable can also be set in conf/dlogenv.sh
 EOF
 }
 
-cd "${DLOG_ROOT}"
-source ./scripts/common.sh
+source "${DLOG_ROOT}/scripts/common.sh"
 
 # get arguments
 COMMAND="${1}"
 shift
 
-#Change to DLOG_HOME to support relative paths
-cd "${DLOG_HOME}"
-
 case "${COMMAND}" in
   run)
     java ${OPTS} $@


Mime
View raw message