predictionio-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From don...@apache.org
Subject incubator-predictionio git commit: [PIO-53] Docker compose based integration tests
Date Wed, 15 Feb 2017 07:10:24 GMT
Repository: incubator-predictionio
Updated Branches:
  refs/heads/develop 9b629800b -> e72d66e20


[PIO-53] Docker compose based integration tests

Remove dependency on Docker hub to enable per commit integration tests


Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/e72d66e2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/e72d66e2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/e72d66e2

Branch: refs/heads/develop
Commit: e72d66e201ddd70cfb679b82e332b851eb7297e1
Parents: 9b62980
Author: Donald Szeto <donald@apache.org>
Authored: Tue Feb 14 23:09:25 2017 -0800
Committer: Donald Szeto <donald@apache.org>
Committed: Tue Feb 14 23:09:25 2017 -0800

----------------------------------------------------------------------
 .gitignore                                 |  3 ++
 .travis.yml                                |  1 -
 make-distribution.sh                       |  1 +
 tests/Dockerfile                           | 56 ++++++-------------------
 tests/after_script.travis.sh               |  2 +
 tests/before_script.travis.sh              |  1 +
 tests/build-docker.sh                      | 15 +++++++
 tests/docker-compose.yml                   | 31 ++++++++++++++
 tests/docker-files/env-conf/hbase-site.xml |  6 +++
 tests/docker-files/env-conf/pio-env.sh     |  4 +-
 tests/docker-files/init.sh                 | 51 +++++++++++-----------
 tests/docker-files/pgpass                  |  1 +
 tests/docker-files/wait-for-postgres.sh    | 16 +++++++
 tests/run_docker.sh                        | 16 ++-----
 tests/script.travis.sh                     |  2 +-
 15 files changed, 122 insertions(+), 84 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 4e3460f..68ad5c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,3 +28,6 @@ quickstartapp/
 /docs/manual/source/gallery/template-gallery.html.md
 test-reports/
 apache-rat-0.11.jar
+tests/dist
+tests/docker-files/*.jar
+tests/docker-files/*.tgz

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 68dee42..5e11d1a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -68,7 +68,6 @@ env:
 
 before_install:
   - unset SBT_OPTS JVM_OPTS
-  - docker pull predictionio/pio-testing
   - service haveged start
 
 before_script:

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index 23769fe..a6accdb 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -54,6 +54,7 @@ touch ${DISTDIR}/RELEASE
 TARNAME="PredictionIO-$VERSION.tar.gz"
 TARDIR="PredictionIO-$VERSION"
 cp -r ${DISTDIR} ${TARDIR}
+cp -r ${DISTDIR} tests/
 
 tar zcvf ${TARNAME} ${TARDIR}
 rm -rf ${TARDIR}

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/Dockerfile
----------------------------------------------------------------------
diff --git a/tests/Dockerfile b/tests/Dockerfile
index 41ef344..fda6682 100644
--- a/tests/Dockerfile
+++ b/tests/Dockerfile
@@ -15,65 +15,49 @@
 # limitations under the License.
 #
 
-from ubuntu
+FROM ubuntu:xenial
 
 ENV SPARK_VERSION 1.4.0
 ENV ELASTICSEARCH_VERSION 1.4.4
 ENV HBASE_VERSION 1.0.0
 
-RUN echo "== Updating system =="
-RUN apt-get update -y
-RUN echo "== Downloading packages =="
-RUN apt-get install -y \
+RUN apt-get update && apt-get install -y \
     wget curl \
     python-pip \
     python3-pip \
-    postgresql postgresql-contrib \
+    postgresql-client \
     openjdk-8-jdk \
     openssh-client openssh-server \
     git
 
 RUN pip install predictionio
-RUN pip3 install --upgrade pip
-RUN pip3 install xmlrunner
-RUN pip3 install --upgrade requests
-RUN pip3 install --upgrade urllib3
+RUN pip3 install --upgrade \
+    pip \
+    xmlrunner \
+    requests \
+    urllib3
 
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/jre
 
-RUN echo "== Installing Spark =="
-RUN mkdir vendors
-RUN wget http://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz
-RUN tar zxvfC spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
-RUN rm spark-${SPARK_VERSION}-bin-hadoop2.6.tgz
+ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
 ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6
 
-RUN echo "== Installing Elasticsearch =="
-RUN wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz
-RUN tar zxvfC elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz /vendors
-RUN rm elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz
 ENV ELASTICSEARCH_HOME /vendors/elasticsearch-${ELASTICSEARCH_VERSION}
 
-RUN echo "== Installing HBase =="
-RUN wget http://archive.apache.org/dist/hbase/hbase-${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz
-RUN tar zxvfC hbase-${HBASE_VERSION}-bin.tar.gz /vendors
-RUN rm hbase-${HBASE_VERSION}-bin.tar.gz
 ENV HBASE_HOME /vendors/hbase-${HBASE_VERSION}
 
-RUN echo "== Downloading database drivers =="
-RUN mkdir drivers
-RUN wget https://jdbc.postgresql.org/download/postgresql-9.4-1204.jdbc41.jar -P /drivers
+COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers
 
-RUN mkdir PredictionIO
 ENV PIO_HOME /PredictionIO
 ENV PATH ${PIO_HOME}/bin/:${PATH}
-ENV HOST_PIO_HOME /pio_host
+COPY dist ${PIO_HOME}
 
-RUN echo "== Setting configs =="
 COPY docker-files/init.sh init.sh
 COPY docker-files/env-conf/spark-env.sh ${SPARK_HOME}/conf/spark-env.sh
 COPY docker-files/env-conf/hbase-site.xml ${HBASE_HOME}/conf/hbase-site.xml
 COPY docker-files/env-conf/pio-env.sh /pio-env.sh
+COPY docker-files/wait-for-postgres.sh /wait-for-postgres.sh
+COPY docker-files/pgpass /root/.pgpass
 
 # Default repositories setup
 ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL
@@ -88,20 +72,6 @@ ENV JVM_OPTS '-Dfile.encoding=UTF8 -Xms2048M -Xmx2048M -Xss8M -XX:MaxPermSize=51
 EXPOSE 8000
 # eventserver
 EXPOSE 7070
-# spark master UI
-EXPOSE 8080
-# spark worker UI
-EXPOSE 8081
-# spark context UI
-EXPOSE 4040
-# HMaster
-EXPOSE 60000
-# HMaster Info Web UI
-EXPOSE 60010
-# Region Server
-Expose 60020
-# Region Server Http
-EXPOSE 60030
 
 ENTRYPOINT ["/init.sh"]
 CMD 'bash'

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/after_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh
index 69e28e0..f41e929 100755
--- a/tests/after_script.travis.sh
+++ b/tests/after_script.travis.sh
@@ -20,4 +20,6 @@ set -e
 
 if [[ $BUILD_TYPE == Unit ]]; then
   ./bin/travis/pio-stop-travis
+else
+  docker-compose -f tests/docker-compose.yml down
 fi

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/before_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/before_script.travis.sh b/tests/before_script.travis.sh
index 9bfecdc..fbc6b0a 100755
--- a/tests/before_script.travis.sh
+++ b/tests/before_script.travis.sh
@@ -37,4 +37,5 @@ if [[ $BUILD_TYPE == Unit ]]; then
 
 else # Integration Tests
   ./make-distribution.sh
+  tests/build-docker.sh
 fi

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
new file mode 100755
index 0000000..fe783cb
--- /dev/null
+++ b/tests/build-docker.sh
@@ -0,0 +1,15 @@
+#!/bin/bash -x
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ ! -f $DIR/docker-files/spark-1.4.0-bin-hadoop2.6.tgz ]; then
+  wget http://d3kbcqa49mib13.cloudfront.net/spark-1.4.0-bin-hadoop2.6.tgz
+  mv spark-1.4.0-bin-hadoop2.6.tgz $DIR/docker-files/
+fi
+
+if [ ! -f $DIR/docker-files/postgresql-9.4-1204.jdbc41.jar ]; then
+  wget https://jdbc.postgresql.org/download/postgresql-9.4-1204.jdbc41.jar
+  mv postgresql-9.4-1204.jdbc41.jar $DIR/docker-files/
+fi
+
+docker build -t predictionio/pio-testing $DIR

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
new file mode 100644
index 0000000..d55834d
--- /dev/null
+++ b/tests/docker-compose.yml
@@ -0,0 +1,31 @@
+version: "3"
+services:
+  elasticsearch:
+    image: elasticsearch:1-alpine
+    ports:
+      - 9200:9200
+      - 9300:9300
+  hbase:
+    image: harisekhon/hbase:1.0
+    ports:
+      - 16000:16000
+      - 16010:16010
+      - 16020:16020
+      - 16030:16030
+  postgres:
+    image: postgres:9
+    ports:
+      - 5432:5432
+    environment:
+      POSTGRES_USER: pio
+      POSTGRES_PASSWORD: pio
+      POSTGRES_INITDB_ARGS: --encoding=UTF8
+  pio-testing:
+    image: predictionio/pio-testing:latest
+    depends_on:
+      - elasticsearch
+      - hbase
+      - postgres
+    volumes:
+      - ./pio_tests:/tests/pio_tests
+      - ~/.ivy2:/root/.ivy2

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/env-conf/hbase-site.xml
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/hbase-site.xml b/tests/docker-files/env-conf/hbase-site.xml
index 12185f1..3c4e24c 100644
--- a/tests/docker-files/env-conf/hbase-site.xml
+++ b/tests/docker-files/env-conf/hbase-site.xml
@@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 -->
 <configuration>
+  <!--
   <property>
     <name>hbase.rootdir</name>
     <value>file:///hbase-files/data</value>
@@ -25,4 +26,9 @@ limitations under the License.
     <name>hbase.zookeeper.property.dataDir</name>
     <value>/hbase-files/zookeeper</value>
   </property>
+  -->
+  <property>
+    <name>hbase.zookeeper.quorum</name>
+    <value>hbase</value>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/env-conf/pio-env.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env.sh b/tests/docker-files/env-conf/pio-env.sh
index 4ffec46..5417d24 100644
--- a/tests/docker-files/env-conf/pio-env.sh
+++ b/tests/docker-files/env-conf/pio-env.sh
@@ -73,7 +73,7 @@ PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_MODELDATA_SO
 # Please change PIO_STORAGE_SOURCES_PGSQL_USERNAME and
 # PIO_STORAGE_SOURCES_PGSQL_PASSWORD accordingly
 PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc
-PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://localhost/pio
+PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://postgres/pio
 PIO_STORAGE_SOURCES_PGSQL_USERNAME=pio
 PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
 
@@ -86,7 +86,7 @@ PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
 # Elasticsearch Example
 PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
 #PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio
-PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=localhost
+PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch
 PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
 PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/init.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/init.sh b/tests/docker-files/init.sh
index 69fd86c..dc74ada 100755
--- a/tests/docker-files/init.sh
+++ b/tests/docker-files/init.sh
@@ -16,40 +16,41 @@
 # limitations under the License.
 #
 
-set -e
+#set -e
 
-echo '== Setting up Postgres... =='
-service postgresql start
-runuser postgres -c 'createuser -s root'
-createdb root
+#echo '== Setting up Postgres... =='
+#service postgresql start
+#runuser postgres -c 'createuser -s root'
+#createdb root
+/wait-for-postgres.sh postgres
 
-psql -c "create user pio with password 'pio'" && createdb pio
+#psql -c "create user pio with password 'pio'" && createdb pio
 
-echo '== Starting SSH... =='
-service ssh start
-ssh-keygen -b 2048 -t rsa -q -f /root/.ssh/id_rsa -N ""
-cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
+#echo '== Starting SSH... =='
+#/etc/init.d/sshd start
+#ssh-keygen -b 2048 -t rsa -q -f /root/.ssh/id_rsa -N ""
+#cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
 
-echo '== Starting HBase... =='
-$HBASE_HOME/bin/start-hbase.sh
+#echo '== Starting HBase... =='
+#$HBASE_HOME/bin/start-hbase.sh
 
-echo '== Starting standalone Spark cluster... =='
-$SPARK_HOME/sbin/start-all.sh
+#echo '== Starting standalone Spark cluster... =='
+#$SPARK_HOME/sbin/start-all.sh
 
-echo '== Starting Elasticsearch... =='
-$ELASTICSEARCH_HOME/bin/elasticsearch -d -p $PIO_HOME/es.pid
+#echo '== Starting Elasticsearch... =='
+#$ELASTICSEARCH_HOME/bin/elasticsearch -d -p $PIO_HOME/es.pid
 
-echo '== Copying distribution to PIO_HOME... =='
-DISTRIBUTION_TAR=`find /pio_host -maxdepth 1 -name PredictionIO-*.tar.gz | head -1`
-tar zxvfC $DISTRIBUTION_TAR /
-DIR_NAME=/`basename $DISTRIBUTION_TAR`
-DIR_NAME=${DIR_NAME%.tar.gz}
-mv $DIR_NAME/* $PIO_HOME/
+#echo '== Copying distribution to PIO_HOME... =='
+#DISTRIBUTION_TAR=`find /pio_host -maxdepth 1 -name PredictionIO-*.tar.gz | head -1`
+#tar zxvfC $DISTRIBUTION_TAR /
+#DIR_NAME=/`basename $DISTRIBUTION_TAR`
+#DIR_NAME=${DIR_NAME%.tar.gz}
+#mv $DIR_NAME/* $PIO_HOME/
 mv /pio-env.sh $PIO_HOME/conf/pio-env.sh
 
-echo '== Copying tests to a separate directory =='
-mkdir /tests
-cp -r /pio_host/tests/pio_tests /tests/pio_tests
+#echo '== Copying tests to a separate directory =='
+#mkdir /tests
+#cp -r /pio_host/tests/pio_tests /tests/pio_tests
 export PYTHONPATH=/tests:$PYTHONPATH
 
 # after initialization run given command

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/pgpass
----------------------------------------------------------------------
diff --git a/tests/docker-files/pgpass b/tests/docker-files/pgpass
new file mode 100644
index 0000000..dcbcb23
--- /dev/null
+++ b/tests/docker-files/pgpass
@@ -0,0 +1 @@
+postgres:5432:pio:pio:pio

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/docker-files/wait-for-postgres.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/wait-for-postgres.sh b/tests/docker-files/wait-for-postgres.sh
new file mode 100755
index 0000000..3fcc27c
--- /dev/null
+++ b/tests/docker-files/wait-for-postgres.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# wait-for-postgres.sh
+
+set -e
+
+host="$1"
+shift
+cmd="$@"
+
+until psql -h "$host" -U "pio" -c '\l'; do
+  >&2 echo "Postgres is unavailable - sleeping"
+  sleep 1
+done
+
+>&2 echo "Postgres is up - executing command"
+exec $cmd

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/run_docker.sh
----------------------------------------------------------------------
diff --git a/tests/run_docker.sh b/tests/run_docker.sh
index a8a79f3..c60fb7e 100755
--- a/tests/run_docker.sh
+++ b/tests/run_docker.sh
@@ -16,12 +16,11 @@
 # limitations under the License.
 #
 
-USAGE=$"Usage: run_docer <meta> <event> <model> <pio> <command>
+USAGE=$"Usage: run_docker <meta> <event> <model> <command>
   Where:
     meta         = [PGSQL,ELASTICSEARCH]
     event        = [PGSQL,HBASE]
     model        = [PGSQL,LOCALFS,HDFS]
-    pio          = path to PredictionIO directory
     command      = command to run in the container"
 
 if ! [[ "$1" =~ ^(PGSQL|ELASTICSEARCH)$ ]]; then
@@ -39,17 +38,10 @@ if ! [[ "$3" =~ ^(PGSQL|LOCALFS|HDFS)$ ]]; then
   exit 1
 fi
 
-if [ ! -d "$4" ]; then
-  echo "Directory $4 does not exist"
-  echo "$USAGE"
-  exit 1
-fi
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-docker run -it -h localhost \
-  -v $4:/pio_host \
-  -v ~/.ivy2:/root/.ivy2 \
+docker-compose -f $DIR/docker-compose.yml run \
   -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$1 \
   -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$2 \
   -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$3 \
-  -p 8000:8000 -p 7070:7070 -p 8080:8080 -p 8081:8081 -p 4040:4040 \
-  -p 60000:60000 -p 60010:60010 -p 60020:60020 -p 60030:60030 predictionio/pio-testing $5
+  pio-testing $4

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e72d66e2/tests/script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/script.travis.sh b/tests/script.travis.sh
index 4adb135..f7f81ac 100755
--- a/tests/script.travis.sh
+++ b/tests/script.travis.sh
@@ -36,5 +36,5 @@ else
   REPO=`pwd`
 
   ./tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
-    $REPO 'python3 /tests/pio_tests/tests.py'
+    'python3 /tests/pio_tests/tests.py'
 fi


Mime
View raw message