predictionio-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From don...@apache.org
Subject incubator-predictionio git commit: Integration tests for both ES1 and ES5
Date Mon, 13 Mar 2017 21:52:37 GMT
Repository: incubator-predictionio
Updated Branches:
  refs/heads/develop 6d92b4576 -> 3da809c7f


Integration tests for both ES1 and ES5


Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/3da809c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/3da809c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/3da809c7

Branch: refs/heads/develop
Commit: 3da809c7f6b82add7007d19707bfd312ec68bb1a
Parents: 6d92b45
Author: Donald Szeto <donald@apache.org>
Authored: Mon Mar 13 14:52:24 2017 -0700
Committer: Donald Szeto <donald@apache.org>
Committed: Mon Mar 13 14:52:24 2017 -0700

----------------------------------------------------------------------
 tests/Dockerfile-es1                       |  57 +++++++++++++
 tests/after_script.travis.sh               |   8 +-
 tests/before_script.travis.sh              |   4 +-
 tests/build-docker.sh                      |   8 +-
 tests/docker-compose-es1.yml               |  38 +++++++++
 tests/docker-files/env-conf/pio-env-es1.sh | 103 ++++++++++++++++++++++++
 tests/run_docker.sh                        |  19 +++--
 7 files changed, 229 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/Dockerfile-es1
----------------------------------------------------------------------
diff --git a/tests/Dockerfile-es1 b/tests/Dockerfile-es1
new file mode 100644
index 0000000..73d4c36
--- /dev/null
+++ b/tests/Dockerfile-es1
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM predictionio/pio
+
+ENV SPARK_VERSION 1.6.3
+ENV ELASTICSEARCH_VERSION 1.7
+ENV HBASE_VERSION 1.0.0
+
+ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
+# WORKAROUND: es-hadoop stops on RDD#take(1)
+RUN echo "spark.locality.wait.node 0s" > /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6/conf/spark-defaults.conf
+ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6
+
+COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar
+COPY docker-files/init.sh init.sh
+COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml
+COPY docker-files/env-conf/pio-env-es1.sh ${PIO_HOME}/conf/pio-env.sh
+COPY docker-files/pgpass /root/.pgpass
+RUN chmod 600 /root/.pgpass
+
+# Python
+RUN pip install python-dateutil
+RUN pip install pytz
+
+# Default repositories setup
+ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL
+ENV PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE PGSQL
+ENV PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE PGSQL
+
+# JVM settings
+ENV JVM_OPTS '-Dfile.encoding=UTF8 -Xms2048M -Xmx2048M -Xss8M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=256M'
+
+# Expose relevant ports
+# pio engine
+EXPOSE 8000
+# eventserver
+EXPOSE 7070
+
+ENV SLEEP_TIME 30
+
+ENTRYPOINT ["/init.sh"]
+CMD 'bash'

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/after_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh
index 005d410..8cbe2c0 100755
--- a/tests/after_script.travis.sh
+++ b/tests/after_script.travis.sh
@@ -16,8 +16,14 @@
 # limitations under the License.
 #
 
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
 # Print a summary of containers used
 docker ps -a
 
 # Clean up used containers
-docker-compose -f tests/docker-compose.yml down
+if [ "$ES_VERSION" = "1" ]; then
+    docker-compose -f $DIR/docker-compose-es1.yml down
+else
+    docker-compose -f $DIR/docker-compose.yml down
+fi

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/before_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/before_script.travis.sh b/tests/before_script.travis.sh
index b6d3966..9ec7a4a 100755
--- a/tests/before_script.travis.sh
+++ b/tests/before_script.travis.sh
@@ -16,4 +16,6 @@
 # limitations under the License.
 #
 
-tests/build-docker.sh
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+$DIR/build-docker.sh

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
index 4438396..ed43715 100755
--- a/tests/build-docker.sh
+++ b/tests/build-docker.sh
@@ -39,6 +39,12 @@ mkdir assembly
 cp dist/lib/*.jar assembly/
 mkdir -p lib/spark
 cp dist/lib/spark/*.jar lib/spark
+rm *.tar.gz
 docker build -t predictionio/pio .
 popd
-docker build -t predictionio/pio-testing $DIR
+
+if [ "$ES_VERSION" = "1" ]; then
+    docker build -t predictionio/pio-testing-es1 -f $DIR/Dockerfile-es1 $DIR
+else
+    docker build -t predictionio/pio-testing $DIR
+fi

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/docker-compose-es1.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose-es1.yml b/tests/docker-compose-es1.yml
new file mode 100644
index 0000000..381f41c
--- /dev/null
+++ b/tests/docker-compose-es1.yml
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "2"
+services:
+  elasticsearch:
+    image: elasticsearch:1
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+  hbase:
+    image: harisekhon/hbase:1.0
+  postgres:
+    image: postgres:9
+    environment:
+      POSTGRES_USER: pio
+      POSTGRES_PASSWORD: pio
+      POSTGRES_INITDB_ARGS: --encoding=UTF8
+  pio-testing:
+    image: predictionio/pio-testing-es1:latest
+    depends_on:
+      - elasticsearch
+      - hbase
+      - postgres
+    volumes:
+      - ~/.ivy2:/root/.ivy2
+      - ~/.sbt:/root/.sbt

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/docker-files/env-conf/pio-env-es1.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env-es1.sh b/tests/docker-files/env-conf/pio-env-es1.sh
new file mode 100644
index 0000000..e1076ba
--- /dev/null
+++ b/tests/docker-files/env-conf/pio-env-es1.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+#
+# Copy this file as pio-env.sh and edit it for your site's configuration.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# PredictionIO Main Configuration
+#
+# This section controls core behavior of PredictionIO. It is very likely that
+# you need to change these to fit your site.
+
+# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
+SPARK_HOME=$SPARK_HOME
+
+POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar
+MYSQL_JDBC_DRIVER=
+
+# ES_CONF_DIR: You must configure this if you have advanced configuration for
+#              your Elasticsearch setup.
+# ES_CONF_DIR=/opt/elasticsearch
+
+# HADOOP_CONF_DIR: You must configure this if you intend to run PredictionIO
+#                  with Hadoop 2.
+# HADOOP_CONF_DIR=/opt/hadoop
+
+# HBASE_CONF_DIR: You must configure this if you intend to run PredictionIO
+#                 with HBase on a remote cluster.
+HBASE_CONF_DIR=$PIO_HOME/conf
+
+# Filesystem paths where PredictionIO uses as block storage.
+PIO_FS_BASEDIR=$HOME/.pio_store
+PIO_FS_ENGINESDIR=$PIO_FS_BASEDIR/engines
+PIO_FS_TMPDIR=$PIO_FS_BASEDIR/tmp
+
+# PredictionIO Storage Configuration
+#
+# This section controls programs that make use of PredictionIO's built-in
+# storage facilities. Default values are shown below.
+#
+# For more information on storage configuration please refer to
+# https://docs.prediction.io/system/anotherdatastore/
+
+# Storage Repositories
+
+# Default is to use PostgreSQL
+PIO_STORAGE_REPOSITORIES_METADATA_NAME=pio_meta
+PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$PIO_STORAGE_REPOSITORIES_METADATA_SOURCE
+
+PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME=pio_event
+PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE
+
+PIO_STORAGE_REPOSITORIES_MODELDATA_NAME=pio_model
+PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE
+
+# Storage Data Sources
+
+# PostgreSQL Default Settings
+# Please change "pio" to your database name in PIO_STORAGE_SOURCES_PGSQL_URL
+# Please change PIO_STORAGE_SOURCES_PGSQL_USERNAME and
+# PIO_STORAGE_SOURCES_PGSQL_PASSWORD accordingly
+PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc
+PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://postgres/pio
+PIO_STORAGE_SOURCES_PGSQL_USERNAME=pio
+PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
+
+# MySQL Example
+# PIO_STORAGE_SOURCES_MYSQL_TYPE=jdbc
+# PIO_STORAGE_SOURCES_MYSQL_URL=jdbc:mysql://localhost/pio
+# PIO_STORAGE_SOURCES_MYSQL_USERNAME=pio
+# PIO_STORAGE_SOURCES_MYSQL_PASSWORD=pio
+
+# Elasticsearch Example
+PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
+#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio
+PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch
+PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
+#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME
+
+# Local File System Example
+PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs
+PIO_STORAGE_SOURCES_LOCALFS_PATH=$PIO_FS_BASEDIR/local_models
+
+# HBase Example
+PIO_STORAGE_SOURCES_HBASE_TYPE=hbase
+#PIO_STORAGE_SOURCES_HBASE_HOME=$HBASE_HOME
+
+# HDFS config
+PIO_STORAGE_SOURCES_HDFS_TYPE=hdfs
+PIO_STORAGE_SOURCES_HDFS_PATH=/hdfs_models

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/run_docker.sh
----------------------------------------------------------------------
diff --git a/tests/run_docker.sh b/tests/run_docker.sh
index fe07957..6257fa2 100755
--- a/tests/run_docker.sh
+++ b/tests/run_docker.sh
@@ -46,8 +46,17 @@ shift
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-docker-compose -f $DIR/docker-compose.yml run \
-  -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
-  -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
-  -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
-  pio-testing $@
+if [ "$ES_VERSION" = "1" ]; then
+    docker-compose -f $DIR/docker-compose-es1.yml run \
+      -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
+      -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
+      -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
+      pio-testing $@
+else
+    docker-compose -f $DIR/docker-compose.yml run \
+      -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
+      -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
+      -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
+      pio-testing $@
+fi
+


Mime
View raw message