predictionio-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From don...@apache.org
Subject [16/50] [abbrv] incubator-predictionio git commit: [PIO-49] Additional improvement to ES5 support
Date Tue, 25 Apr 2017 22:11:39 GMT
[PIO-49] Additional improvement to ES5 support

* Add option to build ES1 or ES5-default distribution
* Aggregate all storage subprojects
* Update deprecated SBT code
* Update Docker Compose to use official Elastic images
* Update Travis to use new build option


Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/31c4bd19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/31c4bd19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/31c4bd19

Branch: refs/heads/livedoc
Commit: 31c4bd1927eb9ae753f82b39b71ff3df7ca779de
Parents: d78b3cb
Author: Donald Szeto <donald@apache.org>
Authored: Tue Mar 7 23:16:33 2017 -0800
Committer: Donald Szeto <donald@apache.org>
Committed: Tue Mar 7 23:16:33 2017 -0800

----------------------------------------------------------------------
 .travis.yml                      |  4 +--
 build.sbt                        |  6 +++++
 make-distribution.sh             | 47 ++++++++++++++++++++++++++++++++++-
 storage/elasticsearch/build.sbt  |  3 +--
 storage/elasticsearch1/build.sbt |  2 +-
 storage/hbase/build.sbt          |  2 +-
 storage/hdfs/build.sbt           |  2 +-
 storage/jdbc/build.sbt           |  2 +-
 storage/localfs/build.sbt        |  2 +-
 tests/build-docker.sh            |  8 ++++--
 tests/docker-compose.yml         |  4 ++-
 11 files changed, 69 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index b763e2d..799ec3d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,8 +46,8 @@ env:
   matrix:
     - BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
     - BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
-    - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
-    - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+    - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
ES_VERSION=5
+    - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
ES_VERSION=5
 
 before_install:
   - unset SBT_OPTS JVM_OPTS

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 98444b9..fa5ba29 100644
--- a/build.sbt
+++ b/build.sbt
@@ -166,6 +166,12 @@ val root = (project in file(".")).
     common,
     core,
     data,
+    dataElasticsearch1,
+    dataElasticsearch,
+    dataHbase,
+    dataHdfs,
+    dataJdbc,
+    dataLocalfs,
     tools,
     e2)
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index 31954c0..c360c0e 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -19,6 +19,45 @@
 
 set -e
 
+usage ()
+{
+    echo "Usage: $0 [-h|--help] [--with-es=x]"
+    echo ""
+    echo "  -h|--help    Show usage"
+    echo ""
+    echo "  --with-es=1  Build distribution with Elasticsearch 1 support as default"
+    echo "  --with-es=5  Build distribution with Elasticsearch 5 support as default"
+}
+
+ES_VERSION=1
+
+for i in "$@"
+do
+case $i in
+    -h|--help)
+    usage
+    shift
+    exit
+    ;;
+    --with-es=*)
+    ES_VERSION="${i#*=}"
+    shift
+    ;;
+    *)
+    usage
+    exit 1
+    ;;
+esac
+done
+
+if [ "$ES_VERSION" = "1" ] || [ "$ES_VERSION" = "5" ]
+then
+    echo -e "\033[0;32mBuilding with Elasticsearch $ES_VERSION support as the default choice\033[0m"
+else
+    usage
+    exit 1
+fi
+
 FWDIR="$(cd `dirname $0`; pwd)"
 DISTDIR="${FWDIR}/dist"
 
@@ -45,7 +84,13 @@ cp ${FWDIR}/project/build.properties ${DISTDIR}/project
 cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt
 cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib
 cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark
-cp ${FWDIR}/assembly/extra/*jar ${DISTDIR}/lib/extra
+
+if [ "$ES_VERSION" = "5" ]
+then
+    mv ${DISTDIR}/lib/spark/pio-data-elasticsearch1-assembly-*.jar ${DISTDIR}/lib/extra
+else
+    mv ${DISTDIR}/lib/spark/pio-data-elasticsearch-assembly-*.jar ${DISTDIR}/lib/extra
+fi
 
 rm -f ${DISTDIR}/lib/*javadoc.jar
 rm -f ${DISTDIR}/lib/*sources.jar

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index b22cbd8..925b66f 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -53,5 +53,4 @@ assemblyShadeRules in assembly := Seq(
 // skip test in assembly
 test in assembly := {}
 
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile
/ "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
-
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly"
/ "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
index 8c29b84..457aba2 100644
--- a/storage/elasticsearch1/build.sbt
+++ b/storage/elasticsearch1/build.sbt
@@ -43,5 +43,5 @@ assemblyMergeStrategy in assembly := {
 // skip test in assembly
 test in assembly := {}
 
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile
/ "assembly" / "extra" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly"
/ "spark" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
index 5856a5e..673697b 100644
--- a/storage/hbase/build.sbt
+++ b/storage/hbase/build.sbt
@@ -52,5 +52,5 @@ assemblyMergeStrategy in assembly := {
 // skip test in assembly
 test in assembly := {}
 
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile
/ "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly"
/ "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index 9f064c6..35c9153 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -40,5 +40,5 @@ assemblyMergeStrategy in assembly := {
 // skip test in assembly
 test in assembly := {}
 
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile
/ "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly"
/ "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
index 63d420b..0a07ee0 100644
--- a/storage/jdbc/build.sbt
+++ b/storage/jdbc/build.sbt
@@ -43,5 +43,5 @@ assemblyMergeStrategy in assembly := {
 // skip test in assembly
 test in assembly := {}
 
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile
/ "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly"
/ "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
index 2cf9977..f94b355 100644
--- a/storage/localfs/build.sbt
+++ b/storage/localfs/build.sbt
@@ -40,5 +40,5 @@ assemblyMergeStrategy in assembly := {
 // skip test in assembly
 test in assembly := {}
 
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile
/ "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly"
/ "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
index 459b929..a899a18 100755
--- a/tests/build-docker.sh
+++ b/tests/build-docker.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/usr/bin/env bash -ex
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -29,7 +29,11 @@ fi
 
 docker pull predictionio/pio-testing-base
 pushd $DIR/..
-./make-distribution.sh
+if [ -z "$ES_VERSION" ]; then
+    ./make-distribution.sh
+else
+    ./make-distribution.sh --with-es=$ES_VERSION
+fi
 sbt/sbt clean
 mkdir assembly
 cp dist/lib/*.jar assembly/

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index b556f7b..b6774d3 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -16,7 +16,9 @@
 version: "2"
 services:
   elasticsearch:
-    image: elasticsearch:5-alpine
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
+    environment:
+      - xpack.security.enabled=false
   hbase:
     image: harisekhon/hbase:1.0
   postgres:


Mime
View raw message