orc-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [orc] 01/02: ORC-560: Improve docker scripts and add centos 8 and debian 10.
Date Wed, 23 Oct 2019 17:49:24 GMT
This is an automated email from the ASF dual-hosted git repository.

omalley pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/orc.git

commit 2ef409f4cd46af71e3018c60e093a9eaefd7cdfb
Author: Owen O'Malley <omalley@apache.org>
AuthorDate: Thu Oct 17 06:57:54 2019 -0700

    ORC-560: Improve docker scripts and add centos 8 and debian 10.
    
    Fixes #437
    
    Signed-off-by: Owen O'Malley <omalley@apache.org>
---
 docker/centos8/Dockerfile         | 53 +++++++++++++++++++++++++++++++++++++++
 docker/debian10/Dockerfile        | 43 +++++++++++++++++++++++++++++++
 docker/os-list.txt                |  3 +++
 docker/run-all.sh                 | 31 ++++++++++++-----------
 docker/{run-all.sh => run-one.sh} | 48 ++++++++++++++++-------------------
 5 files changed, 136 insertions(+), 42 deletions(-)

diff --git a/docker/centos8/Dockerfile b/docker/centos8/Dockerfile
new file mode 100644
index 0000000..4a6d2c6
--- /dev/null
+++ b/docker/centos8/Dockerfile
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ORC compile for CentOS 8
+#
+
+FROM centos:8
+MAINTAINER Owen O'Malley <owen@hortonworks.com>
+
+RUN yum check-update || true
+RUN yum install -y \
+  cmake \
+  curl-devel \
+  cyrus-sasl-devel \
+  expat-devel \
+  gcc \
+  gcc-c++ \
+  gettext-devel \
+  git \
+  java-1.8.0-openjdk \
+  java-1.8.0-openjdk-devel \
+  libtool \
+  make \
+  maven \
+  openssl-devel \
+  tar \
+  wget \
+  which \
+  zlib-devel
+
+ENV TZ=America/Los_Angeles
+WORKDIR /root
+
+VOLUME /root/.m2/repository
+
+CMD git clone https://github.com/apache/orc.git -b master && \
+  mkdir orc/build && \
+  cd orc/build && \
+  cmake .. && \ 
+  make package test-out
diff --git a/docker/debian10/Dockerfile b/docker/debian10/Dockerfile
new file mode 100644
index 0000000..fb73d5b
--- /dev/null
+++ b/docker/debian10/Dockerfile
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ORC compile for Debian 10
+#
+
+FROM debian:buster
+MAINTAINER Owen O'Malley <omalley@apache.org>
+
+RUN apt-get update
+RUN apt-get install -y \
+  cmake \
+  gcc \
+  g++ \
+  git \
+  libsasl2-dev \
+  libssl-dev \
+  openjdk-11-jdk \
+  make \
+  maven
+
+WORKDIR /root
+
+VOLUME /root/.m2/repository
+
+CMD git clone https://github.com/apache/orc.git -b master && \
+  mkdir orc/build && \
+  cd orc/build && \
+  cmake .. && \
+  make package test-out
diff --git a/docker/os-list.txt b/docker/os-list.txt
index dcb50f0..a0bf99d 100644
--- a/docker/os-list.txt
+++ b/docker/os-list.txt
@@ -1,6 +1,9 @@
 centos6
 centos7
+centos8
 debian8
+debian9
+debian10
 ubuntu12
 ubuntu14
 ubuntu16
diff --git a/docker/run-all.sh b/docker/run-all.sh
index d770bae..8f1436b 100755
--- a/docker/run-all.sh
+++ b/docker/run-all.sh
@@ -24,31 +24,32 @@ MAKEDIR="mkdir orc/build && cd orc/build"
 VOLUME="--volume m2cache:/root/.m2/repository"
 mkdir -p logs
 
+function failure {
+    echo "Failed tests"
+    grep -h "FAILED " logs/*-test.log
+    exit 1
+}
+
 start=`date`
 for os in `cat os-list.txt`; do
   echo "Building $os"
   ( cd $os && docker build -t "orc-$os" . ) > logs/$os-build.log 2>&1 ||
exit 1
 done
 testStart=`date`
+rm -f logs/pids.txt
 
 for os in `cat os-list.txt`; do
-  echo "Testing $os"
-  case $os in
-  centos6|ubuntu12)
-     OPTS="-DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local"
-     ;;
-  centos7|debian8|ubuntu14)
-     OPTS="-DSNAPPY_HOME=/usr/local"
-     ;;
-  *)
-     OPTS=""
-     ;;
-  esac
-  docker run $VOLUME "orc-$os" /bin/bash -c \
-	 "$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out"
\
-         > logs/$os-test.log 2>&1 || exit 1
+    ./run-one.sh $1 $2 $os > logs/$os-test.log 2>&1 &
+    echo "$!" >> logs/pids.txt
+    echo "Launching $os as $!"
+done
+
+for job in `cat logs/pids.txt`; do
+    echo "Waiting for $job"
+    wait $job || failure
 done
 
+echo ""
 echo "Build start: $start"
 echo "Test start: $testStart"
 echo "End:" `date`
diff --git a/docker/run-all.sh b/docker/run-one.sh
similarity index 63%
copy from docker/run-all.sh
copy to docker/run-one.sh
index d770bae..7688560 100755
--- a/docker/run-all.sh
+++ b/docker/run-one.sh
@@ -18,37 +18,31 @@
 GITHUB_USER=$1
 URL=https://github.com/$GITHUB_USER/orc.git
 BRANCH=$2
+OS=$3
+
+function failure {
+    echo "FAILED $OS"
+    exit 1
+}
 
 CLONE="git clone $URL -b $BRANCH"
 MAKEDIR="mkdir orc/build && cd orc/build"
 VOLUME="--volume m2cache:/root/.m2/repository"
-mkdir -p logs
 
-start=`date`
-for os in `cat os-list.txt`; do
-  echo "Building $os"
-  ( cd $os && docker build -t "orc-$os" . ) > logs/$os-build.log 2>&1 ||
exit 1
-done
-testStart=`date`
+echo "Started $GITHUB_USER/$BRANCH on $OS at $(date)"
 
-for os in `cat os-list.txt`; do
-  echo "Testing $os"
-  case $os in
-  centos6|ubuntu12)
-     OPTS="-DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local"
-     ;;
-  centos7|debian8|ubuntu14)
-     OPTS="-DSNAPPY_HOME=/usr/local"
-     ;;
-  *)
-     OPTS=""
-     ;;
-  esac
-  docker run $VOLUME "orc-$os" /bin/bash -c \
+case $OS in
+centos6|ubuntu12)
+   OPTS="-DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local"
+   ;;
+centos7|debian8|ubuntu14)
+   OPTS="-DSNAPPY_HOME=/usr/local"
+   ;;
+*)
+   OPTS=""
+   ;;
+esac
+docker run $VOLUME "orc-$OS" /bin/bash -c \
 	 "$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out"
\
-         > logs/$os-test.log 2>&1 || exit 1
-done
-
-echo "Build start: $start"
-echo "Test start: $testStart"
-echo "End:" `date`
+         || failure
+echo "Finished $OS at $(date)"


Mime
View raw message