singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhaoj...@apache.org
Subject [2/3] incubator-singa git commit: SINGA-284 - Add python unittest into Jenkins and link static libs into whl file
Date Sat, 31 Dec 2016 03:59:46 GMT
SINGA-284 - Add python unittest into Jenkins and link static libs into whl file

Update the script to package the whl file into a folder, whose name indicates the version
of OS, cuda and cudnn.
Update the jenkins/docker files to avoid installing libprotobuf and openblas


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/761c463d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/761c463d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/761c463d

Branch: refs/heads/master
Commit: 761c463dd8a2aedd59af7f3a936e359080e6828d
Parents: e44658a
Author: wangwei <wangwei@comp.nus.edu.sg>
Authored: Fri Dec 23 15:16:19 2016 +0800
Committer: wangwei <wangwei@comp.nus.edu.sg>
Committed: Sat Dec 24 09:33:27 2016 +0800

----------------------------------------------------------------------
 tool/jenkins/README.md                     |  8 ++++----
 tool/jenkins/docker/ubuntu14.04/Dockerfile | 11 +++++++----
 tool/jenkins/docker/ubuntu16.04/Dockerfile | 17 +++++++++++------
 tool/jenkins/jenkins_wheel.sh              | 21 +++++++++++++++++----
 4 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/README.md
----------------------------------------------------------------------
diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md
index d20470b..1b5fb7c 100644
--- a/tool/jenkins/README.md
+++ b/tool/jenkins/README.md
@@ -30,19 +30,19 @@ Create a multi-configuration project and configure project as follows:
   * Poll SCM - Schedule - ``H/30 * * * *`` (pull every 30 minutes)
 
 ### Configuration Matrix
-  * User-defined Axis - name ``support`` values ``BLAS CUDNN``
+  * User-defined Axis - name ``lang`` values ``CPP CUDA``
   * Slave - name ``env`` Node/label ``tick all nodes``
 
 ### Build
-  * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $support``
+  * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $lang``
 
 ### Post-build Actions
   * Archive the artifacts - ``build/python/dist/**.whl``
+  * Send build artifacts over SSH - source files ``build/python/dist/*.tar.gz``, Remove prefix
`build/python/dist`, Exec command `tar xf *.tar.gz && rm *.tar.gz`
   * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml``
 
 ## Docker Images
-We provide in `docker` a number of singa docker images for Jenkins to use as
-slaves.
+We provide in `docker` a number of singa docker images for Jenkins to use as slaves.
 
 ## Access Control
 Use `Role Strategy Plugin` to give read access for anonymous users.

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/docker/ubuntu14.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile
index 9b974d2..6d96983 100644
--- a/tool/jenkins/docker/ubuntu14.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu14.04/Dockerfile
@@ -7,10 +7,12 @@
 FROM nvidia/cuda:8.0-cudnn5-devel
 
 # install dependencies
-RUN apt-get update && apt-get install -y \
-    git vim wget build-essential autoconf libtool cmake \
-    libpcre3-dev python-dev python-pip && \
-    pip install numpy protobuf unittest-xml-reporting
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev
protobuf-compiler python-dev python-pip wget\
+    && apt-get clean && apt-get autoremove && apt-get autoclean \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting
protobuf>=2.5.0,<3
+
 
 # install swig 3.0
 RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
@@ -21,6 +23,7 @@ RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz &&
\
 ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
 ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
 ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+ENV OS_VERSION ubuntu14.04
 
 # download singa source
 RUN git clone https://github.com/apache/incubator-singa.git

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/docker/ubuntu16.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile
index 612dd8b..1a9434c 100644
--- a/tool/jenkins/docker/ubuntu16.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu16.04/Dockerfile
@@ -4,21 +4,26 @@
 FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04
 
 # install dependencies
-RUN apt-get update && apt-get install -y \
-    git wget build-essential autoconf libtool cmake \
-    libpcre3-dev python-dev python-pip swig && \
-    pip install numpy protobuf unittest-xml-reporting
+
+MAINTAINER incubator-singa dev@singa.incubator.apache.org
+
+# install dependencies
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev
protobuf-compiler python-dev python-pip swig wget openssh-server\
+    && apt-get clean && apt-get autoremove && apt-get autoclean \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting
protobuf>=2.5.0,<3
 
 # set environment
 ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
 ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
 ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+ENV OS_VERSION ubuntu16.04
 
 # download singa source
 RUN git clone https://github.com/apache/incubator-singa.git
 
-# install ssh service
-RUN apt-get install -y openssh-server
+# config ssh service
 RUN mkdir /var/run/sshd
 RUN echo 'root:singa' | chpasswd
 RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/jenkins_wheel.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_wheel.sh b/tool/jenkins/jenkins_wheel.sh
index c3d075f..f131f68 100644
--- a/tool/jenkins/jenkins_wheel.sh
+++ b/tool/jenkins/jenkins_wheel.sh
@@ -27,26 +27,39 @@ echo OS version: `cat /etc/issue`
 echo kernal version: `uname -a`
 echo CUDA version: $CUDA_VERSION
 echo CUDNN version: $CUDNN_VERSION
+echo OS name: $OS_VERSION
+COMMIT=`git rev-parse HEAD`
+echo COMMIT HASH: $COMMIT
 # set parameters
+CUDA="OFF"
 CUDNN="OFF"
-if [ $1 = "CUDNN" ]; then
+FOLDER=$COMMIT/$OS_VERSION-cpp/
+if [ $1 = "CUDA" ]; then
+  CUDA="ON"
   CUDNN="ON"
+  FOLDER=$COMMIT/$OS_VERSION-cuda$CUDA_VERSION-cudnn$CUDNN_VERSION/
 fi
+echo wheel file folder: build/python/dist/whl/$FOLDER
+
 # setup env
 rm -rf build
 mkdir build
+
 # compile singa c++
 cd build
-cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN -DUSE_MODULES=ON ../
+cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDA -DUSE_MODULES=ON -DUSE_MODULES=ON ../
 make
 # unit test cpp code
 ./bin/test_singa --gtest_output=xml:./gtest.xml
 # compile pysinga
 cd python
 python setup.py bdist_wheel
-# rename dist
+# mv whl file to a folder whose name identifies the OS, CUDA, CUDNN etc.
 cd dist
-mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl
+mkdir -p $FOLDER
+mv *.whl $FOLDER/
+tar czf $FOLDER.tar.gz $FOLDER/*
+
 # unit test python code
 cd ../../../test/python
 PYTHONPATH=../../build/python/ python run.py


Mime
View raw message