singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [2/5] incubator-singa git commit: SINGA-346 Update cudnn from V5 to V7
Date Tue, 24 Apr 2018 10:52:14 GMT
SINGA-346 Update cudnn from V5 to V7

Update conda build scripts to include cudnn (.h and .so) in conda singa package.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/ef5fdaf6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/ef5fdaf6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/ef5fdaf6

Branch: refs/heads/master
Commit: ef5fdaf652b1ac82e0edf0c07fc0499d13238581
Parents: c9f9471
Author: Wang Wei <dcswaw@nus.edu.sg>
Authored: Sun Apr 15 23:29:08 2018 +0800
Committer: Wang Wei <dcswaw@nus.edu.sg>
Committed: Wed Apr 18 22:36:40 2018 +0800

----------------------------------------------------------------------
 tool/conda/README.md                            | 29 ++++++++
 tool/conda/build.sh                             | 17 +++--
 tool/conda/meta.yaml                            |  9 ++-
 tool/jenkins/docker/devel/centos6/Dockerfile    | 64 ------------------
 .../docker/devel/native/centos6/Dockerfile      | 64 ++++++++++++++++++
 .../docker/devel/native/ubuntu/Dockerfile       | 70 ++++++++++++++++++++
 tool/jenkins/docker/devel/ubuntu/Dockerfile     | 70 --------------------
 7 files changed, 178 insertions(+), 145 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/conda/README.md
----------------------------------------------------------------------
diff --git a/tool/conda/README.md b/tool/conda/README.md
new file mode 100644
index 0000000..d30cf84
--- /dev/null
+++ b/tool/conda/README.md
@@ -0,0 +1,29 @@
+# Package SINGA using conda-build
+
+[conda-build](https://conda.io/docs/user-guide/tasks/build-packages/index.html) is a packaging
tool like apt-get, which is associated with [anaconda cloud](https://anaconda.org/) for package
management for both python and cpp libraries.
+
+
+## Environment variables
+
+Depending on the building evironment, export the corresponding build string.
+
+	# for singa with gpu, e.g. cuda8.0-cudnn7.0.5
+    export BUILD_STR=cudax.y-cudnna.b.c
+
+    # for singa running only on cpu
+    export BUILD_STR=cpu
+
+
+To package SINGA with CUDA and CUDNN, 
+
+    export CUDNN_PATH=<path to cudnn folder>
+
+this folder should include a subfolder `include/cudnn.h` for the header file, and another
subfolder `lib64` for the shared libraries. The BUILD_STR and CUDNN_PATH must be consistent.
For example, if CUDNN_PATH is set, then BUILD_STR must be like cudax.y-cudnna.b.c.
+
+## Instruction
+
+After exporting the environment variables, execute the following command to compile SINGA
and package it
+
+    conda-build .
+
+You will see the package path from the screen output.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/conda/build.sh
----------------------------------------------------------------------
diff --git a/tool/conda/build.sh b/tool/conda/build.sh
index 6b3f262..2027725 100644
--- a/tool/conda/build.sh
+++ b/tool/conda/build.sh
@@ -16,19 +16,24 @@
 #
 
 # to compile swig api files which depdend on numpy.i
-export export CPLUS_INCLUDE_PATH=`python -c "from __future__ import print_function; import
numpy; print(numpy.get_include())"`:$CPLUS_INCLUDE_PATH
+export CPLUS_INCLUDE_PATH=`python -c "from __future__ import print_function; import numpy;
print(numpy.get_include())"`:$CPLUS_INCLUDE_PATH
 
 # to let cmake use the dependent libs installed by conda, including python
 export CMAKE_PREFIX_PATH=$PREFIX
-export CMAKE_INCLUDE_PATH=$SINGA_INCLUDE_PATH
-export CMAKE_LIBRARY_PATH=$SINGA_LIBRARY_PATH
+export CMAKE_INCLUDE_PATH=$PREFIX/include
+export CMAKE_LIBRARY_PATH=$PREFIX/lib
 
 mkdir build
 cd build
-USE_CUDA=ON
-# singa with cuda and cudnn has the name as : singa-cudaxx-cudnnxx
-if  [ "$PKG_NAME" == "singa" ]; then USE_CUDA=OFF; fi
+USE_CUDA=OFF
+if [ -z ${CUDNN_PATH+x} ]; then
+	USE_CUDA=ON
+	cp $CUDNN_PATH/include $PREFIX/include 
+	cp -P $CUDNN_PATH/lib64/* $PREFIX/lib/
+fi
+
 PYTHON3=OFF
+# PY3K is set by conda
 if  [ "$PY3K" == "True" ]; then PYTHON3=ON; fi
 cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -DUSE_CUDA=$USE_CUDA -DPYTHON3=$PYTHON3 ..
 make

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/conda/meta.yaml
----------------------------------------------------------------------
diff --git a/tool/conda/meta.yaml b/tool/conda/meta.yaml
index 5ff87a9..de542cd 100644
--- a/tool/conda/meta.yaml
+++ b/tool/conda/meta.yaml
@@ -1,16 +1,15 @@
 package:
-  name: {{ environ.get('SINGA_NAME', 'singa') }}
+  name: singa
   version: "{{ GIT_DESCRIBE_TAG }}"
 
 source:
   git_url: https://github.com/apache/incubator-singa.git
 
 build:
-  number: {{ GIT_DESCRIBE_NUMBER }}
+  number: 0
+  string: py{{py}}_{{ environ.get('BUILD_STR', 'cpu') }}
   script_env:
-    - CONDA_BLD_PATH
-    - SINGA_INCLUDE_PATH
-    - SINGA_LIBRARY_PATH
+    - CUDNN_PATH
 
 requirements:
   build:

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/jenkins/docker/devel/centos6/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/devel/centos6/Dockerfile b/tool/jenkins/docker/devel/centos6/Dockerfile
deleted file mode 100644
index 61b30f6..0000000
--- a/tool/jenkins/docker/devel/centos6/Dockerfile
+++ /dev/null
@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Base unbuntu 14.04 image from nvidia/cuda
-# Change tags to build with different cuda/cudnn versions:
-#   FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04
-#   FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
-#   FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04
-#   FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04
-FROM nvidia/cuda:7.5-cudnn5-devel-centos6
-
-# install dependencies
-RUN yum -y update && yum -y install git wget openssh-server cmake
-
-
-# install conda, conda-build and anaconda-client
-RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
-RUN bash miniconda.sh -b -p /root/miniconda
-ENV PATH /root/miniconda/bin:${PATH}
-RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no
-RUN /root/miniconda/bin/conda update -q conda
-RUN /root/miniconda/bin/conda install conda-build
-RUN /root/miniconda/bin/conda install anaconda-client
-
-# set environment
-ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
-# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
-# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
-
-# download singa source
-# RUN git clone https://github.com/apache/incubator-singa.git
-
-# config ssh service
-RUN mkdir /var/run/sshd
-RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
-RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key
-
-RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
-RUN sed -ri 's/#UsePAM no/UsePAM no/g' /etc/ssh/sshd_config
-RUN echo 'root:singa' | chpasswd
-
-#RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-# SSH login fix. Otherwise user is kicked off after login
-#RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
-
-# dump environment variables into files, so that ssh can see also
-RUN env | grep _ >> /etc/environment
-
-EXPOSE 22
-
-CMD ["/usr/sbin/sshd", "-D"]

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/jenkins/docker/devel/native/centos6/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/devel/native/centos6/Dockerfile b/tool/jenkins/docker/devel/native/centos6/Dockerfile
new file mode 100644
index 0000000..61b30f6
--- /dev/null
+++ b/tool/jenkins/docker/devel/native/centos6/Dockerfile
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Base unbuntu 14.04 image from nvidia/cuda
+# Change tags to build with different cuda/cudnn versions:
+#   FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04
+#   FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
+#   FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04
+#   FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04
+FROM nvidia/cuda:7.5-cudnn5-devel-centos6
+
+# install dependencies
+RUN yum -y update && yum -y install git wget openssh-server cmake
+
+
+# install conda, conda-build and anaconda-client
+RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+RUN bash miniconda.sh -b -p /root/miniconda
+ENV PATH /root/miniconda/bin:${PATH}
+RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no
+RUN /root/miniconda/bin/conda update -q conda
+RUN /root/miniconda/bin/conda install conda-build
+RUN /root/miniconda/bin/conda install anaconda-client
+
+# set environment
+ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
+# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
+# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+
+# download singa source
+# RUN git clone https://github.com/apache/incubator-singa.git
+
+# config ssh service
+RUN mkdir /var/run/sshd
+RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
+RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key
+
+RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+RUN sed -ri 's/#UsePAM no/UsePAM no/g' /etc/ssh/sshd_config
+RUN echo 'root:singa' | chpasswd
+
+#RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# SSH login fix. Otherwise user is kicked off after login
+#RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+# dump environment variables into files, so that ssh can see also
+RUN env | grep _ >> /etc/environment
+
+EXPOSE 22
+
+CMD ["/usr/sbin/sshd", "-D"]

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/jenkins/docker/devel/native/ubuntu/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/devel/native/ubuntu/Dockerfile b/tool/jenkins/docker/devel/native/ubuntu/Dockerfile
new file mode 100644
index 0000000..302c579
--- /dev/null
+++ b/tool/jenkins/docker/devel/native/ubuntu/Dockerfile
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Base unbuntu 14.04 image from nvidia/cuda
+# Change tags to build with different cuda/cudnn versions:
+FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04
+#   FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
+#   FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04
+#   FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04
+# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
+
+# install dependencies
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git build-essential autoconf libtool
cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler
wget openssh-server swig\
+    && apt-get clean && apt-get autoremove && apt-get autoclean \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf
future
+
+
+# install swig > 3.0.10 for ubuntu < 16.04
+# RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
+#     tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \
+#     ./configure && make && make install
+
+# install conda, conda-build and anaconda-client
+RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+RUN bash miniconda.sh -b -p /root/miniconda
+RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no
+RUN /root/miniconda/bin/conda update -q conda
+RUN /root/miniconda/bin/conda install conda-build
+RUN /root/miniconda/bin/conda install anaconda-client
+ENV PATH /root/miniconda/bin:${PATH}
+
+# set environment
+ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
+# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
+# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+
+# download singa source
+# RUN git clone https://github.com/apache/incubator-singa.git
+
+# config ssh service
+RUN mkdir /var/run/sshd
+RUN echo 'root:singa' | chpasswd
+# for ubuntu 14.04
+# RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# for ubuntu 16.04 prohibit
+RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+# dump environment variables into files, so that ssh can see also
+RUN env | grep _ >> /etc/environment
+
+EXPOSE 22
+
+CMD ["/usr/sbin/sshd", "-D"]

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/jenkins/docker/devel/ubuntu/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/devel/ubuntu/Dockerfile b/tool/jenkins/docker/devel/ubuntu/Dockerfile
deleted file mode 100644
index 302c579..0000000
--- a/tool/jenkins/docker/devel/ubuntu/Dockerfile
+++ /dev/null
@@ -1,70 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Base unbuntu 14.04 image from nvidia/cuda
-# Change tags to build with different cuda/cudnn versions:
-FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04
-#   FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
-#   FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04
-#   FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04
-# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
-
-# install dependencies
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends git build-essential autoconf libtool
cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler
wget openssh-server swig\
-    && apt-get clean && apt-get autoremove && apt-get autoclean \
-    && rm -rf /var/lib/apt/lists/* \
-    && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf
future
-
-
-# install swig > 3.0.10 for ubuntu < 16.04
-# RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
-#     tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \
-#     ./configure && make && make install
-
-# install conda, conda-build and anaconda-client
-RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
-RUN bash miniconda.sh -b -p /root/miniconda
-RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no
-RUN /root/miniconda/bin/conda update -q conda
-RUN /root/miniconda/bin/conda install conda-build
-RUN /root/miniconda/bin/conda install anaconda-client
-ENV PATH /root/miniconda/bin:${PATH}
-
-# set environment
-ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
-# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
-# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
-
-# download singa source
-# RUN git clone https://github.com/apache/incubator-singa.git
-
-# config ssh service
-RUN mkdir /var/run/sshd
-RUN echo 'root:singa' | chpasswd
-# for ubuntu 14.04
-# RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-# for ubuntu 16.04 prohibit
-RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-# SSH login fix. Otherwise user is kicked off after login
-RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
-
-# dump environment variables into files, so that ssh can see also
-RUN env | grep _ >> /etc/environment
-
-EXPOSE 22
-
-CMD ["/usr/sbin/sshd", "-D"]


Mime
View raw message