arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-490: Python: Update manylinux1 build scripts
Date Wed, 18 Jan 2017 00:09:31 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 850774efe -> b1472305c


ARROW-490: Python: Update manylinux1 build scripts

Through the usage of the ExternalProject command, a lot has become much
simpler.

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #290 from xhochy/ARROW-490 and squashes the following commits:

64b024b [Uwe L. Korn] ARROW-490: Python: Update manylinux1 build scripts


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b1472305
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b1472305
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b1472305

Branch: refs/heads/master
Commit: b1472305c7262b07f2e9bf3d5db4eeff09ec5dc4
Parents: 850774e
Author: Uwe L. Korn <uwelk@xhochy.com>
Authored: Tue Jan 17 19:09:24 2017 -0500
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Tue Jan 17 19:09:24 2017 -0500

----------------------------------------------------------------------
 cpp/CMakeLists.txt                              |  4 ++--
 cpp/cmake_modules/BuildUtils.cmake              | 12 ++++++++++
 .../Dockerfile-parquet_arrow-base-x86_64        | 23 +-------------------
 python/manylinux1/Dockerfile-x86_64             | 19 +++++-----------
 python/manylinux1/build_arrow.sh                |  8 ++++---
 5 files changed, 25 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 8a2cfc5..885ab19 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -598,7 +598,7 @@ if (ARROW_JEMALLOC)
   find_package(jemalloc)
 
   if(NOT JEMALLOC_FOUND)
-    set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep")
+    set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
     set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
     set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
     set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
@@ -611,7 +611,7 @@ if (ARROW_JEMALLOC)
         CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
         BUILD_IN_SOURCE 1
         BUILD_COMMAND ${MAKE}
-        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}")
+        BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}")
     else()
       ExternalProject_Add(jemalloc_ep
         URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2

http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/cpp/cmake_modules/BuildUtils.cmake
----------------------------------------------------------------------
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index b620de5..9de9de5 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -27,6 +27,18 @@ function(ADD_ARROW_LIB LIB_NAME)
   add_library(${LIB_NAME}_objlib OBJECT
       ${ARG_SOURCES}
   )
+  if (ARG_STATIC_LINK_LIBS)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_LINK_LIBS})
+  endif()
+  if (ARG_STATIC_PRIVATE_LINK_LIBS)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_PRIVATE_LINK_LIBS})
+  endif()
+  if (ARG_SHARED_LINK_LIBS)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_LINK_LIBS})
+  endif()
+  if(ARG_SHARED_PRIVATE_LINK_LIBS)
+    add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_PRIVATE_LINK_LIBS})
+  endif()
 
   # Necessary to make static linking into other shared libraries work properly
   set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)

http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
----------------------------------------------------------------------
diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
index 714fa1a..94f5bc0 100644
--- a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
+++ b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
@@ -13,28 +13,7 @@
 FROM arrow-base-x86_64
 
 WORKDIR /
-ADD http://zlib.net/zlib-1.2.8.tar.gz /zlib-1.2.8.tar.gz
-RUN tar xf zlib-1.2.8.tar.gz
-WORKDIR zlib-1.2.8
-RUN CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DCMAKE_BUILD_TYPE=Release .
-RUN make -j5 install
-
-WORKDIR /
-ADD https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz /snappy-1.1.3.tar.gz
-RUN tar xf snappy-1.1.3.tar.gz
-WORKDIR /snappy-1.1.3
-RUN ./configure --with-pic --prefix=/usr
-RUN make -j5 install
-
-WORKDIR /
-ADD http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz /thrift-0.9.1.tar.gz
-RUN tar xf thrift-0.9.1.tar.gz
-WORKDIR /thrift-0.9.1
-RUN ./configure LDFLAGS='-L/usr/lib64' CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp
--without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl
--without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d
--without-tests --with-cpp --prefix=/usr --disable-shared --enable-static
-RUN make -j5 install
-
-WORKDIR /
 RUN git clone https://github.com/apache/parquet-cpp.git
 WORKDIR /parquet-cpp
-RUN ARROW_HOME=/usr THRIFT_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr
-DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
+RUN ARROW_HOME=/usr /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr
-DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
 RUN make -j5 install

http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/Dockerfile-x86_64
----------------------------------------------------------------------
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index e62a601..29e00b0 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -16,32 +16,23 @@ FROM quay.io/pypa/manylinux1_x86_64:latest
 RUN yum install -y flex openssl-devel
 
 WORKDIR /
-ADD http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz /boost_1_60_0.tar.gz
+RUN wget http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz
-O /boost_1_60_0.tar.gz
 RUN tar xf boost_1_60_0.tar.gz
 WORKDIR /boost_1_60_0
 RUN ./bootstrap.sh
 RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system
install
 
 WORKDIR /
-ADD https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz /cmake-3.5.2.tar.gz
-RUN tar xf cmake-3.5.2.tar.gz
-WORKDIR /cmake-3.5.2
-RUN ./configure --prefix=/usr
-RUN make -j5 install
-
-WORKDIR /
-ADD https://github.com/google/flatbuffers/archive/v1.3.0.tar.gz /flatbuffers-1.3.0.tar.gz
-RUN tar xf flatbuffers-1.3.0.tar.gz
-WORKDIR /flatbuffers-1.3.0
-RUN CXXFLAGS='-fPIC' cmake -DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr
.
-RUN make -j5 install
+# Install cmake manylinux1 package
+RUN /opt/python/cp35-cp35m/bin/pip install cmake
 
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git
 WORKDIR /multibuild
 RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
+
 ADD arrow /arrow
 WORKDIR /arrow/cpp
-RUN FLATBUFFERS_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON
-DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF .
+RUN /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr
-DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF
-DARROW_JEMALLOC=ON .
 RUN make -j5 install

http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/build_arrow.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 0786b6f..7e2ad58 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -30,10 +30,12 @@ source /multibuild/manylinux_utils.sh
 cd /arrow/python
 
 # PyArrow build configuration
-export PYARROW_CMAKE_OPTIONS='-DCMAKE_BUILD_TYPE=Release'
+export PYARROW_BUILD_TYPE='release'
+export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_PARQUET=ON'
 # Need as otherwise arrow_io is sometimes not linked
 export LDFLAGS="-Wl,--no-as-needed"
 export ARROW_HOME="/usr"
+export PARQUET_HOME="/usr"
 
 # Ensure the target directory exists
 mkdir -p /io/dist
@@ -53,7 +55,7 @@ function repair_wheelhouse {
             cp $whl $out_dir
         else
             # Store libraries directly in . not .libs to fix problems with libpyarrow.so
linkage.
-            auditwheel -v repair -L . $whl -w $out_dir/
+            $PY35_BIN/auditwheel -v repair -L . $whl -w $out_dir/
         fi
     done
     chmod -R a+rwX $out_dir
@@ -67,10 +69,10 @@ for PYTHON in ${PYTHON_VERSIONS}; do
 
     $PIPI_IO "numpy==1.9.0"
     $PIPI_IO "cython==0.24"
+    $PIPI_IO "cmake"
 
     PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
 
-    rm_mkdir fixed_wheels
     repair_wheelhouse dist /io/dist
 done
 


Mime
View raw message