arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-738: Fix manylinux1 build
Date Sun, 02 Apr 2017 18:49:26 GMT
Repository: arrow
Updated Branches:
  refs/heads/master c4d535ca1 -> 9f720b117


ARROW-738: Fix manylinux1 build

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #471 from xhochy/fix-manylinux1-build and squashes the following commits:

5259118 [Uwe L. Korn] Instead of copying, symlink to correct ABI versions
cffcca3 [Uwe L. Korn] Don't hardcode ABI versions
98f79af [Uwe L. Korn] Set PKG_CONFIG_PATH
b97bcf4 [Uwe L. Korn] Read SOVERSION from pkg-config
a83dbc8 [Uwe L. Korn] Build arrow-cpp for each Python version


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/9f720b11
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/9f720b11
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/9f720b11

Branch: refs/heads/master
Commit: 9f720b117648f42356bbfd0a36f6275b878305d1
Parents: c4d535c
Author: Uwe L. Korn <uwelk@xhochy.com>
Authored: Sun Apr 2 14:49:20 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Sun Apr 2 14:49:20 2017 -0400

----------------------------------------------------------------------
 python/CMakeLists.txt                | 39 +++++++++++++++----------------
 python/cmake_modules/FindArrow.cmake | 36 ++++++++++++++++++----------
 python/manylinux1/Dockerfile-x86_64  |  4 +++-
 python/manylinux1/build_arrow.sh     | 10 ++++++++
 python/setup.py                      | 18 +++++++++++---
 5 files changed, 71 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/9f720b11/python/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index f315d01..463a29d 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -226,24 +226,27 @@ endif()
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 
-if (PYARROW_BUNDLE_ARROW_CPP)
-  configure_file(${ARROW_SHARED_LIB}
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}
+function(bundle_arrow_lib library_path)
+  get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
+  configure_file(${${library_path}}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
       COPYONLY)
-  SET(ARROW_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
-  configure_file(${ARROW_IO_SHARED_LIB}
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX}
+  configure_file(${${library_path}}.${ARROW_ABI_VERSION}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
       COPYONLY)
-  SET(ARROW_IO_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX})
-  configure_file(${ARROW_IPC_SHARED_LIB}
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX}
+  configure_file(${${library_path}}.${ARROW_SO_VERSION}
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}
       COPYONLY)
-  SET(ARROW_IPC_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_ipc${CMAKE_SHARED_LIBRARY_SUFFIX})
-  SET(ARROW_PYTHON_SHARED_LIB
-      ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+  SET(ARROW_SHARED_LIB
+      ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+endfunction(bundle_arrow_lib)
+
+if (PYARROW_BUNDLE_ARROW_CPP)
+  # arrow
+  bundle_arrow_lib(ARROW_SHARED_LIB)
+  bundle_arrow_lib(ARROW_IO_SHARED_LIB)
+  bundle_arrow_lib(ARROW_IPC_SHARED_LIB)
+  bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
 endif()
 
 ADD_THIRDPARTY_LIB(arrow
@@ -309,11 +312,7 @@ endif()
 
 if (PYARROW_BUILD_JEMALLOC)
   if (PYARROW_BUNDLE_ARROW_CPP)
-    configure_file(${ARROW_JEMALLOC_SHARED_LIB}
-       ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}
-       COPYONLY)
-    SET(ARROW_JEMALLOC_SHARED_LIB
-        ${BUILD_OUTPUT_ROOT_DIRECTORY}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
+    bundle_arrow_lib(ARROW_JEMALLOC_SHARED_LIB)
   endif()
   ADD_THIRDPARTY_LIB(arrow_jemalloc
     SHARED_LIB ${ARROW_JEMALLOC_SHARED_LIB})

http://git-wip-us.apache.org/repos/asf/arrow/blob/9f720b11/python/cmake_modules/FindArrow.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 5030c9c..c2ca0f4 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -23,6 +23,8 @@
 #  ARROW_SHARED_LIB, path to libarrow's shared library
 #  ARROW_FOUND, whether arrow has been found
 
+include(FindPkgConfig)
+
 set(ARROW_SEARCH_HEADER_PATHS
   $ENV{ARROW_HOME}/include
 )
@@ -31,16 +33,27 @@ set(ARROW_SEARCH_LIB_PATH
   $ENV{ARROW_HOME}/lib
 )
 
-find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
-  ${ARROW_SEARCH_HEADER_PATHS}
-  # make sure we don't accidentally pick up a different version
-  NO_DEFAULT_PATH
-)
-
-find_library(ARROW_LIB_PATH NAMES arrow
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
+pkg_check_modules(ARROW arrow)
+if (ARROW_FOUND)
+  pkg_get_variable(ARROW_ABI_VERSION arrow abi_version)
+  message(STATUS "Arrow ABI version: ${ARROW_ABI_VERSION}")
+  pkg_get_variable(ARROW_SO_VERSION arrow so_version)
+  message(STATUS "Arrow SO version: ${ARROW_SO_VERSION}")
+  set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
+  set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
+else()
+  find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
+    ${ARROW_SEARCH_HEADER_PATHS}
+    # make sure we don't accidentally pick up a different version
+    NO_DEFAULT_PATH
+  )
+
+  find_library(ARROW_LIB_PATH NAMES arrow
+    PATHS
+    ${ARROW_SEARCH_LIB_PATH}
+    NO_DEFAULT_PATH)
+  get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
+endif()
 
 find_library(ARROW_IO_LIB_PATH NAMES arrow_io
   PATHS
@@ -62,7 +75,7 @@ find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
-if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
+if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
   set(ARROW_IO_LIB_NAME libarrow_io)
@@ -70,7 +83,6 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc)
   set(ARROW_PYTHON_LIB_NAME libarrow_python)
 
-  set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
   set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/9f720b11/python/manylinux1/Dockerfile-x86_64
----------------------------------------------------------------------
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 820b94e..56b27ad 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -52,7 +52,9 @@ RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
 
 ADD arrow /arrow
 WORKDIR /arrow/cpp
-RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON
-DARROW_JEMALLOC_USE_SHARED=OFF .
+RUN mkdir build-plain
+WORKDIR /arrow/cpp/build-plain
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON
-DARROW_JEMALLOC_USE_SHARED=OFF ..
 RUN make -j5 install
 
 WORKDIR /

http://git-wip-us.apache.org/repos/asf/arrow/blob/9f720b11/python/manylinux1/build_arrow.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 576a983..8bc4e60 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -39,6 +39,7 @@ export PYARROW_BUNDLE_ARROW_CPP=1
 export LDFLAGS="-Wl,--no-as-needed"
 export ARROW_HOME="/arrow-dist"
 export PARQUET_HOME="/usr"
+export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
 
 # Ensure the target directory exists
 mkdir -p /io/dist
@@ -52,6 +53,15 @@ for PYTHON in ${PYTHON_VERSIONS}; do
     echo "=== (${PYTHON}) Installing build dependencies ==="
     $PIPI_IO "numpy==1.9.0"
     $PIPI_IO "cython==0.24"
+    $PIPI_IO "pandas==0.19.2"
+
+    echo "=== (${PYTHON}) Building Arrow C++ libraries ==="
+    ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
+    mkdir -p "${ARROW_BUILD_DIR}"
+    pushd "${ARROW_BUILD_DIR}"
+    PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist
-DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON
-DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON}
..
+    make -j5 install
+    popd
 
     # Clear output directory
     rm -rf dist/

http://git-wip-us.apache.org/repos/asf/arrow/blob/9f720b11/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 12b44e1..ba77e68 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -17,19 +17,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import glob
+import os
 import os.path as osp
 import re
 import shutil
+import sys
+
 from Cython.Distutils import build_ext as _build_ext
 import Cython
 
-import sys
 
 import pkg_resources
 from setuptools import setup, Extension
 
-import os
-
 from os.path import join as pjoin
 
 from distutils.command.clean import clean as _clean
@@ -207,8 +208,19 @@ class build_ext(_build_ext):
         def move_lib(lib_name):
             lib_filename = (shared_library_prefix + lib_name +
                             shared_library_suffix)
+            # Also copy libraries with ABI/SO version suffix
+            libs = glob.glob(pjoin(self.build_type, lib_filename) + '*')
+            # Longest suffix library should be copied, all others symlinked
+            libs.sort(key=lambda s: -len(s))
+            print(libs, libs[0])
+            lib_filename = os.path.basename(libs[0])
             shutil.move(pjoin(self.build_type, lib_filename),
                         pjoin(build_lib, 'pyarrow', lib_filename))
+            for lib in libs[1:]:
+                filename = os.path.basename(lib)
+                link_name = pjoin(build_lib, 'pyarrow', filename)
+                if not os.path.exists(link_name):
+                    os.symlink(lib_filename, link_name)
 
         if self.bundle_arrow_cpp:
             move_lib("arrow")


Mime
View raw message