arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [arrow-dist] branch master updated: ARROW-1270: [Python] Build scripts for creating macOS Python wheels in Travis CI (#2)
Date Thu, 03 Aug 2017 20:17:55 GMT
This is an automated email from the ASF dual-hosted git repository.

wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-dist.git


The following commit(s) were added to refs/heads/master by this push:
     new 91200bd  ARROW-1270: [Python] Build scripts for creating macOS Python wheels in Travis
CI (#2)
91200bd is described below

commit 91200bd6aa8ed2b6ba53bd4c1abf13c7053b0269
Author: Wes McKinney <wesm@users.noreply.github.com>
AuthorDate: Thu Aug 3 16:17:53 2017 -0400

    ARROW-1270: [Python] Build scripts for creating macOS Python wheels in Travis CI (#2)
    
    * Build scripts for creating macOS Python wheels in Travis CI
    
    Contributed by Uwe L Korn, Wes McKinney
    
    * Move arrow submodule to root
    
    * Add Docker manylinux1 build to matrix also
    
    * Remove errant build step
    
    * Send all wheels to a central BinTray repo, wait for ASF INFRA
    
    * Deploy to BinTray from all branches
---
 .gitignore              | 106 +++++++++++++++++++++++++++++++++++++++++++++++
 .gitmodules             |   6 +++
 .travis.yml             | 107 ++++++++++++++++++++++++++++++++----------------
 README.md               |  19 +++++++++
 appveyor.yml            |   2 +-
 arrow                   |   1 +
 descriptor.json         |  27 ++++++++++++
 python-macos/README.md  |  20 +++++++++
 python-macos/config.sh  |  77 ++++++++++++++++++++++++++++++++++
 python-macos/multibuild |   1 +
 10 files changed, 330 insertions(+), 36 deletions(-)

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..044d762
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..ade0454
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "multibuild"]
+	path = python-macos/multibuild
+	url = https://github.com/matthew-brett/multibuild.git
+[submodule "arrow"]
+	path = arrow
+	url = https://github.com/apache/arrow.git
diff --git a/.travis.yml b/.travis.yml
index 22a640d..dad8067 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,38 +1,75 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
+env:
+  global:
+  - PLAT=x86_64
+  - BUILD_COMMIT=e9f76e125b836d0fdc0a533e2fee3fca8bf4c1a1
+language: python
+python: 3.5
 sudo: required
 dist: trusty
-addons:
-  apt:
-    sources:
-    - ubuntu-toolchain-r-test
-    packages:
-    - gcc-4.9   # Needed for C++11
-    - g++-4.9   # Needed for C++11
-    - gdb
-    - ccache
-    - valgrind
-    - libboost-dev
-    - libboost-filesystem-dev
-    - libboost-system-dev
-    - libjemalloc-dev
-    - gtk-doc-tools
-    - autoconf-archive
-    - libgirepository1.0-dev
-
+services:
+- docker
 matrix:
+  exclude:
+  - python: 3.5
+  include:
+  - os: osx
+    language: objective-c
+    env:
+    - MB_PYTHON_VERSION=2.7
+    - NP_BUILD_DEP=1.10.1
+    - NP_TEST_DEP=1.10.1
+    - PANDAS_DEP="0.20.2"
+  - os: osx
+    language: objective-c
+    env:
+    - MB_PYTHON_VERSION=3.5
+    - NP_BUILD_DEP="1.10.1"
+    - NP_TEST_DEP="1.10.1"
+    - PANDAS_DEP="0.20.2"
+  - os: osx
+    language: objective-c
+    env:
+    - MB_PYTHON_VERSION=3.6
+    - NP_BUILD_DEP="1.11.3"
+    - NP_TEST_DEP="1.11.3"
+    - PANDAS_DEP="0.20.2"
+  - language: cpp
+    os: linux
+    before_script:
+    - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
+    before_install:
+    install:
+    script:
+    - pushd arrow/python/manylinux1
+    - git clone ../../ arrow
+    - docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
+    - docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io arrow-base-x86_64
+      /io/build_arrow.sh
+before_install:
+- brew uninstall boost
+- brew update
+- brew upgrade cmake
+- BUILD_DEPENDS="numpy==$NP_BUILD_DEP six cython"
+- TEST_DEPENDS="numpy==$NP_TEST_DEP six pandas==$PANDAS_DEP"
+- pushd python-macos
+- source multibuild/common_utils.sh
+- source multibuild/travis_steps.sh
+- before_install
+- popd
+install:
+- clean_code arrow $BUILD_COMMIT
+- build_wheel arrow $PLAT
+script:
+- echo "SCRIPT"
+- pwd
+after_success:
+- echo "After success"
+- pwd
+deploy:
+  provider: bintray
+  file: descriptor.json
+  user: wesm
+  on:
+    all_branches: true
+  key:
+    secure: nuErTGtf9e3itkAe4gODHMG7jxjW8PiDargOP4df5DntE9c3TtfV7lzIRc0YmARDLPbe4tKbBd9+Sv6deimAd6ZFWYhQALVKWEANVK+ROeH6rusq60rdxiDpdtWqTF+yXqcYk76J1ifdtPtFW1wHkXfEcnmQhP93PijIiT0+oiXN4BjdReSr0FKqDqc3ZlNu6dwx6Bb0P/IJkvAKb0V6cQwfWn2vRuQj2YcOwGPHDUDyH2/b5lCR+3egLu5DiwEvGlXsUtOIMmWoBmkoTAUkE2SO9ZToZHW9+xCIzOlQYqm35t+aknItmcmbGMfdGoo4ImDdLzo4wjFSt+sjRIKvC90ondtom6ZKc3Z+KJFzuAfmnZMT23RjgcohaDoh8g269tew/5dZdQRO5GJrtV/85Ou6a3ka+rOU1bdxQ0OcOJRWsDKm+gqWRwpikUU29+a4sREKhLp8wdOocmTSlL2lcysDppogVAqv
[...]
diff --git a/README.md b/README.md
index dc3c459..df7e50b 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,20 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
 ## Build scripts and Packaging Automation for Apache Arrow
\ No newline at end of file
diff --git a/appveyor.yml b/appveyor.yml
index c8f7c4d..48db293 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -55,7 +55,7 @@ deploy:
     api_key:
       secure: T6rUTOWbyxuculsGAsKosSLEdQnY+B6Vlp7bWXdA7zLYgBFfJ7sA73JOigjA6jW+
     subject: wesm
-    repo: pyarrow-windows-wheels
+    repo: apache-arrow-test
     package: pyarrow
     version: "%pyarrow_version%"
     publish: true
diff --git a/arrow b/arrow
new file mode 160000
index 0000000..21a0191
--- /dev/null
+++ b/arrow
@@ -0,0 +1 @@
+Subproject commit 21a0191579e1065212d2a435a51d9721dd33c176
diff --git a/descriptor.json b/descriptor.json
new file mode 100644
index 0000000..8e9e621
--- /dev/null
+++ b/descriptor.json
@@ -0,0 +1,27 @@
+{
+    "package": {
+        "name": "pyarrow",
+        "repo": "apache-arrow-test",
+        "subject": "wesm"
+    },
+    "version": {
+        "name": "0.5.1.dev1"
+    },
+    "files": [
+        {
+            "includePattern": "arrow/python/dist/(.*\\.whl)",
+            "uploadPattern": "wheels/$1",
+            "matrixParams": {
+                "override": 1
+            }
+        },
+        {
+            "includePattern": "arrow/python/manylinux1/dist/(.*\\.whl)",
+            "uploadPattern": "wheels/$1",
+            "matrixParams": {
+                "override": 1
+            }
+        }
+    ],
+    "publish": true
+}
diff --git a/python-macos/README.md b/python-macos/README.md
new file mode 100644
index 0000000..1bc974f
--- /dev/null
+++ b/python-macos/README.md
@@ -0,0 +1,20 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# MacOS wheels for pyarrow
\ No newline at end of file
diff --git a/python-macos/config.sh b/python-macos/config.sh
new file mode 100755
index 0000000..a723788
--- /dev/null
+++ b/python-macos/config.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+function build_wheel {
+    echo `pwd`
+    echo CFLAGS=${CFLAGS}
+    echo CXXFLAGS=${CXXFLAGS}
+    echo LDFLAGS=${LDFLAGS}
+    pushd $1
+
+	wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz
-O boost_1_60_0.tar.gz
+	tar xf boost_1_60_0.tar.gz
+    pushd boost_1_60_0
+    ./bootstrap.sh
+    ./bjam "cxxflags=-fPIC ${CFLAGS}" cflags="-fPIC ${CXXFLAGS}" --prefix=/usr/local --with-filesystem
--with-date_time --with-system --with-regex install
+    popd
+
+    # Arrow is 64bit-only at the moment
+    export CFLAGS="-arch x86_64"
+    export CXXFLAGS="-arch x86_64"
+    export ARROW_HOME=/usr/local/
+    export PARQUET_HOME=/usr/local/
+    pip install cython==0.25.2 numpy==${NP_TEST_DEP}
+    pushd cpp
+    mkdir build
+    pushd build
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON
-DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DMAKE=make ..
+    make -j5
+    make install
+    popd
+    popd
+
+    git clone https://github.com/apache/parquet-cpp.git
+    pushd parquet-cpp
+    mkdir build
+    pushd build
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_BUILD_TESTS=OFF
-DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF ..
+    make -j5
+    make install
+    popd
+    popd
+
+    unset ARROW_HOME
+    unset PARQUET_HOME
+    export PYARROW_WITH_PARQUET=1
+    export PYARROW_WITH_JEMALLOC=1
+    export PYARROW_BUNDLE_ARROW_CPP=1
+    export PYARROW_BUILD_TYPE='release'
+    pushd python
+    echo "python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp"
+    python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp
+    python setup.py bdist_wheel
+    ls -l dist/
+    popd
+
+    pip install delocate==0.6.4
+    delocate-wheel -L . -v python/dist/*.whl
+    popd
+}
diff --git a/python-macos/multibuild b/python-macos/multibuild
new file mode 160000
index 0000000..413205a
--- /dev/null
+++ b/python-macos/multibuild
@@ -0,0 +1 @@
+Subproject commit 413205af2f6ee6976ac5f94da3e3aa012be91762

-- 
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" <commits@arrow.apache.org>'].

Mime
View raw message