arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-1029: [Python] Fixes for building pyarrow with Parquet support on MSVC. Add to appveyor build
Date Wed, 17 May 2017 17:32:13 GMT
Repository: arrow
Updated Branches:
  refs/heads/master a4f3259b0 -> fee447510


ARROW-1029: [Python] Fixes for building pyarrow with Parquet support on MSVC. Add to appveyor
build

Miscellaneous fixes to build with ``--with-parquet`` and pass the test suite. We still have
a bunch of compiler warnings; not blocking for 0.4.0, but after PARQUET-991 is resolved we
should fix the MSVC compiler warnings in pyarrow

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #700 from wesm/ARROW-1029 and squashes the following commits:

b651169c [Wes McKinney] Fix post rebase issue
0f438488 [Wes McKinney] Fixes for building pyarrow with Parquet support on MSVC. Add to appveyor
build


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/fee44751
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/fee44751
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/fee44751

Branch: refs/heads/master
Commit: fee44751048847ce1e08039ab72c09918b11b31b
Parents: a4f3259
Author: Wes McKinney <wes.mckinney@twosigma.com>
Authored: Wed May 17 13:32:08 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Wed May 17 13:32:08 2017 -0400

----------------------------------------------------------------------
 ci/msvc-build.bat                      | 32 ++++++++++++++----
 python/CMakeLists.txt                  | 16 +++++----
 python/cmake_modules/FindArrow.cmake   |  2 --
 python/cmake_modules/FindParquet.cmake | 51 ++++++++++++++++++-----------
 python/doc/source/development.rst      | 51 +++++++++++++++++++++++++++--
 python/pyarrow/filesystem.py           |  8 +++++
 python/pyarrow/parquet.py              |  2 +-
 python/pyarrow/tests/test_parquet.py   |  6 ++--
 8 files changed, 127 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/fee44751/ci/msvc-build.bat
----------------------------------------------------------------------
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 504da76..f756fc5 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -19,17 +19,19 @@
 
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython
-conda install -n arrow -q -y -c conda-forge flatbuffers rapidjson
+conda install -n arrow -q -y -c conda-forge ^
+      flatbuffers rapidjson ^
+      cmake git boost-cpp thrift-cpp snappy zlib brotli
+
 call activate arrow
 
 set ARROW_HOME=%CONDA_PREFIX%\Library
-set FLATBUFFERS_HOME=%CONDA_PREFIX%\Library
-set RAPIDJSON_HOME=%CONDA_PREFIX%\Library
+set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 
 @rem Build and test Arrow C++ libraries
 
 mkdir cpp\build
-cd cpp\build
+pushd cpp\build
 
 cmake -G "%GENERATOR%" ^
       -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
@@ -44,10 +46,28 @@ cmake --build . --target INSTALL --config Release  || exit /B
 set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%
 
 ctest -VV  || exit /B
+popd
+
+@rem Build parquet-cpp
+
+git clone https://github.com/apache/parquet-cpp.git || exit /B
+mkdir parquet-cpp\build
+pushd parquet-cpp\build
+
+set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+set PARQUET_HOME=%CONDA_PREFIX%\Library
+cmake -G "%GENERATOR%" ^
+     -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
+     -DCMAKE_BUILD_TYPE=Release ^
+     -DPARQUET_ZLIB_VENDORED=off ^
+     -DPARQUET_BUILD_TESTS=off .. || exit /B
+cmake --build . --target INSTALL --config Release || exit /B
+popd
 
 @rem Build and import pyarrow
 set PYTHONPATH=
 
-cd ..\..\python
-python setup.py build_ext --inplace  || exit /B
+pushd python
+python setup.py build_ext --inplace --with-parquet  || exit /B
 py.test pyarrow -v -s || exit /B
+popd

http://git-wip-us.apache.org/repos/asf/arrow/blob/fee44751/python/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index ded85e8..6f48f7f 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -219,12 +219,6 @@ include_directories(SYSTEM
 # Dependencies
 ############################################################
 
-## Parquet
-find_package(Parquet)
-if(PARQUET_FOUND)
-  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
-endif()
-
 ## Arrow
 find_package(Arrow REQUIRED)
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
@@ -286,9 +280,14 @@ set(LINK_LIBS
 )
 
 if (PYARROW_BUILD_PARQUET)
+  ## Parquet
+  find_package(Parquet)
+
   if(NOT (PARQUET_FOUND AND PARQUET_ARROW_FOUND))
     message(FATAL_ERROR "Unable to locate Parquet libraries")
   endif()
+  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
+
   if (PYARROW_BUNDLE_ARROW_CPP)
       get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY)
       get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE)
@@ -333,11 +332,14 @@ if (PYARROW_BUILD_PARQUET)
       #SET(PARQUET_ARROW_SHARED_LIB
       #    ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
   endif()
+  ADD_THIRDPARTY_LIB(parquet
+    SHARED_LIB ${PARQUET_SHARED_LIB})
   ADD_THIRDPARTY_LIB(parquet_arrow
     SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
   set(LINK_LIBS
     ${LINK_LIBS}
-    parquet_arrow)
+    parquet_shared
+    parquet_arrow_shared)
   set(CYTHON_EXTENSIONS
     ${CYTHON_EXTENSIONS}
     _parquet)

http://git-wip-us.apache.org/repos/asf/arrow/blob/fee44751/python/cmake_modules/FindArrow.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index 9fb1355..c16a4bf 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -76,10 +76,8 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   if (MSVC)
     set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
     set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
-    set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH})
     set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
     set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
-    set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB})
   else()
     set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
     set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)

http://git-wip-us.apache.org/repos/asf/arrow/blob/fee44751/python/cmake_modules/FindParquet.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index ffd043d..de53a29 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -1,16 +1,19 @@
-# Copyright 2012 Cloudera Inc.
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 # - Find PARQUET (parquet/parquet.h, libparquet.a, libparquet.so)
 # This module defines
@@ -71,9 +74,14 @@ endif()
 
 if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
   set(PARQUET_FOUND TRUE)
-  set(PARQUET_LIB_NAME libparquet)
-  set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
-  set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  if (MSVC)
+    set(PARQUET_STATIC_LIB "${PARQUET_LIBRARIES}_static")
+    set(PARQUET_SHARED_LIB "${PARQUET_LIBRARIES}")
+  else()
+    set(PARQUET_LIB_NAME libparquet)
+    set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
+    set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
 else ()
   set(PARQUET_FOUND FALSE)
 endif ()
@@ -81,11 +89,16 @@ endif ()
 if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES)
   set(PARQUET_ARROW_FOUND TRUE)
   get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH)
-  set(PARQUET_ARROW_LIB_NAME libparquet_arrow)
-  set(PARQUET_ARROW_STATIC_LIB
-    ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.a)
-  set(PARQUET_ARROW_SHARED_LIB
-    ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  if (MSVC)
+    set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBRARIES}_static")
+    set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_LIBRARIES}")
+  else()
+    set(PARQUET_ARROW_LIB_NAME libparquet_arrow)
+    set(PARQUET_ARROW_STATIC_LIB
+      ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.a)
+    set(PARQUET_ARROW_SHARED_LIB
+      ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
 else ()
   set(PARQUET_ARROW_FOUND FALSE)
 endif ()

http://git-wip-us.apache.org/repos/asf/arrow/blob/fee44751/python/doc/source/development.rst
----------------------------------------------------------------------
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 440c1c4..6ec563e 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -174,14 +174,37 @@ You should be able to run the unit tests with:
 Windows
 =======
 
-First, make sure you can `build the C++ library <https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md>`_.
+First, we bootstrap a conda environment similar to the `C++ build instructions
+<https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md>`_. This
+includes all the dependencies for Arrow and the Apache Parquet C++ libraries.
 
-Now, we need to build and install the C++ libraries someplace.
+First, starting from fresh clones of Apache Arrow and parquet-cpp:
+
+.. code-block:: shell
+
+   git clone https://github.com/apache/arrow.git
+   git clone https://github.com/apache/parquet-cpp.git
+
+.. code-block:: shell
+
+   conda create -n arrow-dev cmake git boost-cpp ^
+         flatbuffers snappy zlib brotli thrift-cpp rapidjson
+   activate arrow-dev
+
+As one git housekeeping item, we must run this command in our Arrow clone:
+
+.. code-block:: shell
+
+   cd arrow
+   git config core.symlinks true
+
+Now, we build and install Arrow C++ libraries
 
 .. code-block:: shell
 
    mkdir cpp\build
    cd cpp\build
+   set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
    set ARROW_HOME=C:\thirdparty
    cmake -G "Visual Studio 14 2015 Win64" ^
          -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
@@ -191,6 +214,22 @@ Now, we need to build and install the C++ libraries someplace.
    cmake --build . --target INSTALL --config Release
    cd ..\..
 
+Now, we build parquet-cpp and install the result in the same place:
+
+.. code-block:: shell
+
+   mkdir ..\parquet-cpp\build
+   pushd ..\parquet-cpp\build
+   set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+   set PARQUET_HOME=C:\thirdparty
+   cmake -G "Visual Studio 14 2015 Win64" ^
+         -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
+         -DCMAKE_BUILD_TYPE=Release ^
+         -DPARQUET_ZLIB_VENDORED=off ^
+         -DPARQUET_BUILD_TESTS=off ..
+   cmake --build . --target INSTALL --config Release
+   popd
+
 After that, we must put the install directory's bin path in our ``%PATH%``:
 
 .. code-block:: shell
@@ -202,7 +241,13 @@ Now, we can build pyarrow:
 .. code-block:: shell
 
    cd python
-   python setup.py build_ext --inplace
+   python setup.py build_ext --inplace --with-parquet
+
+Then run the unit tests with:
+
+.. code-block:: shell
+
+   py.test pyarrow -v
 
 Running C++ unit tests with Python
 ----------------------------------

http://git-wip-us.apache.org/repos/asf/arrow/blob/fee44751/python/pyarrow/filesystem.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py
index ac37fd8..798d96b 100644
--- a/python/pyarrow/filesystem.py
+++ b/python/pyarrow/filesystem.py
@@ -92,6 +92,10 @@ class Filesystem(object):
                                  filesystem=self)
         return dataset.read(columns=columns, nthreads=nthreads)
 
+    @property
+    def pathsep(self):
+        return '/'
+
 
 class LocalFilesystem(Filesystem):
 
@@ -132,6 +136,10 @@ class LocalFilesystem(Filesystem):
         """
         return open(path, mode=mode)
 
+    @property
+    def pathsep(self):
+        return os.path.sep
+
 
 class HdfsClient(lib._HdfsClient, Filesystem):
     """

http://git-wip-us.apache.org/repos/asf/arrow/blob/fee44751/python/pyarrow/parquet.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index e69d85e..f59a719 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -582,7 +582,7 @@ def _make_manifest(path_or_paths, fs, pathsep='/'):
 
     if is_string(path_or_paths) and fs.isdir(path_or_paths):
         manifest = ParquetManifest(path_or_paths, filesystem=fs,
-                                   pathsep=pathsep)
+                                   pathsep=fs.pathsep)
         metadata_path = manifest.metadata_path
         pieces = manifest.pieces
         partitions = manifest.partitions

http://git-wip-us.apache.org/repos/asf/arrow/blob/fee44751/python/pyarrow/tests/test_parquet.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index db446d3..5f65f28 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -192,10 +192,10 @@ def test_pandas_column_selection(tmpdir):
 
 def _random_integers(size, dtype):
     # We do not generate integers outside the int64 range
-    i64_info = np.iinfo('int64')
+    platform_int_info = np.iinfo('int_')
     iinfo = np.iinfo(dtype)
-    return np.random.randint(max(iinfo.min, i64_info.min),
-                             min(iinfo.max, i64_info.max),
+    return np.random.randint(max(iinfo.min, platform_int_info.min),
+                             min(iinfo.max, platform_int_info.max),
                              size=size).astype(dtype)
 
 


Mime
View raw message