arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject arrow git commit: ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor…
Date Mon, 10 Jul 2017 17:32:50 GMT
Repository: arrow
Updated Branches:
  refs/heads/master cab07c2bd -> bc16e0e04


ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor…

… jobs; Windows.md updated.

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #823 from MaxRis/ARROW-1196 and squashes the following commits:

1ca1155 [Max Risuhin] ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor
jobs; Windows.md updated.


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/bc16e0e0
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/bc16e0e0
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/bc16e0e0

Branch: refs/heads/master
Commit: bc16e0e04a3d4ea9c96579ac59cee4b16e771850
Parents: cab07c2
Author: Max Risuhin <risuhin.max@gmail.com>
Authored: Mon Jul 10 19:32:46 2017 +0200
Committer: Uwe L. Korn <uwelk@xhochy.com>
Committed: Mon Jul 10 19:32:46 2017 +0200

----------------------------------------------------------------------
 appveyor.yml                                |  15 +++
 ci/msvc-build.bat                           |  31 ++++-
 cpp/CMakeLists.txt                          |   4 +
 cpp/apidoc/Windows.md                       | 146 ++++++++++++++++++++---
 cpp/cmake_modules/ThirdpartyToolchain.cmake |   7 +-
 5 files changed, 181 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index 802d43b..f3ce5a5 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -20,9 +20,23 @@ os: Visual Studio 2015
 
 environment:
   matrix:
+    - GENERATOR: NMake Makefiles
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Release"
+    - GENERATOR: Visual Studio 14 2015 Win64
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Debug"
     - GENERATOR: Visual Studio 14 2015 Win64
       PYTHON: "3.5"
       ARCH: "64"
+      CONFIGURATION: "Release"
+    - GENERATOR: Visual Studio 14 2015 Win64
+      PYTHON: "3.5"
+      ARCH: "64"
+      CONFIGURATION: "Toolchain"
+
   MSVC_DEFAULT_OPTIONS: ON
   BOOST_ROOT: C:\Libraries\boost_1_63_0
   BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
@@ -30,6 +44,7 @@ environment:
 init:
   - set MINICONDA=C:\Miniconda35-x64
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
+  - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio
14.0\VC\vcvarsall.bat" x86_amd64
 
 build_script:
   - git config core.symlinks true

http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/ci/msvc-build.bat
----------------------------------------------------------------------
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 366fc28..c84d12d 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -17,18 +17,41 @@
 
 @echo on
 
+if "%CONFIGURATION%" == "Debug" (
+  mkdir cpp\build-debug
+  pushd cpp\build-debug
+
+  cmake -G "%GENERATOR%" ^
+        -DARROW_BOOST_USE_SHARED=OFF ^
+        -DCMAKE_BUILD_TYPE=Debug ^
+        -DARROW_CXXFLAGS="/MP" ^
+        ..  || exit /B
+
+  cmake --build . --config Debug || exit /B
+  popd
+
+  @rem Finish Debug build successfully
+  exit /B 0
+)
+
 conda update --yes --quiet conda
 
 conda create -n arrow -q -y python=%PYTHON% ^
       six pytest setuptools numpy pandas cython
-conda install -n arrow -q -y -c conda-forge ^
-      flatbuffers rapidjson ^
-      cmake git boost-cpp thrift-cpp snappy zlib brotli gflags
+
+if "%CONFIGURATION%" == "Toolchain" (
+  conda install -n arrow -q -y -c conda-forge ^
+      flatbuffers rapidjson cmake git boost-cpp ^
+      thrift-cpp snappy zlib brotli gflags lz4-c zstd
+)
 
 call activate arrow
 
+if "%CONFIGURATION%" == "Toolchain" (
+  set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+)
+
 set ARROW_HOME=%CONDA_PREFIX%\Library
-set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 
 @rem Build and test Arrow C++ libraries
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6d01fd9..002a07e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -152,6 +152,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
       "Snappy static lib suffix used on Windows with MSVC (default is empty string)")
     set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING
       "Zlib static lib suffix used on Windows with MSVC (default libstatic)")
+    set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+      "Lz4 static lib suffix used on Windows with MSVC (default _static)")
+    set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+      "ZStd static lib suffix used on Windows with MSVC (default _static)")
   endif()
 endif()
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/apidoc/Windows.md
----------------------------------------------------------------------
diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index b9a6143..6bfb951 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -21,24 +21,13 @@ investing time in maintaining the thirdparty build dependencies for Arrow
and
 related C++ libraries using the conda package manager. Others are free to add
 other development instructions for Windows here.
 
-### Visual Studio
-
-Microsoft provides the free Visual Studio 2017 Community edition. When doing
-development, you must launch the developer command prompt using
-
-```
-"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat"
-arch=amd64
-```
-
-It's easiest to configure a console emulator like [cmder][3] to automatically
-launch this when starting a new development console.
-
 ### conda and package toolchain
 
 [Miniconda][1] is a minimal Python distribution including the conda package
 manager. To get started, download and install a 64-bit distribution.
 
-We recommend using packages from [conda-forge][2]
+We recommend using packages from [conda-forge][2].  
+Launch cmd.exe and run following commands:
 
 ```shell
 conda config --add channels conda-forge
@@ -47,9 +36,92 @@ conda config --add channels conda-forge
 Now, you can bootstrap a build environment
 
 ```shell
-conda create -n arrow-dev cmake git boost
+conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy
zlib brotli gflags lz4-c zstd
 ```
 
+Activate just created conda environment with pre-installed packages from
+previous step:
+
+```shell
+activate arrow-dev
+```
+
+We are using [cmake][4] tool to support Windows builds.  
+To allow cmake to pick up 3rd party dependencies, you should set
+`ARROW_BUILD_TOOLCHAIN` environment variable to contain `Library` folder
+path of new created on previous step `arrow-dev` conda environment.
+For instance, if `Miniconda` was installed to default destination, `Library`
+folder path for `arrow-dev` conda environment will be as following:
+
+```shell
+C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library
+```
+
+To set `ARROW_BUILD_TOOLCHAIN` environment variable visible only for current terminal session
you can run following:
+```shell
+set ARROW_BUILD_TOOLCHAIN=C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library
+```
+
+To validate value of `ARROW_BUILD_TOOLCHAIN` environment variable you can run following terminal
command:
+```shell
+echo %ARROW_BUILD_TOOLCHAIN%
+```
+
+As alternative to `ARROW_BUILD_TOOLCHAIN`, it's possible to configure path
+to each 3rd party dependency separately by setting appropriate environment
+variable:
+
+`FLATBUFFERS_HOME` variable with path to `flatbuffers` installation  
+`RAPIDJSON_HOME` variable with path to `rapidjson` installation  
+`GFLAGS_HOME` variable with path to `gflags` installation  
+`SNAPPY_HOME` variable with path to `snappy` installation  
+`ZLIB_HOME` variable with path to `zlib` installation  
+`BROTLI_HOME` variable with path to `brotli` installation  
+`LZ4_HOME` variable with path to `lz4` installation  
+`ZSTD_HOME` variable with path to `zstd` installation
+
+### Customize static libraries names lookup of 3rd party dependencies 
+
+If you decided to use pre-built 3rd party dependencies libs, it's possible to
+configure Arrow's cmake build script to search for customized names of 3rd
+party static libs.
+
+`zlib`. Pass `-DARROW_ZLIB_VENDORED=OFF` to enable lookup of custom zlib
+build. Set `ZLIB_HOME` environment variable. Pass
+`-DZLIB_MSVC_STATIC_LIB_SUFFIX=%ZLIB_SUFFIX%` to link with z%ZLIB_SUFFIX%.lib
+
+`brotli`. Set `BROTLY_HOME` environment variable. Pass
+`-DBROTLI_MSVC_STATIC_LIB_SUFFIX=%BROTLI_SUFFIX%` to link with
+brotli*%BROTLI_SUFFIX%.lib.
+
+`snappy`. Set `SNAPPY_HOME` environment variable. Pass
+`-DSNAPPY_MSVC_STATIC_LIB_SUFFIX=%SNAPPY_SUFFIX%` to link with
+snappy%SNAPPY_SUFFIX%.lib.
+
+`lz4`. Set `LZ4_HOME` environment variable. Pass
+`-LZ4_MSVC_STATIC_LIB_SUFFIX=%LZ4_SUFFIX%` to link with
+lz4%LZ4_SUFFIX%.lib.
+
+`zstd`. Set `ZSTD_HOME` environment variable. Pass
+`-ZSTD_MSVC_STATIC_LIB_SUFFIX=%ZSTD_SUFFIX%` to link with
+zstd%ZSTD_SUFFIX%.lib.
+
+### Visual Studio
+
+Microsoft provides the free Visual Studio Community edition. When doing
+development, you must launch the developer command prompt using
+
+#### Visual Studio 2015
+
+```"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64```
+
+#### Visual Studio 2017
+
+```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat"
-arch=amd64```
+
+It's easiest to configure a console emulator like [cmder][3] to automatically
+launch this when starting a new development console.
+
 ## Building with NMake
 
 Activate your conda build environment:
@@ -58,7 +130,8 @@ Activate your conda build environment:
 activate arrow-dev
 ```
 
-Now, do an out of source build using `nmake`:
+Change working directory in cmd.exe to the root directory of Arrow and
+do an out of source build using `nmake`:
 
 ```
 cd cpp
@@ -72,14 +145,53 @@ When using conda, only release builds are currently supported.
 
 ## Build using Visual Studio (MSVC) Solution Files
 
-To build on the command line by instead generating a MSVC solution, instead
-run:
+Activate your conda build environment:
 
 ```
+activate arrow-dev
+```
+
+Change working directory in cmd.exe to the root directory of Arrow and
+do an out of source build by generating a MSVC solution:
+
+```
+cd cpp
+mkdir build
+cd build
 cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=Release ..
 cmake --build . --config Release
 ```
 
+## Debug build
+
+To build Debug version of Arrow you should have pre-insalled Debug version of
+boost libs.
+
+It's recommended to configure cmake build with following variables for Debug build:
+
+`-DARROW_BOOST_USE_SHARED=OFF` - enables static linking with boost debug libs and
+simplifies run-time loading of 3rd parties. (Recommended)
+
+`-DBOOST_ROOT` - sets the root directory of boost libs. (Optional)
+
+`-DBOOST_LIBRARYDIR` - sets the directory with boost lib files. (Optional)
+
+Command line to build Arrow in Debug might look as following:
+
+```
+cd cpp
+mkdir build
+cd build
+cmake -G "Visual Studio 14 2015 Win64" -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=OFF
-DCMAKE_BUILD_TYPE=Debug -DBOOST_ROOT=C:/local/boost_1_63_0  -DBOOST_LIBRARYDIR=C:/local/boost_1_63_0/lib64-msvc-14.0
..
+cmake --build . --config Debug
+```
+
+To get the latest build instructions, you can reference [msvc-build.bat][5], which is used
by automated Appveyor builds.
+
+
+
 [1]: https://conda.io/miniconda.html
 [2]: https://conda-forge.github.io/
 [3]: http://cmder.net/
+[4]: https://cmake.org/
+[5]: https://github.com/apache/arrow/blob/master/ci/msvc-build.bat

http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/cmake_modules/ThirdpartyToolchain.cmake
----------------------------------------------------------------------
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 3eef2f7..74939ac 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -346,10 +346,15 @@ if (ARROW_IPC)
   ## Flatbuffers
   if("${FLATBUFFERS_HOME}" STREQUAL "")
     set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
+    if (MSVC)
+      set(FLATBUFFERS_CMAKE_CXX_FLAGS /EHsc)
+    else()
+      set(FLATBUFFERS_CMAKE_CXX_FLAGS -fPIC)
+    endif()
     ExternalProject_Add(flatbuffers_ep
       URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
       CMAKE_ARGS
-      "-DCMAKE_CXX_FLAGS=-fPIC"
+      "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}"
       "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
       "-DFLATBUFFERS_BUILD_TESTS=OFF")
 


Mime
View raw message