parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject parquet-cpp git commit: PARQUET-679: [C++] Resolve unit tests issues on Windows; Run unit tes…
Date Tue, 09 May 2017 17:15:29 GMT
Repository: parquet-cpp
Updated Branches:
  refs/heads/master e6763722b -> a5e6482e2


PARQUET-679: [C++] Resolve unit tests issues on Windows; Run unit tes…

…ts with Appveyor; Appveyor build jobs for Visual Studio cmake generator and Debug configuration.

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #319 from MaxRis/PARQUET-679 and squashes the following commits:

6bf2cd7 [Max Risuhin] PARQUET-679: [C++] Resolve unit tests issues on Windows; Run unit tests
with Appveyor; Appveyor build jobs for Visual Studio cmake generator and Debug configuration.


Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/a5e6482e
Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/a5e6482e
Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/a5e6482e

Branch: refs/heads/master
Commit: a5e6482e2909b89022374c4bddb70fb1b5eb63cf
Parents: e676372
Author: Max Risuhin <risuhin.max@gmail.com>
Authored: Tue May 9 19:15:24 2017 +0200
Committer: Uwe L. Korn <uwe@apache.org>
Committed: Tue May 9 19:15:24 2017 +0200

----------------------------------------------------------------------
 CMakeLists.txt                                | 14 +++++
 appveyor.yml                                  | 10 +++-
 ci/msvc-build.bat                             | 12 +++-
 cmake_modules/ThirdpartyToolchain.cmake       | 69 ++++++++++++++--------
 src/parquet/arrow/arrow-reader-writer-test.cc |  8 +--
 src/parquet/util/bit-util.h                   |  2 +-
 6 files changed, 81 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 46d4394..e18cf7b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -153,6 +153,17 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 
+if (MSVC)
+  # define output directories to overwrite default Visual Studio generator output directories
+  FOREACH(BUILD_CONFIG ${CMAKE_CONFIGURATION_TYPES})
+    string(TOUPPER ${BUILD_CONFIG} UPPERCASE_BUILD_CONFIG)
+    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
+    set(CMAKE_PDB_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
+    set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
+  ENDFOREACH()
+endif()
+
 ############################################################
 # Benchmarking
 ############################################################
@@ -353,6 +364,9 @@ endif()
 #   Debug symbols are stripped for reduced binary size. Add
 #   -DPARQUET_CXXFLAGS="-g" to include them
 set(CXX_FLAGS_DEBUG "-ggdb -O0")
+if (MSVC)
+  set(CXX_FLAGS_DEBUG "${CXX_FLAGS_DEBUG} /bigobj")
+endif()
 set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
 set(CXX_FLAGS_RELEASE "-O3")
 

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index 437cd6d..f3fac11 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -23,14 +23,22 @@ environment:
     - GENERATOR: NMake Makefiles
       PYTHON: "3.5"
       ARCH: "64"
+
+    - GENERATOR: Visual Studio 14 2015 Win64
+      PYTHON: "3.5"
+      ARCH: "64"
   MSVC_DEFAULT_OPTIONS: ON
   BOOST_ROOT: C:\Libraries\boost_1_63_0
   BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
 
+configuration:
+  - Debug
+  - Release
+
 init:
   - set MINICONDA=C:\Miniconda35-x64
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
   - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio
14.0\VC\vcvarsall.bat" x64
 
 build_script:
-  - call ci\msvc-build.bat
\ No newline at end of file
+  - call ci\msvc-build.bat

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/ci/msvc-build.bat
----------------------------------------------------------------------
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 7e723fa..b1b54b4 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -20,8 +20,16 @@
 mkdir build
 cd build
 
+SET PARQUET_TEST_DATA=%APPVEYOR_BUILD_FOLDER%\data
+
 cmake -G "%GENERATOR%" ^
-      -DCMAKE_BUILD_TYPE=Release ^
+      -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DPARQUET_BOOST_USE_SHARED=OFF ^
+      -DPARQUET_CXXFLAGS="/MP" ^
       .. || exit /B
 
-nmake || exit /B
\ No newline at end of file
+cmake --build . --config %CONFIGURATION% || exit /B
+
+if "%CONFIGURATION%" == "Release" (
+  ctest -VV || exit /B
+)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/cmake_modules/ThirdpartyToolchain.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index 44b2619..74b4632 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -82,7 +82,7 @@ if (PARQUET_BOOST_USE_SHARED)
   endif()
 
   find_package(Boost COMPONENTS regex REQUIRED)
-  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
   else()
     set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
@@ -91,7 +91,7 @@ else()
   # Find static Boost libraries.
   set(Boost_USE_STATIC_LIBS ON)
   find_package(Boost COMPONENTS regex REQUIRED)
-  if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
   else()
     set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
@@ -129,7 +129,11 @@ if (NOT ZLIB_FOUND)
   set(ZLIB_HOME "${ZLIB_PREFIX}")
   set(ZLIB_INCLUDE_DIRS "${ZLIB_PREFIX}/include")
   if (MSVC)
-    set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
+    if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+      set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib)
+    else()
+      set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
+    endif()
   else()
     set(ZLIB_STATIC_LIB_NAME libz.a)
   endif()
@@ -175,7 +179,11 @@ if (NOT THRIFT_FOUND)
   set(THRIFT_HOME "${THRIFT_PREFIX}")
   set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include")
   IF (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
-    set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthriftd.a")
+    IF (MSVC)
+      set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/thriftmdd.lib")
+    ELSE()
+      set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthriftd.a")
+    ENDIF()
   ELSE()
     IF (MSVC)
       set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/thriftmd.lib")
@@ -206,7 +214,9 @@ if (NOT THRIFT_FOUND)
                            "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIRS}"
                            "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}"
                            "-DWITH_SHARED_LIB=OFF"
+                           "-DWITH_PLUGIN=OFF"
                            ${THRIFT_CMAKE_ARGS})
+    set (THRIFT_DEPENDENCIES zlib_ep)
   endif()
 
   if (CMAKE_VERSION VERSION_GREATER "3.2")
@@ -215,12 +225,14 @@ if (NOT THRIFT_FOUND)
       URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"
       BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}"
       CMAKE_ARGS ${THRIFT_CMAKE_ARGS}
-      STEP_TARGETS flex_step libevent_step)
+      STEP_TARGETS flex_step libevent_step
+      DEPENDS ${THRIFT_DEPENDENCIES})
   else()
     ExternalProject_Add(thrift_ep
       URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"
       CMAKE_ARGS ${THRIFT_CMAKE_ARGS}
-      STEP_TARGETS flex_step libevent_step)
+      STEP_TARGETS flex_step libevent_step
+      DEPENDS ${THRIFT_DEPENDENCIES})
   endif()
     set(THRIFT_VENDORED 1)
 else()
@@ -235,27 +247,28 @@ if (MSVC)
 
   # Download and configure Windows build of Flex and Bison
   ExternalProject_Add_Step(thrift_ep flex_step
-    COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/dist/winflexbison &&
-            cd thirdparty/dist/winflexbison &&
-            curl -SLO https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip
&&
-            ${CMAKE_COMMAND} -E tar xzf win_flex_bison-${WINFLEXBISON_VERSION}.zip
+    COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/dist/winflexbison
+    COMMAND cd thirdparty/dist/winflexbison
+    COMMAND curl -SLO https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip
+    COMMAND ${CMAKE_COMMAND} -E tar xzf win_flex_bison-${WINFLEXBISON_VERSION}.zip
     DEPENDERS configure
+    DEPENDEES download
     WORKING_DIRECTORY ${SOURCE_DIR})
 
   # Download and build libevent
   ExternalProject_Add_Step(thrift_ep libevent_step
-    COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/src &&
-            cd thirdparty/src &&
-            curl -SLO https://github.com/nmathewson/Libevent/archive/release-${LIBEVENT_VERSION}-rc.zip
&&
-            ${CMAKE_COMMAND} -E tar xzf release-${LIBEVENT_VERSION}-rc.zip &&
-            cd Libevent-release-${LIBEVENT_VERSION}-rc &&
-            nmake -f Makefile.nmake &&
-            ${CMAKE_COMMAND} -E make_directory lib &&
-            copy *.lib lib &&
-            xcopy /E /I /D WIN32-Code\\nmake include\\event2 &&
-            xcopy /E /I /D WIN32-Code\\nmake\\event2 include\\event2 &&
-            copy *.h include
+    COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/src
+    COMMAND cd thirdparty/src
+    COMMAND curl -SLO https://github.com/nmathewson/Libevent/archive/release-${LIBEVENT_VERSION}-rc.zip
+    COMMAND ${CMAKE_COMMAND} -E tar xzf release-${LIBEVENT_VERSION}-rc.zip
+    COMMAND cd Libevent-release-${LIBEVENT_VERSION}-rc
+    COMMAND ${CMAKE_COMMAND} -E make_directory build
+    COMMAND cd build
+    COMMAND ${CMAKE_COMMAND} -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=.. ..
+    COMMAND nmake
+    COMMAND nmake install
     DEPENDERS configure
+    DEPENDEES download
     WORKING_DIRECTORY ${SOURCE_DIR})
 endif()
 
@@ -554,13 +567,16 @@ if (NOT ARROW_FOUND)
     -DARROW_IPC=OFF
     -DARROW_BOOST_USE_SHARED=${PARQUET_BOOST_USE_SHARED}
     -DARROW_BUILD_TESTS=OFF)
-  
+
   if (MSVC)
-    set(ARROW_CMAKE_ARGS -G "NMake Makefiles" ${ARROW_CMAKE_ARGS})
+    set(ARROW_CMAKE_ARGS -G "NMake Makefiles" -DARROW_CXXFLAGS="/WX" ${ARROW_CMAKE_ARGS})
+    if (NOT PARQUET_BOOST_USE_SHARED)
+      set(ARROW_CMAKE_ARGS -DARROW_BOOST_USE_SHARED=OFF ${ARROW_CMAKE_ARGS})
+    endif()
   endif()
 
   if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "")
-    set(ARROW_VERSION "f7ab7270bb07466dabf84c015a6db2a192eb3dad")
+    set(ARROW_VERSION "2c3e111d45c056d429cef312533c9f3f96b08ae8")
   else()
     set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}")
   endif()
@@ -581,7 +597,7 @@ if (NOT ARROW_FOUND)
         CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp
         CMAKE_GENERATOR "NMake Makefiles"
         CMAKE_GENERATOR_PLATFORM "x64"
-        BUILD_COMMAND nmake && nmake install
+        BUILD_COMMAND nmake COMMAND nmake install
         STEP_TARGETS copy_dll_step)
     else()
       ExternalProject_Add(arrow_ep
@@ -589,12 +605,13 @@ if (NOT ARROW_FOUND)
         CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp
         CMAKE_GENERATOR "NMake Makefiles"
         CMAKE_GENERATOR_PLATFORM "x64"
-        BUILD_COMMAND nmake && nmake install
+        BUILD_COMMAND nmake COMMAND nmake install
         STEP_TARGETS copy_dll_step)
     endif()
 
     ExternalProject_Get_Property(arrow_ep SOURCE_DIR)
     ExternalProject_Add_Step(arrow_ep copy_dll_step
+      COMMAND ${CMAKE_COMMAND} -E make_directory ${BUILD_OUTPUT_ROOT_DIRECTORY}
       COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_SHARED_LIB} ${BUILD_OUTPUT_ROOT_DIRECTORY}
       DEPENDEES build
       WORKING_DIRECTORY ${SOURCE_DIR})

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/src/parquet/arrow/arrow-reader-writer-test.cc
----------------------------------------------------------------------
diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc
index 48790d9..64690ae 100644
--- a/src/parquet/arrow/arrow-reader-writer-test.cc
+++ b/src/parquet/arrow/arrow-reader-writer-test.cc
@@ -661,10 +661,10 @@ TEST_F(TestInt96ParquetIO, ReadIntoTimestamp) {
 
   // 2nd January 1970, 11:35min 145738543ns
   Int96 day;
-  day.value[2] = 2440589l;
+  day.value[2] = UINT32_C(2440589);
   int64_t seconds = (11 * 60 + 35) * 60;
   *(reinterpret_cast<int64_t*>(&(day.value))) =
-      seconds * 1000l * 1000l * 1000l + 145738543;
+      seconds * INT64_C(1000) * INT64_C(1000) * INT64_C(1000) + 145738543;
   // Compute the corresponding nanosecond timestamp
   struct tm datetime = {0};
   datetime.tm_year = 70;
@@ -676,7 +676,7 @@ TEST_F(TestInt96ParquetIO, ReadIntoTimestamp) {
   epoch.tm_year = 70;
   epoch.tm_mday = 1;
   // Nanoseconds since the epoch
-  int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000000;
+  int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * INT64_C(1000000000);
   val += 145738543;
 
   std::vector<std::shared_ptr<schema::Node>> fields(
@@ -1064,7 +1064,7 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 TEST(TestArrowWrite, CheckChunkSize) {
   const int num_columns = 2;
   const int num_rows = 128;
-  const int64_t chunk_size = 0; // note the chunk_size is 0
+  const int64_t chunk_size = 0;  // note the chunk_size is 0
   std::shared_ptr<Table> table;
   MakeDoubleTable(num_columns, num_rows, 1, &table);
 

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/src/parquet/util/bit-util.h
----------------------------------------------------------------------
diff --git a/src/parquet/util/bit-util.h b/src/parquet/util/bit-util.h
index 8f0a270..559d5d5 100644
--- a/src/parquet/util/bit-util.h
+++ b/src/parquet/util/bit-util.h
@@ -329,7 +329,7 @@ class BitUtil {
   // Returns the minimum number of bits needed to represent the value of 'x'
   static inline int NumRequiredBits(uint64_t x) {
     for (int i = 63; i >= 0; --i) {
-      if (x & 1L << i) return i + 1;
+      if (x & (UINT64_C(1) << i)) return i + 1;
     }
     return 0;
   }


Mime
View raw message