parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject parquet-cpp git commit: PARQUET-1104: Upgrade to Apache Arrow 0.7.0 RC0
Date Wed, 13 Sep 2017 21:34:56 GMT
Repository: parquet-cpp
Updated Branches:
  refs/heads/master 6466cfd3a -> 005767c1c


PARQUET-1104: Upgrade to Apache Arrow 0.7.0 RC0

This will enable most of the outstanding PRs to be rebased and then merged

Close #384
Close #394

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #397 from wesm/PARQUET-1104 and squashes the following commits:

4106c5b [Wes McKinney] Include boost_system in static toolchain until ARROW-1536 resolved
005ee74 [Wes McKinney] Account for internal API change in ARROW-1511
3cab00d [Wes McKinney] Change MutableBuffer to Buffer to account for deprecation in ARROW-1402
7cd9702 [Wes McKinney] Upgrade to Apache Arrow 0.7.0 RC0


Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/005767c1
Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/005767c1
Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/005767c1

Branch: refs/heads/master
Commit: 005767c1ca69d95d62d5ae6cc47b06c6d99328e3
Parents: 6466cfd
Author: Wes McKinney <wes.mckinney@twosigma.com>
Authored: Wed Sep 13 17:34:52 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Wed Sep 13 17:34:52 2017 -0400

----------------------------------------------------------------------
 CMakeLists.txt                                |  3 ++-
 README.md                                     |  3 ++-
 cmake_modules/ThirdpartyToolchain.cmake       | 20 +++++++++++++++++---
 src/parquet/arrow/arrow-reader-writer-test.cc |  8 ++++----
 src/parquet/arrow/reader.cc                   |  8 ++++----
 src/parquet/statistics-test.cc                | 10 ++++------
 6 files changed, 33 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1ecf877..1deab40 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -499,7 +499,8 @@ if (PARQUET_BOOST_USE_SHARED)
     boost_shared_regex)
 else()
   set(BOOST_LINK_LIBS
-    boost_static_regex)
+    boost_static_regex
+    boost_static_system)
 endif()
 
 #############################################################

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 770f0ca..5407977 100644
--- a/README.md
+++ b/README.md
@@ -66,7 +66,8 @@ Check [Windows developer guide][1] for instructions to build parquet-cpp
on Wind
 
 ## Third Party Dependencies
 
-- Apache Arrow (memory management, compression, IO, optional columnar data adapters)
+- Apache Arrow >= 0.7.0 (memory management, compression, IO, optional columnar
+  data adapters)
 - Thrift 0.7+ [install instructions](https://thrift.apache.org/docs/install/)
 - googletest 1.7.0 (cannot be installed with package managers)
 - Google Benchmark (only required if building benchmarks)

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/cmake_modules/ThirdpartyToolchain.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index b1ea10a..1221765 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -70,20 +70,24 @@ if (PARQUET_BOOST_USE_SHARED)
     add_definitions(-DBOOST_ALL_DYN_LINK)
   endif()
 
-  find_package(Boost COMPONENTS regex REQUIRED)
+  find_package(Boost COMPONENTS regex system REQUIRED)
   if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
+    set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
   else()
     set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
+    set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
   endif()
 else()
   # Find static Boost libraries.
   set(Boost_USE_STATIC_LIBS ON)
-  find_package(Boost COMPONENTS regex REQUIRED)
+  find_package(Boost COMPONENTS regex system REQUIRED)
   if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
     set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
+    set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
   else()
     set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
+    set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
   endif()
 endif()
 
@@ -98,9 +102,19 @@ if (PARQUET_BOOST_USE_SHARED)
     set_target_properties(boost_shared_regex
                           PROPERTIES IMPORTED_LOCATION "${BOOST_SHARED_REGEX_LIBRARY}")
   endif()
+  add_library(boost_shared_system SHARED IMPORTED)
+  if (MSVC)
+    set_target_properties(boost_shared_system
+                          PROPERTIES IMPORTED_IMPLIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
+  else()
+    set_target_properties(boost_shared_system
+                          PROPERTIES IMPORTED_LOCATION "${BOOST_SHARED_SYSTEM_LIBRARY}")
+  endif()
 else()
   add_library(boost_static_regex STATIC IMPORTED)
   set_target_properties(boost_static_regex PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_REGEX_LIBRARY})
+  add_library(boost_static_system STATIC IMPORTED)
+  set_target_properties(boost_static_system PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_SYSTEM_LIBRARY})
 endif()
 
 include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
@@ -339,7 +353,7 @@ if (NOT ARROW_FOUND)
     -DARROW_BUILD_TESTS=OFF)
 
   if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "")
-    set(ARROW_VERSION "939957f33ed0dd02013917b366ff85eb857c3947")
+    set(ARROW_VERSION "97f9029ce835dfc2655ca91b9820a2e6aed89107")
   else()
     set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}")
   endif()

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/src/parquet/arrow/arrow-reader-writer-test.cc
----------------------------------------------------------------------
diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc
index 986adfc..c7ac057 100644
--- a/src/parquet/arrow/arrow-reader-writer-test.cc
+++ b/src/parquet/arrow/arrow-reader-writer-test.cc
@@ -779,10 +779,10 @@ TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compability) {
     }
   }
 
-  const int32_t kOffset = 0;
-  ASSERT_OK(MakePrimitiveArray(std::make_shared<::arrow::Int64Type>(), values->length(),
-                               int64_data, values->null_bitmap(), values->null_count(),
-                               kOffset, &expected_values));
+  std::vector<std::shared_ptr<Buffer>> buffers{values->null_bitmap(), int64_data};
+  auto arr_data = std::make_shared<::arrow::ArrayData>(::arrow::int64(), values->length(),
+                                                       buffers, values->null_count());
+  ASSERT_OK(MakeArray(arr_data, &expected_values));
   this->ReadAndCheckSingleColumnTable(expected_values);
 }
 

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/src/parquet/arrow/reader.cc
----------------------------------------------------------------------
diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc
index 9908280..8d5ea7e 100644
--- a/src/parquet/arrow/reader.cc
+++ b/src/parquet/arrow/reader.cc
@@ -334,7 +334,7 @@ class PARQUET_NO_EXPORT StructImpl : public ColumnReader::Impl {
   std::shared_ptr<Field> field_;
   PoolBuffer def_levels_buffer_;
 
-  Status DefLevelsToNullArray(std::shared_ptr<MutableBuffer>* null_bitmap,
+  Status DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap,
                               int64_t* null_count);
   void InitField(const NodePtr& node, const std::vector<std::shared_ptr<Impl>>&
children);
 };
@@ -1376,9 +1376,9 @@ Status ColumnReader::NextBatch(int batch_size, std::shared_ptr<Array>*
out) {
 
 // StructImpl methods
 
-Status StructImpl::DefLevelsToNullArray(std::shared_ptr<MutableBuffer>* null_bitmap_out,
+Status StructImpl::DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap_out,
                                         int64_t* null_count_out) {
-  std::shared_ptr<MutableBuffer> null_bitmap;
+  std::shared_ptr<Buffer> null_bitmap;
   auto null_count = 0;
   ValueLevelsPtr def_levels_data;
   size_t def_levels_length;
@@ -1460,7 +1460,7 @@ Status StructImpl::GetRepLevels(ValueLevelsPtr* data, size_t* length)
{
 
 Status StructImpl::NextBatch(int batch_size, std::shared_ptr<Array>* out) {
   std::vector<std::shared_ptr<Array>> children_arrays;
-  std::shared_ptr<MutableBuffer> null_bitmap;
+  std::shared_ptr<Buffer> null_bitmap;
   int64_t null_count;
 
   // Gather children arrays and def levels

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/src/parquet/statistics-test.cc
----------------------------------------------------------------------
diff --git a/src/parquet/statistics-test.cc b/src/parquet/statistics-test.cc
index d6c5205..1521cbd 100644
--- a/src/parquet/statistics-test.cc
+++ b/src/parquet/statistics-test.cc
@@ -194,8 +194,9 @@ bool* TestRowGroupStatistics<BooleanType>::GetValuesPointer(std::vector<bool>&
v
 }
 
 template <typename TestType>
-typename std::vector<typename TestType::c_type> TestRowGroupStatistics<
-    TestType>::GetDeepCopy(const std::vector<typename TestType::c_type>& values)
{
+typename std::vector<typename TestType::c_type>
+TestRowGroupStatistics<TestType>::GetDeepCopy(
+    const std::vector<typename TestType::c_type>& values) {
   return values;
 }
 
@@ -478,7 +479,7 @@ class TestStatistics : public ::testing::Test {
 };
 
 using CompareTestTypes = ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType,
-                                   DoubleType, ByteArrayType, FLBAType>;
+                                          DoubleType, ByteArrayType, FLBAType>;
 
 // TYPE::INT32
 template <>
@@ -680,8 +681,5 @@ TEST_F(TestStatisticsFLBA, UnknownSortOrder) {
   ASSERT_FALSE(cc_metadata->is_stats_set());
 }
 
-
-
-
 }  // namespace test
 }  // namespace parquet


Mime
View raw message