parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject parquet-cpp git commit: PARQUET-829: Make use of ARROW-469
Date Wed, 11 Jan 2017 22:17:54 GMT
Repository: parquet-cpp
Updated Branches:
  refs/heads/master 585469c7f -> c0870f7ab


PARQUET-829: Make use of ARROW-469

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #217 from xhochy/PARQUET-829 and squashes the following commits:

d750e5c [Uwe L. Korn] PARQUET-829: Make use of ARROW-469


Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/c0870f7a
Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/c0870f7a
Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/c0870f7a

Branch: refs/heads/master
Commit: c0870f7ab228aa7c8f34b255a2bcfc815ee6c95d
Parents: 585469c
Author: Uwe L. Korn <uwelk@xhochy.com>
Authored: Wed Jan 11 17:17:47 2017 -0500
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Wed Jan 11 17:17:47 2017 -0500

----------------------------------------------------------------------
 cmake_modules/FindArrow.cmake           |  8 ++++++--
 cmake_modules/ThirdpartyToolchain.cmake |  2 +-
 src/parquet/arrow/reader.cc             | 26 +++++++++++++-------------
 3 files changed, 20 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/c0870f7a/cmake_modules/FindArrow.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/FindArrow.cmake b/cmake_modules/FindArrow.cmake
index eec5d66..6c67b7f 100644
--- a/cmake_modules/FindArrow.cmake
+++ b/cmake_modules/FindArrow.cmake
@@ -23,12 +23,16 @@
 #  ARROW_SHARED_LIB, path to libarrow's shared library
 #  ARROW_FOUND, whether arrow has been found
 
+if( NOT "$ENV{ARROW_HOME}" STREQUAL "")
+  set(ARROW_HOME "$ENV{ARROW_HOME}")
+endif()
+
 set(ARROW_SEARCH_HEADER_PATHS
-  $ENV{ARROW_HOME}/include
+  ${ARROW_HOME}/include
 )
 
 set(ARROW_SEARCH_LIB_PATH
-  $ENV{ARROW_HOME}/lib
+  ${ARROW_HOME}/lib
 )
 
 find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/c0870f7a/cmake_modules/ThirdpartyToolchain.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index 54033ec..9a17dcf 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -22,7 +22,7 @@ set(THRIFT_VERSION "0.9.1")
 
 # Brotli 0.5.2 does not install headers/libraries yet, but 0.6.0.dev does
 set(BROTLI_VERSION "5db62dcc9d386579609540cdf8869e95ad334bbd")
-set(ARROW_VERSION "74685f386307171a90a9f97316e25b7f39cdd0a1")
+set(ARROW_VERSION "7d3e2a3ab90324625b738e464a020758379f457a")
 
 # find boost headers and libs
 set(Boost_DEBUG TRUE)

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/c0870f7a/src/parquet/arrow/reader.cc
----------------------------------------------------------------------
diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc
index db281d9..cecbc42 100644
--- a/src/parquet/arrow/reader.cc
+++ b/src/parquet/arrow/reader.cc
@@ -319,7 +319,7 @@ template <typename ArrowType>
 Status FlatColumnReader::Impl::InitDataBuffer(int batch_size) {
   using ArrowCType = typename ArrowType::c_type;
   data_buffer_ = std::make_shared<PoolBuffer>(pool_);
-  RETURN_NOT_OK(data_buffer_->Resize(batch_size * sizeof(ArrowCType)));
+  RETURN_NOT_OK(data_buffer_->Resize(batch_size * sizeof(ArrowCType), false));
   data_buffer_ptr_ = data_buffer_->mutable_data();
 
   return Status::OK();
@@ -328,7 +328,7 @@ Status FlatColumnReader::Impl::InitDataBuffer(int batch_size) {
 template <>
 Status FlatColumnReader::Impl::InitDataBuffer<::arrow::BooleanType>(int batch_size)
{
   data_buffer_ = std::make_shared<PoolBuffer>(pool_);
-  RETURN_NOT_OK(data_buffer_->Resize(::arrow::BitUtil::CeilByte(batch_size) / 8));
+  RETURN_NOT_OK(data_buffer_->Resize(::arrow::BitUtil::CeilByte(batch_size) / 8, false));
   data_buffer_ptr_ = data_buffer_->mutable_data();
   memset(data_buffer_ptr_, 0, data_buffer_->size());
 
@@ -347,16 +347,16 @@ Status FlatColumnReader::Impl::TypedReadBatch(
   if (descr_->max_definition_level() > 0) {
     int valid_bits_size = ::arrow::BitUtil::CeilByte(batch_size) / 8;
     valid_bits_buffer_ = std::make_shared<PoolBuffer>(pool_);
-    RETURN_NOT_OK(valid_bits_buffer_->Resize(valid_bits_size));
+    RETURN_NOT_OK(valid_bits_buffer_->Resize(valid_bits_size, false));
     valid_bits_ptr_ = valid_bits_buffer_->mutable_data();
     memset(valid_bits_ptr_, 0, valid_bits_size);
     null_count_ = 0;
   }
 
   while ((values_to_read > 0) && column_reader_) {
-    RETURN_NOT_OK(values_buffer_.Resize(values_to_read * sizeof(ParquetCType)));
+    RETURN_NOT_OK(values_buffer_.Resize(values_to_read * sizeof(ParquetCType), false));
     if (descr_->max_definition_level() > 0) {
-      RETURN_NOT_OK(def_levels_buffer_.Resize(values_to_read * sizeof(int16_t)));
+      RETURN_NOT_OK(def_levels_buffer_.Resize(values_to_read * sizeof(int16_t), false));
     }
     auto reader = dynamic_cast<TypedColumnReader<ParquetType>*>(column_reader_.get());
     int64_t values_read;
@@ -386,13 +386,13 @@ Status FlatColumnReader::Impl::TypedReadBatch(
       //    should still free some underlying pages to the OS.
 
       auto data_buffer = std::make_shared<PoolBuffer>(pool_);
-      RETURN_NOT_OK(data_buffer->Resize(valid_bits_idx_ * sizeof(ArrowCType)));
+      RETURN_NOT_OK(data_buffer->Resize(valid_bits_idx_ * sizeof(ArrowCType), false));
       memcpy(data_buffer->mutable_data(), data_buffer_->data(), data_buffer->size());
       data_buffer_ = data_buffer;
 
       auto valid_bits_buffer = std::make_shared<PoolBuffer>(pool_);
-      RETURN_NOT_OK(
-          valid_bits_buffer->Resize(::arrow::BitUtil::CeilByte(valid_bits_idx_) / 8));
+      RETURN_NOT_OK(valid_bits_buffer->Resize(
+          ::arrow::BitUtil::CeilByte(valid_bits_idx_) / 8, false));
       memcpy(valid_bits_buffer->mutable_data(), valid_bits_buffer_->data(),
           valid_bits_buffer->size());
       valid_bits_buffer_ = valid_bits_buffer;
@@ -420,16 +420,16 @@ Status FlatColumnReader::Impl::TypedReadBatch<::arrow::BooleanType,
BooleanType>
   if (descr_->max_definition_level() > 0) {
     valid_bits_buffer_ = std::make_shared<PoolBuffer>(pool_);
     int valid_bits_size = ::arrow::BitUtil::CeilByte(batch_size) / 8;
-    RETURN_NOT_OK(valid_bits_buffer_->Resize(valid_bits_size));
+    RETURN_NOT_OK(valid_bits_buffer_->Resize(valid_bits_size, false));
     valid_bits_ptr_ = valid_bits_buffer_->mutable_data();
     memset(valid_bits_ptr_, 0, valid_bits_size);
     null_count_ = 0;
   }
 
   while ((values_to_read > 0) && column_reader_) {
-    RETURN_NOT_OK(values_buffer_.Resize(values_to_read * sizeof(bool)));
+    RETURN_NOT_OK(values_buffer_.Resize(values_to_read * sizeof(bool), false));
     if (descr_->max_definition_level() > 0) {
-      RETURN_NOT_OK(def_levels_buffer_.Resize(values_to_read * sizeof(int16_t)));
+      RETURN_NOT_OK(def_levels_buffer_.Resize(values_to_read * sizeof(int16_t), false));
     }
     auto reader = dynamic_cast<TypedColumnReader<BooleanType>*>(column_reader_.get());
     int64_t values_read;
@@ -492,9 +492,9 @@ Status FlatColumnReader::Impl::ReadByteArrayBatch(
   int values_to_read = batch_size;
   BuilderType builder(pool_, field_->type);
   while ((values_to_read > 0) && column_reader_) {
-    RETURN_NOT_OK(values_buffer_.Resize(values_to_read * sizeof(ByteArray)));
+    RETURN_NOT_OK(values_buffer_.Resize(values_to_read * sizeof(ByteArray), false));
     if (descr_->max_definition_level() > 0) {
-      RETURN_NOT_OK(def_levels_buffer_.Resize(values_to_read * sizeof(int16_t)));
+      RETURN_NOT_OK(def_levels_buffer_.Resize(values_to_read * sizeof(int16_t), false));
     }
     auto reader = dynamic_cast<TypedColumnReader<ByteArrayType>*>(column_reader_.get());
     int64_t values_read;


Mime
View raw message