parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject parquet-cpp git commit: PARQUET-721: benchmarks for reading into Arrow
Date Mon, 26 Sep 2016 02:35:18 GMT
Repository: parquet-cpp
Updated Branches:
  refs/heads/master 5c1d9e94b -> 549a58b01


PARQUET-721: benchmarks for reading into Arrow

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #165 from xhochy/parquet-721 and squashes the following commits:

1c34bb1 [Uwe L. Korn] Fix cmake problems
b4a4448 [Uwe L. Korn] PARQUET-721: benchmarks for reading into Arrow


Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/549a58b0
Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/549a58b0
Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/549a58b0

Branch: refs/heads/master
Commit: 549a58b010df5e5f5dcf601108e62fd1696ef788
Parents: 5c1d9e9
Author: Uwe L. Korn <uwelk@xhochy.com>
Authored: Sun Sep 25 22:35:01 2016 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Sun Sep 25 22:35:01 2016 -0400

----------------------------------------------------------------------
 src/parquet/arrow/CMakeLists.txt                |   9 ++
 .../arrow/arrow-reader-writer-benchmark.cc      | 157 +++++++++++++++++++
 2 files changed, 166 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/549a58b0/src/parquet/arrow/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/parquet/arrow/CMakeLists.txt b/src/parquet/arrow/CMakeLists.txt
index f36abe6..376e84a 100644
--- a/src/parquet/arrow/CMakeLists.txt
+++ b/src/parquet/arrow/CMakeLists.txt
@@ -83,6 +83,15 @@ else()
   ADD_PARQUET_LINK_LIBRARIES(arrow-reader-writer-test parquet_arrow_shared)
 endif()
 
+if(PARQUET_BUILD_BENCHMARKS)
+  ADD_PARQUET_BENCHMARK(arrow-reader-writer-benchmark)
+  if (PARQUET_BUILD_STATIC)
+    ADD_PARQUET_LINK_LIBRARIES(arrow-reader-writer-benchmark parquet_arrow_static)
+  else()
+    ADD_PARQUET_LINK_LIBRARIES(arrow-reader-writer-benchmark parquet_arrow_shared)
+  endif()
+endif()
+
 # Headers: top level
 install(FILES
   io.h

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/549a58b0/src/parquet/arrow/arrow-reader-writer-benchmark.cc
----------------------------------------------------------------------
diff --git a/src/parquet/arrow/arrow-reader-writer-benchmark.cc b/src/parquet/arrow/arrow-reader-writer-benchmark.cc
new file mode 100644
index 0000000..5f0b3f5
--- /dev/null
+++ b/src/parquet/arrow/arrow-reader-writer-benchmark.cc
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "benchmark/benchmark.h"
+
+#include "parquet/arrow/reader.h"
+#include "parquet/arrow/writer.h"
+#include "parquet/file/reader-internal.h"
+#include "parquet/file/writer-internal.h"
+#include "parquet/column/reader.h"
+#include "parquet/column/writer.h"
+#include "parquet/util/input.h"
+
+#include "arrow/column.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
+#include "arrow/types/primitive.h"
+
+using arrow::NumericBuilder;
+
+namespace parquet {
+
+using arrow::FileReader;
+using arrow::WriteFlatTable;
+using schema::PrimitiveNode;
+
+namespace benchmark {
+
+// This should result in multiple pages for most primitive types
+constexpr int64_t BENCHMARK_SIZE = 10 * 1024 * 1024;
+
+template <typename ParquetType>
+struct benchmark_traits {};
+
+template <>
+struct benchmark_traits<Int32Type> {
+  using arrow_type = ::arrow::Int32Type;
+};
+
+template <>
+struct benchmark_traits<Int64Type> {
+  using arrow_type = ::arrow::Int64Type;
+};
+
+template <>
+struct benchmark_traits<DoubleType> {
+  using arrow_type = ::arrow::DoubleType;
+};
+
+template <typename ParquetType>
+using ArrowType = typename benchmark_traits<ParquetType>::arrow_type;
+
+template <typename ParquetType>
+std::shared_ptr<ColumnDescriptor> MakeSchema(Repetition::type repetition) {
+  auto node = PrimitiveNode::Make("int64", repetition, ParquetType::type_num);
+  return std::make_shared<ColumnDescriptor>(
+      node, repetition != Repetition::REQUIRED, repetition == Repetition::REPEATED);
+}
+
+template <bool nullable, typename ParquetType>
+void SetBytesProcessed(::benchmark::State& state) {
+  int64_t bytes_processed =
+      state.iterations() * BENCHMARK_SIZE * sizeof(typename ParquetType::c_type);
+  if (nullable) {
+    bytes_processed += state.iterations() * BENCHMARK_SIZE * sizeof(int16_t);
+  }
+  state.SetBytesProcessed(bytes_processed);
+}
+
+template <bool nullable, typename ParquetType>
+std::shared_ptr<::arrow::Table> TableFromVector(
+    const std::vector<typename ParquetType::c_type>& vec) {
+  ::arrow::TypePtr type = std::make_shared<ArrowType<ParquetType>>();
+  NumericBuilder<ArrowType<ParquetType>> builder(::arrow::default_memory_pool(),
type);
+  if (nullable) {
+    std::vector<uint8_t> valid_bytes(BENCHMARK_SIZE, 0);
+    int n = {0};
+    std::generate(valid_bytes.begin(), valid_bytes.end(), [&n] { return n++ % 2; });
+    builder.Append(vec.data(), vec.size(), valid_bytes.data());
+  } else {
+    builder.Append(vec.data(), vec.size(), nullptr);
+  }
+  std::shared_ptr<::arrow::Array> array = builder.Finish();
+  auto field = std::make_shared<::arrow::Field>("column", type, nullable);
+  auto schema = std::make_shared<::arrow::Schema>(
+      std::vector<std::shared_ptr<::arrow::Field>>({field}));
+  auto column = std::make_shared<::arrow::Column>(field, array);
+  return std::make_shared<::arrow::Table>(
+      "table", schema, std::vector<std::shared_ptr<::arrow::Column>>({column}));
+}
+
+template <bool nullable, typename ParquetType>
+static void BM_WriteColumn(::benchmark::State& state) {
+  format::ColumnChunk thrift_metadata;
+  std::vector<typename ParquetType::c_type> values(BENCHMARK_SIZE, 128);
+  std::shared_ptr<::arrow::Table> table = TableFromVector<nullable, ParquetType>(values);
+
+  while (state.KeepRunning()) {
+    auto output = std::make_shared<InMemoryOutputStream>();
+    WriteFlatTable(table.get(), ::arrow::default_memory_pool(), output, BENCHMARK_SIZE);
+  }
+  SetBytesProcessed<nullable, ParquetType>(state);
+}
+
+BENCHMARK_TEMPLATE(BM_WriteColumn, false, Int32Type);
+BENCHMARK_TEMPLATE(BM_WriteColumn, true, Int32Type);
+
+BENCHMARK_TEMPLATE(BM_WriteColumn, false, Int64Type);
+BENCHMARK_TEMPLATE(BM_WriteColumn, true, Int64Type);
+
+BENCHMARK_TEMPLATE(BM_WriteColumn, false, DoubleType);
+BENCHMARK_TEMPLATE(BM_WriteColumn, true, DoubleType);
+
+template <bool nullable, typename ParquetType>
+static void BM_ReadColumn(::benchmark::State& state) {
+  std::vector<typename ParquetType::c_type> values(BENCHMARK_SIZE, 128);
+  std::shared_ptr<::arrow::Table> table = TableFromVector<nullable, ParquetType>(values);
+  auto output = std::make_shared<InMemoryOutputStream>();
+  WriteFlatTable(table.get(), ::arrow::default_memory_pool(), output, BENCHMARK_SIZE);
+  std::shared_ptr<Buffer> buffer = output->GetBuffer();
+
+  while (state.KeepRunning()) {
+    auto reader = ParquetFileReader::Open(
+        std::unique_ptr<RandomAccessSource>(new BufferReader(buffer)));
+    FileReader filereader(::arrow::default_memory_pool(), std::move(reader));
+    std::shared_ptr<::arrow::Table> table;
+    filereader.ReadFlatTable(&table);
+  }
+  SetBytesProcessed<nullable, ParquetType>(state);
+}
+
+BENCHMARK_TEMPLATE(BM_ReadColumn, false, Int32Type);
+BENCHMARK_TEMPLATE(BM_ReadColumn, true, Int32Type);
+
+BENCHMARK_TEMPLATE(BM_ReadColumn, false, Int64Type);
+BENCHMARK_TEMPLATE(BM_ReadColumn, true, Int64Type);
+
+BENCHMARK_TEMPLATE(BM_ReadColumn, false, DoubleType);
+BENCHMARK_TEMPLATE(BM_ReadColumn, true, DoubleType);
+
+}  // namespace benchmark
+
+}  // namespace parquet


Mime
View raw message