parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [1/4] parquet-cpp git commit: PARQUET-818: Refactoring to utilize common IO, buffer, memory management abstractions and implementations
Date Fri, 30 Dec 2016 16:36:35 GMT
Repository: parquet-cpp
Updated Branches:
  refs/heads/master 1c4012d36 -> 2154e873d


http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/2154e873/src/parquet/util/output.cc
----------------------------------------------------------------------
diff --git a/src/parquet/util/output.cc b/src/parquet/util/output.cc
deleted file mode 100644
index 422000f..0000000
--- a/src/parquet/util/output.cc
+++ /dev/null
@@ -1,118 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/util/output.h"
-
-#include <cstring>
-#include <memory>
-#include <sstream>
-
-#include "parquet/exception.h"
-#include "parquet/util/buffer.h"
-#include "parquet/util/logging.h"
-
-namespace parquet {
-
-// ----------------------------------------------------------------------
-// OutputStream
-
-OutputStream::~OutputStream() {}
-
-// ----------------------------------------------------------------------
-// In-memory output stream
-
-InMemoryOutputStream::InMemoryOutputStream(
-    int64_t initial_capacity, MemoryAllocator* allocator)
-    : size_(0), capacity_(initial_capacity) {
-  if (initial_capacity == 0) { initial_capacity = IN_MEMORY_DEFAULT_CAPACITY; }
-  buffer_.reset(new OwnedMutableBuffer(initial_capacity, allocator));
-}
-
-InMemoryOutputStream::~InMemoryOutputStream() {}
-
-uint8_t* InMemoryOutputStream::Head() {
-  return buffer_->mutable_data() + size_;
-}
-
-void InMemoryOutputStream::Write(const uint8_t* data, int64_t length) {
-  if (size_ + length > capacity_) {
-    int64_t new_capacity = capacity_ * 2;
-    while (new_capacity < size_ + length) {
-      new_capacity *= 2;
-    }
-    buffer_->Resize(new_capacity);
-    capacity_ = new_capacity;
-  }
-  memcpy(Head(), data, length);
-  size_ += length;
-}
-
-int64_t InMemoryOutputStream::Tell() {
-  return size_;
-}
-
-std::shared_ptr<Buffer> InMemoryOutputStream::GetBuffer() {
-  buffer_->Resize(size_);
-  std::shared_ptr<Buffer> result = buffer_;
-  buffer_ = nullptr;
-  return result;
-}
-
-// ----------------------------------------------------------------------
-// local file output stream
-
-LocalFileOutputStream::LocalFileOutputStream(const std::string& path) : is_open_(true)
{
-  file_ = fopen(path.c_str(), "wb");
-  if (file_ == nullptr || ferror(file_)) {
-    std::stringstream ss;
-    ss << "Unable to open file: " << path;
-    throw ParquetException(ss.str());
-  }
-}
-
-LocalFileOutputStream::~LocalFileOutputStream() {
-  CloseFile();
-}
-
-void LocalFileOutputStream::Close() {
-  CloseFile();
-}
-
-int64_t LocalFileOutputStream::Tell() {
-  DCHECK(is_open_);
-  int64_t position = ftell(file_);
-  if (position < 0) { throw ParquetException("ftell failed, did the file disappear?");
}
-  return position;
-}
-
-void LocalFileOutputStream::Write(const uint8_t* data, int64_t length) {
-  DCHECK(is_open_);
-  int64_t bytes_written = fwrite(data, sizeof(uint8_t), length, file_);
-  if (bytes_written != length) {
-    int error_code = ferror(file_);
-    throw ParquetException("fwrite failed, error code: " + std::to_string(error_code));
-  }
-}
-
-void LocalFileOutputStream::CloseFile() {
-  if (is_open_) {
-    fclose(file_);
-    is_open_ = false;
-  }
-}
-
-}  // namespace parquet

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/2154e873/src/parquet/util/output.h
----------------------------------------------------------------------
diff --git a/src/parquet/util/output.h b/src/parquet/util/output.h
deleted file mode 100644
index 9b2c2d3..0000000
--- a/src/parquet/util/output.h
+++ /dev/null
@@ -1,107 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_OUTPUT_H
-#define PARQUET_UTIL_OUTPUT_H
-
-#include <cstdint>
-#include <memory>
-#include <string>
-
-#include "parquet/util/macros.h"
-#include "parquet/util/mem-allocator.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-class Buffer;
-class ResizableBuffer;
-
-// ----------------------------------------------------------------------
-// Output stream classes
-
-// Abstract output stream
-class PARQUET_EXPORT OutputStream {
- public:
-  virtual ~OutputStream();
-
-  // Close the output stream
-  virtual void Close() = 0;
-
-  // Return the current position in the output stream relative to the start
-  virtual int64_t Tell() = 0;
-
-  // Copy bytes into the output stream
-  virtual void Write(const uint8_t* data, int64_t length) = 0;
-};
-
-static constexpr int64_t IN_MEMORY_DEFAULT_CAPACITY = 1024;
-
-// An output stream that is an in-memory
-class PARQUET_EXPORT InMemoryOutputStream : public OutputStream {
- public:
-  explicit InMemoryOutputStream(int64_t initial_capacity = IN_MEMORY_DEFAULT_CAPACITY,
-      MemoryAllocator* allocator = default_allocator());
-
-  virtual ~InMemoryOutputStream();
-
-  // Close is currently a no-op with the in-memory stream
-  virtual void Close() {}
-
-  virtual int64_t Tell();
-
-  virtual void Write(const uint8_t* data, int64_t length);
-
-  // Return complete stream as Buffer
-  std::shared_ptr<Buffer> GetBuffer();
-
- private:
-  // Mutable pointer to the current write position in the stream
-  uint8_t* Head();
-
-  std::shared_ptr<ResizableBuffer> buffer_;
-  int64_t size_;
-  int64_t capacity_;
-
-  DISALLOW_COPY_AND_ASSIGN(InMemoryOutputStream);
-};
-
-class PARQUET_EXPORT LocalFileOutputStream : public OutputStream {
- public:
-  explicit LocalFileOutputStream(const std::string& path);
-
-  virtual ~LocalFileOutputStream();
-
-  // Close the output stream
-  void Close() override;
-
-  // Return the current position in the output stream relative to the start
-  int64_t Tell() override;
-
-  // Copy bytes into the output stream
-  void Write(const uint8_t* data, int64_t length) override;
-
- private:
-  void CloseFile();
-
-  FILE* file_;
-  bool is_open_;
-};
-
-}  // namespace parquet
-
-#endif  // PARQUET_UTIL_OUTPUT_H

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/2154e873/src/parquet/util/rle-encoding.h
----------------------------------------------------------------------
diff --git a/src/parquet/util/rle-encoding.h b/src/parquet/util/rle-encoding.h
index 7aba813..d4be1fc 100644
--- a/src/parquet/util/rle-encoding.h
+++ b/src/parquet/util/rle-encoding.h
@@ -25,8 +25,8 @@
 
 #include "parquet/util/bit-stream-utils.inline.h"
 #include "parquet/util/bit-util.h"
-#include "parquet/util/buffer.h"
 #include "parquet/util/compiler-util.h"
+#include "parquet/util/memory.h"
 
 namespace parquet {
 


Mime
View raw message