arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-788: [C++] Align WriteTensor message
Date Sun, 09 Apr 2017 17:56:20 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 739ed8202 -> b0863cb63


ARROW-788: [C++] Align WriteTensor message

Author: Philipp Moritz <pcmoritz@gmail.com>

Closes #512 from pcmoritz/tensor-alignment and squashes the following commits:

fd20f05 [Philipp Moritz] align WriteTensor to 8 bytes


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b0863cb6
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b0863cb6
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b0863cb6

Branch: refs/heads/master
Commit: b0863cb63d62ae7c4a429164e5a2e350d3c1f21a
Parents: 739ed82
Author: Philipp Moritz <pcmoritz@gmail.com>
Authored: Sun Apr 9 13:56:14 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Sun Apr 9 13:56:14 2017 -0400

----------------------------------------------------------------------
 cpp/src/arrow/ipc/reader.cc |  2 ++
 cpp/src/arrow/ipc/writer.cc | 11 +++++++++++
 2 files changed, 13 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/b0863cb6/cpp/src/arrow/ipc/reader.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 55f632f..a7c4f04 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -495,6 +495,8 @@ Status ReadRecordBatch(const std::shared_ptr<Schema>& schema,
int64_t offset,
 
 Status ReadTensor(
     int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out) {
+  // Respect alignment of Tensor messages (see WriteTensor)
+  offset = PaddedLength(offset);
   std::shared_ptr<Message> message;
   std::shared_ptr<Buffer> data;
   RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message, &data));

http://git-wip-us.apache.org/repos/asf/arrow/blob/b0863cb6/cpp/src/arrow/ipc/writer.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 5330206..9305567 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -470,6 +470,16 @@ class DictionaryWriter : public RecordBatchWriter {
   int64_t dictionary_id_;
 };
 
+// Adds padding bytes if necessary to ensure all memory blocks are written on
+// 8-byte boundaries.
+Status AlignStreamPosition(io::OutputStream* stream) {
+  int64_t position;
+  RETURN_NOT_OK(stream->Tell(&position));
+  int64_t remainder = PaddedLength(position) - position;
+  if (remainder > 0) { return stream->Write(kPaddingBytes, remainder); }
+  return Status::OK();
+}
+
 Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
     io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
     MemoryPool* pool, int max_recursion_depth, bool allow_64bit) {
@@ -486,6 +496,7 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
 
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
     int64_t* body_length) {
+  RETURN_NOT_OK(AlignStreamPosition(dst));
   std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(WriteTensorMessage(tensor, 0, &metadata));
   RETURN_NOT_OK(WriteMessage(*metadata, dst, metadata_length));


Mime
View raw message