arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-794: [C++/Python] Disallow strided tensors in ipc::WriteTensor
Date Mon, 10 Apr 2017 12:11:32 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 72e1e0875 -> acbda1893


ARROW-794: [C++/Python] Disallow strided tensors in ipc::WriteTensor

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #519 from wesm/ARROW-794 and squashes the following commits:

ab82ba7 [Wes McKinney] Typo
6f0a350 [Wes McKinney] Typo
e945298 [Wes McKinney] Raise ValueError if writing strided ndarray with WriteTensor and pyarrow.write_tensor


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/acbda189
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/acbda189
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/acbda189

Branch: refs/heads/master
Commit: acbda1893c55c68b3afd6c4cde1ee11e6926bb75
Parents: 72e1e08
Author: Wes McKinney <wes.mckinney@twosigma.com>
Authored: Mon Apr 10 08:11:25 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Mon Apr 10 08:11:25 2017 -0400

----------------------------------------------------------------------
 cpp/src/arrow/ipc/ipc-read-write-test.cc | 18 ++++++++++++++++++
 cpp/src/arrow/ipc/writer.cc              |  4 ++++
 python/pyarrow/tests/test_tensor.py      | 13 +++++++++++++
 3 files changed, 35 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/acbda189/cpp/src/arrow/ipc/ipc-read-write-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 6807296..1a91ec3 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -642,5 +642,23 @@ TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
   CheckTensorRoundTrip(tzero);
 }
 
+TEST_F(TestTensorRoundTrip, NonContiguous) {
+  std::string path = "test-write-tensor-strided";
+  constexpr int64_t kBufferSize = 1 << 20;
+  ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_));
+
+  std::vector<int64_t> values;
+  test::randint<int64_t>(24, 0, 100, &values);
+
+  auto data = test::GetBufferFromVector(values);
+  Int64Tensor tensor(data, {4, 3}, {48, 16});
+
+  int32_t metadata_length;
+  int64_t body_length;
+  ASSERT_OK(mmap_->Seek(0));
+  ASSERT_RAISES(
+      Invalid, WriteTensor(tensor, mmap_.get(), &metadata_length, &body_length));
+}
+
 }  // namespace ipc
 }  // namespace arrow

http://git-wip-us.apache.org/repos/asf/arrow/blob/acbda189/cpp/src/arrow/ipc/writer.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 9305567..d38a65c 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -496,6 +496,10 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
 
 Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
     int64_t* body_length) {
+  if (!tensor.is_contiguous()) {
+    return Status::Invalid("No support yet for writing non-contiguous tensors");
+  }
+
   RETURN_NOT_OK(AlignStreamPosition(dst));
   std::shared_ptr<Buffer> metadata;
   RETURN_NOT_OK(WriteTensorMessage(tensor, 0, &metadata));

http://git-wip-us.apache.org/repos/asf/arrow/blob/acbda189/python/pyarrow/tests/test_tensor.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_tensor.py b/python/pyarrow/tests/test_tensor.py
index a39064b..327b7f0 100644
--- a/python/pyarrow/tests/test_tensor.py
+++ b/python/pyarrow/tests/test_tensor.py
@@ -98,3 +98,16 @@ def test_tensor_ipc_roundtrip():
         assert result.equals(tensor)
     finally:
         _try_delete(path)
+
+
+def test_tensor_ipc_strided():
+    data = np.random.randn(10, 4)
+    tensor = pa.Tensor.from_numpy(data[::2])
+
+    path = 'pyarrow-tensor-ipc-strided'
+    try:
+        with pytest.raises(ValueError):
+            mmap = pa.create_memory_map(path, 1024)
+            pa.write_tensor(tensor, mmap)
+    finally:
+        _try_delete(path)


Mime
View raw message