arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-1096: [C++] CreateFileMapping maximum size calculation issue
Date Fri, 16 Jun 2017 01:04:56 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 5b66c25c9 -> 3f26dfa36


ARROW-1096: [C++] CreateFileMapping maximum size calculation issue

Author: Max Risuhin <risuhin.max@gmail.com>

Closes #756 from MaxRis/ARROW-1096 and squashes the following commits:

a864fe7f [Max Risuhin] [C++] CreateFileMapping maximum size calculation issue


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/3f26dfa3
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/3f26dfa3
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/3f26dfa3

Branch: refs/heads/master
Commit: 3f26dfa36488c34329e31f44041f5b842248619c
Parents: 5b66c25
Author: Max Risuhin <risuhin.max@gmail.com>
Authored: Thu Jun 15 21:04:51 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Thu Jun 15 21:04:51 2017 -0400

----------------------------------------------------------------------
 cpp/src/arrow/io/io-file-test.cc | 35 +++++++++++++++++++++++++++++++++++
 cpp/src/arrow/io/mman.h          |  9 +++------
 cpp/src/arrow/io/test-common.h   |  2 ++
 3 files changed, 40 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/3f26dfa3/cpp/src/arrow/io/io-file-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index 1b7e052..6550c74 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -462,6 +462,41 @@ TEST_F(TestMemoryMappedFile, ReadOnly) {
   rommap->Close();
 }
 
+TEST_F(TestMemoryMappedFile, DISABLED_ReadWriteOver4GbFile) {
+  // ARROW-1096
+  const int64_t buffer_size = 1000 * 1000;
+  std::vector<uint8_t> buffer(buffer_size);
+
+  test::random_bytes(buffer_size, 0, buffer.data());
+
+  const int64_t reps = 5000;
+
+  std::string path = "ipc-read-over-4gb-file-test";
+  std::shared_ptr<MemoryMappedFile> rwmmap;
+  ASSERT_OK(InitMemoryMap(reps * buffer_size, path, &rwmmap));
+  AppendFile(path);
+
+  int64_t position = 0;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(rwmmap->Write(buffer.data(), buffer_size));
+    position += buffer_size;
+  }
+  rwmmap->Close();
+
+  std::shared_ptr<MemoryMappedFile> rommap;
+  ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READ, &rommap));
+
+  position = 0;
+  std::shared_ptr<Buffer> out_buffer;
+  for (int i = 0; i < reps; ++i) {
+    ASSERT_OK(rommap->ReadAt(position, buffer_size, &out_buffer));
+
+    ASSERT_EQ(0, memcmp(out_buffer->data(), buffer.data(), buffer_size));
+    position += buffer_size;
+  }
+  rommap->Close();
+}
+
 TEST_F(TestMemoryMappedFile, RetainMemoryMapReference) {
   // ARROW-494
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/3f26dfa3/cpp/src/arrow/io/mman.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/mman.h b/cpp/src/arrow/io/mman.h
index 27d9736..e45c88e 100644
--- a/cpp/src/arrow/io/mman.h
+++ b/cpp/src/arrow/io/mman.h
@@ -93,13 +93,10 @@ static void* mmap(void* addr, size_t len, int prot, int flags, int fildes,
off_t
   const DWORD protect = __map_mmap_prot_page(prot);
   const DWORD desiredAccess = __map_mmap_prot_file(prot);
 
-  const off_t maxSize = off + (off_t)len;
+  const size_t maxSize = off + len;
 
-  const DWORD dwMaxSizeLow =
-      (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
-  const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD))
-                                  ? (DWORD)0
-                                  : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
+  const DWORD dwMaxSizeLow = static_cast<DWORD>(maxSize & 0xFFFFFFFFL);
+  const DWORD dwMaxSizeHigh = static_cast<DWORD>((maxSize >> 32) & 0xFFFFFFFFL);
 
 #ifdef _MSC_VER
 #pragma warning(pop)

http://git-wip-us.apache.org/repos/asf/arrow/blob/3f26dfa3/cpp/src/arrow/io/test-common.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index d6ec270..438f378 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -80,6 +80,8 @@ class MemoryMapFixture {
     return Status::OK();
   }
 
+  void AppendFile(const std::string& path) { tmp_files_.push_back(path); }
+
  private:
   std::vector<std::string> tmp_files_;
 };


Mime
View raw message