kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpe...@apache.org
Subject incubator-kudu git commit: env: Implement RWFile::Truncate()
Date Fri, 01 Apr 2016 23:24:51 GMT
Repository: incubator-kudu
Updated Branches:
  refs/heads/master c6e709f6e -> 1f014124d


env: Implement RWFile::Truncate()

This can replace direct calls to truncate(1) and ftruncate(1).

Change-Id: I1f189df33e7c7afdd50d76d60be0acd6ff5d5820
Reviewed-on: http://gerrit.cloudera.org:8080/2697
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <adar@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/1f014124
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/1f014124
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/1f014124

Branch: refs/heads/master
Commit: 1f014124d1dbd4a546976721062cfef251a19187
Parents: c6e709f
Author: Mike Percy <mpercy@apache.org>
Authored: Thu Mar 31 16:24:33 2016 -0700
Committer: Mike Percy <mpercy@apache.org>
Committed: Fri Apr 1 23:24:17 2016 +0000

----------------------------------------------------------------------
 src/kudu/util/env-test.cc      | 38 +++++++++++++++++++++++++++++++++++++
 src/kudu/util/env.h            | 29 ++++++++++++++++++++++------
 src/kudu/util/env_posix.cc     | 26 +++++++++++++++++++++++--
 src/kudu/util/memenv/memenv.cc |  8 ++++++++
 src/kudu/util/pb_util-test.cc  | 10 ++++++----
 5 files changed, 99 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1f014124/src/kudu/util/env-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env-test.cc b/src/kudu/util/env-test.cc
index 03e2503..5079542 100644
--- a/src/kudu/util/env-test.cc
+++ b/src/kudu/util/env-test.cc
@@ -54,6 +54,7 @@ using std::string;
 using std::vector;
 
 static const uint64_t kOneMb = 1024 * 1024;
+static const uint64_t kTwoMb = 2 * kOneMb;
 
 class TestEnv : public KuduTest {
  public:
@@ -334,6 +335,43 @@ TEST_F(TestEnv, TestHolePunch) {
   ASSERT_EQ(size_on_disk - punch_amount, new_size_on_disk);
 }
 
+TEST_F(TestEnv, TestTruncate) {
+  LOG(INFO) << "Testing Truncate()";
+  string test_path = GetTestPath("test_env_wf");
+  gscoped_ptr<RWFile> file;
+  ASSERT_OK(env_->NewRWFile(test_path, &file));
+  uint64_t size;
+  ASSERT_OK(file->Size(&size));
+  ASSERT_EQ(0, size);
+
+  // Truncate to 2 MB (up).
+  ASSERT_OK(file->Truncate(kTwoMb));
+  ASSERT_OK(file->Size(&size));
+  ASSERT_EQ(kTwoMb, size);
+  ASSERT_OK(env_->GetFileSize(test_path, &size));
+  ASSERT_EQ(kTwoMb, size);
+
+  // Truncate to 1 MB (down).
+  ASSERT_OK(file->Truncate(kOneMb));
+  ASSERT_OK(file->Size(&size));
+  ASSERT_EQ(kOneMb, size);
+  ASSERT_OK(env_->GetFileSize(test_path, &size));
+  ASSERT_EQ(kOneMb, size);
+
+  ASSERT_OK(file->Close());
+
+  // Read the whole file. Ensure it is all zeroes.
+  gscoped_ptr<RandomAccessFile> raf;
+  ASSERT_OK(env_->NewRandomAccessFile(test_path, &raf));
+  Slice s;
+  gscoped_ptr<uint8_t[]> scratch(new uint8_t[size]);
+  ASSERT_OK(env_util::ReadFully(raf.get(), 0, size, &s, scratch.get()));
+  const uint8_t* data = s.data();
+  for (int i = 0; i < size; i++) {
+    ASSERT_EQ(0, data[i]) << "Not null at position " << i;
+  }
+}
+
 class ShortReadRandomAccessFile : public RandomAccessFile {
  public:
   explicit ShortReadRandomAccessFile(shared_ptr<RandomAccessFile> wrapped)

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1f014124/src/kudu/util/env.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h
index 0d90ef3..5b3ff77 100644
--- a/src/kudu/util/env.h
+++ b/src/kudu/util/env.h
@@ -350,12 +350,6 @@ class WritableFile {
   WritableFile() { }
   virtual ~WritableFile();
 
-  // Pre-allocates 'size' bytes for the file in the underlying filesystem.
-  // size bytes are added to the current pre-allocated size or to the current
-  // offset, whichever is bigger. In no case is the file truncated by this
-  // operation.
-  virtual Status PreAllocate(uint64_t size) = 0;
-
   virtual Status Append(const Slice& data) = 0;
 
   // If possible, uses scatter-gather I/O to efficiently append
@@ -365,6 +359,18 @@ class WritableFile {
   // implementation source code (e.g., env_posix.cc)
   virtual Status AppendVector(const std::vector<Slice>& data_vector) = 0;
 
+  // Pre-allocates 'size' bytes for the file in the underlying filesystem.
+  // size bytes are added to the current pre-allocated size or to the current
+  // offset, whichever is bigger. In no case is the file truncated by this
+  // operation.
+  //
+  // On some implementations, preallocation is done without initializing the
+  // contents of the data blocks (as opposed to writing zeroes), requiring no
+  // IO to the data blocks.
+  //
+  // In no case is the file truncated by this operation.
+  virtual Status PreAllocate(uint64_t size) = 0;
+
   virtual Status Close() = 0;
 
   // Flush all dirty data (not metadata) to disk.
@@ -437,9 +443,20 @@ class RWFile {
   // beginning at 'offset'. It is safe to preallocate the same range
   // repeatedly; this is an idempotent operation.
   //
+  // On some implementations, preallocation is done without initializing the
+  // contents of the data blocks (as opposed to writing zeroes), requiring no
+  // IO to the data blocks. On such implementations, this is much faster than
+  // using Truncate() to increase the file size.
+  //
   // In no case is the file truncated by this operation.
   virtual Status PreAllocate(uint64_t offset, size_t length) = 0;
 
+  // Truncate the file. If 'new_size' is less than the previous file size, the
+  // extra data will be lost. If 'new_size' is greater than the previous file
+  // size, the file length is extended, and the extended portion will contain
+  // null bytes ('\0').
+  virtual Status Truncate(uint64_t length) = 0;
+
   // Deallocates space given by 'offset' and length' from the file,
   // effectively "punching a hole" in it. The space will be reclaimed by
   // the filesystem and reads to that range will return zeroes. Useful

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1f014124/src/kudu/util/env_posix.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index d568f76..fe964fa 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -65,6 +65,10 @@
 #define fread_unlocked fread
 #endif
 
+#define RETRY_ON_EINTR(ret, expr) do { \
+  ret = (expr); \
+} while ((ret == -1) && (errno == EINTR));
+
 // See KUDU-588 for details.
 DEFINE_bool(writable_file_use_fsync, false,
             "Use fsync(2) instead of fdatasync(2) for synchronizing dirty "
@@ -122,7 +126,9 @@ int fallocate(int fd, int mode, off_t offset, off_t len) {
 
   if (stat.st_size < size) {
     // fcntl does not change the file size, so set it if necessary.
-    return ftruncate(fd, size);
+    int ret;
+    RETRY_ON_EINTR(ret, ftruncate(fd, size));
+    return ret;
   }
   return 0;
 }
@@ -349,7 +355,9 @@ class PosixWritableFile : public WritableFile {
     // If we've allocated more space than we used, truncate to the
     // actual size of the file and perform Sync().
     if (filesize_ < pre_allocated_size_) {
-      if (ftruncate(fd_, filesize_) < 0) {
+      int ret;
+      RETRY_ON_EINTR(ret, ftruncate(fd_, filesize_));
+      if (ret != 0) {
         s = IOError(filename_, errno);
         pending_sync_ = true;
       }
@@ -553,6 +561,20 @@ class PosixRWFile : public RWFile {
     return Status::OK();
   }
 
+  virtual Status Truncate(uint64_t length) OVERRIDE {
+    TRACE_EVENT2("io", "PosixRWFile::Truncate", "path", filename_, "length", length);
+    ThreadRestrictions::AssertIOAllowed();
+    int ret;
+    RETRY_ON_EINTR(ret, ftruncate(fd_, length));
+    if (ret != 0) {
+      int err = errno;
+      return Status::IOError(Substitute("Unable to truncate file $0", filename_),
+                             Substitute("ftruncate() failed: $0", ErrnoToString(err)),
+                             err);
+    }
+    return Status::OK();
+  }
+
   virtual Status PunchHole(uint64_t offset, size_t length) OVERRIDE {
 #if defined(__linux__)
     TRACE_EVENT1("io", "PosixRWFile::PunchHole", "path", filename_);

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1f014124/src/kudu/util/memenv/memenv.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/memenv/memenv.cc b/src/kudu/util/memenv/memenv.cc
index 92b2007..1af79df 100644
--- a/src/kudu/util/memenv/memenv.cc
+++ b/src/kudu/util/memenv/memenv.cc
@@ -94,6 +94,10 @@ class FileState : public RefCountedThreadSafe<FileState> {
     return s;
   }
 
+  Status Truncate(uint64_t length) {
+    return Status::NotSupported("Not implemented");
+  }
+
   Status Append(const Slice& data) {
     return AppendRaw(data.data(), data.size());
   }
@@ -309,6 +313,10 @@ class RWFileImpl : public RWFile {
     return Status::OK();
   }
 
+  virtual Status Truncate(uint64_t length) OVERRIDE {
+    return file_->Truncate(length);
+  }
+
   virtual Status PunchHole(uint64_t offset, size_t length) OVERRIDE {
     return Status::OK();
   }

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/1f014124/src/kudu/util/pb_util-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/pb_util-test.cc b/src/kudu/util/pb_util-test.cc
index 5e746e4..3696f51 100644
--- a/src/kudu/util/pb_util-test.cc
+++ b/src/kudu/util/pb_util-test.cc
@@ -199,10 +199,12 @@ TEST_F(TestPBUtil, TestPBContainerCorruption) {
   ASSERT_OK(CreateKnownGoodContainerFile());
   uint64_t known_good_size = 0;
   ASSERT_OK(env_->GetFileSize(path_, &known_good_size));
-  int ret = truncate(path_.c_str(), known_good_size - 2);
-  if (ret != 0) {
-    PLOG(ERROR) << "truncate() of file " << path_ << " failed";
-    FAIL();
+  {
+    gscoped_ptr<RWFile> file;
+    RWFileOptions opts;
+    opts.mode = Env::OPEN_EXISTING;
+    ASSERT_OK(env_->NewRWFile(opts, path_, &file));
+    ASSERT_OK(file->Truncate(known_good_size - 2));
   }
   s = ReadPBContainerFromPath(env_.get(), path_, &test_pb);
   ASSERT_TRUE(s.IsCorruption()) << "Should be incorrect size: " << path_ <<
": " << s.ToString();


Mime
View raw message