kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [1/2] incubator-kudu git commit: env: add GetFileSizeOnDiskRecursively
Date Thu, 14 Jul 2016 18:03:24 GMT
Repository: incubator-kudu
Updated Branches:
  refs/heads/master 4e99544d6 -> 71a5bb507


env: add GetFileSizeOnDiskRecursively

This was originally needed by a new integration test. Since then, it's no
longer needed, but the new method might prove useful in the future.

Change-Id: Ic49629595e776ce5c755e15b04c6509053ff361f
Reviewed-on: http://gerrit.cloudera.org:8080/3644
Tested-by: Adar Dembo <adar@cloudera.com>
Reviewed-by: Dan Burkert <dan@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/8c68d350
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/8c68d350
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/8c68d350

Branch: refs/heads/master
Commit: 8c68d35061aece52203c6eb0bb8de75474245a95
Parents: 4e99544
Author: Adar Dembo <adar@cloudera.com>
Authored: Wed Jul 13 10:14:29 2016 -0700
Committer: Adar Dembo <adar@cloudera.com>
Committed: Thu Jul 14 18:01:14 2016 +0000

----------------------------------------------------------------------
 src/kudu/tablet/compaction-test.cc | 35 ++++-----------------------------
 src/kudu/util/env.h                |  8 ++++++++
 src/kudu/util/env_posix.cc         | 32 ++++++++++++++++++++++++++++++
 3 files changed, 44 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/8c68d350/src/kudu/tablet/compaction-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/compaction-test.cc b/src/kudu/tablet/compaction-test.cc
index 3383866..695040f 100644
--- a/src/kudu/tablet/compaction-test.cc
+++ b/src/kudu/tablet/compaction-test.cc
@@ -383,13 +383,6 @@ class TestCompaction : public KuduRowSetTest {
     }
   }
 
-  Status GetDataDiskSpace(uint64_t* bytes_used) {
-    *bytes_used = 0;
-    return env_->Walk(fs_manager()->GetDataRootDirs().at(0),
-                      Env::PRE_ORDER, Bind(&TestCompaction::GetDataDiskSpaceCb,
-                                           Unretained(this), bytes_used));
-  }
-
  protected:
   OpId op_id_;
 
@@ -398,28 +391,6 @@ class TestCompaction : public KuduRowSetTest {
   MvccManager mvcc_;
 
   scoped_refptr<LogAnchorRegistry> log_anchor_registry_;
-
- private:
-
-  Status GetDataDiskSpaceCb(uint64_t* bytes_used,
-                            Env::FileType type,
-                            const string& dirname, const string& basename) {
-    uint64_t file_bytes_used = 0;
-    switch (type) {
-      case Env::FILE_TYPE:
-        RETURN_NOT_OK(env_->GetFileSizeOnDisk(
-            JoinPathSegments(dirname, basename), &file_bytes_used));
-        *bytes_used += file_bytes_used;
-        break;
-      case Env::DIRECTORY_TYPE:
-        // Ignore directory space consumption; it varies from filesystem to
-        // filesystem and isn't interesting for this test.
-        break;
-      default:
-        LOG(FATAL) << "Unknown file type: " << type;
-    }
-    return Status::OK();
-  }
 };
 
 TEST_F(TestCompaction, TestMemRowSetInput) {
@@ -771,7 +742,8 @@ TEST_F(TestCompaction, TestCompactionFreesDiskSpace) {
   }
 
   uint64_t bytes_before;
-  ASSERT_NO_FATAL_FAILURE(GetDataDiskSpace(&bytes_before));
+  ASSERT_OK(env_->GetFileSizeOnDiskRecursively(
+      fs_manager()->GetDataRootDirs().at(0), &bytes_before));
 
   ASSERT_OK(tablet()->Compact(Tablet::FORCE_COMPACT_ALL));
 
@@ -781,7 +753,8 @@ TEST_F(TestCompaction, TestCompactionFreesDiskSpace) {
   deadline.AddDelta(MonoDelta::FromSeconds(30));
   while (true) {
     uint64_t bytes_after;
-    ASSERT_NO_FATAL_FAILURE(GetDataDiskSpace(&bytes_after));
+    ASSERT_OK(env_->GetFileSizeOnDiskRecursively(
+        fs_manager()->GetDataRootDirs().at(0), &bytes_after));
     LOG(INFO) << Substitute("Data disk space: $0 (before), $1 (after) ",
                             bytes_before, bytes_after);
     if (bytes_after < bytes_before) {

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/8c68d350/src/kudu/util/env.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h
index f62f85b..7eb97e7 100644
--- a/src/kudu/util/env.h
+++ b/src/kudu/util/env.h
@@ -165,6 +165,10 @@ class Env {
   // of space consumed by the file, not the user-facing file size.
   virtual Status GetFileSizeOnDisk(const std::string& fname, uint64_t* file_size) = 0;
 
+  // Walk 'root' recursively, looking up the amount of space used by each file
+  // as reported by GetFileSizeOnDisk(), storing the grand total in 'bytes_used'.
+  virtual Status GetFileSizeOnDiskRecursively(const std::string& root, uint64_t* bytes_used)
= 0;
+
   // Store the block size of the filesystem where fname resides in
   // *block_size. fname must exist but it may be a file or a directory.
   virtual Status GetBlockSize(const std::string& fname, uint64_t* block_size) = 0;
@@ -264,6 +268,7 @@ class Env {
 
   // Get the total amount of RAM installed on this machine.
   virtual Status GetTotalRAMBytes(int64_t* ram) = 0;
+
  private:
   // No copying allowed
   Env(const Env&);
@@ -585,6 +590,9 @@ class EnvWrapper : public Env {
   Status GetFileSizeOnDisk(const std::string& f, uint64_t* s) OVERRIDE {
     return target_->GetFileSizeOnDisk(f, s);
   }
+  Status GetFileSizeOnDiskRecursively(const std::string& root, uint64_t* bytes_used)
OVERRIDE {
+    return target_->GetFileSizeOnDiskRecursively(root, bytes_used);
+  }
   Status GetBlockSize(const std::string& f, uint64_t* s) OVERRIDE {
     return target_->GetBlockSize(f, s);
   }

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/8c68d350/src/kudu/util/env_posix.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index 465e7aa..773971f 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -898,6 +898,17 @@ class PosixEnv : public Env {
     return s;
   }
 
+  virtual Status GetFileSizeOnDiskRecursively(const string& root,
+                                              uint64_t* bytes_used) OVERRIDE {
+    TRACE_EVENT1("io", "PosixEnv::GetFileSizeOnDiskRecursively", "path", root);
+    uint64_t total = 0;
+    RETURN_NOT_OK(Walk(root, Env::PRE_ORDER,
+                       Bind(&PosixEnv::GetFileSizeOnDiskRecursivelyCb,
+                            Unretained(this), &total)));
+    *bytes_used = total;
+    return Status::OK();
+  }
+
   virtual Status GetBlockSize(const string& fname, uint64_t* block_size) OVERRIDE {
     TRACE_EVENT1("io", "PosixEnv::GetBlockSize", "path", fname);
     ThreadRestrictions::AssertIOAllowed();
@@ -1207,6 +1218,27 @@ class PosixEnv : public Env {
         return Status::OK();
     }
   }
+
+  Status GetFileSizeOnDiskRecursivelyCb(uint64_t* bytes_used,
+                                        Env::FileType type,
+                                        const string& dirname,
+                                        const string& basename) {
+    uint64_t file_bytes_used = 0;
+    switch (type) {
+      case Env::FILE_TYPE:
+        RETURN_NOT_OK(GetFileSizeOnDisk(
+            JoinPathSegments(dirname, basename), &file_bytes_used));
+        *bytes_used += file_bytes_used;
+        break;
+      case Env::DIRECTORY_TYPE:
+        // Ignore directory space consumption as it varies from filesystem to
+        // filesystem.
+        break;
+      default:
+        LOG(FATAL) << "Unknown file type: " << type;
+    }
+    return Status::OK();
+  }
 };
 
 PosixEnv::PosixEnv() {}


Mime
View raw message