kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [1/2] kudu git commit: fs: add filesystem type detection to data dirs
Date Fri, 23 Jun 2017 23:48:41 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 44bd94268 -> 410763afa


fs: add filesystem type detection to data dirs

This patch moves some detection code out of LogBlockManager and into
DataDirManager. It also adds detection of xfs filesystems.

To verify that I didn't regress the existing ext4 on el6 detection behavior,
I ran log_block_manager-test on an ext4 on CentOS 6.6 and verified that the
log output included mention of how the LBM was limiting the containers' max
blocks due to buggy ext4.

Change-Id: I2df259d2fbbc1e950b33005c5da02a3e2cd076f2
Reviewed-on: http://gerrit.cloudera.org:8080/7278
Reviewed-by: Todd Lipcon <todd@apache.org>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9fdf77a6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9fdf77a6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9fdf77a6

Branch: refs/heads/master
Commit: 9fdf77a6a50905819a77efa1a1a2ab74c3b6aff9
Parents: 44bd942
Author: Adar Dembo <adar@cloudera.com>
Authored: Fri Jun 23 11:30:38 2017 -0700
Committer: Adar Dembo <adar@cloudera.com>
Committed: Fri Jun 23 23:48:11 2017 +0000

----------------------------------------------------------------------
 src/kudu/fs/data_dirs.cc         | 17 ++++++++++++++++-
 src/kudu/fs/data_dirs.h          | 16 ++++++++++++++++
 src/kudu/fs/log_block_manager.cc |  4 +---
 src/kudu/util/env.h              |  5 +++++
 src/kudu/util/env_posix.cc       | 24 +++++++++++++++++++++++-
 5 files changed, 61 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/9fdf77a6/src/kudu/fs/data_dirs.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc
index a9fe498..84f1f7a 100644
--- a/src/kudu/fs/data_dirs.cc
+++ b/src/kudu/fs/data_dirs.cc
@@ -169,11 +169,13 @@ DataDirMetrics::DataDirMetrics(const scoped_refptr<MetricEntity>&
entity)
 
 DataDir::DataDir(Env* env,
                  DataDirMetrics* metrics,
+                 DataDirFsType fs_type,
                  string dir,
                  unique_ptr<PathInstanceMetadataFile> metadata_file,
                  unique_ptr<ThreadPool> pool)
     : env_(env),
       metrics_(metrics),
+      fs_type_(fs_type),
       dir_(std::move(dir)),
       metadata_file_(std::move(metadata_file)),
       pool_(std::move(pool)),
@@ -366,9 +368,22 @@ Status DataDirManager::Open(int max_data_dirs, LockMode mode) {
                   .set_max_threads(1)
                   .Build(&pool));
 
+    // Figure out what filesystem the data directory is on.
+    DataDirFsType fs_type = DataDirFsType::OTHER;
+    bool result;
+    RETURN_NOT_OK(env_->IsOnExtFilesystem(p, &result));
+    if (result) {
+      fs_type = DataDirFsType::EXT;
+    } else {
+      RETURN_NOT_OK(env_->IsOnXfsFilesystem(p, &result));
+      if (result) {
+        fs_type = DataDirFsType::XFS;
+      }
+    }
+
     // Create the data directory in-memory structure itself.
     unique_ptr<DataDir> dd(new DataDir(
-        env_, metrics_.get(), p,
+        env_, metrics_.get(), fs_type, p,
         unique_ptr<PathInstanceMetadataFile>(instance.release()),
         unique_ptr<ThreadPool>(pool.release())));
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/9fdf77a6/src/kudu/fs/data_dirs.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/data_dirs.h b/src/kudu/fs/data_dirs.h
index 597e9fd..6b21892 100644
--- a/src/kudu/fs/data_dirs.h
+++ b/src/kudu/fs/data_dirs.h
@@ -97,6 +97,18 @@ class DataDirGroup {
 
 }  // namespace internal
 
+// Detected type of filesystem.
+enum class DataDirFsType {
+  // ext2, ext3, or ext4.
+  EXT,
+
+  // SGI xfs.
+  XFS,
+
+  // None of the above.
+  OTHER
+};
+
 struct DataDirMetrics {
   explicit DataDirMetrics(const scoped_refptr<MetricEntity>& entity);
 
@@ -108,6 +120,7 @@ class DataDir {
  public:
   DataDir(Env* env,
           DataDirMetrics* metrics,
+          DataDirFsType fs_type,
           std::string dir,
           std::unique_ptr<PathInstanceMetadataFile> metadata_file,
           std::unique_ptr<ThreadPool> pool);
@@ -141,6 +154,8 @@ class DataDir {
   };
   Status RefreshIsFull(RefreshMode mode);
 
+  DataDirFsType fs_type() const { return fs_type_; }
+
   const std::string& dir() const { return dir_; }
 
   const PathInstanceMetadataFile* instance() const {
@@ -155,6 +170,7 @@ class DataDir {
  private:
   Env* env_;
   DataDirMetrics* metrics_;
+  const DataDirFsType fs_type_;
   const std::string dir_;
   const std::unique_ptr<PathInstanceMetadataFile> metadata_file_;
   const std::unique_ptr<ThreadPool> pool_;

http://git-wip-us.apache.org/repos/asf/kudu/blob/9fdf77a6/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index 9565b5f..bc2ae7c 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -1471,9 +1471,7 @@ Status LogBlockManager::Open(FsReport* report) {
       // The log block manager requires hole punching and, of the ext
       // filesystems, only ext4 supports it. Thus, if this is an ext
       // filesystem, it's ext4 by definition.
-      bool is_on_ext4;
-      RETURN_NOT_OK(env_->IsOnExtFilesystem(dd->dir(), &is_on_ext4));
-      if (buggy_el6_kernel_ && is_on_ext4) {
+      if (buggy_el6_kernel_ && dd->fs_type() == DataDirFsType::EXT) {
         uint64_t fs_block_size =
             dd->instance()->metadata()->filesystem_block_size_bytes();
         bool untested_block_size =

http://git-wip-us.apache.org/repos/asf/kudu/blob/9fdf77a6/src/kudu/util/env.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h
index 443e295..c2a1bdc 100644
--- a/src/kudu/util/env.h
+++ b/src/kudu/util/env.h
@@ -316,6 +316,11 @@ class Env {
   // On success, 'result' contains the answer. On failure, 'result' is unset.
   virtual Status IsOnExtFilesystem(const std::string& path, bool* result) = 0;
 
+  // Checks whether the given path resides on an xfs filesystem.
+  //
+  // On success, 'result' contains the answer. On failure, 'result' is unset.
+  virtual Status IsOnXfsFilesystem(const std::string& path, bool* result) = 0;
+
   // Gets the kernel release string for this machine.
   virtual std::string GetKernelRelease() = 0;
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/9fdf77a6/src/kudu/util/env_posix.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index 966e80e..9f18de7 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -1513,7 +1513,7 @@ class PosixEnv : public Env {
   }
 
   virtual Status IsOnExtFilesystem(const string& path, bool* result) OVERRIDE {
-    TRACE_EVENT0("io", "PosixEnv::IsOnExtFilesystem");
+    TRACE_EVENT1("io", "PosixEnv::IsOnExtFilesystem", "path", path);
     MAYBE_RETURN_EIO(path, IOError(Env::kInjectedFailureStatusMsg, EIO));
     ThreadRestrictions::AssertIOAllowed();
 
@@ -1531,6 +1531,28 @@ class PosixEnv : public Env {
     return Status::OK();
   }
 
+  virtual Status IsOnXfsFilesystem(const string& path, bool* result) OVERRIDE {
+    TRACE_EVENT1("io", "PosixEnv::IsOnXfsFilesystem", "path", path);
+    MAYBE_RETURN_EIO(path, IOError(Env::kInjectedFailureStatusMsg, EIO));
+    ThreadRestrictions::AssertIOAllowed();
+
+#ifdef __APPLE__
+    *result = false;
+#else
+    struct statfs buf;
+    int ret;
+    RETRY_ON_EINTR(ret, statfs(path.c_str(), &buf));
+    if (ret == -1) {
+      return IOError(Substitute("statfs: $0", path), errno);
+    }
+
+    // This magic number isn't defined in any header but is the value of the
+    // US-ASCII string 'XFSB' expressed in hexadecimal.
+    *result = (buf.f_type == 0x58465342);
+#endif
+    return Status::OK();
+  }
+
   virtual string GetKernelRelease() OVERRIDE {
     // There's no reason for this to ever fail.
     struct utsname u;


Mime
View raw message