kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jdcry...@apache.org
Subject [1/2] kudu git commit: fs: ensure the existence of important directories on open
Date Mon, 30 Oct 2017 16:20:40 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 13908bd9f -> c08f27fc5


fs: ensure the existence of important directories on open

Seems like a pretty easy way to improve robustness, especially for CLI tools
that may not access these directories directly but nevertheless want to
guarantee their existence before making destructive changes.

Change-Id: I5238987273886421f8e6f87f850183dc81c8c4ec
Reviewed-on: http://gerrit.cloudera.org:8080/8399
Tested-by: Adar Dembo <adar@cloudera.com>
Reviewed-by: Andrew Wong <awong@cloudera.com>
Reviewed-by: Todd Lipcon <todd@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9a4ea1c7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9a4ea1c7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9a4ea1c7

Branch: refs/heads/master
Commit: 9a4ea1c7fa1cc6c7345f012f35e7681364ed3534
Parents: 13908bd
Author: Adar Dembo <adar@cloudera.com>
Authored: Thu Oct 26 10:32:05 2017 -0700
Committer: Todd Lipcon <todd@apache.org>
Committed: Thu Oct 26 23:04:50 2017 +0000

----------------------------------------------------------------------
 src/kudu/fs/fs_manager-test.cc   | 16 ++++++++++++++++
 src/kudu/fs/fs_manager.cc        | 14 ++++++++++++++
 src/kudu/tools/kudu-tool-test.cc | 14 ++++++++------
 3 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/9a4ea1c7/src/kudu/fs/fs_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager-test.cc b/src/kudu/fs/fs_manager-test.cc
index 6ed4f93..2d011d9 100644
--- a/src/kudu/fs/fs_manager-test.cc
+++ b/src/kudu/fs/fs_manager-test.cc
@@ -400,4 +400,20 @@ TEST_F(FsManagerTestBase, TestUmask) {
   EXPECT_EQ("700", FilePermsAsString(root));
 }
 
+TEST_F(FsManagerTestBase, TestOpenFailsWhenMissingImportantDir) {
+  const string kWalRoot = fs_manager()->GetWalsRootDir();
+
+  ASSERT_OK(env_->DeleteDir(kWalRoot));
+  ReinitFsManager();
+  Status s = fs_manager()->Open();
+  ASSERT_TRUE(s.IsNotFound());
+  ASSERT_STR_CONTAINS(s.ToString(), "could not verify required directory");
+
+  unique_ptr<WritableFile> f;
+  ASSERT_OK(env_->NewWritableFile(kWalRoot, &f));
+  s = fs_manager()->Open();
+  ASSERT_TRUE(s.IsCorruption());
+  ASSERT_STR_CONTAINS(s.ToString(), "exists but is not a directory");
+}
+
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/9a4ea1c7/src/kudu/fs/fs_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc
index 5f62858..e9df4e3 100644
--- a/src/kudu/fs/fs_manager.cc
+++ b/src/kudu/fs/fs_manager.cc
@@ -307,6 +307,20 @@ Status FsManager::Open(FsReport* report) {
     }
   }
 
+  // Ensure all of the ancillary directories exist.
+  vector<string> ancillary_dirs = { GetWalsRootDir(),
+                                    GetTabletMetadataDir(),
+                                    GetConsensusMetadataDir() };
+  for (const auto& d : ancillary_dirs) {
+    bool is_dir;
+    RETURN_NOT_OK_PREPEND(env_->IsDirectory(d, &is_dir),
+                          Substitute("could not verify required directory $0", d));
+    if (!is_dir) {
+      return Status::Corruption(
+          Substitute("Required directory $0 exists but is not a directory", d));
+    }
+  }
+
   // Remove leftover temporary files from the WAL root and fix permissions.
   //
   // Temporary files in the data directory roots will be removed by the block

http://git-wip-us.apache.org/repos/asf/kudu/blob/9a4ea1c7/src/kudu/tools/kudu-tool-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 0df6dc2..0edd1e3 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -67,6 +67,7 @@
 #include "kudu/gutil/port.h"
 #include "kudu/gutil/ref_counted.h"
 #include "kudu/gutil/stl_util.h"
+#include "kudu/gutil/strings/join.h"
 #include "kudu/gutil/strings/numbers.h"
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/strcat.h"
@@ -639,8 +640,9 @@ TEST_F(ToolTest, TestFsCheck) {
 
 TEST_F(ToolTest, TestFsCheckLiveServer) {
   NO_FATALS(StartExternalMiniCluster());
-  string master_data_dir = cluster_->GetDataPath("master-0");
-  string args = Substitute("fs check --fs_wal_dir $0", master_data_dir);
+  string args = Substitute("fs check --fs_wal_dir $0 --fs_data_dirs $1",
+                           cluster_->GetWalPath("master-0"),
+                           JoinStrings(cluster_->GetDataPaths("master-0"), ","));
   NO_FATALS(RunFsCheck(args, 0, "", {}, 0));
   args += " --repair";
   string stdout;
@@ -1421,11 +1423,11 @@ TEST_F(ToolTest, TestRemoteReplicaCopy) {
   // Shut down the destination server and delete one tablet from
   // local fs on destination tserver while it is offline.
   cluster_->tablet_server(kDstTsIndex)->Shutdown();
-  const string& tserver_dir = cluster_->tablet_server(kDstTsIndex)->data_dir();
   const string& deleted_tablet_id = tablets[1].tablet_status().tablet_id();
-  NO_FATALS(RunActionStdoutNone(Substitute("local_replica delete $0 --fs_wal_dir=$1 "
-                                           "--fs_data_dirs=$1 --clean_unsafe",
-                                           deleted_tablet_id, tserver_dir)));
+  NO_FATALS(RunActionStdoutNone(Substitute(
+      "local_replica delete $0 --fs_wal_dir=$1 --fs_data_dirs=$2 --clean_unsafe",
+      deleted_tablet_id, cluster_->tablet_server(kDstTsIndex)->wal_dir(),
+      JoinStrings(cluster_->tablet_server(kDstTsIndex)->data_dirs(), ","))));
 
   // At this point, we expect only 2 tablets to show up on destination when
   // we restart the destination tserver. deleted_tablet_id should not be found on


Mime
View raw message