kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [5/5] kudu git commit: tool: add ksck
Date Thu, 25 Aug 2016 21:51:06 GMT
tool: add ksck

This patch adds ksck functionality to the new CLI tool. ksck is still
available as a library used by several integration tests.

Change-Id: Iad9aaffa5c0c77080dcaeb2cdfa572dcbeeb1407
Reviewed-on: http://gerrit.cloudera.org:8080/4121
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <todd@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/86afadc7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/86afadc7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/86afadc7

Branch: refs/heads/master
Commit: 86afadc7fc6f9274402edcb3599c243ec2de0ccb
Parents: 7321b38
Author: Adar Dembo <adar@cloudera.com>
Authored: Wed Aug 24 20:24:08 2016 -0700
Committer: Todd Lipcon <todd@apache.org>
Committed: Thu Aug 25 21:46:47 2016 +0000

----------------------------------------------------------------------
 docs/release_notes.adoc               |   3 +
 src/kudu/tools/CMakeLists.txt         |   7 +-
 src/kudu/tools/kudu-ksck.cc           | 151 -----------------------------
 src/kudu/tools/kudu-tool-test.cc      |   6 ++
 src/kudu/tools/tool_action.h          |   1 +
 src/kudu/tools/tool_action_cluster.cc | 149 ++++++++++++++++++++++++++++
 src/kudu/tools/tool_main.cc           |   1 +
 7 files changed, 162 insertions(+), 156 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/docs/release_notes.adoc
----------------------------------------------------------------------
diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc
index 38c3d7a..b629a2f 100644
--- a/docs/release_notes.adoc
+++ b/docs/release_notes.adoc
@@ -43,6 +43,9 @@ detailed below.
 - The `kudu-pbc-dump` tool has been removed. The same functionality is now
   implemented as `kudu pbc dump`.
 
+- The `kudu-ksck` tool has been removed. The same functionality is now
+  implemented as `kudu cluster ksck`.
+
 - KuduSession methods in the C++ library are no longer advertised as thread-safe
   to have one set of semantics for both C++ and Java Kudu client libraries.
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/tools/CMakeLists.txt b/src/kudu/tools/CMakeLists.txt
index ce361f0..614bf11 100644
--- a/src/kudu/tools/CMakeLists.txt
+++ b/src/kudu/tools/CMakeLists.txt
@@ -82,13 +82,9 @@ target_link_libraries(ksck
   ${KUDU_BASE_LIBS}
 )
 
-add_executable(kudu-ksck kudu-ksck.cc)
-target_link_libraries(kudu-ksck
-  ksck
-)
-
 add_executable(kudu
   tool_action.cc
+  tool_action_cluster.cc
   tool_action_fs.cc
   tool_action_pbc.cc
   tool_action_tablet.cc
@@ -100,6 +96,7 @@ target_link_libraries(kudu
   kudu_common
   kudu_fs
   kudu_util
+  ksck
   master
   tserver
   ${KUDU_BASE_LIBS}

http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/kudu-ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-ksck.cc b/src/kudu/tools/kudu-ksck.cc
deleted file mode 100644
index 70a6ce1..0000000
--- a/src/kudu/tools/kudu-ksck.cc
+++ /dev/null
@@ -1,151 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-// Command line tool to run Ksck against a cluster. Defaults to running against a local Master
-// on the default RPC port. It verifies that all the reported Tablet Servers are running
and that
-// the tablets are in a consistent state.
-
-#include <gflags/gflags.h>
-#include <glog/logging.h>
-#include <iostream>
-
-#include "kudu/gutil/strings/split.h"
-#include "kudu/gutil/strings/substitute.h"
-#include "kudu/tools/ksck_remote.h"
-#include "kudu/util/flags.h"
-#include "kudu/util/logging.h"
-
-#define PUSH_PREPEND_NOT_OK(s, statuses, msg) do { \
-  ::kudu::Status _s = (s); \
-  if (PREDICT_FALSE(!_s.ok())) { \
-    statuses->push_back(string(msg) + ": " + _s.ToString()); \
-  } \
-} while (0);
-
-using std::cerr;
-using std::cout;
-using std::endl;
-using std::shared_ptr;
-using std::vector;
-using strings::Substitute;
-
-DEFINE_string(master_address, "",
-              "Address of master server to run against.");
-
-DEFINE_bool(checksum_scan, false,
-            "Perform a checksum scan on data in the cluster.");
-
-DEFINE_string(tables, "",
-              "Tables to check (comma-separated list of names). "
-              "If not specified, checks all tables.");
-
-DEFINE_string(tablets, "",
-              "Tablets to check (comma-separated list of IDs) "
-              "If not specified, checks all tablets.");
-
-namespace kudu {
-namespace tools {
-
-static string GetKsckUsage(const char* progname) {
-  string msg = Substitute("Usage: $0 --master_address=<addr> <flags>\n\n", progname);
-  msg += "Check the health of a Kudu cluster.\n\n"
-         "By default, ksck checks that master and tablet server processes are running,\n"
-         "and that table metadata is consistent. Use the 'checksum' flag to check that\n"
-         "tablet data is consistent (also see the 'tables' and 'tablets' flags below).\n"
-         "Use the 'checksum_snapshot' along with 'checksum' if the table or tablets are\n"
-         "actively receiving inserts or updates.";
-  return msg;
-}
-
-// Run ksck.
-// Error information is appended to the provided vector.
-// If the vector is empty upon completion, ksck ran successfully.
-static void RunKsck(vector<string>* error_messages) {
-  vector<Sockaddr> master_addrs;
-  PUSH_PREPEND_NOT_OK(ParseAddressList(FLAGS_master_address,
-                                       master::Master::kDefaultPort,
-                                       &master_addrs),
-                      error_messages, "Unable to parse master address");
-  // return early if the addresses could not be resolved
-  if (!error_messages->empty()) return;
-
-  shared_ptr<KsckMaster> master;
-  PUSH_PREPEND_NOT_OK(RemoteKsckMaster::Build(master_addrs[0], &master),
-                      error_messages, "Unable to build KsckMaster");
-  if (!error_messages->empty()) return;
-  shared_ptr<KsckCluster> cluster(new KsckCluster(master));
-  shared_ptr<Ksck> ksck(new Ksck(cluster));
-
-  ksck->set_table_filters(strings::Split(FLAGS_tables, ",", strings::SkipEmpty()));
-  ksck->set_tablet_id_filters(strings::Split(FLAGS_tablets, ",", strings::SkipEmpty()));
-
-  // This is required for everything below.
-  PUSH_PREPEND_NOT_OK(ksck->CheckMasterRunning(), error_messages,
-                      "Master aliveness check error");
-  if (!error_messages->empty()) return;
-
-  // This is also required for everything below.
-  PUSH_PREPEND_NOT_OK(ksck->FetchTableAndTabletInfo(), error_messages,
-                      "Error fetching the cluster metadata from the Master server");
-  if (!error_messages->empty()) return;
-
-  PUSH_PREPEND_NOT_OK(ksck->FetchInfoFromTabletServers(), error_messages,
-                      "Error fetching info from tablet servers");
-
-  // TODO: Add support for tables / tablets filter in the consistency check.
-  PUSH_PREPEND_NOT_OK(ksck->CheckTablesConsistency(), error_messages,
-                      "Table consistency check error");
-
-  if (FLAGS_checksum_scan) {
-    PUSH_PREPEND_NOT_OK(ksck->ChecksumData(ChecksumOptions()),
-                        error_messages, "Checksum scan error");
-  }
-}
-
-} // namespace tools
-} // namespace kudu
-
-int main(int argc, char** argv) {
-  google::SetUsageMessage(kudu::tools::GetKsckUsage(argv[0]));
-  if (argc < 2) {
-    google::ShowUsageWithFlagsRestrict(argv[0], __FILE__);
-    exit(1);
-  }
-  kudu::ParseCommandLineFlags(&argc, &argv, true);
-  FLAGS_logtostderr = true;
-  kudu::InitGoogleLoggingSafe(argv[0]);
-
-  vector<string> error_messages;
-  kudu::tools::RunKsck(&error_messages);
-
-  // All good.
-  if (error_messages.empty()) {
-    cout << "OK" << endl;
-    return 0;
-  }
-
-  // Something went wrong.
-  cerr << "==================" << endl;
-  cerr << "Errors:" << endl;
-  cerr << "==================" << endl;
-  for (const string& s : error_messages) {
-    cerr << s << endl;
-  }
-  cerr << endl;
-  cerr << "FAILED" << endl;
-  return 1;
-}

http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/kudu-tool-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 2700fb1..ba748aa 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -137,6 +137,12 @@ TEST_F(ToolTest, TestModeHelp) {
     };
     NO_FATALS(RunTestHelp("tablet cmeta", kCmetaModeRegexes));
   }
+  {
+    const vector<string> kClusterModeRegexes = {
+        "ksck.*Check the health of a Kudu cluster",
+    };
+    NO_FATALS(RunTestHelp("cluster", kClusterModeRegexes));
+  }
 }
 
 TEST_F(ToolTest, TestActionHelp) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_action.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action.h b/src/kudu/tools/tool_action.h
index da5d1e9..7d79a4f 100644
--- a/src/kudu/tools/tool_action.h
+++ b/src/kudu/tools/tool_action.h
@@ -247,6 +247,7 @@ class Action {
 };
 
 // Returns new nodes for each major mode.
+std::unique_ptr<Mode> BuildClusterMode();
 std::unique_ptr<Mode> BuildFsMode();
 std::unique_ptr<Mode> BuildPbcMode();
 std::unique_ptr<Mode> BuildTabletMode();

http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_action_cluster.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_cluster.cc b/src/kudu/tools/tool_action_cluster.cc
new file mode 100644
index 0000000..d198a9b
--- /dev/null
+++ b/src/kudu/tools/tool_action_cluster.cc
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/tools/tool_action.h"
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include <gflags/gflags.h>
+
+#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/strings/split.h"
+#include "kudu/master/master.h"
+#include "kudu/tools/ksck.h"
+#include "kudu/tools/ksck_remote.h"
+#include "kudu/util/net/net_util.h"
+#include "kudu/util/status.h"
+
+#define PUSH_PREPEND_NOT_OK(s, statuses, msg) do { \
+  ::kudu::Status _s = (s); \
+  if (PREDICT_FALSE(!_s.ok())) { \
+    statuses.push_back(string(msg) + ": " + _s.ToString()); \
+  } \
+} while (0);
+
+
+DEFINE_bool(checksum_scan, false,
+            "Perform a checksum scan on data in the cluster.");
+
+DEFINE_string(tables, "",
+              "Tables to check (comma-separated list of names). "
+              "If not specified, checks all tables.");
+
+DEFINE_string(tablets, "",
+              "Tablets to check (comma-separated list of IDs) "
+              "If not specified, checks all tablets.");
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::shared_ptr;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace kudu {
+namespace tools {
+
+namespace {
+
+Status RunKsck(const RunnerContext& context) {
+  vector<Sockaddr> master_addrs;
+  string master_address = FindOrDie(context.required_args, "master_address");
+  RETURN_NOT_OK_PREPEND(ParseAddressList(master_address,
+                                         master::Master::kDefaultPort,
+                                         &master_addrs),
+                        "unable to parse master address");
+
+  shared_ptr<KsckMaster> master;
+  RETURN_NOT_OK_PREPEND(RemoteKsckMaster::Build(master_addrs[0], &master),
+                        "unable to build KsckMaster");
+
+  shared_ptr<KsckCluster> cluster(new KsckCluster(master));
+  shared_ptr<Ksck> ksck(new Ksck(cluster));
+
+  ksck->set_table_filters(strings::Split(
+      FLAGS_tables, ",", strings::SkipEmpty()));
+  ksck->set_tablet_id_filters(strings::Split(
+      FLAGS_tablets, ",", strings::SkipEmpty()));
+
+  RETURN_NOT_OK_PREPEND(ksck->CheckMasterRunning(),
+                        "master liveness check error");
+  RETURN_NOT_OK_PREPEND(ksck->FetchTableAndTabletInfo(),
+                        "error fetching the cluster metadata from the Master server");
+
+  vector<string> error_messages;
+  PUSH_PREPEND_NOT_OK(ksck->FetchInfoFromTabletServers(), error_messages,
+                      "error fetching info from tablet servers");
+
+  // TODO: Add support for tables / tablets filter in the consistency check.
+  PUSH_PREPEND_NOT_OK(ksck->CheckTablesConsistency(), error_messages,
+                      "table consistency check error");
+
+  if (FLAGS_checksum_scan) {
+    PUSH_PREPEND_NOT_OK(ksck->ChecksumData(ChecksumOptions()),
+                        error_messages, "checksum scan error");
+  }
+
+  // All good.
+  if (error_messages.empty()) {
+    cout << "OK" << endl;
+    return Status::OK();
+  }
+
+  // Something went wrong.
+  cerr << "==================" << endl;
+  cerr << "Errors:" << endl;
+  cerr << "==================" << endl;
+  for (const auto& s : error_messages) {
+    cerr << s << endl;
+  }
+  cerr << endl;
+  cerr << "FAILED" << endl;
+  return Status::RuntimeError("ksck discovered errors");
+}
+
+} // anonymous namespace
+
+unique_ptr<Mode> BuildClusterMode() {
+  // TODO: the long description looks pretty horrible, need to fix.
+  string desc = "Check the health of a Kudu cluster. By default, ksck checks "
+      "that master and tablet server processes are running, and that table "
+      "metadata is consistent. Use the 'checksum' flag to check that tablet "
+      "data is consistent (also see the 'tables' and 'tablets' flags). Use the "
+      "'checksum_snapshot' along with 'checksum' if the table or tablets are "
+      "actively receiving inserts or updates.";
+  unique_ptr<Action> ksck = ActionBuilder(
+      { "ksck", desc }, &RunKsck)
+    .AddRequiredParameter({ "master_address", "Kudu Master RPC address of form hostname:port"
})
+    .AddOptionalParameter("checksum_scan")
+    .AddOptionalParameter("checksum_snapshot")
+    .AddOptionalParameter("tables")
+    .AddOptionalParameter("tablets")
+    .Build();
+
+  return ModeBuilder({ "cluster", "Operate on a Kudu cluster" })
+      .AddAction(std::move(ksck))
+      .Build();
+}
+
+} // namespace tools
+} // namespace kudu
+

http://git-wip-us.apache.org/repos/asf/kudu/blob/86afadc7/src/kudu/tools/tool_main.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_main.cc b/src/kudu/tools/tool_main.cc
index 18137a1..70fdfc0 100644
--- a/src/kudu/tools/tool_main.cc
+++ b/src/kudu/tools/tool_main.cc
@@ -111,6 +111,7 @@ int DispatchCommand(const vector<Mode*>& chain,
 int RunTool(int argc, char** argv, bool show_help) {
   unique_ptr<Mode> root =
       ModeBuilder({ argv[0], "" }) // root mode description isn't printed
+      .AddMode(BuildClusterMode())
       .AddMode(BuildFsMode())
       .AddMode(BuildPbcMode())
       .AddMode(BuildTabletMode())


Mime
View raw message