kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ale...@apache.org
Subject kudu git commit: [tools] Implement a manual leader_step_down for a tablet
Date Fri, 07 Oct 2016 22:39:30 GMT
Repository: kudu
Updated Branches:
  refs/heads/master fc5e6a3e7 -> 833abea78


[tools] Implement a manual leader_step_down for a tablet

This change introduces a leader_step_down functionality
under 'kudu tablet'. This tool may be handy to recover from
situations when a single tablet server is overloaded and we want
to kick off a new election to balance the load across the clusters.
Although it is not guaranteed that a different replica will be elected
as the leader, this is an optimistic effort to elect a new tablet
server as the leader for the given tablet in the cluster.

Test: Ran 8000 iterations of leader_step_down test on dist_test.

Also snuck in a small change to display host:port details with
'kudu table list <master_addresses> --list_tablets' command.

Change-Id: Ia046a28a2008f4f5d1e955f57752a32a1ddc5ab8
Reviewed-on: http://gerrit.cloudera.org:8080/4533
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <aserbin@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/833abea7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/833abea7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/833abea7

Branch: refs/heads/master
Commit: 833abea787d3ac4bde3e5292c82b3408aad9ba92
Parents: fc5e6a3
Author: Dinesh Bhat <dinesh@cloudera.com>
Authored: Thu Sep 22 09:56:34 2016 -0700
Committer: Alexey Serbin <aserbin@cloudera.com>
Committed: Fri Oct 7 22:11:16 2016 +0000

----------------------------------------------------------------------
 src/kudu/tools/kudu-admin-test.cc            | 91 +++++++++++++++++++++++
 src/kudu/tools/kudu-tool-test.cc             |  3 +-
 src/kudu/tools/tool_action_cluster.cc        |  8 +-
 src/kudu/tools/tool_action_common.cc         |  6 ++
 src/kudu/tools/tool_action_common.h          |  6 ++
 src/kudu/tools/tool_action_local_replica.cc  | 32 ++++----
 src/kudu/tools/tool_action_remote_replica.cc | 10 +--
 src/kudu/tools/tool_action_table.cc          | 15 ++--
 src/kudu/tools/tool_action_tablet.cc         | 81 ++++++++++++++------
 src/kudu/tools/tool_action_test.cc           |  4 +-
 10 files changed, 190 insertions(+), 66 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/kudu-admin-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc
index b5aac2c..b5c5f83 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -37,6 +37,7 @@ using client::KuduClientBuilder;
 using client::KuduSchema;
 using client::KuduTableCreator;
 using client::sp::shared_ptr;
+using consensus::ConsensusStatePB;
 using itest::TabletServerMap;
 using itest::TServerDetails;
 using std::string;
@@ -153,6 +154,96 @@ TEST_F(AdminCliTest, TestChangeConfig) {
                                                 MonoDelta::FromSeconds(10)));
 }
 
+Status GetTermFromConsensus(const vector<TServerDetails*>& tservers,
+                            const string& tablet_id,
+                            int64 *current_term) {
+  ConsensusStatePB cstate;
+  for (auto& ts : tservers) {
+    RETURN_NOT_OK(
+        itest::GetConsensusState(ts, tablet_id,
+                                 consensus::CONSENSUS_CONFIG_COMMITTED,
+                                 MonoDelta::FromSeconds(10), &cstate));
+    if (cstate.has_leader_uuid() && cstate.has_current_term()) {
+      *current_term = cstate.current_term();
+      return Status::OK();
+    }
+  }
+  return Status::NotFound(Substitute(
+      "No leader replica found for tablet $0", tablet_id));
+}
+
+TEST_F(AdminCliTest, TestLeaderStepDown) {
+  FLAGS_num_tablet_servers = 3;
+  FLAGS_num_replicas = 3;
+  BuildAndStart({}, {});
+
+  vector<TServerDetails*> tservers;
+  AppendValuesFromMap(tablet_servers_, &tservers);
+  ASSERT_EQ(FLAGS_num_tablet_servers, tservers.size());
+  for (auto& ts : tservers) {
+    ASSERT_OK(itest::WaitUntilTabletRunning(ts,
+                                            tablet_id_,
+                                            MonoDelta::FromSeconds(10)));
+  }
+
+  int64 current_term;
+  ASSERT_OK(GetTermFromConsensus(tservers, tablet_id_,
+                                 &current_term));
+
+  // The leader for the given tablet may change anytime, resulting in
+  // the command returning an error code. Hence checking for term advancement
+  // only if the leader_step_down succeeds. It is also unsafe to check
+  // the term advancement without honoring status of the command since
+  // there may not have been another election in the meanwhile.
+  string stderr;
+  Status s = Subprocess::Call({GetKuduCtlAbsolutePath(),
+                               "tablet", "leader_step_down",
+                               cluster_->master()->bound_rpc_addr().ToString(),
+                               tablet_id_}, nullptr, &stderr);
+  bool not_currently_leader = stderr.find(
+      Status::IllegalState("").CodeAsString()) != string::npos;
+  ASSERT_TRUE(s.ok() || not_currently_leader);
+  if (s.ok()) {
+    int64 new_term;
+    AssertEventually([&]() {
+        ASSERT_OK(GetTermFromConsensus(tservers, tablet_id_,
+                                       &new_term));
+        ASSERT_GT(new_term, current_term);
+      });
+  }
+}
+
+TEST_F(AdminCliTest, TestLeaderStepDownWhenNotPresent) {
+  FLAGS_num_tablet_servers = 3;
+  FLAGS_num_replicas = 3;
+  BuildAndStart(
+      { "--enable_leader_failure_detection=false" },
+      { "--catalog_manager_wait_for_new_tablets_to_elect_leader=false" });
+  vector<TServerDetails*> tservers;
+  AppendValuesFromMap(tablet_servers_, &tservers);
+  ASSERT_EQ(FLAGS_num_tablet_servers, tservers.size());
+  for (auto& ts : tservers) {
+    ASSERT_OK(itest::WaitUntilTabletRunning(ts,
+                                            tablet_id_,
+                                            MonoDelta::FromSeconds(10)));
+  }
+
+  int64 current_term;
+  ASSERT_TRUE(GetTermFromConsensus(tservers, tablet_id_,
+                                   &current_term).IsNotFound());
+  string stdout;
+  ASSERT_OK(Subprocess::Call({
+    GetKuduCtlAbsolutePath(),
+    "tablet",
+    "leader_step_down",
+    cluster_->master()->bound_rpc_addr().ToString(),
+    tablet_id_
+  }, &stdout));
+  ASSERT_STR_CONTAINS(stdout,
+                      Substitute("No leader replica found for tablet $0",
+                                 tablet_id_));
+}
+
 TEST_F(AdminCliTest, TestDeleteTable) {
   FLAGS_num_tablet_servers = 1;
   FLAGS_num_replicas = 1;

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/kudu-tool-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 34e0a59..94c5160 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -282,7 +282,8 @@ TEST_F(ToolTest, TestModeHelp) {
   }
   {
     const vector<string> kTabletModeRegexes = {
-        "change_config.*Change.*Raft configuration"
+        "change_config.*Change.*Raft configuration",
+        "leader_step_down.*Force the tablet's leader replica to step down"
     };
     NO_FATALS(RunTestHelp("tablet", kTabletModeRegexes));
   }

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/tool_action_cluster.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_cluster.cc b/src/kudu/tools/tool_action_cluster.cc
index 513d75b..e5351f2 100644
--- a/src/kudu/tools/tool_action_cluster.cc
+++ b/src/kudu/tools/tool_action_cluster.cc
@@ -28,6 +28,7 @@
 #include "kudu/gutil/strings/split.h"
 #include "kudu/tools/ksck.h"
 #include "kudu/tools/ksck_remote.h"
+#include "kudu/tools/tool_action_common.h"
 #include "kudu/util/status.h"
 
 #define PUSH_PREPEND_NOT_OK(s, statuses, msg) do { \
@@ -62,8 +63,6 @@ using std::vector;
 
 namespace {
 
-const char* const kMasterAddressesArg = "master_addresses";
-
 Status RunKsck(const RunnerContext& context) {
   const string& master_addresses_str = FindOrDie(context.required_args,
                                                  kMasterAddressesArg);
@@ -130,10 +129,7 @@ unique_ptr<Mode> BuildClusterMode() {
       ActionBuilder("ksck", &RunKsck)
       .Description(desc)
       .ExtraDescription(extra_desc)
-      .AddRequiredParameter({
-        kMasterAddressesArg,
-        "Comma-separated list of Kudu Master addressess where each address is "
-        "of form 'hostname:port'" })
+      .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
       .AddOptionalParameter("checksum_scan")
       .AddOptionalParameter("checksum_scan_concurrency")
       .AddOptionalParameter("checksum_snapshot")

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/tool_action_common.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_common.cc b/src/kudu/tools/tool_action_common.cc
index 867cc2a..bd0db19 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ b/src/kudu/tools/tool_action_common.cc
@@ -97,6 +97,12 @@ using tserver::TabletServerAdminServiceProxy;
 using tserver::TabletServerServiceProxy;
 using tserver::WriteRequestPB;
 
+const char* const kMasterAddressesArg = "master_addresses";
+const char* const kMasterAddressesArgDesc = "Comma-separated list of Kudu "
+    "Master addresses where each address is of form 'hostname:port'";
+const char* const kTabletIdArg = "tablet_id";
+const char* const kTabletIdArgDesc = "Tablet Identifier";
+
 namespace {
 
 enum PrintEntryType {

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/tool_action_common.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_common.h b/src/kudu/tools/tool_action_common.h
index d51f656..27284f1 100644
--- a/src/kudu/tools/tool_action_common.h
+++ b/src/kudu/tools/tool_action_common.h
@@ -35,6 +35,12 @@ class ServerStatusPB;
 
 namespace tools {
 
+// Constants for parameters and descriptions.
+extern const char* const kMasterAddressesArg;
+extern const char* const kMasterAddressesArgDesc;
+extern const char* const kTabletIdArg;
+extern const char* const kTabletIdArgDesc;
+
 // Utility methods used by multiple actions across different modes.
 
 // Builds a proxy to a Kudu server running at 'address', returning it in

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/tool_action_local_replica.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_local_replica.cc b/src/kudu/tools/tool_action_local_replica.cc
index b0455a7..461235e 100644
--- a/src/kudu/tools/tool_action_local_replica.cc
+++ b/src/kudu/tools/tool_action_local_replica.cc
@@ -116,8 +116,8 @@ using tserver::WriteRequestPB;
 
 namespace {
 
-static const char* const kSeparatorLine =
-  "----------------------------------------------------------------------\n";
+const char* const kSeparatorLine =
+    "----------------------------------------------------------------------\n";
 
 string Indent(int indent) {
   return string(indent, ' ');
@@ -172,7 +172,7 @@ Status ParsePeerString(const string& peer_str,
 Status PrintReplicaUuids(const RunnerContext& context) {
   unique_ptr<FsManager> fs_manager;
   RETURN_NOT_OK(FsInit(&fs_manager));
-  const string& tablet_id = FindOrDie(context.required_args, "tablet_id");
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
 
   // Load the cmeta file and print all peer uuids.
   unique_ptr<ConsensusMetadata> cmeta;
@@ -186,7 +186,7 @@ Status PrintReplicaUuids(const RunnerContext& context) {
 
 Status RewriteRaftConfig(const RunnerContext& context) {
   // Parse tablet ID argument.
-  const string& tablet_id = FindOrDie(context.required_args, "tablet_id");
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
   if (tablet_id != master::SysCatalogTable::kSysCatalogTabletId) {
     LOG(WARNING) << "Master will not notice rewritten Raft config of regular "
                  << "tablets. A regular Raft config change must occur.";
@@ -238,7 +238,7 @@ Status RewriteRaftConfig(const RunnerContext& context) {
 
 Status CopyFromRemote(const RunnerContext& context) {
   // Parse the tablet ID and source arguments.
-  const string& tablet_id = FindOrDie(context.required_args, "tablet_id");
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
   const string& rpc_address = FindOrDie(context.required_args, "source");
 
   HostPort hp;
@@ -259,7 +259,7 @@ Status CopyFromRemote(const RunnerContext& context) {
 Status DumpWals(const RunnerContext& context) {
   unique_ptr<FsManager> fs_manager;
   RETURN_NOT_OK(FsInit(&fs_manager));
-  const string& tablet_id = FindOrDie(context.required_args, "tablet_id");
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
 
   shared_ptr<LogReader> reader;
   RETURN_NOT_OK(LogReader::Open(fs_manager.get(),
@@ -309,7 +309,7 @@ Status ListBlocksInRowSet(const Schema& schema,
 Status DumpBlockIdsForLocalReplica(const RunnerContext& context) {
   unique_ptr<FsManager> fs_manager;
   RETURN_NOT_OK(FsInit(&fs_manager));
-  const string& tablet_id = FindOrDie(context.required_args, "tablet_id");
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
 
   scoped_refptr<TabletMetadata> meta;
   RETURN_NOT_OK(TabletMetadata::Load(fs_manager.get(), tablet_id, &meta));
@@ -567,7 +567,7 @@ Status DumpRowSetInternal(FsManager* fs_manager,
 Status DumpRowSet(const RunnerContext& context) {
   unique_ptr<FsManager> fs_manager;
   RETURN_NOT_OK(FsInit(&fs_manager));
-  const string& tablet_id = FindOrDie(context.required_args, "tablet_id");
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
 
   scoped_refptr<TabletMetadata> meta;
   RETURN_NOT_OK(TabletMetadata::Load(fs_manager.get(), tablet_id, &meta));
@@ -607,7 +607,7 @@ Status DumpRowSet(const RunnerContext& context) {
 Status DumpMeta(const RunnerContext& context) {
   unique_ptr<FsManager> fs_manager;
   RETURN_NOT_OK(FsInit(&fs_manager));
-  const string& tablet_id = FindOrDie(context.required_args, "tablet_id");
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
   RETURN_NOT_OK(DumpTabletMeta(fs_manager.get(), tablet_id, 0));
   return Status::OK();
 }
@@ -616,7 +616,7 @@ unique_ptr<Mode> BuildDumpMode() {
   unique_ptr<Action> dump_block_ids =
       ActionBuilder("block_ids", &DumpBlockIdsForLocalReplica)
       .Description("Dump the IDs of all blocks belonging to a local replica")
-      .AddRequiredParameter({ "tablet_id", "tablet identifier" })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddOptionalParameter("fs_wal_dir")
       .AddOptionalParameter("fs_data_dirs")
       .Build();
@@ -624,7 +624,7 @@ unique_ptr<Mode> BuildDumpMode() {
   unique_ptr<Action> dump_meta =
       ActionBuilder("meta", &DumpMeta)
       .Description("Dump the metadata of a local replica")
-      .AddRequiredParameter({ "tablet_id", "tablet identifier" })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddOptionalParameter("fs_wal_dir")
       .AddOptionalParameter("fs_data_dirs")
       .Build();
@@ -632,7 +632,7 @@ unique_ptr<Mode> BuildDumpMode() {
   unique_ptr<Action> dump_rowset =
       ActionBuilder("rowset", &DumpRowSet)
       .Description("Dump the rowset contents of a local replica")
-      .AddRequiredParameter({ "tablet_id", "tablet identifier" })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddOptionalParameter("dump_data")
       .AddOptionalParameter("fs_wal_dir")
       .AddOptionalParameter("fs_data_dirs")
@@ -645,7 +645,7 @@ unique_ptr<Mode> BuildDumpMode() {
       ActionBuilder("wals", &DumpWals)
       .Description("Dump all WAL (write-ahead log) segments of "
         "a local replica")
-      .AddRequiredParameter({ "tablet_id", "Tablet identifier" })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddOptionalParameter("fs_wal_dir")
       .AddOptionalParameter("fs_data_dirs")
       .AddOptionalParameter("print_entries")
@@ -669,7 +669,7 @@ unique_ptr<Mode> BuildLocalReplicaMode() {
       ActionBuilder("print_replica_uuids", &PrintReplicaUuids)
       .Description("Print all replica UUIDs found in a "
         "tablet's Raft configuration")
-      .AddRequiredParameter({ "tablet_id", "Tablet identifier" })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddOptionalParameter("fs_wal_dir")
       .AddOptionalParameter("fs_data_dirs")
       .Build();
@@ -677,7 +677,7 @@ unique_ptr<Mode> BuildLocalReplicaMode() {
   unique_ptr<Action> rewrite_raft_config =
       ActionBuilder("rewrite_raft_config", &RewriteRaftConfig)
       .Description("Rewrite a replica's Raft configuration")
-      .AddRequiredParameter({ "tablet_id", "Tablet identifier" })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddRequiredVariadicParameter({
         "peers", "List of peers where each peer is of "
         "form 'uuid:hostname:port'" })
@@ -696,7 +696,7 @@ unique_ptr<Mode> BuildLocalReplicaMode() {
   unique_ptr<Action> copy_from_remote =
       ActionBuilder("copy_from_remote", &CopyFromRemote)
       .Description("Copy a replica from a remote server")
-      .AddRequiredParameter({ "tablet_id", "Tablet identifier" })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddRequiredParameter({ "source", "Source RPC address of "
         "form hostname:port" })
       .AddOptionalParameter("fs_wal_dir")

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/tool_action_remote_replica.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_remote_replica.cc b/src/kudu/tools/tool_action_remote_replica.cc
index e305d4c..0e117a9 100644
--- a/src/kudu/tools/tool_action_remote_replica.cc
+++ b/src/kudu/tools/tool_action_remote_replica.cc
@@ -136,8 +136,6 @@ class ReplicaDumper {
 namespace {
 
 const char* const kReasonArg = "reason";
-const char* const kTabletArg = "tablet_id";
-const char* const kTabletDesc = "Tablet identifier";
 const char* const kTServerAddressArg = "tserver_address";
 const char* const kTServerAddressDesc = "Address of a Kudu Tablet Server of "
     "form 'hostname:port'. Port may be omitted if the Tablet Server is bound "
@@ -189,7 +187,7 @@ Status CheckReplicas(const RunnerContext& context) {
 
 Status DeleteReplica(const RunnerContext& context) {
   const string& address = FindOrDie(context.required_args, kTServerAddressArg);
-  const string& tablet_id = FindOrDie(context.required_args, kTabletArg);
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
   const string& reason = FindOrDie(context.required_args, kReasonArg);
 
   ServerStatusPB status;
@@ -220,7 +218,7 @@ Status DeleteReplica(const RunnerContext& context) {
 
 Status DumpReplica(const RunnerContext& context) {
   const string& address = FindOrDie(context.required_args, kTServerAddressArg);
-  const string& tablet_id = FindOrDie(context.required_args, kTabletArg);
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
 
   unique_ptr<TabletServerServiceProxy> proxy;
   RETURN_NOT_OK(BuildProxy(address, tserver::TabletServer::kDefaultPort,
@@ -295,7 +293,7 @@ unique_ptr<Mode> BuildRemoteReplicaMode() {
       ActionBuilder("delete", &DeleteReplica)
       .Description("Delete a replica from a Kudu Tablet Server")
       .AddRequiredParameter({ kTServerAddressArg, kTServerAddressDesc })
-      .AddRequiredParameter({ kTabletArg, kTabletDesc })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddRequiredParameter({ kReasonArg, "Reason for deleting the replica" })
       .Build();
 
@@ -303,7 +301,7 @@ unique_ptr<Mode> BuildRemoteReplicaMode() {
       ActionBuilder("dump", &DumpReplica)
       .Description("Dump the data of a replica on a Kudu Tablet Server")
       .AddRequiredParameter({ kTServerAddressArg, kTServerAddressDesc })
-      .AddRequiredParameter({ kTabletArg, kTabletDesc })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .Build();
 
   unique_ptr<Action> list =

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/tool_action_table.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc
index c50e3bd..5296886 100644
--- a/src/kudu/tools/tool_action_table.cc
+++ b/src/kudu/tools/tool_action_table.cc
@@ -28,6 +28,7 @@
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/stl_util.h"
 #include "kudu/gutil/strings/split.h"
+#include "kudu/tools/tool_action_common.h"
 #include "kudu/util/status.h"
 
 DEFINE_bool(list_tablets, false,
@@ -49,7 +50,6 @@ using std::vector;
 
 namespace {
 
-const char* const kMasterAddressesArg = "master_addresses";
 const char* const kTableNameArg = "table_name";
 
 Status DeleteTable(const RunnerContext& context) {
@@ -93,7 +93,8 @@ Status ListTables(const RunnerContext& context) {
       cout << "T " << token->tablet().id() << "\t";
       for (const auto* replica : token->tablet().replicas()) {
         cout << "P" << (replica->is_leader() ? "(L) " : "    ")
-             << replica->ts().uuid() << "    ";
+             << replica->ts().uuid() << "(" << replica->ts().hostname()
+             << ":" << replica->ts().port() << ")" << "    ";
       }
       cout << endl;
     }
@@ -108,20 +109,14 @@ unique_ptr<Mode> BuildTableMode() {
   unique_ptr<Action> delete_table =
       ActionBuilder("delete", &DeleteTable)
       .Description("Delete a table")
-      .AddRequiredParameter({
-        kMasterAddressesArg,
-        "Comma-separated list of Kudu Master addresses where each address is "
-        "of form 'hostname:port'" })
+      .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
       .AddRequiredParameter({ kTableNameArg, "Name of the table to delete" })
       .Build();
 
   unique_ptr<Action> list_tables =
       ActionBuilder("list", &ListTables)
       .Description("List all tables")
-      .AddRequiredParameter({
-        kMasterAddressesArg,
-        "Comma-separated list of Kudu Master addresses where each address is "
-        "of form 'hostname:port'" })
+      .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
       .AddOptionalParameter("list_tablets")
       .Build();
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/tool_action_tablet.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_tablet.cc b/src/kudu/tools/tool_action_tablet.cc
index 0e57d08..0ebc710 100644
--- a/src/kudu/tools/tool_action_tablet.cc
+++ b/src/kudu/tools/tool_action_tablet.cc
@@ -18,6 +18,7 @@
 #include "kudu/tools/tool_action.h"
 
 #include <algorithm>
+#include <iostream>
 #include <memory>
 #include <string>
 #include <vector>
@@ -31,6 +32,7 @@
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/rpc/rpc_controller.h"
+#include "kudu/server/server_base.pb.h"
 #include "kudu/tools/tool_action_common.h"
 #include "kudu/util/net/net_util.h"
 #include "kudu/util/status.h"
@@ -47,6 +49,8 @@ using consensus::ChangeConfigType;
 using consensus::ConsensusServiceProxy;
 using consensus::RaftPeerPB;
 using rpc::RpcController;
+using std::cout;
+using std::endl;
 using std::string;
 using std::unique_ptr;
 using std::vector;
@@ -54,10 +58,8 @@ using strings::Substitute;
 
 namespace {
 
-const char* const kMasterAddressesArg = "master_addresses";
 const char* const kReplicaTypeArg = "replica_type";
 const char* const kReplicaUuidArg = "replica_uuid";
-const char* const kTabletIdArg = "tablet_id";
 
 Status GetRpcAddressForTS(const client::sp::shared_ptr<KuduClient>& client,
                           const string& uuid,
@@ -94,6 +96,7 @@ Status GetTabletLeader(const client::sp::shared_ptr<KuduClient>&
client,
       return Status::OK();
     }
   }
+
   return Status::NotFound(Substitute(
       "No leader replica found for tablet $0", tablet_id));
 }
@@ -134,13 +137,6 @@ Status ChangeConfig(const RunnerContext& context, ChangeConfigType
cc_type) {
   string leader_uuid;
   HostPort leader_hp;
   RETURN_NOT_OK(GetTabletLeader(client, tablet_id, &leader_uuid, &leader_hp));
-  vector<Sockaddr> leader_addrs;
-  RETURN_NOT_OK(leader_hp.ResolveAddresses(&leader_addrs));
-  if (leader_addrs.empty()) {
-    return Status::NotFound(
-        "Unable to resolve IP address for tablet leader host",
-        leader_hp.ToString());
-  }
 
   unique_ptr<ConsensusServiceProxy> proxy;
   RETURN_NOT_OK(BuildProxy(leader_hp.host(), leader_hp.port(), &proxy));
@@ -172,17 +168,52 @@ Status RemoveReplica(const RunnerContext& context) {
   return ChangeConfig(context, consensus::REMOVE_SERVER);
 }
 
+Status LeaderStepDown(const RunnerContext& context) {
+  const string& master_addresses_str = FindOrDie(context.required_args,
+                                                 kMasterAddressesArg);
+  vector<string> master_addresses = strings::Split(master_addresses_str, ",");
+  const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
+
+  client::sp::shared_ptr<KuduClient> client;
+  RETURN_NOT_OK(KuduClientBuilder()
+                .master_server_addrs(master_addresses)
+                .Build(&client));
+
+  // If leader is not present, command can gracefully return.
+  string leader_uuid;
+  HostPort leader_hp;
+  Status s = GetTabletLeader(client, tablet_id, &leader_uuid, &leader_hp);
+  if (s.IsNotFound()) {
+    cout << s.ToString() << endl;
+    return Status::OK();
+  }
+  RETURN_NOT_OK(s);
+
+  unique_ptr<ConsensusServiceProxy> proxy;
+  RETURN_NOT_OK(BuildProxy(leader_hp.host(), leader_hp.port(), &proxy));
+
+  consensus::LeaderStepDownRequestPB req;
+  consensus::LeaderStepDownResponsePB resp;
+  RpcController rpc;
+  rpc.set_timeout(client->default_admin_operation_timeout());
+  req.set_dest_uuid(leader_uuid);
+  req.set_tablet_id(tablet_id);
+
+  RETURN_NOT_OK(proxy->LeaderStepDown(req, &resp, &rpc));
+  if (resp.has_error()) {
+    return StatusFromPB(resp.error().status());
+  }
+  return Status::OK();
+}
+
 } // anonymous namespace
 
 unique_ptr<Mode> BuildTabletMode() {
   unique_ptr<Action> add_replica =
       ActionBuilder("add_replica", &AddReplica)
       .Description("Add a new replica to a tablet's Raft configuration")
-      .AddRequiredParameter({
-        kMasterAddressesArg,
-        "Comma-separated list of Kudu Master addresses where each address is "
-        "of form 'hostname:port'" })
-      .AddRequiredParameter({ kTabletIdArg, "Tablet Identifier" })
+      .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddRequiredParameter({ kReplicaUuidArg, "New replica's UUID" })
       .AddRequiredParameter(
           { kReplicaTypeArg, "New replica's type. Must be VOTER or NON-VOTER."
@@ -193,11 +224,8 @@ unique_ptr<Mode> BuildTabletMode() {
       ActionBuilder("change_replica_type", &ChangeReplicaType)
       .Description(
           "Change the type of an existing replica in a tablet's Raft configuration")
-      .AddRequiredParameter({
-        kMasterAddressesArg,
-        "Comma-separated list of Kudu Master addresses where each address is "
-        "of 'form hostname:port'" })
-      .AddRequiredParameter({ kTabletIdArg, "Tablet Identifier" })
+      .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddRequiredParameter({ kReplicaUuidArg, "Existing replica's UUID" })
       .AddRequiredParameter(
           { kReplicaTypeArg, "Existing replica's new type. Must be VOTER or NON-VOTER."
@@ -207,14 +235,18 @@ unique_ptr<Mode> BuildTabletMode() {
   unique_ptr<Action> remove_replica =
       ActionBuilder("remove_replica", &RemoveReplica)
       .Description("Remove an existing replica from a tablet's Raft configuration")
-      .AddRequiredParameter({
-        kMasterAddressesArg,
-        "Comma-separated list of Kudu Master addresses where each address is "
-        "of form 'hostname:port'" })
-      .AddRequiredParameter({ kTabletIdArg, "Tablet Identifier" })
+      .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
       .AddRequiredParameter({ kReplicaUuidArg, "Existing replica's UUID" })
       .Build();
 
+  unique_ptr<Action> leader_step_down =
+      ActionBuilder("leader_step_down", &LeaderStepDown)
+      .Description("Force the tablet's leader replica to step down")
+      .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
+      .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc })
+      .Build();
+
   unique_ptr<Mode> change_config =
       ModeBuilder("change_config")
       .Description("Change a tablet's Raft configuration")
@@ -226,6 +258,7 @@ unique_ptr<Mode> BuildTabletMode() {
   return ModeBuilder("tablet")
       .Description("Operate on remote Kudu tablets")
       .AddMode(std::move(change_config))
+      .AddAction(std::move(leader_step_down))
       .Build();
 }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/833abea7/src/kudu/tools/tool_action_test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_test.cc b/src/kudu/tools/tool_action_test.cc
index 29b8950..5166025 100644
--- a/src/kudu/tools/tool_action_test.cc
+++ b/src/kudu/tools/tool_action_test.cc
@@ -105,6 +105,7 @@
 #include "kudu/common/types.h"
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/tools/tool_action_common.h"
 #include "kudu/util/oid_generator.h"
 #include "kudu/util/random.h"
 #include "kudu/util/stopwatch.h"
@@ -204,11 +205,8 @@ DEFINE_bool(use_random, false,
 namespace kudu {
 namespace tools {
 
-
 namespace {
 
-const char* const kMasterAddressesArg = "master_addresses";
-
 class Generator {
  public:
   enum Mode {


Mime
View raw message