kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [3/3] kudu git commit: KUDU-2529: Add a "-tables=<tables>" flag to the "kudu table list".
Date Wed, 05 Sep 2018 16:00:39 GMT
KUDU-2529: Add a "-tables=<tables>" flag to the "kudu table list".

"-tables=<tables>" flag can help to filter the tables that you want
  while running "kudu table list <master> -list_tables".

Change-Id: I23006a3f3e4f157e6c6c10968b8692e89892d1ce
Reviewed-on: http://gerrit.cloudera.org:8080/11360
Reviewed-by: Adar Dembo <adar@cloudera.com>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4fceeae3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4fceeae3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4fceeae3

Branch: refs/heads/master
Commit: 4fceeae3f314ef794bd58c47439033941a1aa568
Parents: 6b70c27
Author: helifu <hzhelifu@corp.netease.com>
Authored: Fri Aug 24 17:14:19 2018 +0800
Committer: Adar Dembo <adar@cloudera.com>
Committed: Wed Sep 5 16:00:18 2018 +0000

----------------------------------------------------------------------
 src/kudu/tools/ksck.cc                |  14 +---
 src/kudu/tools/kudu-tool-test.cc      | 127 ++++++++++++++++++++++++++++-
 src/kudu/tools/tool_action_cluster.cc |   6 +-
 src/kudu/tools/tool_action_common.cc  |  13 +++
 src/kudu/tools/tool_action_common.h   |   4 +
 src/kudu/tools/tool_action_table.cc   |   7 +-
 6 files changed, 152 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc
index eab1ac1..c426d38 100644
--- a/src/kudu/tools/ksck.cc
+++ b/src/kudu/tools/ksck.cc
@@ -39,9 +39,9 @@
 #include "kudu/gutil/strings/human_readable.h"
 #include "kudu/gutil/strings/join.h"
 #include "kudu/gutil/strings/substitute.h"
-#include "kudu/gutil/strings/util.h"
 #include "kudu/tablet/tablet.pb.h"
 #include "kudu/tools/color.h"
+#include "kudu/tools/tool_action_common.h"
 #include "kudu/util/atomic.h"
 #include "kudu/util/blocking_queue.h"
 #include "kudu/util/countdown_latch.h"
@@ -99,18 +99,6 @@ namespace kudu {
 namespace tools {
 
 namespace {
-// Return true if 'str' matches any of the patterns in 'patterns', or if
-// 'patterns' is empty.
-bool MatchesAnyPattern(const vector<string>& patterns, const string& str) {
-  // Consider no filter a wildcard.
-  if (patterns.empty()) return true;
-
-  for (const auto& p : patterns) {
-    if (MatchPattern(str, p)) return true;
-  }
-  return false;
-}
-
 void BuildKsckConsensusStateForConfigMember(const consensus::ConsensusStatePB& cstate,
                                             KsckConsensusState* ksck_cstate) {
   CHECK(ksck_cstate);

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/kudu-tool-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 28180c4..8669695 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -140,6 +140,8 @@ using kudu::client::KuduClient;
 using kudu::client::KuduClientBuilder;
 using kudu::client::KuduSchema;
 using kudu::client::KuduSchemaBuilder;
+using kudu::client::KuduScanToken;
+using kudu::client::KuduScanTokenBuilder;
 using kudu::client::KuduTable;
 using kudu::client::sp::shared_ptr;
 using kudu::cluster::ExternalMiniCluster;
@@ -176,6 +178,7 @@ using kudu::tserver::MiniTabletServer;
 using kudu::tserver::WriteRequestPB;
 using std::back_inserter;
 using std::copy;
+using std::map;
 using std::make_pair;
 using std::ostringstream;
 using std::pair;
@@ -581,7 +584,9 @@ TEST_F(ToolTest, TestModeHelp) {
   {
     const vector<string> kTableModeRegexes = {
         "delete.*Delete a table",
-        "list.*List all tables",
+        "rename_table.*Rename a table",
+        "rename_column.*Rename a column",
+        "list.*List tables",
     };
     NO_FATALS(RunTestHelp("table", kTableModeRegexes));
   }
@@ -2117,6 +2122,39 @@ TEST_F(ToolTest, TestMasterList) {
   ASSERT_STR_CONTAINS(out, master->bound_rpc_hostport().ToString());
 }
 
+// Operate on Kudu tables:
+// (1)delete a table
+// (2)rename a table
+// (3)rename a column
+// (4)list tables
+TEST_F(ToolTest, TestDeleteTable) {
+  NO_FATALS(StartExternalMiniCluster());
+  shared_ptr<KuduClient> client;
+  ASSERT_OK(cluster_->CreateClient(nullptr, &client));
+  string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+
+  const string& kTableName = "kudu.table";
+
+  // Create a table.
+  TestWorkload workload(cluster_.get());
+  workload.set_table_name(kTableName);
+  workload.set_num_replicas(1);
+  workload.Setup();
+
+  // Check that the table exists.
+  bool exist = false;
+  ASSERT_OK(client->TableExists(kTableName, &exist));
+  ASSERT_EQ(exist, true);
+
+  // Delete the table.
+  NO_FATALS(RunActionStdoutNone(Substitute("table delete $0 $1",
+                                           master_addr, kTableName)));
+
+  // Check that the table does not exist.
+  ASSERT_OK(client->TableExists(kTableName, &exist));
+  ASSERT_EQ(exist, false);
+}
+
 TEST_F(ToolTest, TestRenameTable) {
   NO_FATALS(StartExternalMiniCluster());
   shared_ptr<KuduClient> client;
@@ -2183,6 +2221,93 @@ TEST_F(ToolTest, TestRenameColumn) {
   ASSERT_STR_CONTAINS(table->schema().ToString(), kNewColumnName);
 }
 
+TEST_F(ToolTest, TestListTables) {
+  NO_FATALS(StartExternalMiniCluster());
+  shared_ptr<KuduClient> client;
+  ASSERT_OK(cluster_->CreateClient(nullptr, &client));
+  string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+
+  // Replica's format.
+  string ts_uuid = cluster_->tablet_server(0)->uuid();
+  string ts_addr = cluster_->tablet_server(0)->bound_rpc_addr().ToString();
+  string expect_replica = Substitute("    L $0 $1", ts_uuid, ts_addr);
+
+  // Create some tables.
+  const int kNumTables = 10;
+  vector<string> table_names;
+  for (int i = 0; i < kNumTables; ++i) {
+    string table_name = Substitute("kudu.table_$0", i);
+    table_names.push_back(table_name);
+
+    TestWorkload workload(cluster_.get());
+    workload.set_table_name(table_name);
+    workload.set_num_replicas(1);
+    workload.Setup();
+  }
+  std::sort(table_names.begin(), table_names.end());
+
+  const auto& ProcessTables = [&] (const int num) {
+    ASSERT_GE(num, 1);
+    ASSERT_LE(num, kNumTables);
+
+    vector<string> expected;
+    expected.insert(expected.end(), table_names.begin(), table_names.begin() + num);
+
+    string filter = "";
+    if (kNumTables != num) {
+      filter = Substitute("-tables=$0", JoinStrings(expected, ","));
+    }
+    vector<string> lines;
+    NO_FATALS(RunActionStdoutLines(
+        Substitute("table list $0 $1", master_addr, filter), &lines));
+
+    std::sort(lines.begin(), lines.end());
+    ASSERT_EQ(expected, lines);
+  };
+
+  const auto& ProcessTablets = [&] (const int num) {
+    ASSERT_GE(num, 1);
+    ASSERT_LE(num, kNumTables);
+
+    string filter = "";
+    if (kNumTables != num) {
+      filter = Substitute("-tables=$0",
+        JoinStringsIterator(table_names.begin(), table_names.begin() + num, ","));
+    }
+    vector<string> lines;
+    NO_FATALS(RunActionStdoutLines(
+        Substitute("table list $0 $1 -list_tablets", master_addr, filter), &lines));
+
+    map<string, pair<string, string>> output;
+    for (int i = 0; i < lines.size(); ++i) {
+      if (lines[i].empty()) continue;
+      ASSERT_LE(i + 2, lines.size());
+      output[lines[i]] = pair<string, string>(lines[i + 1], lines[i + 2]);
+      i += 2;
+    }
+
+    for (const auto& e : output) {
+      shared_ptr<KuduTable> table;
+      ASSERT_OK(client->OpenTable(e.first, &table));
+      vector<KuduScanToken*> tokens;
+      ElementDeleter deleter(&tokens);
+      KuduScanTokenBuilder builder(table.get());
+      ASSERT_OK(builder.Build(&tokens));
+      ASSERT_EQ(1, tokens.size()); // Only one partition(tablet) under table.
+      // Tablet's format.
+      string expect_tablet = Substitute("  T $0", tokens[0]->tablet().id());
+      ASSERT_EQ(expect_tablet, e.second.first);
+      ASSERT_EQ(expect_replica, e.second.second);
+    }
+  };
+
+  // List the tables and tablets.
+  for (int i = 1; i <= kNumTables; ++i) {
+    ProcessTables(i);
+    ProcessTablets(i);
+  }
+}
+
 Status CreateLegacyHmsTable(HmsClient* client,
                             const string& hms_database_name,
                             const string& hms_table_name,

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/tool_action_cluster.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_cluster.cc b/src/kudu/tools/tool_action_cluster.cc
index 8fccd84..b6a6981 100644
--- a/src/kudu/tools/tool_action_cluster.cc
+++ b/src/kudu/tools/tool_action_cluster.cc
@@ -28,6 +28,7 @@
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/optional/optional.hpp>
 #include <gflags/gflags.h>
+#include <gflags/gflags_declare.h>
 #include <glog/logging.h>
 
 #include "kudu/gutil/basictypes.h"
@@ -62,10 +63,7 @@ using strings::Substitute;
   } \
 } while (0);
 
-DEFINE_string(tables, "",
-              "Tables to check (comma-separated list of names). "
-              "If not specified, checks all tables.");
-
+DECLARE_string(tables);
 DEFINE_string(tablets, "",
               "Tablets to check (comma-separated list of IDs) "
               "If not specified, checks all tablets.");

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/tool_action_common.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_common.cc b/src/kudu/tools/tool_action_common.cc
index db86b02..e615915 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ b/src/kudu/tools/tool_action_common.cc
@@ -56,6 +56,7 @@
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/stringpiece.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/strings/util.h"
 #include "kudu/master/master.proxy.h" // IWYU pragma: keep
 #include "kudu/rpc/messenger.h"
 #include "kudu/rpc/rpc_controller.h"
@@ -101,6 +102,8 @@ DEFINE_string(flag_tags, "", "Comma-separated list of tags used to restrict
whic
                              "flags are returned. An empty value matches all tags");
 DEFINE_bool(all_flags, false, "Whether to return all flags, or only flags that "
                               "were explicitly set.");
+DEFINE_string(tables, "", "Tables to include (comma-separated list of table names)"
+                          "If not specified, includes all tables.");
 
 namespace boost {
 template <typename Signature>
@@ -449,6 +452,16 @@ string GetMasterAddresses(const client::KuduClient& client) {
   return HostPort::ToCommaSeparatedString(client.data_->master_hostports());
 }
 
+bool MatchesAnyPattern(const vector<string>& patterns, const string& str) {
+  // Consider no filter a wildcard.
+  if (patterns.empty()) return true;
+
+  for (const auto& p : patterns) {
+    if (MatchPattern(str, p)) return true;
+  }
+  return false;
+}
+
 Status PrintServerStatus(const string& address, uint16_t default_port) {
   ServerStatusPB status;
   RETURN_NOT_OK(GetServerStatus(address, default_port, &status));

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/tool_action_common.h
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_common.h b/src/kudu/tools/tool_action_common.h
index 3043bd8..d3a4524 100644
--- a/src/kudu/tools/tool_action_common.h
+++ b/src/kudu/tools/tool_action_common.h
@@ -140,6 +140,10 @@ void SetAlterExternalCatalogs(client::KuduTableAlterer* alterer, bool
alter_exte
 // Get the configured master addresses on the most recently connected to leader master.
 std::string GetMasterAddresses(const client::KuduClient& client);
 
+// Return true if 'str' matches any of the patterns in 'patterns', or if
+// 'patterns' is empty.
+bool MatchesAnyPattern(const std::vector<std::string>& patterns, const std::string&
str);
+
 // A table of data to present to the user.
 //
 // Supports formatting based on the --format flag.

http://git-wip-us.apache.org/repos/asf/kudu/blob/4fceeae3/src/kudu/tools/tool_action_table.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc
index 75d3a61..69521c2 100644
--- a/src/kudu/tools/tool_action_table.cc
+++ b/src/kudu/tools/tool_action_table.cc
@@ -23,6 +23,7 @@
 #include <vector>
 
 #include <gflags/gflags.h>
+#include <gflags/gflags_declare.h>
 
 #include "kudu/client/client.h"
 #include "kudu/client/replica_controller-internal.h"
@@ -36,6 +37,7 @@
 #include "kudu/tools/tool_action_common.h"
 #include "kudu/util/status.h"
 
+DECLARE_string(tables);
 DEFINE_bool(alter_external_catalogs, true,
             "Whether to alter external catalogs, such as the Hive Metastore, "
             "when renaming a table.");
@@ -73,7 +75,9 @@ class TableLister {
     vector<string> table_names;
     RETURN_NOT_OK(client->ListTables(&table_names));
 
+    vector<string> table_filters = Split(FLAGS_tables, ",", strings::SkipEmpty());
     for (const auto& tname : table_names) {
+      if (!MatchesAnyPattern(table_filters, tname)) continue;
       cout << tname << endl;
       if (!FLAGS_list_tablets) {
         continue;
@@ -186,8 +190,9 @@ unique_ptr<Mode> BuildTableMode() {
 
   unique_ptr<Action> list_tables =
       ActionBuilder("list", &ListTables)
-      .Description("List all tables")
+      .Description("List tables")
       .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
+      .AddOptionalParameter("tables")
       .AddOptionalParameter("list_tablets")
       .Build();
 


Mime
View raw message