kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [kudu] branch master updated: [hms] Add a tool to list all Kudu HMS entries
Date Sun, 02 Jun 2019 02:17:55 GMT
This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new 11a29a9  [hms] Add a tool to list all Kudu HMS entries
11a29a9 is described below

commit 11a29a9e8d6c095a0200dfabe0afb6d203559342
Author: Grant Henke <granthenke@apache.org>
AuthorDate: Fri May 31 18:45:47 2019 -0500

    [hms] Add a tool to list all Kudu HMS entries
    
    Adds a tool to list the Kudu HMS entries with optional
    columns in various formats.
    
    Change-Id: Ib18cde6e92e799390bb977562fd8d2eac2dae026
    Reviewed-on: http://gerrit.cloudera.org:8080/13490
    Tested-by: Kudu Jenkins
    Reviewed-by: Alexey Serbin <aserbin@cloudera.com>
---
 src/kudu/tools/kudu-tool-test.cc  | 75 +++++++++++++++++++++++++++++++++++
 src/kudu/tools/tool_action_hms.cc | 83 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 155 insertions(+), 3 deletions(-)

diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index d872724..727046a 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -950,6 +950,7 @@ TEST_F(ToolTest, TestTopLevelHelp) {
       "cluster.*Kudu cluster",
       "diagnose.*Diagnostic tools.*",
       "fs.*Kudu filesystem",
+      "hms.*Hive Metastores",
       "local_replica.*tablet replicas",
       "master.*Kudu Master",
       "pbc.*protobuf container",
@@ -970,6 +971,18 @@ TEST_F(ToolTest, TestTopLevelHelp) {
 TEST_F(ToolTest, TestModeHelp) {
   {
     const vector<string> kFsModeRegexes = {
+        "check.*Check metadata consistency",
+        "downgrade.*Downgrade the metadata",
+        "fix.*Fix automatically-repairable metadata",
+        "list.*List the Kudu table HMS entries",
+        "precheck.*Check that the Kudu cluster is prepared",
+    };
+    NO_FATALS(RunTestHelp("hms", kFsModeRegexes));
+    NO_FATALS(RunTestHelp("hms not_a_mode", kFsModeRegexes,
+                          Status::InvalidArgument("unknown command 'not_a_mode'")));
+  }
+  {
+    const vector<string> kFsModeRegexes = {
         "check.*Kudu filesystem for inconsistencies",
         "dump.*Dump a Kudu filesystem",
         "format.*new Kudu filesystem",
@@ -3949,6 +3962,68 @@ TEST_F(ToolTest, TestHmsPrecheck) {
   }), tables);
 }
 
+TEST_F(ToolTest, TestHmsList) {
+  ExternalMiniClusterOptions opts;
+  opts.hms_mode = HmsMode::ENABLE_HIVE_METASTORE;
+  opts.enable_kerberos = EnableKerberos();
+  NO_FATALS(StartExternalMiniCluster(std::move(opts)));
+
+  string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+  thrift::ClientOptions hms_opts;
+  hms_opts.enable_kerberos = EnableKerberos();
+  hms_opts.service_principal = "hive";
+  HmsClient hms_client(cluster_->hms()->address(), hms_opts);
+  ASSERT_OK(hms_client.Start());
+  ASSERT_TRUE(hms_client.IsConnected());
+
+  FLAGS_hive_metastore_uris = cluster_->hms()->uris();
+  FLAGS_hive_metastore_sasl_enabled = EnableKerberos();
+  HmsCatalog hms_catalog(master_addr);
+  ASSERT_OK(hms_catalog.Start());
+
+  shared_ptr<KuduClient> kudu_client;
+  ASSERT_OK(cluster_->CreateClient(nullptr, &kudu_client));
+
+  // Create a simple Kudu table so we can use the schema.
+  shared_ptr<KuduTable> simple_table;
+  ASSERT_OK(CreateKuduTable(kudu_client, "default.simple"));
+  ASSERT_OK(kudu_client->OpenTable("default.simple", &simple_table));
+
+  string kUsername = "alice";
+  ASSERT_OK(hms_catalog.CreateTable(
+      "1", "default.table1", kUsername, KuduSchema::ToSchema(simple_table->schema()),
+          hms::HmsClient::kManagedTable));
+  ASSERT_OK(hms_catalog.CreateTable(
+      "2", "default.table2", boost::none, KuduSchema::ToSchema(simple_table->schema()),
+      hms::HmsClient::kExternalTable));
+
+  // Test the output when HMS integration is disabled.
+  string err;
+  RunActionStderrString(Substitute("hms list $0", master_addr), &err);
+  ASSERT_STR_CONTAINS(err,
+      "Configuration error: the Kudu leader master is not configured with "
+      "the Hive Metastore integration");
+
+  // Enable the HMS integration.
+  cluster_->ShutdownNodes(cluster::ClusterNodes::MASTERS_ONLY);
+  cluster_->EnableMetastoreIntegration();
+  ASSERT_OK(cluster_->Restart());
+
+  // Run the list tool with the defaults. atabase,table,type,$0
+  string out;
+  RunActionStdoutString(Substitute("hms list $0", master_addr), &out);
+  ASSERT_STR_CONTAINS(out,
+      "default  | table1 | MANAGED_TABLE  | default.table1");
+  ASSERT_STR_CONTAINS(out,
+      "default  | table2 | EXTERNAL_TABLE | default.table2");
+
+  // Run the list tool filtering columns and using a different format.
+  RunActionStdoutString(Substitute("hms list --columns table,owner,kudu.table_id, --format=csv
$0",
+                                   master_addr), &out);
+  ASSERT_STR_CONTAINS(out, "table1,alice,1");
+  ASSERT_STR_CONTAINS(out, "table2,,");
+}
+
 // This test is parameterized on the serialization mode and Kerberos.
 class ControlShellToolTest :
     public ToolTest,
diff --git a/src/kudu/tools/tool_action_hms.cc b/src/kudu/tools/tool_action_hms.cc
index f96d462..dfd2f69 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -27,6 +27,7 @@
 #include <utility>
 #include <vector>
 
+#include <boost/algorithm/string/predicate.hpp>
 #include <boost/optional/optional.hpp>
 #include <gflags/gflags.h>
 #include <gflags/gflags_declare.h>
@@ -38,6 +39,7 @@
 #include "kudu/common/schema.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/strings/split.h"
+#include "kudu/gutil/strings/stringpiece.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/hms/hive_metastore_types.h"
 #include "kudu/hms/hms_catalog.h"
@@ -48,6 +50,7 @@
 #include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 
+DECLARE_string(columns);
 DECLARE_bool(force);
 DECLARE_bool(hive_metastore_sasl_enabled);
 DECLARE_int64(timeout_ms);
@@ -212,10 +215,10 @@ Status PrintTables(const string& master_addrs,
       "HMS database",
       "HMS table",
       "HMS table type",
-      Substitute("HMS $0", HmsClient::kStorageHandlerKey),
       Substitute("HMS $0", HmsClient::kKuduTableNameKey),
       Substitute("HMS $0", HmsClient::kKuduTableIdKey),
       Substitute("HMS $0", HmsClient::kKuduMasterAddrsKey),
+      Substitute("HMS $0", HmsClient::kStorageHandlerKey),
   });
   for (auto& pair : tables) {
     vector<string> row;
@@ -232,10 +235,10 @@ Status PrintTables(const string& master_addrs,
       row.emplace_back(hms_table.dbName);
       row.emplace_back(hms_table.tableName);
       row.emplace_back(hms_table.tableType);
-      row.emplace_back(hms_table.parameters[HmsClient::kStorageHandlerKey]);
       row.emplace_back(hms_table.parameters[HmsClient::kKuduTableNameKey]);
       row.emplace_back(hms_table.parameters[HmsClient::kKuduTableIdKey]);
       row.emplace_back(hms_table.parameters[HmsClient::kKuduMasterAddrsKey]);
+      row.emplace_back(hms_table.parameters[HmsClient::kStorageHandlerKey]);
     } else {
       row.resize(10);
     }
@@ -244,6 +247,51 @@ Status PrintTables(const string& master_addrs,
   return table.PrintTo(out);
 }
 
+// Prints catalog information about Kudu HMS tables in data table format to 'out'.
+Status PrintHMSTables(vector<hive::Table> tables, ostream& out) {
+  DataTable table({});
+  for (const auto& column : strings::Split(FLAGS_columns, ",", strings::SkipEmpty()))
{
+    vector<string> values;
+    if (boost::iequals(column, "database")) {
+      for (auto& hms_table : tables) {
+        values.emplace_back(hms_table.dbName);
+      }
+    } else if (boost::iequals(column, "table")) {
+      for (auto& hms_table : tables) {
+        values.emplace_back(hms_table.tableName);
+      }
+    } else if (boost::iequals(column, "type")) {
+      for (auto& hms_table : tables) {
+        values.emplace_back(hms_table.tableType);
+      }
+    } else if (boost::iequals(column, "owner")) {
+      for (auto& hms_table : tables) {
+        values.emplace_back(hms_table.owner);
+      }
+    } else if (boost::iequals(column, HmsClient::kKuduTableNameKey)) {
+      for (auto& hms_table : tables) {
+        values.emplace_back(hms_table.parameters[HmsClient::kKuduTableNameKey]);
+      }
+    } else if (boost::iequals(column, HmsClient::kKuduTableIdKey)) {
+      for (auto& hms_table : tables) {
+        values.emplace_back(hms_table.parameters[HmsClient::kKuduTableIdKey]);
+      }
+    } else if (boost::iequals(column, HmsClient::kKuduMasterAddrsKey)) {
+      for (auto& hms_table : tables) {
+        values.emplace_back(hms_table.parameters[HmsClient::kKuduMasterAddrsKey]);
+      }
+    } else if (boost::iequals(column, HmsClient::kStorageHandlerKey)) {
+      for (auto& hms_table : tables) {
+        values.emplace_back(hms_table.parameters[HmsClient::kStorageHandlerKey]);
+      }
+    } else {
+      return Status::InvalidArgument("unknown column (--columns)", column);
+    }
+    table.AddColumn(column.ToString(), std::move(values));
+  }
+  return table.PrintTo(out);
+}
+
 // A report of inconsistent tables in Kudu and the HMS catalogs.
 struct CatalogReport {
   // Kudu tables in the HMS catalog which have no corresponding table in the
@@ -717,6 +765,18 @@ Status FixHmsMetadata(const RunnerContext& context) {
   return Status::RuntimeError("Failed to fix some catalog metadata inconsistencies");
 }
 
+Status List(const RunnerContext& context) {
+  shared_ptr<KuduClient> kudu_client;
+  unique_ptr<HmsCatalog> hms_catalog;
+  string master_addrs;
+  RETURN_NOT_OK(Init(context, &kudu_client, &hms_catalog, &master_addrs));
+
+  vector<hive::Table> hms_tables;
+  RETURN_NOT_OK(hms_catalog->GetKuduTables(&hms_tables));
+
+  return PrintHMSTables(hms_tables, cout);
+}
+
 Status Precheck(const RunnerContext& context) {
   string master_addrs;
   RETURN_NOT_OK(ParseMasterAddressesStr(context, &master_addrs));
@@ -771,7 +831,6 @@ Status Precheck(const RunnerContext& context) {
   return Status::IllegalState("found tables in Kudu with case-conflicting names");
 }
 
-// TODO(ghenke): Add dump tool that prints the Kudu HMS entries.
 unique_ptr<Mode> BuildHmsMode() {
   const string kHmsUrisDesc =
     "Address of the Hive Metastore instance(s). If not set, the configuration "
@@ -820,6 +879,23 @@ unique_ptr<Mode> BuildHmsMode() {
         .AddOptionalParameter("ignore_other_clusters")
         .Build();
 
+  unique_ptr<Action> hms_list =
+      ActionBuilder("list", &List)
+          .Description("List the Kudu table HMS entries")
+          .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
+          .AddOptionalParameter("columns",
+                                Substitute("database,table,type,$0",
+                                                  HmsClient::kKuduTableNameKey),
+                                Substitute("Comma-separated list of HMS entry fields to "
+                                           "include in output.\nPossible values: database,
"
+                                           "table, type, owner, $0, $1, $2, $3",
+                                           HmsClient::kKuduTableNameKey,
+                                           HmsClient::kKuduTableIdKey,
+                                           HmsClient::kKuduMasterAddrsKey,
+                                           HmsClient::kStorageHandlerKey))
+          .AddOptionalParameter("format")
+          .Build();
+
   unique_ptr<Action> hms_precheck =
     ActionBuilder("precheck", &Precheck)
         .Description("Check that the Kudu cluster is prepared to enable the Hive "
@@ -831,6 +907,7 @@ unique_ptr<Mode> BuildHmsMode() {
                            .AddAction(std::move(hms_check))
                            .AddAction(std::move(hms_downgrade))
                            .AddAction(std::move(hms_fix))
+                           .AddAction(std::move(hms_list))
                            .AddAction(std::move(hms_precheck))
                            .Build();
 }


Mime
View raw message