kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [2/2] kudu git commit: KUDU-2191: allow metadata upgrade tool to run with HMS integration
Date Fri, 15 Jun 2018 21:40:05 GMT
KUDU-2191: allow metadata upgrade tool to run with HMS integration

Previously the metadata upgrade tool requires the HMS integration
feature being disabled. This commit updates the tool to only alter
tables in Kudu but not in the HMS during upgrade, so that the
restriction no longer applies.

Change-Id: I51d4cabeb1a9defc51f4e307a116419da5588f2d
Reviewed-on: http://gerrit.cloudera.org:8080/10582
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <danburkert@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f2089195
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f2089195
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f2089195

Branch: refs/heads/master
Commit: f2089195d635ea73e7268a9b46db2feac2eeed2d
Parents: e3474f4
Author: hahao <hao.hao@cloudera.com>
Authored: Wed Jun 6 23:58:49 2018 -0700
Committer: Hao Hao <hao.hao@cloudera.com>
Committed: Fri Jun 15 20:07:45 2018 +0000

----------------------------------------------------------------------
 src/kudu/client/client.h          | 16 +++++++++++++++-
 src/kudu/tools/kudu-tool-test.cc  |  5 +++++
 src/kudu/tools/tool_action_hms.cc | 21 ++++++++++++---------
 3 files changed, 32 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/client/client.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index 3404b9a..f03c0e3 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -37,6 +37,7 @@
 #include "kudu/client/shared_ptr.h" // IWYU pragma: keep
 #ifdef KUDU_HEADERS_NO_STUBS
 #include <gtest/gtest_prod.h>
+
 #include "kudu/gutil/macros.h"
 #include "kudu/gutil/port.h"
 #else
@@ -55,13 +56,19 @@ class MonoDelta;
 class PartitionSchema;
 class SecurityUnknownTskTest;
 
+namespace client {
+class KuduClient;
+}
+
 namespace tools {
 class LeaderMasterProxy;
+
+Status AlterKuduTable(const client::sp::shared_ptr<client::KuduClient>& kudu_client,
+                      const std::string& name, const std::string& new_name);
 } // namespace tools
 
 namespace client {
 
-class KuduClient;
 class KuduDelete;
 class KuduInsert;
 class KuduLoggingCallback;
@@ -1181,8 +1188,14 @@ class KUDU_EXPORT KuduTableAlterer {
 
  private:
   class KUDU_NO_EXPORT Data;
+
   friend class KuduClient;
 
+  friend Status tools::AlterKuduTable(
+      const client::sp::shared_ptr<client::KuduClient>& kudu_client,
+      const std::string& name,
+      const std::string& new_name);
+
   FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable);
 
   KuduTableAlterer(KuduClient* client,
@@ -2391,6 +2404,7 @@ class KUDU_EXPORT KuduPartitioner {
   Status PartitionRow(const KuduPartialRow& row, int* partition);
  private:
   class KUDU_NO_EXPORT Data;
+
   friend class KuduPartitionerBuilder;
 
   explicit KuduPartitioner(Data* data);

http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/tools/kudu-tool-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 12bd979..57bc295 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -2075,6 +2075,11 @@ TEST_F(ToolTest, TestHmsUpgrade) {
     ASSERT_EQ(2, table_names.size());
   }
 
+  // Restart external mini cluster to enable Hive Metastore integration.
+  cluster_->EnableMetastoreIntegration();
+  cluster_->ShutdownNodes(cluster::ClusterNodes::ALL);
+  ASSERT_OK(cluster_->Restart());
+
   // Upgrade the historical metadata in both Hive Metastore and Kudu.
   string out;
   NO_FATALS(RunActionStdinStdoutString(

http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/tools/tool_action_hms.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_hms.cc b/src/kudu/tools/tool_action_hms.cc
index 1f141c0..5c90c44 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -102,6 +102,16 @@ string RenameHiveIncompatibleTable(const string& name) {
   return table_name;
 }
 
+// Only alter the table in Kudu but not in the Hive Metastore.
+Status AlterKuduTable(const client::sp::shared_ptr<KuduClient>& kudu_client,
+                      const string& name,
+                      const string& new_name) {
+  unique_ptr<KuduTableAlterer> alterer(kudu_client->NewTableAlterer(name));
+  return alterer->RenameTo(new_name)
+                ->alter_external_catalogs(false)
+                ->Alter();
+}
+
 // Alter legacy tables (which includes non-Impala tables, Impala managed/external
 // tables) to follow the format 'database_name.table_name' in table naming in Kudu.
 // Also, create HMS entries for non-Impala tables.
@@ -126,13 +136,6 @@ Status AlterLegacyKuduTables(const client::sp::shared_ptr<KuduClient>&
kudu_clie
     }
   }
 
-  auto alter_kudu_table = [&](const string& name,
-                              const string& new_name) -> Status {
-    unique_ptr<KuduTableAlterer> alterer(kudu_client->NewTableAlterer(name));
-    return alterer->RenameTo(new_name)
-                  ->Alter();
-  };
-
   unordered_map<string, Status> failures;
   for (const auto& table_name : table_names) {
     hive::Table* hms_table = FindOrNull(hms_tables_map, table_name);
@@ -150,7 +153,7 @@ Status AlterLegacyKuduTables(const client::sp::shared_ptr<KuduClient>&
kudu_clie
         RETURN_NOT_OK(kudu_client->TableExists(new_table_name, &exist));
         if (!exist) {
           // TODO(Hao): Use notification listener to avoid race conditions.
-          s = alter_kudu_table(table_name, new_table_name).AndThen([&] {
+          s = AlterKuduTable(kudu_client, table_name, new_table_name).AndThen([&] {
             return hms_catalog->UpgradeLegacyImpalaTable(kudu_table->id(),
                 hms_table->dbName, hms_table->tableName,
                 client::SchemaFromKuduSchema(kudu_table->schema()));
@@ -175,7 +178,7 @@ Status AlterLegacyKuduTables(const client::sp::shared_ptr<KuduClient>&
kudu_clie
         s = hms_catalog->CreateTable(kudu_table->id(), new_table_name, schema);
       }
       s = s.AndThen([&] {
-        return alter_kudu_table(table_name, new_table_name);
+        return AlterKuduTable(kudu_client, table_name, new_table_name);
       });
     }
 


Mime
View raw message