kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [1/2] kudu git commit: KUDU-2191: Add a method to decide the scope of AlterTable
Date Fri, 15 Jun 2018 21:40:04 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 5f1ca32f3 -> f2089195d


KUDU-2191: Add a method to decide the scope of AlterTable

This commit adds a private method in KuduTableAlterer to indicate
whether the table alteration should be applied to external catalogs,
such as the Hive Metastore, which the Kudu master may has been
configured to integrate with.

This method will be useful to the HMS integration tools, such as the
metadata upgrade tool which may need to alter tables in Kudu only.

Change-Id: I703f12b95bb7ca2d65455f0f0602520332b3c678
Reviewed-on: http://gerrit.cloudera.org:8080/10581
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao <hao.hao@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e3474f4c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e3474f4c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e3474f4c

Branch: refs/heads/master
Commit: e3474f4ce7b1ebc88d12c8ffe1837df8e8211603
Parents: 5f1ca32
Author: hahao <hao.hao@cloudera.com>
Authored: Fri Jun 1 15:54:28 2018 -0700
Committer: Hao Hao <hao.hao@cloudera.com>
Committed: Fri Jun 15 18:18:40 2018 +0000

----------------------------------------------------------------------
 src/kudu/client/client.cc                      | 10 ++++++++--
 src/kudu/client/client.h                       |  8 ++++++++
 src/kudu/client/table_alterer-internal.cc      |  1 +
 src/kudu/client/table_alterer-internal.h       |  5 ++++-
 src/kudu/integration-tests/master_hms-itest.cc | 10 ++++++++++
 src/kudu/master/catalog_manager.cc             | 11 ++++++-----
 src/kudu/master/master.proto                   |  4 ++++
 7 files changed, 41 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/client.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc
index 87851e0..912e71a 100644
--- a/src/kudu/client/client.cc
+++ b/src/kudu/client/client.cc
@@ -396,8 +396,8 @@ Status KuduClient::DeleteTable(const string& table_name) {
   return data_->DeleteTable(this, table_name, deadline);
 }
 
-KuduTableAlterer* KuduClient::NewTableAlterer(const string& name) {
-  return new KuduTableAlterer(this, name);
+KuduTableAlterer* KuduClient::NewTableAlterer(const string& table_name) {
+  return new KuduTableAlterer(this, table_name);
 }
 
 Status KuduClient::IsAlterTableInProgress(const string& table_name,
@@ -1140,6 +1140,12 @@ KuduTableAlterer* KuduTableAlterer::wait(bool wait) {
   return this;
 }
 
+KuduTableAlterer* KuduTableAlterer::alter_external_catalogs(
+    bool alter_external_catalogs) {
+  data_->alter_external_catalogs_ = alter_external_catalogs;
+  return this;
+}
+
 Status KuduTableAlterer::Alter() {
   AlterTableRequestPB req;
   AlterTableResponsePB resp;

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/client.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index 852c7be..3404b9a 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -50,6 +50,7 @@ namespace kudu {
 
 class ClientStressTest_TestUniqueClientIds_Test;
 class KuduPartialRow;
+class MasterHmsTest_TestAlterTable_Test;
 class MonoDelta;
 class PartitionSchema;
 class SecurityUnknownTskTest;
@@ -1182,9 +1183,16 @@ class KUDU_EXPORT KuduTableAlterer {
   class KUDU_NO_EXPORT Data;
   friend class KuduClient;
 
+  FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable);
+
   KuduTableAlterer(KuduClient* client,
                    const std::string& name);
 
+  // Whether to apply the alteration to external catalogs, such as the Hive
+  // Metastore, which the Kudu master has been configured to integrate with.
+  // This method returns a raw pointer to this alterer object.
+  KuduTableAlterer* alter_external_catalogs(bool alter_external_catalogs);
+
   // Owned.
   Data* data_;
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/table_alterer-internal.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/table_alterer-internal.cc b/src/kudu/client/table_alterer-internal.cc
index 2baebdf..12adcaf 100644
--- a/src/kudu/client/table_alterer-internal.cc
+++ b/src/kudu/client/table_alterer-internal.cc
@@ -61,6 +61,7 @@ Status KuduTableAlterer::Data::ToRequest(AlterTableRequestPB* req) {
   }
 
   req->Clear();
+  req->set_alter_external_catalogs(alter_external_catalogs_);
   req->mutable_table()->set_table_name(table_name_);
   if (rename_to_.is_initialized()) {
     req->set_new_table_name(rename_to_.get());

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/table_alterer-internal.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/table_alterer-internal.h b/src/kudu/client/table_alterer-internal.h
index f847ace..6f2e4c3 100644
--- a/src/kudu/client/table_alterer-internal.h
+++ b/src/kudu/client/table_alterer-internal.h
@@ -44,7 +44,6 @@ class KuduTableAlterer::Data {
   ~Data();
   Status ToRequest(master::AlterTableRequestPB* req);
 
-
   KuduClient* const client_;
   const std::string table_name_;
 
@@ -78,6 +77,10 @@ class KuduTableAlterer::Data {
   // Schema of add/drop range partition bound rows.
   const Schema* schema_;
 
+  // Whether to apply the alteration to external catalogs, such as the Hive
+  // Metastore. The default value is true.
+  bool alter_external_catalogs_ = true;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(Data);
 };

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/integration-tests/master_hms-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/master_hms-itest.cc b/src/kudu/integration-tests/master_hms-itest.cc
index 68c6728..160121d 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -347,6 +347,16 @@ TEST_F(MasterHmsTest, TestAlterTable) {
     ASSERT_OK(table_alterer->Alter());
   });
   NO_FATALS(CheckTable("default", "a"));
+
+  // Only alter the table in Kudu, the corresponding table in the HMS will not be altered.
+  table_alterer.reset(client_->NewTableAlterer("default.a")->RenameTo("default.b")
+                             ->alter_external_catalogs(false));
+  ASSERT_OK(table_alterer->Alter());
+  bool exists;
+  ASSERT_OK(client_->TableExists("default.b", &exists));
+  ASSERT_TRUE(exists);
+  hive::Table hms_table;
+  ASSERT_OK(hms_client_->GetTable("default", "a", &hms_table));
 }
 
 TEST_F(MasterHmsTest, TestDeleteTable) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/master/catalog_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 9e8d692..49e368b 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -2136,12 +2136,13 @@ Status CatalogManager::AlterTableRpc(const AlterTableRequestPB&
req,
   LOG(INFO) << Substitute("Servicing AlterTable request from $0:\n$1",
                           RequestorString(rpc), SecureShortDebugString(req));
 
-  // If the HMS integration is enabled and the alteration includes a table
-  // rename, then don't directly rename the table in the Kudu catalog. Instead,
-  // rename the table in the HMS and wait for the notification log listener to
-  // apply that event to the catalog. By 'serializing' the rename through the
+  // If the HMS integration is enabled, the alteration includes a table
+  // rename and the table should be altered in the HMS, then don't directly
+  // rename the table in the Kudu catalog. Instead, rename the table
+  // in the HMS and wait for the notification log listener to apply
+  // that event to the catalog. By 'serializing' the rename through the
   // HMS, race conditions are avoided.
-  if (hms_catalog_ && req.has_new_table_name()) {
+  if (hms_catalog_ && req.has_new_table_name() && req.alter_external_catalogs())
{
     // Look up the table, lock it, and mark it as removed.
     scoped_refptr<TableInfo> table;
     TableMetadataLock l;

http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/master/master.proto
----------------------------------------------------------------------
diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto
index 1b59ef0..6fba23b 100644
--- a/src/kudu/master/master.proto
+++ b/src/kudu/master/master.proto
@@ -564,6 +564,10 @@ message AlterTableRequestPB {
   // The table schema to use when decoding the range bound row operations. Only
   // necessary when partitions are being added or dropped.
   optional SchemaPB schema = 4;
+
+  // Whether to apply the alteration to external catalogs, such as the Hive Metastore,
+  // which the Kudu master has been configured to integrate with.
+  optional bool alter_external_catalogs = 5 [default = true];
 }
 
 message AlterTableResponsePB {


Mime
View raw message