From commits-return-6050-archive-asf-public=cust-asf.ponee.io@kudu.apache.org Fri Jun 15 23:40:06 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 4853E180636 for ; Fri, 15 Jun 2018 23:40:05 +0200 (CEST) Received: (qmail 68396 invoked by uid 500); 15 Jun 2018 21:40:04 -0000 Mailing-List: contact commits-help@kudu.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kudu.apache.org Delivered-To: mailing list commits@kudu.apache.org Received: (qmail 68386 invoked by uid 99); 15 Jun 2018 21:40:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Jun 2018 21:40:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3A763E1134; Fri, 15 Jun 2018 21:40:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: granthenke@apache.org To: commits@kudu.apache.org Date: Fri, 15 Jun 2018 21:40:04 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] kudu git commit: KUDU-2191: Add a method to decide the scope of AlterTable Repository: kudu Updated Branches: refs/heads/master 5f1ca32f3 -> f2089195d KUDU-2191: Add a method to decide the scope of AlterTable This commit adds a private method in KuduTableAlterer to indicate whether the table alteration should be applied to external catalogs, such as the Hive Metastore, which the Kudu master may has been configured to integrate with. This method will be useful to the HMS integration tools, such as the metadata upgrade tool which may need to alter tables in Kudu only. Change-Id: I703f12b95bb7ca2d65455f0f0602520332b3c678 Reviewed-on: http://gerrit.cloudera.org:8080/10581 Tested-by: Kudu Jenkins Reviewed-by: Hao Hao Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e3474f4c Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e3474f4c Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e3474f4c Branch: refs/heads/master Commit: e3474f4ce7b1ebc88d12c8ffe1837df8e8211603 Parents: 5f1ca32 Author: hahao Authored: Fri Jun 1 15:54:28 2018 -0700 Committer: Hao Hao Committed: Fri Jun 15 18:18:40 2018 +0000 ---------------------------------------------------------------------- src/kudu/client/client.cc | 10 ++++++++-- src/kudu/client/client.h | 8 ++++++++ src/kudu/client/table_alterer-internal.cc | 1 + src/kudu/client/table_alterer-internal.h | 5 ++++- src/kudu/integration-tests/master_hms-itest.cc | 10 ++++++++++ src/kudu/master/catalog_manager.cc | 11 ++++++----- src/kudu/master/master.proto | 4 ++++ 7 files changed, 41 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/client.cc ---------------------------------------------------------------------- diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc index 87851e0..912e71a 100644 --- a/src/kudu/client/client.cc +++ b/src/kudu/client/client.cc @@ -396,8 +396,8 @@ Status KuduClient::DeleteTable(const string& table_name) { return data_->DeleteTable(this, table_name, deadline); } -KuduTableAlterer* KuduClient::NewTableAlterer(const string& name) { - return new KuduTableAlterer(this, name); +KuduTableAlterer* KuduClient::NewTableAlterer(const string& table_name) { + return new KuduTableAlterer(this, table_name); } Status KuduClient::IsAlterTableInProgress(const string& table_name, @@ -1140,6 +1140,12 @@ KuduTableAlterer* KuduTableAlterer::wait(bool wait) { return this; } +KuduTableAlterer* KuduTableAlterer::alter_external_catalogs( + bool alter_external_catalogs) { + data_->alter_external_catalogs_ = alter_external_catalogs; + return this; +} + Status KuduTableAlterer::Alter() { AlterTableRequestPB req; AlterTableResponsePB resp; http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/client.h ---------------------------------------------------------------------- diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h index 852c7be..3404b9a 100644 --- a/src/kudu/client/client.h +++ b/src/kudu/client/client.h @@ -50,6 +50,7 @@ namespace kudu { class ClientStressTest_TestUniqueClientIds_Test; class KuduPartialRow; +class MasterHmsTest_TestAlterTable_Test; class MonoDelta; class PartitionSchema; class SecurityUnknownTskTest; @@ -1182,9 +1183,16 @@ class KUDU_EXPORT KuduTableAlterer { class KUDU_NO_EXPORT Data; friend class KuduClient; + FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable); + KuduTableAlterer(KuduClient* client, const std::string& name); + // Whether to apply the alteration to external catalogs, such as the Hive + // Metastore, which the Kudu master has been configured to integrate with. + // This method returns a raw pointer to this alterer object. + KuduTableAlterer* alter_external_catalogs(bool alter_external_catalogs); + // Owned. Data* data_; http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/table_alterer-internal.cc ---------------------------------------------------------------------- diff --git a/src/kudu/client/table_alterer-internal.cc b/src/kudu/client/table_alterer-internal.cc index 2baebdf..12adcaf 100644 --- a/src/kudu/client/table_alterer-internal.cc +++ b/src/kudu/client/table_alterer-internal.cc @@ -61,6 +61,7 @@ Status KuduTableAlterer::Data::ToRequest(AlterTableRequestPB* req) { } req->Clear(); + req->set_alter_external_catalogs(alter_external_catalogs_); req->mutable_table()->set_table_name(table_name_); if (rename_to_.is_initialized()) { req->set_new_table_name(rename_to_.get()); http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/client/table_alterer-internal.h ---------------------------------------------------------------------- diff --git a/src/kudu/client/table_alterer-internal.h b/src/kudu/client/table_alterer-internal.h index f847ace..6f2e4c3 100644 --- a/src/kudu/client/table_alterer-internal.h +++ b/src/kudu/client/table_alterer-internal.h @@ -44,7 +44,6 @@ class KuduTableAlterer::Data { ~Data(); Status ToRequest(master::AlterTableRequestPB* req); - KuduClient* const client_; const std::string table_name_; @@ -78,6 +77,10 @@ class KuduTableAlterer::Data { // Schema of add/drop range partition bound rows. const Schema* schema_; + // Whether to apply the alteration to external catalogs, such as the Hive + // Metastore. The default value is true. + bool alter_external_catalogs_ = true; + private: DISALLOW_COPY_AND_ASSIGN(Data); }; http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/integration-tests/master_hms-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/master_hms-itest.cc b/src/kudu/integration-tests/master_hms-itest.cc index 68c6728..160121d 100644 --- a/src/kudu/integration-tests/master_hms-itest.cc +++ b/src/kudu/integration-tests/master_hms-itest.cc @@ -347,6 +347,16 @@ TEST_F(MasterHmsTest, TestAlterTable) { ASSERT_OK(table_alterer->Alter()); }); NO_FATALS(CheckTable("default", "a")); + + // Only alter the table in Kudu, the corresponding table in the HMS will not be altered. + table_alterer.reset(client_->NewTableAlterer("default.a")->RenameTo("default.b") + ->alter_external_catalogs(false)); + ASSERT_OK(table_alterer->Alter()); + bool exists; + ASSERT_OK(client_->TableExists("default.b", &exists)); + ASSERT_TRUE(exists); + hive::Table hms_table; + ASSERT_OK(hms_client_->GetTable("default", "a", &hms_table)); } TEST_F(MasterHmsTest, TestDeleteTable) { http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/master/catalog_manager.cc ---------------------------------------------------------------------- diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc index 9e8d692..49e368b 100644 --- a/src/kudu/master/catalog_manager.cc +++ b/src/kudu/master/catalog_manager.cc @@ -2136,12 +2136,13 @@ Status CatalogManager::AlterTableRpc(const AlterTableRequestPB& req, LOG(INFO) << Substitute("Servicing AlterTable request from $0:\n$1", RequestorString(rpc), SecureShortDebugString(req)); - // If the HMS integration is enabled and the alteration includes a table - // rename, then don't directly rename the table in the Kudu catalog. Instead, - // rename the table in the HMS and wait for the notification log listener to - // apply that event to the catalog. By 'serializing' the rename through the + // If the HMS integration is enabled, the alteration includes a table + // rename and the table should be altered in the HMS, then don't directly + // rename the table in the Kudu catalog. Instead, rename the table + // in the HMS and wait for the notification log listener to apply + // that event to the catalog. By 'serializing' the rename through the // HMS, race conditions are avoided. - if (hms_catalog_ && req.has_new_table_name()) { + if (hms_catalog_ && req.has_new_table_name() && req.alter_external_catalogs()) { // Look up the table, lock it, and mark it as removed. scoped_refptr table; TableMetadataLock l; http://git-wip-us.apache.org/repos/asf/kudu/blob/e3474f4c/src/kudu/master/master.proto ---------------------------------------------------------------------- diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto index 1b59ef0..6fba23b 100644 --- a/src/kudu/master/master.proto +++ b/src/kudu/master/master.proto @@ -564,6 +564,10 @@ message AlterTableRequestPB { // The table schema to use when decoding the range bound row operations. Only // necessary when partitions are being added or dropped. optional SchemaPB schema = 4; + + // Whether to apply the alteration to external catalogs, such as the Hive Metastore, + // which the Kudu master has been configured to integrate with. + optional bool alter_external_catalogs = 5 [default = true]; } message AlterTableResponsePB {