From commits-return-7528-archive-asf-public=cust-asf.ponee.io@kudu.apache.org Mon Jun 10 18:29:45 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 8934818067E for ; Mon, 10 Jun 2019 20:29:45 +0200 (CEST) Received: (qmail 50301 invoked by uid 500); 10 Jun 2019 18:29:45 -0000 Mailing-List: contact commits-help@kudu.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kudu.apache.org Delivered-To: mailing list commits@kudu.apache.org Received: (qmail 50226 invoked by uid 99); 10 Jun 2019 18:29:44 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Jun 2019 18:29:44 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id D35A387A97; Mon, 10 Jun 2019 18:29:44 +0000 (UTC) Date: Mon, 10 Jun 2019 18:29:46 +0000 To: "commits@kudu.apache.org" Subject: [kudu] 02/02: [hms] Adjust storage handler package MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: granthenke@apache.org In-Reply-To: <156019138475.18227.13520384925875726099@gitbox.apache.org> References: <156019138475.18227.13520384925875726099@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: kudu X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Rev: 71b094000cb3569d49f0fdf016ae5d17ff1f119e X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20190610182944.D35A387A97@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. granthenke pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git commit 71b094000cb3569d49f0fdf016ae5d17ff1f119e Author: Grant Henke AuthorDate: Thu Jun 6 11:50:19 2019 -0500 [hms] Adjust storage handler package This patch changes the new KuduStorageHandler package from “org.apache.kudu.hive” to “org.apache.hadoop.hive.kudu”. This is being done to ensure the stand-in storage handler can be a real storage handler when a Hive integration is added in the future. The “org.apache.hadoop.hive” package is the standard package all Hive storage handlers lives under. Note: For the sake of including this into the Impala project without breaking things, we temporarily maintain support for the old storage handler. It will be remvoed in a follow up commit before a release. Change-Id: I66b314c1c8b56785005d6e0d8b679e19219494fe Reviewed-on: http://gerrit.cloudera.org:8080/13540 Reviewed-by: Hao Hao Tested-by: Kudu Jenkins --- .../kudu/hive/metastore/KuduMetastorePlugin.java | 10 +++++++--- src/kudu/hms/hms_catalog.cc | 5 ++--- src/kudu/hms/hms_client.cc | 19 ++++++++++++++++++- src/kudu/hms/hms_client.h | 5 +++++ src/kudu/master/hms_notification_log_listener.cc | 6 +++--- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java index f1a1f19..d40238e 100644 --- a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java +++ b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java @@ -66,8 +66,11 @@ import org.apache.hadoop.hive.metastore.events.ListenerEvent; */ public class KuduMetastorePlugin extends MetaStoreEventListener { + // TODO(ghenke): Remove this after Impala integration of the adjusted KUDU_STORAGE_HANDLER. @VisibleForTesting - static final String KUDU_STORAGE_HANDLER = "org.apache.kudu.hive.KuduStorageHandler"; + static final String TEMP_KUDU_STORAGE_HANDLER = "org.apache.kudu.hive.KuduStorageHandler"; + @VisibleForTesting + static final String KUDU_STORAGE_HANDLER = "org.apache.hadoop.hive.kudu.KuduStorageHandler"; @VisibleForTesting static final String LEGACY_KUDU_STORAGE_HANDLER = "com.cloudera.kudu.hive.KuduStorageHandler"; @VisibleForTesting @@ -237,8 +240,9 @@ public class KuduMetastorePlugin extends MetaStoreEventListener { * @return {@code true} if the table is a Kudu table, otherwise {@code false} */ private boolean isKuduTable(Table table) { - return KUDU_STORAGE_HANDLER.equals(table.getParameters() - .get(hive_metastoreConstants.META_TABLE_STORAGE)); + String storageHandler = table.getParameters().get(hive_metastoreConstants.META_TABLE_STORAGE); + return KUDU_STORAGE_HANDLER.equals(storageHandler) || + TEMP_KUDU_STORAGE_HANDLER.equals(storageHandler); } /** diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc index 6f03f5c..6785adc 100644 --- a/src/kudu/hms/hms_catalog.cc +++ b/src/kudu/hms/hms_catalog.cc @@ -195,8 +195,7 @@ Status HmsCatalog::DowngradeToLegacyImpalaTable(const string& name) { return ha_client_.Execute([&] (HmsClient* client) { hive::Table table; RETURN_NOT_OK(client->GetTable(hms_database.ToString(), hms_table.ToString(), &table)); - if (table.parameters[HmsClient::kStorageHandlerKey] != - HmsClient::kKuduStorageHandler) { + if (!hms::HmsClient::IsKuduTable(table)) { return Status::IllegalState("non-Kudu table cannot be downgraded"); } // Downgrade the storage handler. @@ -267,7 +266,7 @@ Status HmsCatalog::AlterTable(const string& id, RETURN_NOT_OK(client->GetTable(hms_database.ToString(), hms_table.ToString(), &table)); // Check that the HMS entry belongs to the table being altered. - if (table.parameters[HmsClient::kStorageHandlerKey] != HmsClient::kKuduStorageHandler || + if (!hms::HmsClient::IsKuduTable(table) || (check_id && table.parameters[HmsClient::kKuduTableIdKey] != id)) { // The original table isn't a Kudu table, or isn't the same Kudu table. return Status::NotFound("the HMS entry for the table being " diff --git a/src/kudu/hms/hms_client.cc b/src/kudu/hms/hms_client.cc index 13e2237..b4a88d7 100644 --- a/src/kudu/hms/hms_client.cc +++ b/src/kudu/hms/hms_client.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include #include +#include "kudu/gutil/map-util.h" #include "kudu/gutil/strings/split.h" #include "kudu/gutil/strings/strip.h" #include "kudu/gutil/strings/substitute.h" @@ -100,7 +102,9 @@ const char* const HmsClient::kKuduTableNameKey = "kudu.table_name"; const char* const HmsClient::kKuduMasterAddrsKey = "kudu.master_addresses"; const char* const HmsClient::kKuduMasterEventKey = "kudu.master_event"; const char* const HmsClient::kKuduCheckIdKey = "kudu.check_id"; -const char* const HmsClient::kKuduStorageHandler = "org.apache.kudu.hive.KuduStorageHandler"; +const char* const HmsClient::kKuduStorageHandler = + "org.apache.hadoop.hive.kudu.KuduStorageHandler"; +const char* const HmsClient::kOldKuduStorageHandler = "org.apache.kudu.hive.KuduStorageHandler"; const char* const HmsClient::kTransactionalEventListeners = "hive.metastore.transactional.event.listeners"; @@ -376,5 +380,18 @@ Status HmsClient::DeserializeJsonTable(Slice json, hive::Table* table) { return Status::OK(); } +bool HmsClient::IsKuduTable(const hive::Table& table) { + const string* storage_handler = + FindOrNull(table.parameters, hms::HmsClient::kStorageHandlerKey); + if (!storage_handler) { + return false; + } + + // TODO(ghenke): Remove special kOldKuduStorageHandler handling after Impala integration + // of the adjusted kKuduStorageHandler. + return *storage_handler == hms::HmsClient::kKuduStorageHandler || + *storage_handler == hms::HmsClient::kOldKuduStorageHandler; +} + } // namespace hms } // namespace kudu diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h index c79f401..9d10a6c 100644 --- a/src/kudu/hms/hms_client.h +++ b/src/kudu/hms/hms_client.h @@ -66,6 +66,8 @@ class HmsClient { static const char* const kKuduCheckIdKey; static const char* const kStorageHandlerKey; static const char* const kKuduStorageHandler; + // TODO(ghenke): Remove this after Impala integration of the adjusted kKuduStorageHandler. + static const char* const kOldKuduStorageHandler; static const char* const kHiveFilterFieldParams; static const char* const kTransactionalEventListeners; @@ -180,6 +182,9 @@ class HmsClient { const std::string& table_name, std::vector* partitions) WARN_UNUSED_RESULT; + // Returns true if the HMS table is a Kudu table. + static bool IsKuduTable(const hive::Table& table) WARN_UNUSED_RESULT; + // Deserializes a JSON encoded table. // // Notification event log messages often include table objects serialized as diff --git a/src/kudu/master/hms_notification_log_listener.cc b/src/kudu/master/hms_notification_log_listener.cc index 63a5eb5..5b19815 100644 --- a/src/kudu/master/hms_notification_log_listener.cc +++ b/src/kudu/master/hms_notification_log_listener.cc @@ -342,7 +342,8 @@ Status HmsNotificationLogListenerTask::HandleAlterTableEvent(const hive::Notific const string* storage_handler = FindOrNull(before_table.parameters, hms::HmsClient::kStorageHandlerKey); - if (!storage_handler || *storage_handler != hms::HmsClient::kKuduStorageHandler) { + + if (!hms::HmsClient::IsKuduTable(before_table)) { // Not a Kudu table; skip it. VLOG(2) << Substitute("Ignoring alter event for non-Kudu table $0", before_table.tableName); @@ -400,8 +401,7 @@ Status HmsNotificationLogListenerTask::HandleDropTableEvent(const hive::Notifica return Status::OK(); } - const string* storage_handler = FindOrNull(table.parameters, hms::HmsClient::kStorageHandlerKey); - if (!storage_handler || *storage_handler != hms::HmsClient::kKuduStorageHandler) { + if (!hms::HmsClient::IsKuduTable(table)) { // Not a Kudu table; skip it. VLOG(2) << Substitute("Ignoring drop event for non-Kudu table $0", table.tableName); return Status::OK();