kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [kudu] branch master updated: KUDU-2841. Expose HMS DB UUID to clients in HiveMetastoreConfig
Date Tue, 11 Jun 2019 22:17:42 GMT
This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new 21067ff  KUDU-2841. Expose HMS DB UUID to clients in HiveMetastoreConfig
21067ff is described below

commit 21067ffcd1be94dc73f289631fd59ce8d76cfe33
Author: Todd Lipcon <todd@apache.org>
AuthorDate: Thu Jun 6 23:48:03 2019 -0700

    KUDU-2841. Expose HMS DB UUID to clients in HiveMetastoreConfig
    
    This fills in the hms_uuid field in HiveMetastoreConfig by fetching the
    UUID from the HMS when the HMSCatalog starts up. The new API call is not
    available in Hive 2.3 (in our thirdparty build) but has been backported
    in some vendor distros and is available upstream in Hive 3.x. In order
    to support it where available, this patch manually adds the call to the
    thrift definition file, and catches the UNKNOWN_METHOD error code that
    comes back when running against Hive 2.3.
    
    I manually tested this against a Hive 3 master and verified that the
    VLOG message at startup properly indicated the UUID of the HMS.
    
    Change-Id: I393df4dd281c3fb63a836db346d7b33fb6edc6a2
    Reviewed-on: http://gerrit.cloudera.org:8080/13556
    Reviewed-by: Hao Hao <hao.hao@cloudera.com>
    Reviewed-by: Andrew Wong <awong@cloudera.com>
    Tested-by: Andrew Wong <awong@cloudera.com>
---
 .../apache/kudu/client/HiveMetastoreConfig.java    |  3 +++
 src/kudu/hms/hive_metastore.thrift                 |  4 ++++
 src/kudu/hms/hms_catalog-test.cc                   | 10 +++++++++
 src/kudu/hms/hms_catalog.cc                        | 26 +++++++++++++++++++++-
 src/kudu/hms/hms_catalog.h                         | 10 +++++++++
 src/kudu/hms/hms_client.cc                         | 17 ++++++++++++++
 src/kudu/hms/hms_client.h                          |  3 +++
 src/kudu/master/master_service.cc                  |  5 ++++-
 src/kudu/thrift/client.cc                          |  1 +
 src/kudu/thrift/client.h                           |  2 ++
 10 files changed, 79 insertions(+), 2 deletions(-)

diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/HiveMetastoreConfig.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/HiveMetastoreConfig.java
index 4da7bd9..77ba8b8 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/HiveMetastoreConfig.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/HiveMetastoreConfig.java
@@ -20,6 +20,8 @@ package org.apache.kudu.client;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.yetus.audience.InterfaceStability;
 
+import javax.annotation.CheckForNull;
+
 /**
  * The Hive Metastore configuration of a Kudu cluster.
  */
@@ -46,6 +48,7 @@ public class HiveMetastoreConfig {
     return hiveMetastoreSaslEnabled;
   }
 
+  @CheckForNull
   public String getHiveMetastoreUuid() {
     return hiveMetastoreUuid;
   }
diff --git a/src/kudu/hms/hive_metastore.thrift b/src/kudu/hms/hive_metastore.thrift
index 969f4f8..5ea4dec 100644
--- a/src/kudu/hms/hive_metastore.thrift
+++ b/src/kudu/hms/hive_metastore.thrift
@@ -22,6 +22,7 @@
 # https://raw.githubusercontent.com/apache/hive/rel/release-2.3.0/metastore/if/hive_metastore.thrift
 # With backports:
 #   - HIVE-16993
+#   - Backport get_metastore_db_uuid() API from HIVE-16555
 #
 # With edits:
 #   - Change cpp namespace to 'hive' to match the Kudu codebase style.
@@ -1498,6 +1499,9 @@ service ThriftHiveMetastore extends fb303.FacebookService
   ClearFileMetadataResult clear_file_metadata(1:ClearFileMetadataRequest req)
   CacheFileMetadataResult cache_file_metadata(1:CacheFileMetadataRequest req)
 
+  // Metastore DB properties
+  string get_metastore_db_uuid() throws (1:MetaException o1)
+
 }
 
 // * Note about the DDL_TIME: When creating or altering a table or a partition,
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index 3630ac8..a8cc0a9 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -452,5 +452,15 @@ TEST_F(HmsCatalogTest, TestReconnect) {
   NO_FATALS(CheckTableDoesNotExist(kHmsDatabase, "a"));
 }
 
+TEST_F(HmsCatalogTest, TestMetastoreUuid) {
+  string uuid;
+  Status s = hms_catalog_->GetUuid(&uuid);
+  if (s.ok()) {
+    ASSERT_EQ(36, uuid.size());
+  } else {
+    ASSERT_TRUE(s.IsNotSupported()) << s.ToString();
+  }
+}
+
 } // namespace hms
 } // namespace kudu
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index 6785adc..0fb46a7 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -124,7 +124,23 @@ Status HmsCatalog::Start(HmsClientVerifyKuduSyncConfig verify_service_config)
{
   options.retry_count = FLAGS_hive_metastore_retry_count;
   options.verify_service_config = verify_service_config == VERIFY;
 
-  return ha_client_.Start(std::move(addresses), std::move(options));
+  RETURN_NOT_OK(ha_client_.Start(std::move(addresses), std::move(options)));
+
+  // Fetch the UUID of the HMS DB, if available.
+  string uuid;
+  Status uuid_status = ha_client_.Execute([&] (HmsClient* client) {
+    return client->GetUuid(&uuid);
+  });
+  if (uuid_status.ok()) {
+    VLOG(1) << "Connected to HMS with uuid " << uuid;
+    uuid_ = std::move(uuid);
+  } else if (uuid_status.IsNotSupported()) {
+    VLOG(1) << "Unable to fetch UUID for HMS: " << uuid_status.ToString();
+  } else {
+    // If we couldn't connect to the HMS at all, fail.
+    return uuid_status;
+  }
+  return Status::OK();
 }
 
 void HmsCatalog::Stop() {
@@ -288,6 +304,14 @@ Status HmsCatalog::GetNotificationEvents(int64_t last_event_id, int max_events,
   });
 }
 
+Status HmsCatalog::GetUuid(string* uuid) {
+  if (!uuid_) {
+    return Status::NotSupported("No HMS UUID available");
+  }
+  *uuid = uuid_.value();
+  return Status::OK();
+}
+
 namespace {
 
 string column_to_field_type(const ColumnSchema& column) {
diff --git a/src/kudu/hms/hms_catalog.h b/src/kudu/hms/hms_catalog.h
index e878f4c..981da96 100644
--- a/src/kudu/hms/hms_catalog.h
+++ b/src/kudu/hms/hms_catalog.h
@@ -132,6 +132,14 @@ class HmsCatalog {
   Status GetNotificationEvents(int64_t last_event_id, int max_events,
                                std::vector<hive::NotificationEvent>* events) WARN_UNUSED_RESULT;
 
+  // Get the UUID associated with the remote HMS instance. This is an identifier
+  // stored in the HMS's backing database which does not change even if the
+  // HMS itself changes hostnames, etc.
+  //
+  // NOTE: this is not implemented in Hive 2.x but may be backported into some
+  // vendor releases (eg CDH6). This function may return Status::NotSupported().
+  Status GetUuid(std::string* uuid) WARN_UNUSED_RESULT;
+
   // Validates the hive_metastore_uris gflag.
   static bool ValidateUris(const char* flag_name, const std::string& metastore_uris);
 
@@ -188,6 +196,8 @@ class HmsCatalog {
   const std::string master_addresses_;
 
   thrift::HaClient<hms::HmsClient> ha_client_;
+
+  boost::optional<std::string> uuid_;
 };
 
 } // namespace hms
diff --git a/src/kudu/hms/hms_client.cc b/src/kudu/hms/hms_client.cc
index b4a88d7..0fa8ba1 100644
--- a/src/kudu/hms/hms_client.cc
+++ b/src/kudu/hms/hms_client.cc
@@ -26,6 +26,7 @@
 
 #include <boost/algorithm/string/predicate.hpp>
 #include <glog/logging.h>
+#include <thrift/TApplicationException.h>
 #include <thrift/Thrift.h>
 #include <thrift/protocol/TJSONProtocol.h>
 #include <thrift/protocol/TProtocol.h>
@@ -44,6 +45,7 @@
 #include "kudu/util/status.h"
 #include "kudu/util/stopwatch.h"
 
+using apache::thrift::TApplicationException;
 using apache::thrift::TException;
 using apache::thrift::protocol::TJSONProtocol;
 using apache::thrift::transport::TMemoryBuffer;
@@ -81,6 +83,12 @@ namespace hms {
     return Status::RemoteError((msg), e.what()); \
   } catch (const SaslException& e) { \
     return e.status().CloneAndPrepend((msg)); \
+  } catch (const TApplicationException& e) { \
+    switch (e.getType()) { \
+      case TApplicationException::UNKNOWN_METHOD: \
+        return Status::NotSupported((msg), e.what()); \
+      default: return Status::RemoteError((msg), e.what()); \
+    } \
   } catch (const TTransportException& e) { \
     switch (e.getType()) { \
       case TTransportException::TIMED_OUT: return Status::TimedOut((msg), e.what()); \
@@ -346,6 +354,15 @@ Status HmsClient::GetNotificationEvents(int64_t last_event_id,
   return Status::OK();
 }
 
+Status HmsClient::GetUuid(string* uuid) {
+  DCHECK(uuid);
+  SCOPED_LOG_SLOW_EXECUTION(WARNING, kSlowExecutionWarningThresholdMs,
+                            "get HMS database UUID");
+  HMS_RET_NOT_OK(client_.get_metastore_db_uuid(*uuid),
+                 "failed to get HMS DB UUID");
+  return Status::OK();
+}
+
 Status HmsClient::AddPartitions(const string& database_name,
                                 const string& table_name,
                                 vector<hive::Partition> partitions) {
diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h
index 9d10a6c..aa4ef92 100644
--- a/src/kudu/hms/hms_client.h
+++ b/src/kudu/hms/hms_client.h
@@ -172,6 +172,9 @@ class HmsClient {
                                int32_t max_events,
                                std::vector<hive::NotificationEvent>* events) WARN_UNUSED_RESULT;
 
+  // Get the HMS's UUID (see HmsCatalog::GetUuid).
+  Status GetUuid(std::string* uuid) WARN_UNUSED_RESULT;
+
   // Adds partitions to an HMS table.
   Status AddPartitions(const std::string& database_name,
                        const std::string& table_name,
diff --git a/src/kudu/master/master_service.cc b/src/kudu/master/master_service.cc
index eaf5364..683809c 100644
--- a/src/kudu/master/master_service.cc
+++ b/src/kudu/master/master_service.cc
@@ -557,7 +557,10 @@ void MasterServiceImpl::ConnectToMaster(const ConnectToMasterRequestPB*
/*req*/,
     auto* metastore_config = resp->mutable_hms_config();
     metastore_config->set_hms_uris(FLAGS_hive_metastore_uris);
     metastore_config->set_hms_sasl_enabled(FLAGS_hive_metastore_sasl_enabled);
-    // TODO(dan): set the hms_uuid field.
+    string uuid;
+    if (server_->catalog_manager()->HmsCatalog()->GetUuid(&uuid).ok()) {
+      metastore_config->set_hms_uuid(std::move(uuid));
+    }
   }
 
   // Assign a location to the client if needed.
diff --git a/src/kudu/thrift/client.cc b/src/kudu/thrift/client.cc
index bce384e..59204ae 100644
--- a/src/kudu/thrift/client.cc
+++ b/src/kudu/thrift/client.cc
@@ -93,6 +93,7 @@ bool IsFatalError(const Status& error) {
         || error.IsNotFound()
         || error.IsInvalidArgument()
         || error.IsIllegalState()
+        || error.IsNotSupported()
         || error.IsRemoteError());
 }
 } // namespace thrift
diff --git a/src/kudu/thrift/client.h b/src/kudu/thrift/client.h
index 49ac621..61c7974 100644
--- a/src/kudu/thrift/client.h
+++ b/src/kudu/thrift/client.h
@@ -30,9 +30,11 @@
 
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/port.h"
+#include "kudu/gutil/ref_counted.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/thrift/ha_client_metrics.h"
 #include "kudu/util/async_util.h"
+#include "kudu/util/metrics.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/net/net_util.h"
 #include "kudu/util/status.h"


Mime
View raw message