kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From danburk...@apache.org
Subject kudu git commit: Bump Sentry and Hadoop versions
Date Mon, 08 Oct 2018 17:59:37 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 2d6f8ffa9 -> 8d6cfe10d


Bump Sentry and Hadoop versions

The motivation is to get access to SENTRY-2371, which introduces a new
Thrift interface that will be used by Kudu to retrieve user privileges.
SENTRY-2371 has not been released yet, so the new Sentry version is
based on the latest commit of the master branch. The new Sentry version
appears to have an incompatibility with Hadoop 2.8.2, so this also
includes a bump to the latest Hadoop 2.8.x version. I suspect the fix
between 2.8.2 and 2.8.5 was introduced in [1], but the commit message is
vague and I haven't dug any further. The incompatiblity manifests as an
exception during Sentry startup:

java.lang.NoSuchMethodError: org.apache.hadoop.conf.Configuration.addResource(Ljava/net/URL;Z)V
	at org.apache.sentry.service.thrift.SentryService.loadConfig(SentryService.java:576)
	at org.apache.sentry.service.thrift.SentryService$CommandImpl.run(SentryService.java:600)
	at org.apache.sentry.SentryMain.main(SentryMain.java:120)

[1]: https://github.com/apache/hadoop/commit/7af9b8ad1e993ef791aa38740b6aabc4c233a30f

Change-Id: I8bcc4ff6fac0435b037b984f45da75bed6ff4be5
Reviewed-on: http://gerrit.cloudera.org:8080/11601
Reviewed-by: Hao Hao <hao.hao@cloudera.com>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8d6cfe10
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8d6cfe10
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8d6cfe10

Branch: refs/heads/master
Commit: 8d6cfe10d47bdb422c6cc0e770afbd70d6ec7adf
Parents: 2d6f8ff
Author: Dan Burkert <danburkert@apache.org>
Authored: Fri Oct 5 14:10:32 2018 -0700
Committer: Dan Burkert <danburkert@apache.org>
Committed: Mon Oct 8 17:59:17 2018 +0000

----------------------------------------------------------------------
 src/kudu/sentry/sentry_common_service.thrift |   3 +-
 src/kudu/sentry/sentry_policy_service.thrift | 100 ++++++++++++++++++++--
 thirdparty/vars.sh                           |   6 +-
 3 files changed, 99 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/8d6cfe10/src/kudu/sentry/sentry_common_service.thrift
----------------------------------------------------------------------
diff --git a/src/kudu/sentry/sentry_common_service.thrift b/src/kudu/sentry/sentry_common_service.thrift
index e37f4f6..b8a730e 100644
--- a/src/kudu/sentry/sentry_common_service.thrift
+++ b/src/kudu/sentry/sentry_common_service.thrift
@@ -19,7 +19,7 @@
  */
 
 # DO NOT MODIFY! Copied from
-# https://raw.githubusercontent.com/apache/sentry/release-2.0.1/sentry-provider/sentry-provider-db/src/main/resources/sentry_common_service.thrift
+# https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_common_service.thrift
 #
 # With edits:
 #   - Change cpp namespace to 'sentry' to match the Kudu codebase style.
@@ -47,4 +47,3 @@ struct TSentryResponseStatus {
 2: required string message
 3: optional string stack
 }
-

http://git-wip-us.apache.org/repos/asf/kudu/blob/8d6cfe10/src/kudu/sentry/sentry_policy_service.thrift
----------------------------------------------------------------------
diff --git a/src/kudu/sentry/sentry_policy_service.thrift b/src/kudu/sentry/sentry_policy_service.thrift
index a11872b..d32e39b 100644
--- a/src/kudu/sentry/sentry_policy_service.thrift
+++ b/src/kudu/sentry/sentry_policy_service.thrift
@@ -19,7 +19,7 @@
  */
 
 # DO NOT MODIFY! Copied from
-# https://raw.githubusercontent.com/apache/sentry/release-2.0.1/sentry-provider/sentry-provider-db/src/main/resources/sentry_policy_service.thrift
+# https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_policy_service.thrift
 #
 # With edits:
 #   - Change cpp namespace to 'sentry' to match the Kudu codebase style.
@@ -30,8 +30,8 @@
 
 include "sentry_common_service.thrift"
 
-namespace java org.apache.sentry.provider.db.service.thrift
-namespace php sentry.provider.db.service.thrift
+namespace java org.apache.sentry.api.service.thrift
+namespace php sentry.api.service.thrift
 namespace cpp sentry
 
 enum TSentryGrantOption {
@@ -44,6 +44,12 @@ enum TSentryGrantOption {
   UNSET = -1
 }
 
+enum TSentryPrincipalType {
+  NONE = 0,
+  ROLE = 1,
+  USER = 2
+}
+
 # Represents a Privilege in transport from the client to the server
 struct TSentryPrivilege {
 1: required string privilegeScope, # Valid values are SERVER, DATABASE, TABLE, COLUMN, URI
@@ -190,9 +196,18 @@ struct TSentryAuthorizable {
 struct TListSentryPrivilegesRequest {
 1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
 2: required string requestorUserName, # user on whose behalf the request is issued
+
+# @Deprecated Use principalName instead to set role names or user names. This parameter will
be
+# removed in the next major version of Sentry 3.0
 4: required string roleName, # get privileges assigned for this role
-5: optional TSentryAuthorizable authorizableHierarchy # get privileges assigned for this
role
+5: optional TSentryAuthorizable authorizableHierarchy, # get privileges assigned for this
role
+
+# Get privileges assigned for this principal name. This principalName should be set to a
role name
+# or user name depending of which function you call, either list_sentry_privileges_by_role
or
+# list_sentry_privileges_by_user
+6: optional string principalName
 }
+
 struct TListSentryPrivilegesResponse {
 1: required sentry_common_service.TSentryResponseStatus status
 2: optional set<TSentryPrivilege> privileges
@@ -250,11 +265,19 @@ struct TListSentryPrivilegesByAuthRequest {
 2: required string requestorUserName, # user on whose behalf the request is issued
 3: required set<TSentryAuthorizable> authorizableSet,
 4: optional set<string> groups,
-5: optional TSentryActiveRoleSet roleSet
+5: optional TSentryActiveRoleSet roleSet,
+6: optional set<string> users
 }
 struct TListSentryPrivilegesByAuthResponse {
 1: required sentry_common_service.TSentryResponseStatus status,
-2: optional map<TSentryAuthorizable, TSentryPrivilegeMap> privilegesMapByAuth # will
not be set in case of an error
+
+# privilegesMapByAuth (legacy & compatible parameter) contains role privileges
+# (will not be set in case of an error)
+2: optional map<TSentryAuthorizable, TSentryPrivilegeMap> privilegesMapByAuth,
+
+# privilegesMapByAuthForUsers contains user privileges
+# (will not be set in case of an error)
+3: optional map<TSentryAuthorizable, TSentryPrivilegeMap> privilegesMapByAuthForUsers
 }
 
 # Obtain a config value from the Sentry service
@@ -329,6 +352,50 @@ struct TSentrySyncIDResponse {
 2: required i64 id // Most recent processed ID
 }
 
+/*
+ * This request is an extension to TSentrySyncIDRequest. Additionally this request
+ * is used to update the HMS events and the owner changes associated with events.
+ * To be backward compatible, TSentrySyncIDRequest is not updated. Instead new request
+ * is created extending it.
+*/
+
+struct TSentryHmsEventNotification {
+1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
+2: required string requestorUserName, # user on whose behalf the request is issued
+3: required i64 id, # Requested ID
+#  Constructed from enum org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType
+4: required string eventType, # Type of the event which resulted in owner update request
+5: required TSentryAuthorizable authorizable, # Authorizable object
+6: optional TSentryPrincipalType ownerType, # Type of the owner
+7: optional string ownerName # owner name
+
+}
+
+struct TSentryHmsEventNotificationResponse {
+1: required sentry_common_service.TSentryResponseStatus status
+2: required i64 id // Most recent processed ID
+}
+
+/**
+* API that requests all roles and users privileges from the Sentry server.
+**/
+struct TSentryPrivilegesRequest {
+1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
+2: required string requestorUserName # user on whose behalf the request is issued
+}
+
+/**
+* API that returns either all users or roles privileges found on the Sentry server.
+*
+* The response returns a mapping object that maps the role or user name to the privileges
+* they have in the server. An empty set of privileges may be returned to each role or user
+* name. Null values are not returned.
+**/
+struct TSentryPrivilegesResponse {
+1: required sentry_common_service.TSentryResponseStatus status
+2: required map<string, set<TSentryPrivilege>> privilegesMap;
+}
+
 service SentryPolicyService
 {
   TCreateSentryRoleResponse create_sentry_role(1:TCreateSentryRoleRequest request)
@@ -346,7 +413,16 @@ service SentryPolicyService
   TListSentryRolesResponse list_sentry_roles_by_group(1:TListSentryRolesRequest request)
   TListSentryRolesResponse list_sentry_roles_by_user(1:TListSentryRolesForUserRequest request)
 
+  # List sentry privileges granted to the given role, filterted
+  # based on authorization hierarchy if present.
   TListSentryPrivilegesResponse list_sentry_privileges_by_role(1:TListSentryPrivilegesRequest
request)
+  # List sentry privileges granted to the given user, filterted
+  # based on authorization hierarchy if present.
+  TListSentryPrivilegesResponse list_sentry_privileges_by_user(1:TListSentryPrivilegesRequest
request)
+  # List sentry privileges granted to the given user and the groups
+  # the user associated with, filterted based on authorization
+  # hierarchy if present.
+  TListSentryPrivilegesResponse list_sentry_privileges_by_user_and_itsgroups(1:TListSentryPrivilegesRequest
request)
 
   # For use with ProviderBackend.getPrivileges only
   TListSentryPrivilegesForProviderResponse list_sentry_privileges_for_provider(1:TListSentryPrivilegesForProviderRequest
request)
@@ -367,4 +443,16 @@ service SentryPolicyService
 
   # Synchronize between HMS notifications and Sentry
   TSentrySyncIDResponse sentry_sync_notifications(1:TSentrySyncIDRequest request);
+
+  # Notify Sentry about new events in HMS. Currently used to synchronize between HMS/Sentry
+  # and also update sentry with the owner information.
+  TSentryHmsEventNotificationResponse sentry_notify_hms_event(1:TSentryHmsEventNotification
request);
+
+  # Returns a map of all roles and their privileges that exist in the Sentry server.
+  # The mapping object returned will be in the form of [roleName, set<privileges>]
+  TSentryPrivilegesResponse list_roles_privileges(1:TSentryPrivilegesRequest request);
+
+  # Returns a map of all users and their privileges that exist in the Sentry server.
+  # The mapping object returned will be in the form of [userName, set<privileges>]
+  TSentryPrivilegesResponse list_users_privileges(1:TSentryPrivilegesRequest request);
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/8d6cfe10/thirdparty/vars.sh
----------------------------------------------------------------------
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index fe25293..fe4cd10 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -222,10 +222,12 @@ HIVE_SOURCE=$TP_SOURCE_DIR/$HIVE_NAME
 
 # Note: The Hadoop release tarball is stripped of unnecessary jars before being
 # uploaded. See thirdparty/package-hadoop.sh for details.
-HADOOP_VERSION=2.8.2
+HADOOP_VERSION=2.8.5
 HADOOP_NAME=hadoop-$HADOOP_VERSION
 HADOOP_SOURCE=$TP_SOURCE_DIR/$HADOOP_NAME
 
-SENTRY_VERSION=2.0.1
+# TODO(dan): bump to a release version once SENTRY-2371 is published. The SHA
+# below is the current head of the master branch.
+SENTRY_VERSION=2c9a927a9e87cba0e4c0f34fc0b55887c6636927
 SENTRY_NAME=apache-sentry-$SENTRY_VERSION-bin
 SENTRY_SOURCE=$TP_SOURCE_DIR/$SENTRY_NAME


Mime
View raw message