kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [kudu] 01/02: Bump Sentry version
Date Sun, 21 Apr 2019 20:58:41 GMT
This is an automated email from the ASF dual-hosted git repository.

adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 345ce27da0d7d16ec37a56502d0939aef87f671f
Author: Hao Hao <hao.hao@cloudera.com>
AuthorDate: Mon Apr 15 22:43:32 2019 -0700

    Bump Sentry version
    
    The motivation is to get access to SENTRY-2471, which includes a fix for
    syncing Sentry privilege changes upon table renaming even for tables
    without HDFS path information. SENTRY-2471 has not been released yet,
    so the new Sentry version is based on the latest commit of the master
    branch.
    
    Note that after bumping the Sentry version, the hms tests start to fail
    with:
    java.lang.UnsatisfiedLinkError: org.apache.hadoop.io.nativeio.NativeIO$POSIX.stat
    (Ljava/lang/String;)Lorg/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat;
      at org.apache.hadoop.io.nativeio.NativeIO$POSIX.stat(Native Method)
      at org.apache.hadoop.io.nativeio.NativeIO$POSIX.getStat(NativeIO.java:451)
    This is due to hadoop 3.1.1 (SENTRY-2329) from Sentry package is loaded
    into the HMS classpath. Since we have no reasons to use hadoop jars in
    Sentry package, we can safely remove them.
    
    Change-Id: I13f05345d2e6dcdadd74503d0d4524947b137d43
    Reviewed-on: http://gerrit.cloudera.org:8080/13059
    Reviewed-by: Adar Dembo <adar@cloudera.com>
    Tested-by: Hao Hao <hao.hao@cloudera.com>
    Reviewed-by: Andrew Wong <awong@cloudera.com>
    Tested-by: Kudu Jenkins
    Reviewed-by: Alexey Serbin <aserbin@cloudera.com>
---
 src/kudu/integration-tests/master_sentry-itest.cc | 11 ++---------
 src/kudu/sentry/mini_sentry.cc                    | 11 +++++++++++
 src/kudu/sentry/sentry_common_service.thrift      |  2 +-
 src/kudu/sentry/sentry_policy_service.thrift      | 18 +++++++++++++++---
 thirdparty/build-thirdparty.sh                    |  4 ++++
 thirdparty/vars.sh                                |  9 ++++++---
 6 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/src/kudu/integration-tests/master_sentry-itest.cc b/src/kudu/integration-tests/master_sentry-itest.cc
index b0f711d..0ada81f 100644
--- a/src/kudu/integration-tests/master_sentry-itest.cc
+++ b/src/kudu/integration-tests/master_sentry-itest.cc
@@ -447,15 +447,13 @@ TEST_F(MasterSentryTest, TestTableOwnership) {
   NO_FATALS(CheckTableDoesNotExist(kDatabaseName, "new_table"));
 }
 
-// TODO(hao): enable the following tests after SENTRY-2471 is fixed.
-TEST_F(MasterSentryTest, DISABLED_TestRenameTablePrivilegeTransfer) {
+// Checks Sentry privileges are synchronized upon table rename in the HMS.
+TEST_F(MasterSentryTest, TestRenameTablePrivilegeTransfer) {
   ASSERT_OK(GrantRenameTablePrivilege(kDatabaseName, kTableName));
   ASSERT_OK(RenameTable(Substitute("$0.$1", kDatabaseName, kTableName),
                         Substitute("$0.$1", kDatabaseName, "b")));
   NO_FATALS(CheckTable(kDatabaseName, "b", make_optional<const string &>(kAdminUser)));
 
-
-  // Checks table rename in the HMS is synchronized with the Sentry privileges.
   unique_ptr<KuduTableAlterer> table_alterer;
   table_alterer.reset(client_->NewTableAlterer(Substitute("$0.$1", kDatabaseName, "b"))
                              ->DropColumn("int16_val"));
@@ -470,11 +468,6 @@ TEST_F(MasterSentryTest, DISABLED_TestRenameTablePrivilegeTransfer) {
     ASSERT_OK(table_alterer->Alter());
   });
   NO_FATALS(CheckTable(kDatabaseName, "b", make_optional<const string&>(kAdminUser)));
-
-  table_alterer.reset(client_->NewTableAlterer(Substitute("$0.$1", kDatabaseName, "b"))
-                             ->RenameTo(Substitute("$0.$1", kDatabaseName, "c")));
-  ASSERT_OK(table_alterer->Alter());
-  NO_FATALS(CheckTable(kDatabaseName, "c", make_optional<const string&>(kAdminUser)));
 }
 
 class TestAuthzTable :
diff --git a/src/kudu/sentry/mini_sentry.cc b/src/kudu/sentry/mini_sentry.cc
index 2477b8a..caecfa9 100644
--- a/src/kudu/sentry/mini_sentry.cc
+++ b/src/kudu/sentry/mini_sentry.cc
@@ -215,6 +215,12 @@ Status MiniSentry::CreateSentryConfigs(const string& tmp_dir) const
{
   //    The interval to run the "store-cleaner" Sentry's thread. Setting to a
   //    negative value means Sentry will not run the "store-cleaner" thread
   //    at all and that allows for faster start-up times of the Sentry service.
+  //
+  // - hive.sentry.server
+  //    Server namespace the HMS instance belongs to for defining server-level
+  //    privileges in Sentry. Sentry uses it to synchronize privileges upon
+  //    receipt of HMS events (such as table rename). Must match with Kudu
+  //    master's flag 'server_name'.
   static const string kFileTemplate = R"(
 <configuration>
 
@@ -293,6 +299,11 @@ Status MiniSentry::CreateSentryConfigs(const string& tmp_dir) const
{
     <value>-1</value>
   </property>
 
+  <property>
+    <name>hive.sentry.server</name>
+    <value>server1</value>
+  </property>
+
 </configuration>
   )";
 
diff --git a/src/kudu/sentry/sentry_common_service.thrift b/src/kudu/sentry/sentry_common_service.thrift
index b8a730e..a375348 100644
--- a/src/kudu/sentry/sentry_common_service.thrift
+++ b/src/kudu/sentry/sentry_common_service.thrift
@@ -19,7 +19,7 @@
  */
 
 # DO NOT MODIFY! Copied from
-# https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_common_service.thrift
+# https://raw.githubusercontent.com/apache/sentry/505b42e81a9d85c4ebe8db3f48ad7a6e824a5db5/sentry-service/sentry-service-api/src/main/resources/sentry_common_service.thrift
 #
 # With edits:
 #   - Change cpp namespace to 'sentry' to match the Kudu codebase style.
diff --git a/src/kudu/sentry/sentry_policy_service.thrift b/src/kudu/sentry/sentry_policy_service.thrift
index dc2c13b..0d8c981 100644
--- a/src/kudu/sentry/sentry_policy_service.thrift
+++ b/src/kudu/sentry/sentry_policy_service.thrift
@@ -19,7 +19,7 @@
  */
 
 # DO NOT MODIFY! Copied from
-# https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_policy_service.thrift
+# https://raw.githubusercontent.com/apache/sentry/505b42e81a9d85c4ebe8db3f48ad7a6e824a5db5/sentry-service/sentry-service-api/src/main/resources/sentry_policy_service.thrift
 #
 # With edits:
 #   - Change cpp namespace to 'sentry' to match the Kudu codebase style.
@@ -70,6 +70,15 @@ struct TSentryGroup {
 1: required string groupName
 }
 
+struct TIsSentryAdminRequest {
+1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
+2: required string userName,
+}
+struct TIsSentryAdminResponse {
+1: required sentry_common_service.TSentryResponseStatus status,
+2: required bool isAdmin,
+}
+
 # CREATE ROLE r1
 struct TCreateSentryRoleRequest {
 1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
@@ -303,7 +312,7 @@ struct TSentryMappingData {
 struct TSentryExportMappingDataRequest {
 1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V1,
 2: required string requestorUserName, # user on whose behalf the request is issued
-3: optional string objectPath # for specific auth object
+3: optional set<TSentryAuthorizable> authorizables # for which permission information
needs to be exported.
 }
 
 struct TSentryExportMappingDataResponse {
@@ -314,7 +323,7 @@ struct TSentryExportMappingDataResponse {
 struct TSentryImportMappingDataRequest {
 1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V1,
 2: required string requestorUserName, # user on whose behalf the request is issued
-3: required bool overwriteRole = false, # if overwrite the exist role with the imported privileges,
default is false 
+3: required bool overwriteRole = false, # if overwrite the exist role with the imported privileges,
default is false
 4: required TSentryMappingData mappingData
 }
 
@@ -400,6 +409,9 @@ struct TSentryPrivilegesResponse {
 
 service SentryPolicyService
 {
+  # Check if the given user is in the Sentry admin group.
+  TIsSentryAdminResponse is_sentry_admin(1:TIsSentryAdminRequest request)
+
   TCreateSentryRoleResponse create_sentry_role(1:TCreateSentryRoleRequest request)
   TDropSentryRoleResponse drop_sentry_role(1:TDropSentryRoleRequest request)
 
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index c13c95c..0fe31ca 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -259,6 +259,10 @@ fi
 
 if [ -n "$F_COMMON" -o -n "$F_SENTRY" ]; then
   mkdir -p $PREFIX/opt
+  # Remove any hadoop jars included in the Sentry package to avoid unexpected
+  # runtime behavior, due to different versions of hadoop jars are loaded
+  # (one from Kudu's third-party dependency, the other from the Sentry package).
+  rm -rf $SENTRY_SOURCE/lib/hadoop-[a-z-]*.jar
   ln -nsf $SENTRY_SOURCE $PREFIX/opt/sentry
 fi
 
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 1c142a7..ba0b700 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -226,8 +226,11 @@ HADOOP_VERSION=2.8.5
 HADOOP_NAME=hadoop-$HADOOP_VERSION
 HADOOP_SOURCE=$TP_SOURCE_DIR/$HADOOP_NAME
 
-# TODO(dan): bump to a release version once SENTRY-2371 is published. The SHA
-# below is the current head of the master branch.
-SENTRY_VERSION=2c9a927a9e87cba0e4c0f34fc0b55887c6636927
+# TODO(dan): bump to a release version once SENTRY-2371, SENTRY-2440 and SENTRY-2471
+# are published. The SHA below is the current head of the master branch.
+# Note: Sentry releases source code only. To build the binary tarball, use `dist`
+# maven profile. For example, `mvn clean install -Pdist`. After a successful build,
+# the tarball will be available under sentry-dist/target.
+SENTRY_VERSION=505b42e81a9d85c4ebe8db3f48ad7a6e824a5db5
 SENTRY_NAME=apache-sentry-$SENTRY_VERSION-bin
 SENTRY_SOURCE=$TP_SOURCE_DIR/$SENTRY_NAME


Mime
View raw message