sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject [1/2] sentry git commit: SENTRY-1536: Refactor SentryStore transaction management to allow for extra transactions for a single permission update (Hao Hao, Reviewed by: Alexander Kolbasov, Kalyan Kumar Kalvagadda and Lei Xu)
Date Fri, 03 Feb 2017 07:15:40 GMT
Repository: sentry
Updated Branches:
  refs/heads/sentry-ha-redesign b98b587e9 -> 2911c532b


http://git-wip-us.apache.org/repos/asf/sentry/blob/2911c532/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index 1f05ba9..ecd1175 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -32,8 +32,11 @@ import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hive.hcatalog.messaging.HCatEventMessage;
 import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
 import org.apache.sentry.core.common.exception.*;
+import org.apache.sentry.hdfs.PermissionsUpdate;
 import org.apache.sentry.hdfs.UpdateableAuthzPaths;
 import org.apache.sentry.hdfs.FullUpdateInitializer;
+import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
+import org.apache.sentry.provider.db.SentryPolicyStorePlugin;
 import org.apache.sentry.provider.db.service.persistent.SentryStore;
 import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
 import org.apache.thrift.TException;
@@ -51,6 +54,7 @@ import java.util.List;
 
 import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE;
 import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE;
+import static org.apache.sentry.hdfs.Updateable.Update;
 
 /**
  * HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry.
@@ -437,14 +441,16 @@ public class HMSFollower implements Runnable {
   private void dropSentryDbPrivileges(String dbName) throws Exception {
     TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
     authorizable.setDb(dbName);
-    sentryStore.dropPrivilege(authorizable);
+    sentryStore.dropPrivilege(authorizable, onDropSentryPrivilege(authorizable));
   }
+
   private void dropSentryTablePrivileges(String dbName, String tableName) throws Exception
{
     TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
     authorizable.setDb(dbName);
     authorizable.setTable(tableName);
-    sentryStore.dropPrivilege(authorizable);
+    sentryStore.dropPrivilege(authorizable, onDropSentryPrivilege(authorizable));
   }
+
   private void renamePrivileges(String oldDbName, String oldTableName, String newDbName,
String newTableName) throws
       Exception {
     TSentryAuthorizable oldAuthorizable = new TSentryAuthorizable(hiveInstance);
@@ -453,6 +459,43 @@ public class HMSFollower implements Runnable {
     TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
     newAuthorizable.setDb(newDbName);
     newAuthorizable.setTable(newTableName);
-    sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable);
+    Update update =
+        onRenameSentryPrivilege(oldAuthorizable, newAuthorizable);
+    sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable, update);
+  }
+
+  @VisibleForTesting
+  static Update onDropSentryPrivilege(TSentryAuthorizable authorizable) {
+    PermissionsUpdate update = new PermissionsUpdate(SentryStore.INIT_CHANGE_ID, false);
+    String authzObj = getAuthzObj(authorizable);
+    update.addPrivilegeUpdate(authzObj).putToDelPrivileges(PermissionsUpdate.ALL_ROLES, PermissionsUpdate.ALL_ROLES);
+    return update;
+  }
+
+  @VisibleForTesting
+  static Update onRenameSentryPrivilege(TSentryAuthorizable oldAuthorizable,
+            TSentryAuthorizable newAuthorizable)
+          throws SentryPolicyStorePlugin.SentryPluginException {
+    String oldAuthz = getAuthzObj(oldAuthorizable);
+    String newAuthz = getAuthzObj(newAuthorizable);
+    PermissionsUpdate update = new PermissionsUpdate(SentryStore.INIT_CHANGE_ID, false);
+    TPrivilegeChanges privUpdate = update.addPrivilegeUpdate(PermissionsUpdate.RENAME_PRIVS);
+    privUpdate.putToAddPrivileges(newAuthz, newAuthz);
+    privUpdate.putToDelPrivileges(oldAuthz, oldAuthz);
+    return update;
+  }
+
+  public static String getAuthzObj(TSentryAuthorizable authzble) {
+    String authzObj = null;
+    if (!SentryStore.isNULL(authzble.getDb())) {
+      String dbName = authzble.getDb();
+      String tblName = authzble.getTable();
+      if (SentryStore.isNULL(tblName)) {
+        authzObj = dbName;
+      } else {
+        authzObj = dbName + "." + tblName;
+      }
+    }
+    return authzObj == null ? null : authzObj.toLowerCase();
   }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/2911c532/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index a35c8d7..dfaac15 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -35,6 +35,10 @@ import org.apache.sentry.core.model.db.AccessConstants;
 import org.apache.sentry.core.common.exception.SentryAlreadyExistsException;
 import org.apache.sentry.core.common.exception.SentryGrantDeniedException;
 import org.apache.sentry.core.common.exception.SentryNoSuchObjectException;
+import org.apache.sentry.hdfs.PermissionsUpdate;
+import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
+import org.apache.sentry.hdfs.service.thrift.TRoleChanges;
+import org.apache.sentry.provider.db.service.model.MSentryPermChange;
 import org.apache.sentry.provider.db.service.model.MSentryPrivilege;
 import org.apache.sentry.provider.db.service.model.MSentryRole;
 import org.apache.sentry.provider.db.service.thrift.TSentryActiveRoleSet;
@@ -57,6 +61,8 @@ import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.io.Files;
 
+import static org.apache.sentry.hdfs.Updateable.Update;
+
 public class TestSentryStore extends org.junit.Assert {
 
   private static File dataDir;
@@ -2263,6 +2269,208 @@ public class TestSentryStore extends org.junit.Assert {
     assertTrue(names.containsAll(result));
   }
 
+  @Test
+  public void testPrivilegesWithPermUpdate() throws Exception {
+    String roleName = "test-privilege";
+    String grantor = "g1";
+    String server = "server1";
+    String db = "db1";
+    String table = "tbl1";
+    String authzObj = "db1.tbl1";
+    createRole(roleName);
+
+    TSentryPrivilege privilege = new TSentryPrivilege();
+    privilege.setPrivilegeScope("Column");
+    privilege.setServerName(server);
+    privilege.setDbName(db);
+    privilege.setTableName(table);
+    privilege.setAction(AccessConstants.SELECT);
+    privilege.setCreateTime(System.currentTimeMillis());
+
+    // Generate the permission add update authzObj "db1.tbl1"
+    PermissionsUpdate addUpdate = new PermissionsUpdate(0, false);
+    addUpdate.addPrivilegeUpdate(authzObj).putToAddPrivileges(
+        roleName, privilege.getAction().toUpperCase());
+
+    // Grant the privilege to role test-privilege and verify it has been persisted.
+    Map<TSentryPrivilege, Update> addPrivilegesUpdateMap = Maps.newHashMap();
+    addPrivilegesUpdateMap.put(privilege, addUpdate);
+    sentryStore.alterSentryRoleGrantPrivileges(grantor, roleName, Sets.newHashSet(privilege),
addPrivilegesUpdateMap);
+    MSentryRole role = sentryStore.getMSentryRoleByName(roleName);
+    Set<MSentryPrivilege> privileges = role.getPrivileges();
+    assertEquals(privileges.toString(), 1, privileges.size());
+
+    // Query the persisted perm change and ensure it equals to the original one
+    long lastChangeID = sentryStore.getLastProcessedPermChangeID();
+    MSentryPermChange addPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID);
+    assertEquals(addUpdate.JSONSerialize(), addPermChange.getPermChange());
+
+    // Generate the permission delete update authzObj "db1.tbl1"
+    PermissionsUpdate delUpdate = new PermissionsUpdate(0, false);
+    delUpdate.addPrivilegeUpdate(authzObj).putToDelPrivileges(
+        roleName, privilege.getAction().toUpperCase());
+
+    // Revoke the same privilege and verify it has been removed.
+    Map<TSentryPrivilege, Update> delPrivilegesUpdateMap = Maps.newHashMap();
+    delPrivilegesUpdateMap.put(privilege, delUpdate);
+    sentryStore.alterSentryRoleRevokePrivileges(grantor, roleName,
+        Sets.newHashSet(privilege), delPrivilegesUpdateMap);
+    role = sentryStore.getMSentryRoleByName(roleName);
+    privileges = role.getPrivileges();
+    assertEquals(0, privileges.size());
+
+    // Query the persisted perm change and ensure it equals to the original one
+    MSentryPermChange delPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID +
1);
+    assertEquals(delUpdate.JSONSerialize(), delPermChange.getPermChange());
+  }
+
+  @Test
+  public void testAddDeleteGroupsWithPermUpdate() throws Exception {
+    String roleName = "test-groups";
+    String grantor = "g1";
+    createRole(roleName);
+
+    Set<TSentryGroup> groups = Sets.newHashSet();
+    TSentryGroup group = new TSentryGroup();
+    group.setGroupName("test-groups-g1");
+    groups.add(group);
+    group = new TSentryGroup();
+    group.setGroupName("test-groups-g2");
+    groups.add(group);
+
+    // Generate the permission add update for role "test-groups"
+    PermissionsUpdate addUpdate = new PermissionsUpdate(0, false);
+    TRoleChanges addrUpdate = addUpdate.addRoleUpdate(roleName);
+    for (TSentryGroup g : groups) {
+      addrUpdate.addToAddGroups(g.getGroupName());
+    }
+
+    // Assign the role "test-groups" to the groups and verify.
+    sentryStore.alterSentryRoleAddGroups(grantor, roleName, groups, addUpdate);
+    MSentryRole role = sentryStore.getMSentryRoleByName(roleName);
+    assertEquals(2, role.getGroups().size());
+
+    // Query the persisted perm change and ensure it equals to the original one
+    long lastChangeID = sentryStore.getLastProcessedPermChangeID();
+    MSentryPermChange addPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID);
+    assertEquals(addUpdate.JSONSerialize(), addPermChange.getPermChange());
+
+    // Generate the permission add update for role "test-groups"
+    PermissionsUpdate delUpdate = new PermissionsUpdate(0, false);
+    TRoleChanges delrUpdate = delUpdate.addRoleUpdate(roleName);
+    for (TSentryGroup g : groups) {
+      delrUpdate.addToDelGroups(g.getGroupName());
+    }
+
+    // Revoke the role "test-groups" to the groups and verify.
+    sentryStore.alterSentryRoleDeleteGroups(roleName, groups, delUpdate);
+    role = sentryStore.getMSentryRoleByName(roleName);
+    assertEquals(Collections.emptySet(), role.getGroups());
+
+    // Query the persisted perm change and ensure it equals to the original one
+    MSentryPermChange delPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID +
1);
+    assertEquals(delUpdate.JSONSerialize(), delPermChange.getPermChange());
+  }
+
+  @Test
+  public void testCreateDropRoleWithPermUpdate() throws Exception {
+    String roleName = "test-drop-role";
+    createRole(roleName);
+
+    // Generate the permission del update for dropping role "test-drop-role"
+    PermissionsUpdate delUpdate = new PermissionsUpdate(0, false);
+    delUpdate.addPrivilegeUpdate(PermissionsUpdate.ALL_AUTHZ_OBJ).putToDelPrivileges(
+            roleName, PermissionsUpdate.ALL_AUTHZ_OBJ);
+    delUpdate.addRoleUpdate(roleName).addToDelGroups(PermissionsUpdate.ALL_GROUPS);
+
+    // Drop the role and verify.
+    sentryStore.dropSentryRole(roleName, delUpdate);
+    checkRoleDoesNotExist(roleName);
+
+    // Query the persisted perm change and ensure it equals to the original one
+    long lastChangeID = sentryStore.getLastProcessedPermChangeID();
+    MSentryPermChange delPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID);
+    assertEquals(delUpdate.JSONSerialize(), delPermChange.getPermChange());
+  }
+
+  @Test
+  public void testDropObjWithPermUpdate() throws Exception {
+    String roleName1 = "list-privs-r1", roleName2 = "list-privs-r2";
+    String grantor = "g1";
+    sentryStore.createSentryRole(roleName1);
+    sentryStore.createSentryRole(roleName2);
+
+    String authzObj = "db1.tbl1";
+    TSentryPrivilege privilege_tbl1 = new TSentryPrivilege();
+    privilege_tbl1.setPrivilegeScope("TABLE");
+    privilege_tbl1.setServerName("server1");
+    privilege_tbl1.setDbName("db1");
+    privilege_tbl1.setTableName("tbl1");
+    privilege_tbl1.setCreateTime(System.currentTimeMillis());
+    privilege_tbl1.setAction("SELECT");
+
+    sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName1, privilege_tbl1);
+
+    // Generate the permission drop update for dropping privilege for "db1.tbl1"
+    PermissionsUpdate dropUpdate = new PermissionsUpdate(0, false);
+    dropUpdate.addPrivilegeUpdate(authzObj).putToDelPrivileges(PermissionsUpdate.ALL_ROLES,
+            PermissionsUpdate.ALL_ROLES);
+
+    // Drop the privilege and verify.
+    sentryStore.dropPrivilege(toTSentryAuthorizable(privilege_tbl1), dropUpdate);
+    assertEquals(0, sentryStore.getAllTSentryPrivilegesByRoleName(roleName1).size());
+    assertEquals(0, sentryStore.getAllTSentryPrivilegesByRoleName(roleName2).size());
+
+    // Query the persisted perm change and ensure it equals to the original one
+    long lastChangeID = sentryStore.getLastProcessedPermChangeID();
+    MSentryPermChange dropPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID);
+    assertEquals(dropUpdate.JSONSerialize(), dropPermChange.getPermChange());
+  }
+
+  @Test
+  public void testRenameObjWithPermUpdate() throws Exception {
+    String roleName1 = "role1", roleName2 = "role2", roleName3 = "role3";
+    String grantor = "g1";
+    String table1 = "tbl1", table2 = "tbl2";
+
+    sentryStore.createSentryRole(roleName1);
+
+    TSentryPrivilege privilege_tbl1 = new TSentryPrivilege();
+    privilege_tbl1.setPrivilegeScope("TABLE");
+    privilege_tbl1.setServerName("server1");
+    privilege_tbl1.setDbName("db1");
+    privilege_tbl1.setTableName(table1);
+    privilege_tbl1.setCreateTime(System.currentTimeMillis());
+    privilege_tbl1.setAction(AccessConstants.ALL);
+
+    sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName1, privilege_tbl1);
+
+    // Generate the permission rename update for renaming privilege for "db1.tbl1"
+    String oldAuthz = "db1.tbl1";
+    String newAuthz = "db1.tbl2";
+    PermissionsUpdate renameUpdate = new PermissionsUpdate(0, false);
+    TPrivilegeChanges privUpdate = renameUpdate.addPrivilegeUpdate(PermissionsUpdate.RENAME_PRIVS);
+    privUpdate.putToAddPrivileges(newAuthz, newAuthz);
+    privUpdate.putToDelPrivileges(oldAuthz, oldAuthz);
+
+    // Rename the privilege and verify.
+    TSentryAuthorizable oldTable = toTSentryAuthorizable(privilege_tbl1);
+    TSentryAuthorizable newTable = toTSentryAuthorizable(privilege_tbl1);
+    newTable.setTable(table2);
+    sentryStore.renamePrivilege(oldTable, newTable, renameUpdate);
+
+    Set<TSentryPrivilege> privilegeSet = sentryStore.getAllTSentryPrivilegesByRoleName(roleName1);
+    assertEquals(1, privilegeSet.size());
+    for (TSentryPrivilege privilege : privilegeSet) {
+      assertTrue(table2.equalsIgnoreCase(privilege.getTableName()));
+    }
+
+    // Query the persisted perm change and ensure it equals to the original one
+    long lastChangeID = sentryStore.getLastProcessedPermChangeID();
+    MSentryPermChange renamePermChange = sentryStore.getMSentryPermChangeByID(lastChangeID);
+    assertEquals(renameUpdate.JSONSerialize(), renamePermChange.getPermChange());
+  }
+
   protected static void addGroupsToUser(String user, String... groupNames) {
     policyFile.addGroupsToUser(user, groupNames);
   }

http://git-wip-us.apache.org/repos/asf/sentry/blob/2911c532/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java
b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java
index d601b1e..fd97936 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java
@@ -17,13 +17,11 @@
 package org.apache.sentry.service.thrift;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.*;
 import org.apache.hive.hcatalog.messaging.HCatEventMessage;
 import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
 import org.apache.sentry.provider.db.service.persistent.SentryStore;
 import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
 
@@ -56,8 +54,9 @@ public class TestHMSFollower {
     authorizable.setServer(hiveInstance);
     authorizable.setDb("db1");
 
-    verify(sentryStore, times(1)).dropPrivilege(authorizable);
+    verify(sentryStore, times(1)).dropPrivilege(authorizable, HMSFollower.onDropSentryPrivilege(authorizable));
   }
+
   @Test
   public void testDropDatabase() throws Exception {
     String dbName = "db1";
@@ -76,7 +75,7 @@ public class TestHMSFollower {
     authorizable.setServer(hiveInstance);
     authorizable.setDb("db1");
 
-    verify(sentryStore, times(1)).dropPrivilege(authorizable);
+    verify(sentryStore, times(1)).dropPrivilege(authorizable, HMSFollower.onDropSentryPrivilege(authorizable))
;
   }
   @Test
   public void testCreateTable() throws Exception {
@@ -100,7 +99,7 @@ public class TestHMSFollower {
     authorizable.setDb("db1");
     authorizable.setTable(tableName);
 
-    verify(sentryStore, times(1)).dropPrivilege(authorizable);
+    verify(sentryStore, times(1)).dropPrivilege(authorizable, HMSFollower.onDropSentryPrivilege(authorizable));
   }
   @Test
   public void testDropTable() throws Exception {
@@ -124,7 +123,7 @@ public class TestHMSFollower {
     authorizable.setDb("db1");
     authorizable.setTable(tableName);
 
-    verify(sentryStore, times(1)).dropPrivilege(authorizable);
+    verify(sentryStore, times(1)).dropPrivilege(authorizable, HMSFollower.onDropSentryPrivilege(authorizable));
   }
   @Test
   public void testRenameTable() throws Exception {
@@ -160,6 +159,6 @@ public class TestHMSFollower {
     newAuthorizable.setDb(newDbName);
     newAuthorizable.setTable(newTableName);
 
-    verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable);
+    verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable, HMSFollower.onRenameSentryPrivilege(authorizable,
newAuthorizable));
   }
 }


Mime
View raw message