From commits-return-12720-archive-asf-public=cust-asf.ponee.io@sentry.apache.org Thu May 31 05:32:09 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 41864180778 for ; Thu, 31 May 2018 05:32:07 +0200 (CEST) Received: (qmail 39380 invoked by uid 500); 31 May 2018 03:32:06 -0000 Mailing-List: contact commits-help@sentry.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@sentry.apache.org Delivered-To: mailing list commits@sentry.apache.org Received: (qmail 34656 invoked by uid 99); 31 May 2018 03:31:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 31 May 2018 03:31:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 954D8E118A; Thu, 31 May 2018 03:31:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: spena@apache.org To: commits@sentry.apache.org Date: Thu, 31 May 2018 03:32:36 -0000 Message-Id: <64d33cb8802a4ab7a8b0256d067c5e13@git.apache.org> In-Reply-To: <5efebc2704cd43a891f304274777fee8@git.apache.org> References: <5efebc2704cd43a891f304274777fee8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [58/86] sentry git commit: SENTRY-2208: Refactor out Sentry service into own module from sentry-provider-db (Anthony Young-Garner, reviewed by Sergio Pena, Steve Moist, Na Li) http://git-wip-us.apache.org/repos/asf/sentry/blob/7db84b2f/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java new file mode 100644 index 0000000..5932335 --- /dev/null +++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -0,0 +1,4796 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.persistent; + +import static org.apache.sentry.core.common.utils.SentryConstants.AUTHORIZABLE_JOINER; +import static org.apache.sentry.core.common.utils.SentryConstants.KV_JOINER; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import javax.jdo.FetchGroup; +import javax.jdo.JDODataStoreException; +import javax.jdo.JDOHelper; +import javax.jdo.PersistenceManager; +import javax.jdo.PersistenceManagerFactory; +import javax.jdo.Query; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.sentry.core.common.exception.SentryAccessDeniedException; +import org.apache.sentry.core.common.exception.SentryAlreadyExistsException; +import org.apache.sentry.core.common.exception.SentryGrantDeniedException; +import org.apache.sentry.core.common.exception.SentryInvalidInputException; +import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; +import org.apache.sentry.core.common.exception.SentrySiteConfigurationException; +import org.apache.sentry.core.common.exception.SentryUserException; +import org.apache.sentry.core.common.utils.PathUtils; +import org.apache.sentry.core.common.utils.SentryConstants; +import org.apache.sentry.core.model.db.AccessConstants; +import org.apache.sentry.core.model.db.DBModelAuthorizable.AuthorizableType; +import org.apache.sentry.hdfs.PathsUpdate; +import org.apache.sentry.hdfs.UniquePathsUpdate; +import org.apache.sentry.hdfs.UpdateableAuthzPaths; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeEntityType; +import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping; +import org.apache.sentry.provider.db.service.model.MAuthzPathsSnapshotId; +import org.apache.sentry.provider.db.service.model.MSentryChange; +import org.apache.sentry.provider.db.service.model.MSentryGroup; +import org.apache.sentry.provider.db.service.model.MSentryHmsNotification; +import org.apache.sentry.provider.db.service.model.MSentryPathChange; +import org.apache.sentry.provider.db.service.model.MSentryPermChange; +import org.apache.sentry.provider.db.service.model.MSentryPrivilege; +import org.apache.sentry.provider.db.service.model.MSentryGMPrivilege; +import org.apache.sentry.provider.db.service.model.MSentryRole; +import org.apache.sentry.provider.db.service.model.MSentryUser; +import org.apache.sentry.provider.db.service.model.MSentryVersion; +import org.apache.sentry.provider.db.service.model.MSentryUtil; +import org.apache.sentry.provider.db.service.model.MPath; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeEntity; +import org.apache.sentry.api.common.ApiConstants.PrivilegeScope; +import org.apache.sentry.api.service.thrift.SentryPolicyStoreProcessor; +import org.apache.sentry.api.service.thrift.TSentryActiveRoleSet; +import org.apache.sentry.api.service.thrift.TSentryAuthorizable; +import org.apache.sentry.api.service.thrift.TSentryGrantOption; +import org.apache.sentry.api.service.thrift.TSentryGroup; +import org.apache.sentry.api.service.thrift.TSentryMappingData; +import org.apache.sentry.api.service.thrift.TSentryPrivilege; +import org.apache.sentry.api.service.thrift.TSentryPrivilegeMap; +import org.apache.sentry.api.service.thrift.TSentryRole; +import org.apache.sentry.service.common.ServiceConstants.SentryEntityType; +import org.apache.sentry.service.common.ServiceConstants.ServerConfig; +import org.datanucleus.store.rdbms.exceptions.MissingTableException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.Gauge; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import static org.apache.sentry.hdfs.Updateable.Update; + +/** + * SentryStore is the data access object for Sentry data. Strings + * such as role and group names will be normalized to lowercase + * in addition to starting and ending whitespace. + *

+ * We have several places where we rely on transactions to support + * read/modify/write semantics for incrementing IDs. + * This works but using DB support is rather expensive and we can + * user in-core serializations to help with this a least within a + * single node and rely on DB for multi-node synchronization. + *

+ * This isn't much of a problem for path updates since they are + * driven by HMSFollower which usually runs on a single leader + * node, but permission updates originate from clients + * directly and may be highly concurrent. + *

+ * We are internally serializing all permissions update anyway, so doing + * partial serialization on every node helps. For this reason all + * SentryStore calls that affect permission deltas are serialized. + *

+ * See SENTRY-1824 + * for more detail. + */ +public class SentryStore { + private static final Logger LOGGER = LoggerFactory + .getLogger(SentryStore.class); + + public static final String NULL_COL = "__NULL__"; + public static final int INDEX_GROUP_ROLES_MAP = 0; + public static final int INDEX_USER_ROLES_MAP = 1; + + // String constants for field names + public static final String SERVER_NAME = "serverName"; + public static final String DB_NAME = "dbName"; + public static final String TABLE_NAME = "tableName"; + public static final String COLUMN_NAME = "columnName"; + public static final String ACTION = "action"; + public static final String URI = "URI"; + public static final String GRANT_OPTION = "grantOption"; + public static final String ROLE_NAME = "roleName"; + + // Initial change ID for permission/path change. Auto increment + // is starting from 1. + public static final long INIT_CHANGE_ID = 1L; + + private static final long EMPTY_CHANGE_ID = 0L; + + public static final long EMPTY_NOTIFICATION_ID = 0L; + + // Representation for empty HMS snapshots not found on MAuthzPathsSnapshotId + public static final long EMPTY_PATHS_SNAPSHOT_ID = 0L; + + // For counters, representation of the "unknown value" + private static final long COUNT_VALUE_UNKNOWN = -1L; + + // Representation for unknown HMS notification ID + private static final long NOTIFICATION_UNKNOWN = -1L; + + private static final String EMPTY_GRANTOR_PRINCIPAL = "--"; + + + private static final Set ALL_ACTIONS = Sets.newHashSet( + AccessConstants.ALL, AccessConstants.ACTION_ALL, + AccessConstants.SELECT, AccessConstants.INSERT, AccessConstants.ALTER, + AccessConstants.CREATE, AccessConstants.DROP, AccessConstants.INDEX, + AccessConstants.LOCK); + + // Now partial revoke just support action with SELECT,INSERT and ALL. + // Now partial revoke just support action with SELECT,INSERT, and ALL. + // e.g. If we REVOKE SELECT from a privilege with action ALL, it will leads to INSERT + // e.g. If we REVOKE SELECT from a privilege with action ALL, it will leads to others individual + // Otherwise, if we revoke other privilege(e.g. ALTER,DROP...), we will remove it from a role directly. + private static final Set PARTIAL_REVOKE_ACTIONS = Sets.newHashSet(AccessConstants.ALL, + AccessConstants.ACTION_ALL.toLowerCase(), AccessConstants.SELECT, AccessConstants.INSERT); + + // Datanucleus property controlling whether query results are loaded at commit time + // to make query usable post-commit + private static final String LOAD_RESULTS_AT_COMMIT = "datanucleus.query.loadResultsAtCommit"; + + private final PersistenceManagerFactory pmf; + private Configuration conf; + private final TransactionManager tm; + + // When it is true, execute DeltaTransactionBlock to persist delta changes. + // When it is false, do not execute DeltaTransactionBlock + private boolean persistUpdateDeltas; + + /** + * counterWait is used to synchronize notifications between Thrift and HMSFollower. + * Technically it doesn't belong here, but the only thing that connects HMSFollower + * and Thrift API is SentryStore. An alternative could be a singleton CounterWait or + * some factory that returns CounterWait instances keyed by name, but this complicates + * things unnecessary. + *

+ * Keeping it here isn't ideal but serves the purpose until we find a better home. + */ + private final CounterWait counterWait; + + public static Properties getDataNucleusProperties(Configuration conf) + throws SentrySiteConfigurationException, IOException { + Properties prop = new Properties(); + prop.putAll(ServerConfig.SENTRY_STORE_DEFAULTS); + String jdbcUrl = conf.get(ServerConfig.SENTRY_STORE_JDBC_URL, "").trim(); + Preconditions.checkArgument(!jdbcUrl.isEmpty(), "Required parameter " + + ServerConfig.SENTRY_STORE_JDBC_URL + " is missed"); + String user = conf.get(ServerConfig.SENTRY_STORE_JDBC_USER, ServerConfig. + SENTRY_STORE_JDBC_USER_DEFAULT).trim(); + //Password will be read from Credential provider specified using property + // CREDENTIAL_PROVIDER_PATH("hadoop.security.credential.provider.path" in sentry-site.xml + // it falls back to reading directly from sentry-site.xml + char[] passTmp = conf.getPassword(ServerConfig.SENTRY_STORE_JDBC_PASS); + if (passTmp == null) { + throw new SentrySiteConfigurationException("Error reading " + + ServerConfig.SENTRY_STORE_JDBC_PASS); + } + String pass = new String(passTmp); + + String driverName = conf.get(ServerConfig.SENTRY_STORE_JDBC_DRIVER, + ServerConfig.SENTRY_STORE_JDBC_DRIVER_DEFAULT); + prop.setProperty(ServerConfig.JAVAX_JDO_URL, jdbcUrl); + prop.setProperty(ServerConfig.JAVAX_JDO_USER, user); + prop.setProperty(ServerConfig.JAVAX_JDO_PASS, pass); + prop.setProperty(ServerConfig.JAVAX_JDO_DRIVER_NAME, driverName); + + /* + * Oracle doesn't support "repeatable-read" isolation level and testing + * showed issues with "serializable" isolation level for Oracle 12, + * so we use "read-committed" instead. + * + * JDBC URL always looks like jdbc:oracle::@ + * we look at the second component. + * + * The isolation property can be overwritten via configuration property. + */ + final String oracleDb = "oracle"; + if (prop.getProperty(ServerConfig.DATANUCLEUS_ISOLATION_LEVEL, ""). + equals(ServerConfig.DATANUCLEUS_REPEATABLE_READ) && + jdbcUrl.contains(oracleDb)) { + String[] parts = jdbcUrl.split(":"); + if ((parts.length > 1) && parts[1].equals(oracleDb)) { + // For Oracle JDBC driver, replace "repeatable-read" with "read-committed" + prop.setProperty(ServerConfig.DATANUCLEUS_ISOLATION_LEVEL, + "read-committed"); + } + } + + for (Map.Entry entry : conf) { + String key = entry.getKey(); + if (key.startsWith(ServerConfig.SENTRY_JAVAX_JDO_PROPERTY_PREFIX) || + key.startsWith(ServerConfig.SENTRY_DATANUCLEUS_PROPERTY_PREFIX)) { + key = StringUtils.removeStart(key, ServerConfig.SENTRY_DB_PROPERTY_PREFIX); + prop.setProperty(key, entry.getValue()); + } + } + // Disallow operations outside of transactions + prop.setProperty("datanucleus.NontransactionalRead", "false"); + prop.setProperty("datanucleus.NontransactionalWrite", "false"); + return prop; + } + + public SentryStore(Configuration conf) throws Exception { + this.conf = conf; + Properties prop = getDataNucleusProperties(conf); + boolean checkSchemaVersion = conf.get( + ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, + ServerConfig.SENTRY_VERIFY_SCHEM_VERSION_DEFAULT).equalsIgnoreCase( + "true"); + + // Schema verification should be set to false only for testing. + // If it is set to false, appropriate datanucleus properties will be set so that + // database schema is automatically created. This is desirable only for running tests. + // Sentry uses SentrySchemaTool to create schema with the help of sql scripts. + + if (!checkSchemaVersion) { + prop.setProperty("datanucleus.schema.autoCreateAll", "true"); + } + pmf = JDOHelper.getPersistenceManagerFactory(prop); + tm = new TransactionManager(pmf, conf); + verifySentryStoreSchema(checkSchemaVersion); + long notificationTimeout = conf.getInt(ServerConfig.SENTRY_NOTIFICATION_SYNC_TIMEOUT_MS, + ServerConfig.SENTRY_NOTIFICATION_SYNC_TIMEOUT_DEFAULT); + counterWait = new CounterWait(notificationTimeout, TimeUnit.MILLISECONDS); + } + + public void setPersistUpdateDeltas(boolean persistUpdateDeltas) { + this.persistUpdateDeltas = persistUpdateDeltas; + } + + + public TransactionManager getTransactionManager() { + return tm; + } + + public CounterWait getCounterWait() { + return counterWait; + } + + // ensure that the backend DB schema is set + void verifySentryStoreSchema(boolean checkVersion) throws Exception { + if (!checkVersion) { + setSentryVersion(SentryStoreSchemaInfo.getSentryVersion(), + "Schema version set implicitly"); + } else { + String currentVersion = getSentryVersion(); + if (!SentryStoreSchemaInfo.getSentryVersion().equals(currentVersion)) { + throw new SentryAccessDeniedException( + "The Sentry store schema version " + currentVersion + + " is different from distribution version " + + SentryStoreSchemaInfo.getSentryVersion()); + } + } + } + + public synchronized void stop() { + if (pmf != null) { + pmf.close(); + } + } + + /** + * Get a single role with the given name inside a transaction + * @param pm Persistence Manager instance + * @param roleName Role name (should not be null) + * @return single role with the given name + */ + public MSentryRole getRole(PersistenceManager pm, String roleName) { + Query query = pm.newQuery(MSentryRole.class); + query.addExtension(LOAD_RESULTS_AT_COMMIT, "false"); + query.setFilter("this.roleName == :roleName"); + query.setUnique(true); + return (MSentryRole) query.execute(roleName); + } + + /** + * Get list of all roles. Should be called inside transaction. + * @param pm Persistence manager instance + * @return List of all roles + */ + @SuppressWarnings("unchecked") + private List getAllRoles(PersistenceManager pm) { + Query query = pm.newQuery(MSentryRole.class); + query.addExtension(LOAD_RESULTS_AT_COMMIT, "false"); + return (List) query.execute(); + } + + /** + * Get a single user with the given name inside a transaction + * @param pm Persistence Manager instance + * @param userName User name (should not be null) + * @return single user with the given name + */ + public MSentryUser getUser(PersistenceManager pm, String userName) { + Query query = pm.newQuery(MSentryUser.class); + query.addExtension(LOAD_RESULTS_AT_COMMIT, "false"); + query.setFilter("this.userName == :userName"); + query.setUnique(true); + return (MSentryUser) query.execute(userName); + } + + /** + * Create a sentry user and persist it. User name is the primary key for the + * user, so an attempt to create a user which exists fails with JDO exception. + * + * @param userName: Name of the user being persisted. + * The name is normalized. + * @throws Exception + */ + public void createSentryUser(final String userName) throws Exception { + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedUserName = trimAndLower(userName); + if (getUser(pm, trimmedUserName) != null) { + throw new SentryAlreadyExistsException("User: " + trimmedUserName); + } + pm.makePersistent( + new MSentryUser(trimmedUserName, System.currentTimeMillis(), Sets.newHashSet())); + return null; + }); + } + + /** + * Normalize the string values - remove leading and trailing whitespaces and + * convert to lower case + * @return normalized input + */ + private String trimAndLower(String input) { + return input.trim().toLowerCase(); + } + + /** + * Create a sentry role and persist it. Role name is the primary key for the + * role, so an attempt to create a role which exists fails with JDO exception. + * + * @param roleName: Name of the role being persisted. + * The name is normalized. + * @throws Exception + */ + public void createSentryRole(final String roleName) throws Exception { + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedRoleName = trimAndLower(roleName); + if (getRole(pm, trimmedRoleName) != null) { + throw new SentryAlreadyExistsException("Role: " + trimmedRoleName); + } + pm.makePersistent(new MSentryRole(trimmedRoleName)); + return null; + }); + } + + /** + * Get count of object of the given class + * @param tClass Class to count + * @param Class type + * @return count of objects or -1 in case of error + */ + private Long getCount(final Class tClass) { + try { + return tm.executeTransaction( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + Query query = pm.newQuery(); + query.addExtension(LOAD_RESULTS_AT_COMMIT, "false"); + query.setClass(tClass); + query.setResult("count(this)"); + Long result = (Long)query.execute(); + return result; + }); + } catch (Exception e) { + return COUNT_VALUE_UNKNOWN; + } + } + + /** + * @return number of roles + */ + public Gauge getRoleCountGauge() { + return () -> getCount(MSentryRole.class); + } + + /** + * @return Number of privileges + */ + public Gauge getPrivilegeCountGauge() { + return () -> getCount(MSentryPrivilege.class); + } + + /** + * @return number of groups + */ + public Gauge getGroupCountGauge() { + return () -> getCount(MSentryGroup.class); + } + + /** + * @return Number of users + */ + Gauge getUserCountGauge() { + return () -> getCount(MSentryUser.class); + } + + /** + * @return number of threads waiting for HMS notifications to be processed + */ + public Gauge getHMSWaitersCountGauge() { + return () -> counterWait.waitersCount(); + } + + /** + * @return current value of last processed notification ID + */ + public Gauge getLastNotificationIdGauge() { + return () -> { + try { + return getLastProcessedNotificationID(); + } catch (Exception e) { + LOGGER.error("Can not read current notificationId", e); + return NOTIFICATION_UNKNOWN; + } + }; + } + + /** + * @return ID of the path snapshot + */ + public Gauge getLastPathsSnapshotIdGauge() { + return () -> { + try { + return getCurrentAuthzPathsSnapshotID(); + } catch (Exception e) { + LOGGER.error("Can not read current paths snapshot ID", e); + return NOTIFICATION_UNKNOWN; + } + }; + } + + /** + * @return Permissions change ID + */ + public Gauge getPermChangeIdGauge() { + return new Gauge() { + @Override + public Long getValue() { + try { + return tm.executeTransaction( + pm -> getLastProcessedChangeIDCore(pm, MSentryPermChange.class) + ); + } catch (Exception e) { + LOGGER.error("Can not read current permissions change ID", e); + return NOTIFICATION_UNKNOWN; + } + } + }; + } + + /** + * @return Path change id + */ + public Gauge getPathChangeIdGauge() { + return () -> { + try { + return tm.executeTransaction( + pm -> getLastProcessedChangeIDCore(pm, MSentryPathChange.class) + ); + } catch (Exception e) { + LOGGER.error("Can not read current path change ID", e); + return NOTIFICATION_UNKNOWN; + } + }; + } + + /** + * Lets the test code know how many privs are in the db, so that we know + * if they are in fact being cleaned up when not being referenced any more. + * @return The number of rows in the db priv table. + */ + @VisibleForTesting + long countMSentryPrivileges() { + return getCount(MSentryPrivilege.class); + } + + @VisibleForTesting + void clearAllTables() { + try { + tm.executeTransaction( + pm -> { + pm.newQuery(MSentryRole.class).deletePersistentAll(); + pm.newQuery(MSentryGroup.class).deletePersistentAll(); + pm.newQuery(MSentryUser.class).deletePersistentAll(); + pm.newQuery(MSentryPrivilege.class).deletePersistentAll(); + pm.newQuery(MSentryPermChange.class).deletePersistentAll(); + pm.newQuery(MSentryPathChange.class).deletePersistentAll(); + pm.newQuery(MAuthzPathsMapping.class).deletePersistentAll(); + pm.newQuery(MPath.class).deletePersistentAll(); + pm.newQuery(MSentryHmsNotification.class).deletePersistentAll(); + pm.newQuery(MAuthzPathsSnapshotId.class).deletePersistentAll(); + return null; + }); + } catch (Exception e) { + // the method only for test, log the error and ignore the exception + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Removes all the information related to HMS Objects from sentry store. + */ + @VisibleForTesting + public void clearHmsPathInformation() throws Exception { + tm.executeTransactionWithRetry( + pm -> { + // Data in MAuthzPathsSnapshotId.class is not cleared intentionally. + // This data will help sentry retain the history of snapshots taken before + // and help in picking appropriate ID even when hdfs sync is enabled/disabled. + pm.newQuery(MSentryPathChange.class).deletePersistentAll(); + pm.newQuery(MAuthzPathsMapping.class).deletePersistentAll(); + pm.newQuery(MPath.class).deletePersistentAll(); + return null; + }); + } + + /** + * Purge a given delta change table, with a specified number of changes to be kept. + * + * @param cls the class of a perm/path delta change {@link MSentryPermChange} or + * {@link MSentryPathChange}. + * @param pm a {@link PersistenceManager} instance. + * @param changesToKeep the number of changes the caller want to keep. + * @param the type of delta change class. + */ + @VisibleForTesting + void purgeDeltaChangeTableCore( + Class cls, PersistenceManager pm, long changesToKeep) { + Preconditions.checkArgument(changesToKeep >= 0, + "changes to keep must be a non-negative number"); + long lastChangedID = getLastProcessedChangeIDCore(pm, cls); + long maxIDDeleted = lastChangedID - changesToKeep; + + Query query = pm.newQuery(cls); + query.addExtension(LOAD_RESULTS_AT_COMMIT, "false"); + + // It is an approximation of "SELECT ... LIMIT CHANGE_TO_KEEP" in SQL, because JDO w/ derby + // does not support "LIMIT". + // See: http://www.datanucleus.org/products/datanucleus/jdo/jdoql_declarative.html + query.setFilter("changeID <= maxChangedIdDeleted"); + query.declareParameters("long maxChangedIdDeleted"); + long numDeleted = query.deletePersistentAll(maxIDDeleted); + if (numDeleted > 0) { + LOGGER.info(String.format("Purged %d of %s to changeID=%d", + numDeleted, cls.getSimpleName(), maxIDDeleted)); + } + } + + /** + * Purge notification id table, keeping a specified number of entries. + * @param pm a {@link PersistenceManager} instance. + * @param changesToKeep the number of changes the caller want to keep. + */ + @VisibleForTesting + protected void purgeNotificationIdTableCore(PersistenceManager pm, + long changesToKeep) { + Preconditions.checkArgument(changesToKeep > 0, + "You need to keep at least one entry in SENTRY_HMS_NOTIFICATION_ID table"); + long lastNotificationID = getLastProcessedNotificationIDCore(pm); + Query query = pm.newQuery(MSentryHmsNotification.class); + query.addExtension(LOAD_RESULTS_AT_COMMIT, "false"); + + // It is an approximation of "SELECT ... LIMIT CHANGE_TO_KEEP" in SQL, because JDO w/ derby + // does not support "LIMIT". + // See: http://www.datanucleus.org/products/datanucleus/jdo/jdoql_declarative.html + query.setFilter("notificationId <= maxNotificationIdDeleted"); + query.declareParameters("long maxNotificationIdDeleted"); + long numDeleted = query.deletePersistentAll(lastNotificationID - changesToKeep); + if (numDeleted > 0) { + LOGGER.info("Purged {} of {}", numDeleted, MSentryHmsNotification.class.getSimpleName()); + } + } + + /** + * Purge delta change tables, {@link MSentryPermChange} and {@link MSentryPathChange}. + * The number of deltas to keep is configurable + */ + public void purgeDeltaChangeTables() { + final int changesToKeep = conf.getInt(ServerConfig.SENTRY_DELTA_KEEP_COUNT, + ServerConfig.SENTRY_DELTA_KEEP_COUNT_DEFAULT); + LOGGER.info("Purging MSentryPathUpdate and MSentyPermUpdate tables, leaving {} entries", + changesToKeep); + try { + tm.executeTransaction(pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + purgeDeltaChangeTableCore(MSentryPermChange.class, pm, changesToKeep); + LOGGER.info("MSentryPermChange table has been purged."); + purgeDeltaChangeTableCore(MSentryPathChange.class, pm, changesToKeep); + LOGGER.info("MSentryPathUpdate table has been purged."); + return null; + }); + } catch (Exception e) { + LOGGER.error("Delta change cleaning process encountered an error", e); + } + } + + /** + * Purge hms notification id table , {@link MSentryHmsNotification}. + * The number of notifications id's to be kept is based on configuration + * sentry.server.delta.keep.count + */ + public void purgeNotificationIdTable() { + final int changesToKeep = conf.getInt(ServerConfig.SENTRY_HMS_NOTIFICATION_ID_KEEP_COUNT, + ServerConfig.SENTRY_HMS_NOTIFICATION_ID_KEEP_COUNT_DEFAULT); + LOGGER.debug("Purging MSentryHmsNotification table, leaving {} entries", + changesToKeep); + try { + tm.executeTransaction(pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + purgeNotificationIdTableCore(pm, changesToKeep); + return null; + }); + } catch (Exception e) { + LOGGER.error("MSentryHmsNotification cleaning process encountered an error", e); + } + } + /** + * Alter a given sentry role to grant a privilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param privilege the given privilege + * @throws Exception + */ + void alterSentryRoleGrantPrivilege(final String grantorPrincipal, + final String roleName, final TSentryPrivilege privilege) throws Exception { + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedRoleName = trimAndLower(roleName); + // first do grant check + grantOptionCheck(pm, grantorPrincipal, privilege); + + // Alter sentry Role and grant Privilege. + MSentryPrivilege mPrivilege = alterSentryRoleGrantPrivilegeCore( + pm, trimmedRoleName, privilege); + + if (mPrivilege != null) { + // update the privilege to be the one actually updated. + convertToTSentryPrivilege(mPrivilege, privilege); + } + return null; + }); + } + + /** + * Alter a given sentry role to grant a set of privileges. + * Internally calls alterSentryRoleGrantPrivilege. + * + * @param grantorPrincipal User name + * @param roleName Role name + * @param privileges Set of privileges + * @throws Exception + */ + public void alterSentryRoleGrantPrivileges(final String grantorPrincipal, + final String roleName, final Set privileges) throws Exception { + for (TSentryPrivilege privilege : privileges) { + alterSentryRoleGrantPrivilege(grantorPrincipal, roleName, privilege); + } + } + + /** + * Alter a given sentry role to grant a privilege, as well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param privilege the given privilege + * @param update the corresponding permission delta update. + * @throws Exception + * + */ + synchronized void alterSentryRoleGrantPrivilege(final String grantorPrincipal, + final String roleName, final TSentryPrivilege privilege, + final Update update) throws Exception { + + execute(update, pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedRoleName = trimAndLower(roleName); + // first do grant check + grantOptionCheck(pm, grantorPrincipal, privilege); + + // Alter sentry Role and grant Privilege. + MSentryPrivilege mPrivilege = alterSentryRoleGrantPrivilegeCore(pm, + trimmedRoleName, privilege); + + if (mPrivilege != null) { + // update the privilege to be the one actually updated. + convertToTSentryPrivilege(mPrivilege, privilege); + } + return null; + }); + } + + /** + * Alter a given sentry role to grant a set of privileges, as well as persist the + * corresponding permission change to MSentryPermChange table in a single transaction. + * Internally calls alterSentryRoleGrantPrivilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param privileges a Set of privileges + * @param privilegesUpdateMap the corresponding map + * @throws Exception + * + */ + public void alterSentryRoleGrantPrivileges(final String grantorPrincipal, + final String roleName, final Set privileges, + final Map privilegesUpdateMap) throws Exception { + + Preconditions.checkNotNull(privilegesUpdateMap); + for (TSentryPrivilege privilege : privileges) { + Update update = privilegesUpdateMap.get(privilege); + if (update != null) { + alterSentryRoleGrantPrivilege(grantorPrincipal, roleName, privilege, + update); + } else { + alterSentryRoleGrantPrivilege(grantorPrincipal, roleName, privilege); + } + } + } + + private MSentryPrivilege alterSentryRoleGrantPrivilegeCore(PersistenceManager pm, + String roleName, TSentryPrivilege privilege) + throws SentryNoSuchObjectException, SentryInvalidInputException { + MSentryPrivilege mPrivilege = null; + MSentryRole mRole = getRole(pm, roleName); + if (mRole == null) { + throw noSuchRole(roleName); + } + + if(privilege.getPrivilegeScope().equalsIgnoreCase(PrivilegeScope.URI.name()) + && StringUtils.isBlank(privilege.getURI())) { + throw new SentryInvalidInputException("cannot grant URI privileges to Null or EMPTY location"); + } + + if (!isNULL(privilege.getColumnName()) || !isNULL(privilege.getTableName()) + || !isNULL(privilege.getDbName())) { + // If Grant is for ALL and Either INSERT/SELECT already exists.. + // need to remove it and GRANT ALL.. + if (AccessConstants.ALL.equalsIgnoreCase(privilege.getAction()) + || AccessConstants.ACTION_ALL.equalsIgnoreCase(privilege.getAction())) { + TSentryPrivilege tNotAll = new TSentryPrivilege(privilege); + tNotAll.setAction(AccessConstants.SELECT); + MSentryPrivilege mSelect = getMSentryPrivilege(tNotAll, pm); + tNotAll.setAction(AccessConstants.INSERT); + MSentryPrivilege mInsert = getMSentryPrivilege(tNotAll, pm); + if ((mSelect != null) && mRole.getPrivileges().contains(mSelect)) { + mSelect.removeRole(mRole); + pm.makePersistent(mSelect); + } + if ((mInsert != null) && mRole.getPrivileges().contains(mInsert)) { + mInsert.removeRole(mRole); + pm.makePersistent(mInsert); + } + } else { + // If Grant is for Either INSERT/SELECT and ALL already exists.. + // do nothing.. + TSentryPrivilege tAll = new TSentryPrivilege(privilege); + tAll.setAction(AccessConstants.ALL); + MSentryPrivilege mAll1 = getMSentryPrivilege(tAll, pm); + tAll.setAction(AccessConstants.ACTION_ALL); + MSentryPrivilege mAll2 = getMSentryPrivilege(tAll, pm); + if (mAll1 != null && mRole.getPrivileges().contains(mAll1)) { + return null; + } + if (mAll2 != null && mRole.getPrivileges().contains(mAll2)) { + return null; + } + } + } + + mPrivilege = getMSentryPrivilege(privilege, pm); + if (mPrivilege == null) { + mPrivilege = convertToMSentryPrivilege(privilege); + } + mPrivilege.appendRole(mRole); + pm.makePersistent(mPrivilege); + return mPrivilege; + } + + /** + * Alter a given sentry user to grant a set of privileges. + * Internally calls alterSentryUserGrantPrivilege. + * + * @param grantorPrincipal User name + * @param userName User name + * @param privileges Set of privileges + * @throws Exception + */ + public void alterSentryUserGrantPrivileges(final String grantorPrincipal, + final String userName, final Set privileges) throws Exception { + + try { + MSentryUser userEntry = getMSentryUserByName(userName, false); + if (userEntry == null) { + createSentryUser(userName); + } + } catch (SentryAlreadyExistsException e) { + // the user may be created by other thread, so swallow the exception and proceed + } + + for (TSentryPrivilege privilege : privileges) { + alterSentryUserGrantPrivilege(grantorPrincipal, userName, privilege); + } + } + + /** + * Alter a given sentry user to grant a privilege. + * + * @param grantorPrincipal User name + * @param userName the given user name + * @param privilege the given privilege + * @throws Exception + */ + void alterSentryUserGrantPrivilege(final String grantorPrincipal, + final String userName, final TSentryPrivilege privilege) throws Exception { + tm.executeTransactionWithRetry( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedUserName = trimAndLower(userName); + // first do grant check + grantOptionCheck(pm, grantorPrincipal, privilege); + + // Alter sentry User and grant Privilege. + MSentryPrivilege mPrivilege = alterSentryUserGrantPrivilegeCore( + pm, trimmedUserName, privilege); + + if (mPrivilege != null) { + // update the privilege to be the one actually updated. + convertToTSentryPrivilege(mPrivilege, privilege); + } + return null; + } + }); + } + + /** + * Alter a given sentry user to grant a privilege, as well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param grantorPrincipal User name + * @param userName the given user name + * @param privilege the given privilege + * @param update the corresponding permission delta update. + * @throws Exception + * + */ + synchronized void alterSentryUserGrantPrivilege(final String grantorPrincipal, + final String userName, final TSentryPrivilege privilege, + final Update update) throws Exception { + + execute(update, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedUserName = trimAndLower(userName); + // first do grant check + grantOptionCheck(pm, grantorPrincipal, privilege); + + // Alter sentry User and grant Privilege. + MSentryPrivilege mPrivilege = alterSentryUserGrantPrivilegeCore(pm, + trimmedUserName, privilege); + + if (mPrivilege != null) { + // update the privilege to be the one actually updated. + convertToTSentryPrivilege(mPrivilege, privilege); + } + return null; + } + }); + } + + /** + * Alter a given sentry user to grant a set of privileges, as well as persist the + * corresponding permission change to MSentryPermChange table in a single transaction. + * Internally calls alterSentryUserGrantPrivilege. + * + * @param grantorPrincipal User name + * @param userName the given user name + * @param privileges a Set of privileges + * @param privilegesUpdateMap the corresponding map + * @throws Exception + * + */ + public void alterSentryUserGrantPrivileges(final String grantorPrincipal, + final String userName, final Set privileges, + final Map privilegesUpdateMap) throws Exception { + + try { + MSentryUser userEntry = getMSentryUserByName(userName, false); + if (userEntry == null) { + createSentryUser(userName); + } + } catch (SentryAlreadyExistsException e) { + // the user may be created by other thread, so swallow the exception and proeed + } + + Preconditions.checkNotNull(privilegesUpdateMap); + for (TSentryPrivilege privilege : privileges) { + Update update = privilegesUpdateMap.get(privilege); + if (update != null) { + alterSentryUserGrantPrivilege(grantorPrincipal, userName, privilege, + update); + } else { + alterSentryUserGrantPrivilege(grantorPrincipal, userName, privilege); + } + } + } + + /** + * Get the user entry by user name + * @param userName the name of the user + * @return the user entry + * @throws Exception if the specified user does not exist + */ + @VisibleForTesting + public MSentryUser getMSentryUserByName(final String userName) throws Exception { + return getMSentryUserByName(userName, true); + } + + /** + * Get the user entry by user name + * @param userName the name of the user + * @param throwExceptionIfNotExist true: throw exception if user does not exist; false: return null + * @return the user entry or null + * @throws Exception if the specified user does not exist and throwExceptionIfNotExist is true + */ + MSentryUser getMSentryUserByName(final String userName, boolean throwExceptionIfNotExist) throws Exception { + return tm.executeTransaction( + new TransactionBlock() { + public MSentryUser execute(PersistenceManager pm) throws Exception { + String trimmedUserName = trimAndLower(userName); + MSentryUser sentryUser = getUser(pm, trimmedUserName); + if (sentryUser == null) { + if (throwExceptionIfNotExist) { + throw noSuchUser(trimmedUserName); + } + else { + return null; + } + } + return sentryUser; + } + }); + } + + private MSentryPrivilege alterSentryUserGrantPrivilegeCore(PersistenceManager pm, + String userName, TSentryPrivilege privilege) + throws SentryNoSuchObjectException, SentryInvalidInputException { + MSentryPrivilege mPrivilege = null; + MSentryUser mUser = getUser(pm, userName); + if (mUser == null) { + throw noSuchUser(userName); + } + + if(privilege.getPrivilegeScope().equalsIgnoreCase(PrivilegeScope.URI.name()) + && StringUtils.isBlank(privilege.getURI())) { + throw new SentryInvalidInputException("cannot grant URI privileges to Null or EMPTY location"); + } + + if (!isNULL(privilege.getColumnName()) || !isNULL(privilege.getTableName()) + || !isNULL(privilege.getDbName())) { + // If Grant is for ALL and Either INSERT/SELECT already exists.. + // need to remove it and GRANT ALL.. + if (AccessConstants.ALL.equalsIgnoreCase(privilege.getAction()) + || AccessConstants.ACTION_ALL.equalsIgnoreCase(privilege.getAction())) { + TSentryPrivilege tNotAll = new TSentryPrivilege(privilege); + tNotAll.setAction(AccessConstants.SELECT); + MSentryPrivilege mSelect = getMSentryPrivilege(tNotAll, pm); + tNotAll.setAction(AccessConstants.INSERT); + MSentryPrivilege mInsert = getMSentryPrivilege(tNotAll, pm); + if ((mSelect != null) && mUser.getPrivileges().contains(mSelect)) { + mSelect.removeUser(mUser); + pm.makePersistent(mSelect); + } + if ((mInsert != null) && mUser.getPrivileges().contains(mInsert)) { + mInsert.removeUser(mUser); + pm.makePersistent(mInsert); + } + } else { + // If Grant is for Either INSERT/SELECT and ALL already exists.. + // do nothing.. + TSentryPrivilege tAll = new TSentryPrivilege(privilege); + tAll.setAction(AccessConstants.ALL); + MSentryPrivilege mAll1 = getMSentryPrivilege(tAll, pm); + tAll.setAction(AccessConstants.ACTION_ALL); + MSentryPrivilege mAll2 = getMSentryPrivilege(tAll, pm); + if (mAll1 != null && mUser.getPrivileges().contains(mAll1)) { + return null; + } + if (mAll2 != null && mUser.getPrivileges().contains(mAll2)) { + return null; + } + } + } + + mPrivilege = getMSentryPrivilege(privilege, pm); + if (mPrivilege == null) { + mPrivilege = convertToMSentryPrivilege(privilege); + } + mPrivilege.appendUser(mUser); + pm.makePersistent(mPrivilege); + return mPrivilege; + } + + /** + * Alter a given sentry user to revoke a privilege. + * + * @param grantorPrincipal User name + * @param userName the given user name + * @param tPrivilege the given privilege + * @throws Exception + * + */ + void alterSentryUserRevokePrivilege(final String grantorPrincipal, + final String userName, final TSentryPrivilege tPrivilege) throws Exception { + + tm.executeTransactionWithRetry( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedUserName = safeTrimLower(userName); + // first do revoke check + grantOptionCheck(pm, grantorPrincipal, tPrivilege); + + alterSentryUserRevokePrivilegeCore(pm, trimmedUserName, tPrivilege); + return null; + } + }); + } + + /** + * Alter a given sentry user to revoke a set of privileges. + * Internally calls alterSentryUserRevokePrivilege. + * + * @param grantorPrincipal User name + * @param userName the given user name + * @param tPrivileges a Set of privileges + * @throws Exception + * + */ + public void alterSentryUserRevokePrivileges(final String grantorPrincipal, + final String userName, final Set tPrivileges) throws Exception { + for (TSentryPrivilege tPrivilege : tPrivileges) { + alterSentryUserRevokePrivilege(grantorPrincipal, userName, tPrivilege); + } + } + + /** + * Alter a given sentry user to revoke a set of privileges, as well as persist the + * corresponding permission change to MSentryPermChange table in a single transaction. + * Internally calls alterSentryUserRevokePrivilege. + * + * @param grantorPrincipal User name + * @param userName the given user name + * @param tPrivileges a Set of privileges + * @param privilegesUpdateMap the corresponding map + * @throws Exception + * + */ + public void alterSentryUserRevokePrivileges(final String grantorPrincipal, + final String userName, final Set tPrivileges, + final Map privilegesUpdateMap) + throws Exception { + + Preconditions.checkNotNull(privilegesUpdateMap); + for (TSentryPrivilege tPrivilege : tPrivileges) { + Update update = privilegesUpdateMap.get(tPrivilege); + if (update != null) { + alterSentryUserRevokePrivilege(grantorPrincipal, userName, + tPrivilege, update); + } else { + alterSentryUserRevokePrivilege(grantorPrincipal, userName, + tPrivilege); + } + } + } + + /** + * Alter a given sentry user to revoke a privilege, as well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param grantorPrincipal User name + * @param userName the given user name + * @param tPrivilege the given privilege + * @param update the corresponding permission delta update transaction block + * @throws Exception + * + */ + private synchronized void alterSentryUserRevokePrivilege(final String grantorPrincipal, + final String userName, final TSentryPrivilege tPrivilege, + final Update update) throws Exception { + execute(update, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedUserName = safeTrimLower(userName); + // first do revoke check + grantOptionCheck(pm, grantorPrincipal, tPrivilege); + + alterSentryUserRevokePrivilegeCore(pm, trimmedUserName, tPrivilege); + return null; + } + }); + } + + private void alterSentryUserRevokePrivilegeCore(PersistenceManager pm, + String userName, TSentryPrivilege tPrivilege) + throws SentryNoSuchObjectException, SentryInvalidInputException { + MSentryUser mUser = getUser(pm, userName); + if (mUser == null) { + throw noSuchUser(userName); + } + if(tPrivilege.getPrivilegeScope().equalsIgnoreCase(PrivilegeScope.URI.name()) + && StringUtils.isBlank(tPrivilege.getURI())) { + throw new SentryInvalidInputException("cannot revoke URI privileges from Null or EMPTY location"); + } + + MSentryPrivilege mPrivilege = getMSentryPrivilege(tPrivilege, pm); + if (mPrivilege == null) { + mPrivilege = convertToMSentryPrivilege(tPrivilege); + } else { + mPrivilege = pm.detachCopy(mPrivilege); + } + + Set privilegeGraph = new HashSet<>(); + if (mPrivilege.getGrantOption() != null) { + privilegeGraph.add(mPrivilege); + } else { + MSentryPrivilege mTure = new MSentryPrivilege(mPrivilege); + mTure.setGrantOption(true); + privilegeGraph.add(mTure); + MSentryPrivilege mFalse = new MSentryPrivilege(mPrivilege); + mFalse.setGrantOption(false); + privilegeGraph.add(mFalse); + } + // Get the privilege graph + populateChildren(pm, SentryEntityType.USER, Sets.newHashSet(userName), mPrivilege, privilegeGraph); + for (MSentryPrivilege childPriv : privilegeGraph) { + revokePrivilegeFromUser(pm, tPrivilege, mUser, childPriv); + } + pm.makePersistent(mUser); + } + + /** + * Alter a given sentry role to revoke a privilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param tPrivilege the given privilege + * @throws Exception + * + */ + void alterSentryRoleRevokePrivilege(final String grantorPrincipal, + final String roleName, final TSentryPrivilege tPrivilege) throws Exception { + + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedRoleName = safeTrimLower(roleName); + // first do revoke check + grantOptionCheck(pm, grantorPrincipal, tPrivilege); + + alterSentryRoleRevokePrivilegeCore(pm, trimmedRoleName, tPrivilege); + return null; + }); + } + + /** + * Alter a given sentry role to revoke a set of privileges. + * Internally calls alterSentryRoleRevokePrivilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param tPrivileges a Set of privileges + * @throws Exception + * + */ + public void alterSentryRoleRevokePrivileges(final String grantorPrincipal, + final String roleName, final Set tPrivileges) throws Exception { + for (TSentryPrivilege tPrivilege : tPrivileges) { + alterSentryRoleRevokePrivilege(grantorPrincipal, roleName, tPrivilege); + } + } + + /** + * Alter a given sentry role to revoke a privilege, as well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param tPrivilege the given privilege + * @param update the corresponding permission delta update transaction block + * @throws Exception + * + */ + private synchronized void alterSentryRoleRevokePrivilege(final String grantorPrincipal, + final String roleName, final TSentryPrivilege tPrivilege, + final Update update) throws Exception { + execute(update, pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedRoleName = safeTrimLower(roleName); + // first do revoke check + grantOptionCheck(pm, grantorPrincipal, tPrivilege); + + alterSentryRoleRevokePrivilegeCore(pm, trimmedRoleName, tPrivilege); + return null; + }); + } + + /** + * Alter a given sentry role to revoke a set of privileges, as well as persist the + * corresponding permission change to MSentryPermChange table in a single transaction. + * Internally calls alterSentryRoleRevokePrivilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param tPrivileges a Set of privileges + * @param privilegesUpdateMap the corresponding map + * @throws Exception + * + */ + public void alterSentryRoleRevokePrivileges(final String grantorPrincipal, + final String roleName, final Set tPrivileges, + final Map privilegesUpdateMap) + throws Exception { + + Preconditions.checkNotNull(privilegesUpdateMap); + for (TSentryPrivilege tPrivilege : tPrivileges) { + Update update = privilegesUpdateMap.get(tPrivilege); + if (update != null) { + alterSentryRoleRevokePrivilege(grantorPrincipal, roleName, + tPrivilege, update); + } else { + alterSentryRoleRevokePrivilege(grantorPrincipal, roleName, + tPrivilege); + } + } + } + + private void alterSentryRoleRevokePrivilegeCore(PersistenceManager pm, + String roleName, TSentryPrivilege tPrivilege) + throws SentryNoSuchObjectException, SentryInvalidInputException { + MSentryRole mRole = getRole(pm, roleName); + if (mRole == null) { + throw noSuchRole(roleName); + } + if(tPrivilege.getPrivilegeScope().equalsIgnoreCase(PrivilegeScope.URI.name()) + && StringUtils.isBlank(tPrivilege.getURI())) { + throw new SentryInvalidInputException("cannot revoke URI privileges from Null or EMPTY location"); + } + + MSentryPrivilege mPrivilege = getMSentryPrivilege(tPrivilege, pm); + if (mPrivilege == null) { + mPrivilege = convertToMSentryPrivilege(tPrivilege); + } else { + mPrivilege = pm.detachCopy(mPrivilege); + } + + Set privilegeGraph = new HashSet<>(); + if (mPrivilege.getGrantOption() != null) { + privilegeGraph.add(mPrivilege); + } else { + MSentryPrivilege mTure = new MSentryPrivilege(mPrivilege); + mTure.setGrantOption(true); + privilegeGraph.add(mTure); + MSentryPrivilege mFalse = new MSentryPrivilege(mPrivilege); + mFalse.setGrantOption(false); + privilegeGraph.add(mFalse); + } + // Get the privilege graph + populateChildren(pm, SentryEntityType.ROLE, Sets.newHashSet(roleName), mPrivilege, privilegeGraph); + for (MSentryPrivilege childPriv : privilegeGraph) { + revokePrivilegeFromRole(pm, tPrivilege, mRole, childPriv); + } + pm.makePersistent(mRole); + } + + /** + * Roles can be granted ALL, SELECT, and INSERT on tables. When + * a role has ALL and SELECT or INSERT are revoked, we need to remove the ALL + * privilege and add SELECT (INSERT was revoked) or INSERT (SELECT was revoked). + */ + private void revokePartial(PersistenceManager pm, + TSentryPrivilege requestedPrivToRevoke, + MSentryRole mRole, MSentryUser mUser, + MSentryPrivilege currentPrivilege) throws SentryInvalidInputException { + MSentryPrivilege persistedPriv = + getMSentryPrivilege(convertToTSentryPrivilege(currentPrivilege), pm); + if (persistedPriv == null) { + // The privilege corresponding to the currentPrivilege doesn't exist in the persistent + // store, so we create a fake one for the code below. The fake one is not associated with + // any role and shouldn't be stored in the persistent storage. + persistedPriv = convertToMSentryPrivilege(convertToTSentryPrivilege(currentPrivilege)); + } + + if (requestedPrivToRevoke.getAction().equalsIgnoreCase(AccessConstants.ALL) || + requestedPrivToRevoke.getAction().equalsIgnoreCase(AccessConstants.ACTION_ALL)) { + if (!persistedPriv.getRoles().isEmpty()) { + if (mRole != null) { + persistedPriv.removeRole(mRole); + } + if (mUser != null) { + persistedPriv.removeUser(mUser); + } + + if (isPrivilegeStall(persistedPriv)) { + pm.deletePersistent(persistedPriv); + } else { + pm.makePersistent(persistedPriv); + } + } + } else { + + Set addActions = new HashSet(); + for (String actionToAdd : PARTIAL_REVOKE_ACTIONS) { + if( !requestedPrivToRevoke.getAction().equalsIgnoreCase(actionToAdd) && + !currentPrivilege.getAction().equalsIgnoreCase(actionToAdd) && + !AccessConstants.ALL.equalsIgnoreCase(actionToAdd) && + !AccessConstants.ACTION_ALL.equalsIgnoreCase(actionToAdd)) { + addActions.add(actionToAdd); + } + } + + if (mRole != null) { + revokeRolePartial(pm, mRole, currentPrivilege, persistedPriv, addActions); + } + + if (mUser != null) { + revokeUserPartial(pm, mUser, currentPrivilege, persistedPriv, addActions); + } + } + } + + private boolean isPrivilegeStall(MSentryPrivilege privilege) { + if (privilege.getUsers().isEmpty() && privilege.getRoles().isEmpty()) { + return true; + } + + return false; + } + + private boolean isPrivilegeStall(MSentryGMPrivilege privilege) { + if (privilege.getRoles().isEmpty()) { + return true; + } + + return false; + } + + private void revokeRolePartial(PersistenceManager pm, MSentryRole mRole, + MSentryPrivilege currentPrivilege, + MSentryPrivilege persistedPriv, + Set addActions) throws SentryInvalidInputException { + // If table / URI, remove ALL + persistedPriv = getMSentryPrivilege(convertToTSentryPrivilege(persistedPriv), pm); + if (persistedPriv != null && !persistedPriv.getRoles().isEmpty()) { + persistedPriv.removeRole(mRole); + if (isPrivilegeStall(persistedPriv)) { + pm.deletePersistent(persistedPriv); + } else { + pm.makePersistent(persistedPriv); + } + } + currentPrivilege.setAction(AccessConstants.ALL); + persistedPriv = getMSentryPrivilege(convertToTSentryPrivilege(currentPrivilege), pm); + if (persistedPriv != null && mRole.getPrivileges().contains(persistedPriv)) { + persistedPriv.removeRole(mRole); + if (isPrivilegeStall(persistedPriv)) { + pm.deletePersistent(persistedPriv); + } else { + pm.makePersistent(persistedPriv); + } + + // add decomposted actions + for (String addAction : addActions) { + currentPrivilege.setAction(addAction); + TSentryPrivilege tSentryPrivilege = convertToTSentryPrivilege(currentPrivilege); + persistedPriv = getMSentryPrivilege(tSentryPrivilege, pm); + if (persistedPriv == null) { + persistedPriv = convertToMSentryPrivilege(tSentryPrivilege); + } + mRole.appendPrivilege(persistedPriv); + } + persistedPriv.appendRole(mRole); + pm.makePersistent(persistedPriv); + } + } + + private void revokeUserPartial(PersistenceManager pm, MSentryUser mUser, + MSentryPrivilege currentPrivilege, + MSentryPrivilege persistedPriv, + Set addActions) throws SentryInvalidInputException { + // If table / URI, remove ALL + persistedPriv = getMSentryPrivilege(convertToTSentryPrivilege(persistedPriv), pm); + if (persistedPriv != null && !persistedPriv.getUsers().isEmpty()) { + persistedPriv.removeUser(mUser); + if (isPrivilegeStall(persistedPriv)) { + pm.deletePersistent(persistedPriv); + } else { + pm.makePersistent(persistedPriv); + } + } + currentPrivilege.setAction(AccessConstants.ALL); + persistedPriv = getMSentryPrivilege(convertToTSentryPrivilege(currentPrivilege), pm); + if (persistedPriv != null && mUser.getPrivileges().contains(persistedPriv)) { + persistedPriv.removeUser(mUser); + if (isPrivilegeStall(persistedPriv)) { + pm.deletePersistent(persistedPriv); + } else { + pm.makePersistent(persistedPriv); + } + + // add decomposted actions + for (String addAction : addActions) { + currentPrivilege.setAction(addAction); + TSentryPrivilege tSentryPrivilege = convertToTSentryPrivilege(currentPrivilege); + persistedPriv = getMSentryPrivilege(tSentryPrivilege, pm); + if (persistedPriv == null) { + persistedPriv = convertToMSentryPrivilege(tSentryPrivilege); + } + mUser.appendPrivilege(persistedPriv); + } + persistedPriv.appendUser(mUser); + pm.makePersistent(persistedPriv); + } + } + + /** + * Revoke privilege from role + */ + private void revokePrivilegeFromRole(PersistenceManager pm, TSentryPrivilege tPrivilege, + MSentryRole mRole, MSentryPrivilege mPrivilege) + throws SentryInvalidInputException { + if (PARTIAL_REVOKE_ACTIONS.contains(mPrivilege.getAction())) { + // if this privilege is in parital revoke actions + // we will do partial revoke + revokePartial(pm, tPrivilege, mRole, null, mPrivilege); + } else { + // otherwise, + // we will revoke it from role directly + MSentryPrivilege persistedPriv = getMSentryPrivilege(convertToTSentryPrivilege(mPrivilege), pm); + if (persistedPriv != null && !persistedPriv.getRoles().isEmpty()) { + persistedPriv.removeRole(mRole); + if (isPrivilegeStall(persistedPriv)) { + pm.deletePersistent(persistedPriv); + } else { + pm.makePersistent(persistedPriv); + } + } + } + } + + /** + * Revoke privilege from user + */ + private void revokePrivilegeFromUser(PersistenceManager pm, TSentryPrivilege tPrivilege, + MSentryUser mUser, MSentryPrivilege mPrivilege) + throws SentryInvalidInputException { + if (PARTIAL_REVOKE_ACTIONS.contains(mPrivilege.getAction())) { + // if this privilege is in parital revoke actions + // we will do partial revoke + revokePartial(pm, tPrivilege, null, mUser, mPrivilege); + } else { + // otherwise, + // we will revoke it from user directly + MSentryPrivilege persistedPriv = getMSentryPrivilege(convertToTSentryPrivilege(mPrivilege), pm); + if (persistedPriv != null && !persistedPriv.getUsers().isEmpty()) { + persistedPriv.removeUser(mUser); + if (isPrivilegeStall(persistedPriv)) { + pm.deletePersistent(persistedPriv); + } else { + pm.makePersistent(persistedPriv); + } + } + } + } + + /** + * Explore Privilege graph and collect child privileges. + * The responsibility to commit/rollback the transaction should be handled by the caller. + */ + private void populateChildren(PersistenceManager pm, SentryEntityType entityType, Set entityNames, MSentryPrivilege priv, + Collection children) throws SentryInvalidInputException { + Preconditions.checkNotNull(pm); + if (!isNULL(priv.getServerName()) || !isNULL(priv.getDbName()) + || !isNULL(priv.getTableName())) { + // Get all TableLevel Privs + Set childPrivs = getChildPrivileges(pm, entityType, entityNames, priv); + for (MSentryPrivilege childPriv : childPrivs) { + // Only recurse for table level privs.. + if (!isNULL(childPriv.getDbName()) && !isNULL(childPriv.getTableName()) + && !isNULL(childPriv.getColumnName())) { + populateChildren(pm, entityType, entityNames, childPriv, children); + } + // The method getChildPrivileges() didn't do filter on "action", + // if the action is not "All", it should judge the action of children privilege. + // For example: a user has a privilege “All on Col1”, + // if the operation is “REVOKE INSERT on table” + // the privilege should be the child of table level privilege. + // but the privilege may still have other meaning, likes "SELECT, CREATE etc. on Col1". + // and the privileges like "SELECT, CREATE etc. on Col1" should not be revoke. + if (!priv.isActionALL()) { + if (childPriv.isActionALL()) { + // If the child privilege is All, we should convert it to the same + // privilege with parent + childPriv.setAction(priv.getAction()); + } + // Only include privilege that imply the parent privilege. + if (!priv.implies(childPriv)) { + continue; + } + } + children.add(childPriv); + } + } + } + + private Set getChildPrivileges(PersistenceManager pm, SentryEntityType entityType, Set entityNames, + MSentryPrivilege parent) throws SentryInvalidInputException { + // Column and URI do not have children + if (!isNULL(parent.getColumnName()) || !isNULL(parent.getURI())) { + return Collections.emptySet(); + } + + Query query = pm.newQuery(MSentryPrivilege.class); + QueryParamBuilder paramBuilder = null; + if (entityType == SentryEntityType.ROLE) { + paramBuilder = QueryParamBuilder.addRolesFilter(query, null, entityNames).add(SERVER_NAME, parent.getServerName()); + } else if (entityType == SentryEntityType.USER) { + paramBuilder = QueryParamBuilder.addUsersFilter(query, null, entityNames).add(SERVER_NAME, parent.getServerName()); + } else { + throw new SentryInvalidInputException("entityType" + entityType + " is not valid"); + } + + if (!isNULL(parent.getDbName())) { + paramBuilder.add(DB_NAME, parent.getDbName()); + if (!isNULL(parent.getTableName())) { + paramBuilder.add(TABLE_NAME, parent.getTableName()) + .addNotNull(COLUMN_NAME); + } else { + paramBuilder.addNotNull(TABLE_NAME); + } + } else { + // Add condition dbName != NULL || URI != NULL + paramBuilder.newChild() + .addNotNull(DB_NAME) + .addNotNull(URI); + } + + query.setFilter(paramBuilder.toString()); + query.setResult("privilegeScope, serverName, dbName, tableName, columnName," + + " URI, action, grantOption"); + List privObjects = + (List) query.executeWithMap(paramBuilder.getArguments()); + Set privileges = new HashSet<>(privObjects.size()); + for (Object[] privObj : privObjects) { + String scope = (String)privObj[0]; + String serverName = (String)privObj[1]; + String dbName = (String)privObj[2]; + String tableName = (String) privObj[3]; + String columnName = (String) privObj[4]; + String URI = (String) privObj[5]; + String action = (String) privObj[6]; + Boolean grantOption = (Boolean) privObj[7]; + MSentryPrivilege priv = + new MSentryPrivilege(scope, serverName, dbName, tableName, + columnName, URI, action, grantOption); + privileges.add(priv); + } + return privileges; + } + + /** + * Drop a given sentry user. + * + * @param userName the given user name + * @throws Exception + */ + public void dropSentryUser(final String userName) throws Exception { + tm.executeTransactionWithRetry( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + pm.setDetachAllOnCommit(false); // No need to detach objects + dropSentryUserCore(pm, userName); + return null; + } + }); + } + + /** + * Drop a given sentry user. As well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param userName the given user name + * @param update the corresponding permission delta update + * @throws Exception + */ + public synchronized void dropSentryUser(final String userName, + final Update update) throws Exception { + execute(update, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + pm.setDetachAllOnCommit(false); // No need to detach objects + dropSentryUserCore(pm, userName); + return null; + } + }); + } + + private void dropSentryUserCore(PersistenceManager pm, String userName) + throws SentryNoSuchObjectException { + String lUserName = trimAndLower(userName); + MSentryUser sentryUser = getUser(pm, lUserName); + if (sentryUser == null) { + throw noSuchUser(lUserName); + } + removePrivilegesForUser(pm, sentryUser); + pm.deletePersistent(sentryUser); + } + + /** + * Removes all the privileges associated with + * a particular user. After this dis-association if the + * privilege doesn't have any users associated it will be + * removed from the underlying persistence layer. + * @param pm Instance of PersistenceManager + * @param sentryUser User for which all the privileges are to be removed. + */ + private void removePrivilegesForUser(PersistenceManager pm, MSentryUser sentryUser) { + List privilegesCopy = new ArrayList<>(sentryUser.getPrivileges()); + + sentryUser.removePrivileges(); + + removeStaledPrivileges(pm, privilegesCopy); + } + + @SuppressWarnings("unchecked") + private List getMSentryPrivileges(TSentryPrivilege tPriv, PersistenceManager pm) { + Query query = pm.newQuery(MSentryPrivilege.class); + QueryParamBuilder paramBuilder = QueryParamBuilder.newQueryParamBuilder(); + paramBuilder + .add(SERVER_NAME, tPriv.getServerName()) + .add("action", tPriv.getAction()); + + if (!isNULL(tPriv.getDbName())) { + paramBuilder.add(DB_NAME, tPriv.getDbName()); + if (!isNULL(tPriv.getTableName())) { + paramBuilder.add(TABLE_NAME, tPriv.getTableName()); + if (!isNULL(tPriv.getColumnName())) { + paramBuilder.add(COLUMN_NAME, tPriv.getColumnName()); + } + } + } else if (!isNULL(tPriv.getURI())) { + // if db is null, uri is not null + paramBuilder.add(URI, tPriv.getURI(), true); + } + + query.setFilter(paramBuilder.toString()); + return (List) query.executeWithMap(paramBuilder.getArguments()); + } + + private MSentryPrivilege getMSentryPrivilege(TSentryPrivilege tPriv, PersistenceManager pm) { + Boolean grantOption = null; + if (tPriv.getGrantOption().equals(TSentryGrantOption.TRUE)) { + grantOption = true; + } else if (tPriv.getGrantOption().equals(TSentryGrantOption.FALSE)) { + grantOption = false; + } + + QueryParamBuilder paramBuilder = QueryParamBuilder.newQueryParamBuilder(); + paramBuilder.add(SERVER_NAME, tPriv.getServerName()) + .add(DB_NAME, tPriv.getDbName()) + .add(TABLE_NAME, tPriv.getTableName()) + .add(COLUMN_NAME, tPriv.getColumnName()) + .add(URI, tPriv.getURI(), true) + .addObject(GRANT_OPTION, grantOption) + .add(ACTION, tPriv.getAction()); + + Query query = pm.newQuery(MSentryPrivilege.class); + query.setUnique(true); + query.setFilter(paramBuilder.toString()); + return (MSentryPrivilege)query.executeWithMap(paramBuilder.getArguments()); + } + + /** + * Drop a given sentry role. + * + * @param roleName the given role name + * @throws Exception + */ + public void dropSentryRole(final String roleName) throws Exception { + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + dropSentryRoleCore(pm, roleName); + return null; + }); + } + + /** + * Drop a given sentry role. As well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param roleName the given role name + * @param update the corresponding permission delta update + * @throws Exception + */ + public synchronized void dropSentryRole(final String roleName, + final Update update) throws Exception { + execute(update, pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + dropSentryRoleCore(pm, roleName); + return null; + }); + } + + private void dropSentryRoleCore(PersistenceManager pm, String roleName) + throws SentryNoSuchObjectException { + String lRoleName = trimAndLower(roleName); + MSentryRole sentryRole = getRole(pm, lRoleName); + if (sentryRole == null) { + throw noSuchRole(lRoleName); + } + removePrivileges(pm, sentryRole); + pm.deletePersistent(sentryRole); + } + + /** + * Removes all the privileges associated with + * a particular role. After this dis-association if the + * privilege doesn't have any roles associated it will be + * removed from the underlying persistence layer. + * @param pm Instance of PersistenceManager + * @param sentryRole Role for which all the privileges are to be removed. + */ + private void removePrivileges(PersistenceManager pm, MSentryRole sentryRole) { + List privilegesCopy = new ArrayList<>(sentryRole.getPrivileges()); + List gmPrivilegesCopy = new ArrayList<>(sentryRole.getGmPrivileges()); + + sentryRole.removePrivileges(); + // with SENTRY-398 generic model + sentryRole.removeGMPrivileges(); + + removeStaledPrivileges(pm, privilegesCopy); + removeStaledGMPrivileges(pm, gmPrivilegesCopy); + } + + private void removeStaledPrivileges(PersistenceManager pm, List privilegesCopy) { + List stalePrivileges = new ArrayList<>(0); + for (MSentryPrivilege privilege : privilegesCopy) { + if (isPrivilegeStall(privilege)) { + stalePrivileges.add(privilege); + } + } + if(!stalePrivileges.isEmpty()) { + pm.deletePersistentAll(stalePrivileges); + } + } + + private void removeStaledGMPrivileges(PersistenceManager pm, List privilegesCopy) { + List stalePrivileges = new ArrayList<>(0); + for (MSentryGMPrivilege privilege : privilegesCopy) { + if (isPrivilegeStall(privilege)) { + stalePrivileges.add(privilege); + } + } + if(!stalePrivileges.isEmpty()) { + pm.deletePersistentAll(stalePrivileges); + } + } + + /** + * Assign a given role to a set of groups. + * + * @param grantorPrincipal grantorPrincipal currently is not used. + * @param roleName the role to be assigned to the groups. + * @param groupNames the list of groups to be added to the role, + * @throws Exception + */ + public void alterSentryRoleAddGroups(final String grantorPrincipal, + final String roleName, final Set groupNames) throws Exception { + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + alterSentryRoleAddGroupsCore(pm, roleName, groupNames); + return null; + }); + } + + /** + * Assign a given role to a set of groups. As well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param grantorPrincipal grantorPrincipal currently is not used. + * @param roleName the role to be assigned to the groups. + * @param groupNames the list of groups to be added to the role, + * @param update the corresponding permission delta update + * @throws Exception + */ + public synchronized void alterSentryRoleAddGroups(final String grantorPrincipal, + final String roleName, final Set groupNames, + final Update update) throws Exception { + + execute(update, pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + alterSentryRoleAddGroupsCore(pm, roleName, groupNames); + return null; + }); + } + + private void alterSentryRoleAddGroupsCore(PersistenceManager pm, String roleName, + Set groupNames) throws SentryNoSuchObjectException { + + // All role names are stored in lowercase. + String lRoleName = trimAndLower(roleName); + MSentryRole role = getRole(pm, lRoleName); + if (role == null) { + throw noSuchRole(lRoleName); + } + + // Add the group to the specified role if it does not belong to the role yet. + Query query = pm.newQuery(MSentryGroup.class); + query.setFilter("this.groupName == :groupName"); + query.setUnique(true); + List groups = Lists.newArrayList(); + for (TSentryGroup tGroup : groupNames) { + String groupName = tGroup.getGroupName().trim(); + MSentryGroup group = (MSentryGroup) query.execute(groupName); + if (group == null) { + group = new MSentryGroup(groupName, System.currentTimeMillis(), Sets.newHashSet(role)); + } + group.appendRole(role); + groups.add(group); + } + pm.makePersistentAll(groups); + } + + public void alterSentryRoleAddUsers(final String roleName, + final Set userNames) throws Exception { + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + alterSentryRoleAddUsersCore(pm, roleName, userNames); + return null; + }); + } + + private void alterSentryRoleAddUsersCore(PersistenceManager pm, String roleName, + Set userNames) throws SentryNoSuchObjectException { + String trimmedRoleName = trimAndLower(roleName); + MSentryRole role = getRole(pm, trimmedRoleName); + if (role == null) { + throw noSuchRole(trimmedRoleName); + } + Query query = pm.newQuery(MSentryUser.class); + query.setFilter("this.userName == :userName"); + query.setUnique(true); + List users = Lists.newArrayList(); + for (String userName : userNames) { + userName = userName.trim(); + MSentryUser user = (MSentryUser) query.execute(userName); + if (user == null) { + user = new MSentryUser(userName, System.currentTimeMillis(), Sets.newHashSet(role)); + } + user.appendRole(role); + users.add(user); + } + pm.makePersistentAll(users); + } + + public void alterSentryRoleDeleteUsers(final String roleName, + final Set userNames) throws Exception { + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedRoleName = trimAndLower(roleName); + MSentryRole role = getRole(pm, trimmedRoleName); + if (role == null) { + throw noSuchRole(trimmedRoleName); + } else { + Query query = pm.newQuery(MSentryUser.class); + query.setFilter("this.userName == :userName"); + query.setUnique(true); + List users = Lists.newArrayList(); + for (String userName : userNames) { + userName = userName.trim(); + MSentryUser user = (MSentryUser) query.execute(userName); + if (user != null) { + user.removeRole(role); + users.add(user); + } + } + pm.makePersistentAll(users); + } + return null; + }); + } + + /** + * Revoke a given role to a set of groups. + * + * @param roleName the role to be assigned to the groups. + * @param groupNames the list of groups to be added to the role, + * @throws Exception + */ + public void alterSentryRoleDeleteGroups(final String roleName, + final Set groupNames) throws Exception { + tm.executeTransactionWithRetry( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedRoleName = trimAndLower(roleName); + MSentryRole role = getRole(pm, trimmedRoleName); + if (role == null) { + throw noSuchRole(trimmedRoleName); + } + Query query = pm.newQuery(MSentryGroup.class); + query.setFilter("this.groupName == :groupName"); + query.setUnique(true); + List groups = Lists.newArrayList(); + for (TSentryGroup tGroup : groupNames) { + String groupName = tGroup.getGroupName().trim(); + MSentryGroup group = (MSentryGroup) query.execute(groupName); + if (group != null) { + group.removeRole(role); + groups.add(group); + } + } + pm.makePersistentAll(groups); + return null; + }); + } + + /** + * Revoke a given role to a set of groups. As well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param roleName the role to be assigned to the groups. + * @param groupNames the list of groups to be added to the role, + * @param update the corresponding permission delta update + * @throws Exception + */ + public synchronized void alterSentryRoleDeleteGroups(final String roleName, + final Set groupNames, final Update update) + throws Exception { + execute(update, pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + String trimmedRoleName = trimAndLower(roleName); + MSentryRole role = getRole(pm, trimmedRoleName); + if (role == null) { + throw noSuchRole(trimmedRoleName); + } + + // Remove the group from the specified role if it belongs to the role. + Query query = pm.newQuery(MSentryGroup.class); + query.setFilter("this.groupName == :groupName"); + query.setUnique(true); + List groups = Lists.newArrayList(); + for (TSentryGroup tGroup : groupNames) { + String groupName = tGroup.getGroupName().trim(); + MSentryGroup group = (MSentryGroup) query.execute(groupName); + if (group != null) { + group.removeRole(role); + groups.add(group); + } + } + pm.makePersistentAll(groups); + return null; + }); + } + + @VisibleForTesting + public MSentryRole getMSentryRoleByName(final String roleName) throws Exception { + return tm.executeTransaction( + pm -> { + String trimmedRoleName = trimAndLower(roleName); + MSentryRole sentryRole = getRole(pm, trimmedRoleName); + if (sentryRole == null) { + throw noSuchRole(trimmedRoleName); + } + return sentryRole; + }); + } + + /** + * Gets the MSentryPrivilege from sentry persistent storage based on TSentryPrivilege + * provided + * + * Method is currently used only in test framework + * @param tPrivilege + * @return MSentryPrivilege if the privilege is found in the storage + * null, if the privilege is not found in the storage. + * @throws Exception + */ + @VisibleForTesting + MSentryPrivilege findMSentryPrivilegeFromTSentryPrivilege(final TSentryPrivilege tPrivilege) throws Exception { + return tm.executeTransaction( + pm -> getMSentryPrivilege(tPrivilege, pm)); + } + + /** + * Returns a list with all the privileges in the sentry persistent storage + * + * Method is currently used only in test framework + * @return List of all sentry privileges in the store + * @throws Exception + */ + @VisibleForTesting + List getAllMSentryPrivileges () throws Exception { + return tm.executeTransaction( + pm -> getAllMSentryPrivilegesCore(pm)); + } + + /** + * Method Returns all the privileges present in the persistent store as a list. + * @param pm PersistenceManager + * @returns list of all the privileges in the persistent store + */ + private List getAllMSentryPrivilegesCore (PersistenceManager pm) { + Query query = pm.newQuery(MSentryPrivilege.class); + return (List) query.execute(); + } + + private boolean hasAnyServerPrivileges(final Set roleNames, final String serverName) throws Exception { + if (roleNames == null || roleNames.isEmpty()) { + return false; + } + return tm.executeTransaction( + pm -> { + pm.setDetachAllOnCommit(false); // No need to detach objects + Query query = pm.newQuery(MSentryPrivilege.class); + query.addExtension(LOAD_RESULTS_AT_COMMIT, "false"); + QueryParamBuilder paramBuilder = QueryParamBuilder.addRolesFilter(query,null, roleNames); + paramBuilder.add(SERVER_NAME, serverName); + query.setFilter(paramBuilder.toString()); + query.setResult("count(this)"); + Long numPrivs = (Long) query.executeWithMap(paramBuilder.getArguments()); + return numPrivs > 0; + }); + } + + private List getMSentryPrivileges(final SentryEntityType entityType, final Set entityNames, + final TSentryAuthorizable authHierarchy) + throws Exception { + if (entityNames == null || entityNames.isEmpty()) { + return Collections.emptyList(); + } + + return tm.executeTransaction( + pm -> { + Query query = pm.newQuery(MSentryPrivilege.class); + QueryParamBuilder paramBuilder = null; + if (entityType == SentryEntityType.ROLE) { + paramBuilder = QueryParamBuilder.addRolesFilter(query, null, entityNames); + } else if (entityType == SentryEntityType.USER) { + paramBuilder = QueryParamBuilder.addUsersFilter(query, null, entityNames); + } else { + throw new SentryInvalidInputException("entityType" + entityType + " is not valid"); + } + + if (authHierarchy != null && authHierarchy.getServer() != null) { + paramBuilder.add(SERVER_NAME, authHierarchy.getServer()); + if (authHierarchy.getDb() != null) { + paramBuilder.addNull(URI) + .newChild() + .add(DB_NAME, authHierarchy.getDb()) + .addNull(DB_NAME); + if (authHierarchy.getTable() != null + && !AccessConstants.ALL.equalsIgnoreCase(authHierarchy.getTable())) { + if (!AccessConstants.SOME.equalsIgnoreCase(authHierarchy.getTable())) { + paramBuilder.addNull(URI) + .newChild() + .add(TABLE_NAME, authHierarchy.getTable()) + .addNull(TABLE_NAME); + } + if (authHierarchy.getColumn() != null + && !AccessConstants.ALL.equalsIgnoreCase(authHierarchy.getColumn()) + && !AccessConstants.SOME.equalsIgnoreCase(authHierarchy.getColumn())) { + paramBuilder.addNull(URI) + .newChild() + .add(COLUMN_NAME, authHierarchy.getColumn()) + .addNull(COLUMN_NAME); + } + } + } + if (authHierarchy.getUri() != null) { + paramBuilder.addNull(DB_NAME) + .newChild() + .addNull(URI) + .newChild() + .addNotNull(URI) + .addCustomParam("\"" + authHierarchy.getUri() + + "\".startsWith(:URI)", URI, authHierarchy.getUri()); + } + } + + query.setFilter(paramBuilder.toString()); + @SuppressWarnings("unchecked") + List result = + (List) + query.executeWithMap(paramBuilder.getArguments()); + return result; + }); + } + + private List getMSentryPrivilegesByAuth( + final SentryEntityType entityType, + final Set entityNames, + final TSentryAuthorizable + authHierarchy) throws Exception { + return tm.executeTransaction( + pm -> { + Query query = pm.newQuery(MSentryPrivilege.class); + QueryParamBuilder paramBuilder = QueryParamBuilder.newQueryParamBuilder(); + if (entityNames == null || entityNames.isEmpty()) { + if (entityType == SentryEntityType.ROLE) { + paramBuilder.addString("!roles.isEmpty()"); + } else if (entityType == SentryEntityType.USER) { + paramBuilder.addString("!users.isEmpty()"); + } else { + throw new SentryInvalidInputException("entityType: " + entityType + " is invalid"); + } + } else { + if (entityType == SentryEntityType.ROLE) { + QueryParamBuilder.addRolesFilter(query, paramBuilder, entityNames); + } else if (entityType ==