From commits-return-12476-archive-asf-public=cust-asf.ponee.io@sentry.apache.org Mon Apr 30 18:35:42 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 8298A1807A1 for ; Mon, 30 Apr 2018 18:35:40 +0200 (CEST) Received: (qmail 25799 invoked by uid 500); 30 Apr 2018 16:35:39 -0000 Mailing-List: contact commits-help@sentry.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@sentry.apache.org Delivered-To: mailing list commits@sentry.apache.org Received: (qmail 25536 invoked by uid 99); 30 Apr 2018 16:35:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 30 Apr 2018 16:35:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0F96CF667E; Mon, 30 Apr 2018 16:35:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: coheigea@apache.org To: commits@sentry.apache.org Date: Mon, 30 Apr 2018 16:35:44 -0000 Message-Id: In-Reply-To: <8edfe1188b9a4cca8447ff2059da159f@git.apache.org> References: <8edfe1188b9a4cca8447ff2059da159f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [8/9] sentry git commit: SENTRY-2207 Refactor out Sentry CLI from sentry-provider-db into own module. Steve Moist, reviewed by Colm O hEigeartaigh. http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaHelper.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaHelper.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaHelper.java deleted file mode 100644 index cf1c725..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaHelper.java +++ /dev/null @@ -1,315 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.provider.db.tools; - -import java.util.IllegalFormatException; - -public final class SentrySchemaHelper { - public static final String DB_DERBY = "derby"; - public static final String DB_MYSQL = "mysql"; - public static final String DB_POSTGRACE = "postgres"; - public static final String DB_ORACLE = "oracle"; - public static final String DB_DB2 = "db2"; - - public interface NestedScriptParser { - - public enum CommandType { - PARTIAL_STATEMENT, - TERMINATED_STATEMENT, - COMMENT - } - - String DEFAUTL_DELIMITER = ";"; - /*** - * Find the type of given command - * @param dbCommand - * @return - */ - boolean isPartialCommand(String dbCommand) throws IllegalArgumentException; - - /** Parse the DB specific nesting format and extract the inner script name if any - * @param dbCommand command from parent script - * @return - * @throws IllegalFormatException - */ - String getScriptName(String dbCommand) throws IllegalArgumentException; - - /*** - * Find if the given command is a nested script execution - * @param dbCommand - * @return - */ - boolean isNestedScript(String dbCommand); - - /*** - * Find if the given command is should be passed to DB - * @param dbCommand - * @return - */ - boolean isNonExecCommand(String dbCommand); - - /*** - * Get the SQL statement delimiter - * @return - */ - String getDelimiter(); - - /*** - * Clear any client specific tags - * @return - */ - String cleanseCommand(String dbCommand); - - /*** - * Does the DB required table/column names quoted - * @return - */ - boolean needsQuotedIdentifier(); - - /*** - * Set DB specific options if any - * @param dbOps - */ - void setDbOpts(String dbOps); - } - - - /*** - * Base implemenation of NestedScriptParser - * abstractCommandParser. - * - */ - private static abstract class AbstractCommandParser implements NestedScriptParser { - private String dbOpts = null; - - @Override - public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ - if (dbCommand == null || dbCommand.isEmpty()) { - throw new IllegalArgumentException("invalid command line " + dbCommand); - } - String trimmedDbCommand = dbCommand.trim(); - return !(trimmedDbCommand.endsWith(getDelimiter()) || isNonExecCommand(trimmedDbCommand)); - } - - @Override - public boolean isNonExecCommand(String dbCommand) { - return dbCommand.startsWith("--") || dbCommand.startsWith("#"); - } - - @Override - public String getDelimiter() { - return DEFAUTL_DELIMITER; - } - - @Override - public String cleanseCommand(String dbCommand) { - // strip off the delimiter - if (dbCommand.endsWith(getDelimiter())) { - dbCommand = dbCommand.substring(0, - dbCommand.length() - getDelimiter().length()); - } - return dbCommand; - } - - @Override - public boolean needsQuotedIdentifier() { - return false; - } - - @Override - public void setDbOpts(String dbOpts) { - this.dbOpts = dbOpts; - } - - protected String getDbOpts() { - return dbOpts; - } - } - - - // Derby commandline parser - public static class DerbyCommandParser extends AbstractCommandParser { - private static final String DERBY_NESTING_TOKEN = "RUN"; - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - - if (!isNestedScript(dbCommand)) { - throw new IllegalArgumentException("Not a script format " + dbCommand); - } - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - return tokens[1].replace(";", "").replaceAll("'", ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - // Derby script format is RUN '' - return dbCommand.startsWith(DERBY_NESTING_TOKEN); - } - } - - - // MySQL parser - public static class MySqlCommandParser extends AbstractCommandParser { - private static final String MYSQL_NESTING_TOKEN = "SOURCE"; - private static final String DELIMITER_TOKEN = "DELIMITER"; - private String delimiter = DEFAUTL_DELIMITER; - - @Override - public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ - boolean isPartial = super.isPartialCommand(dbCommand); - // if this is a delimiter directive, reset our delimiter - if (dbCommand.startsWith(DELIMITER_TOKEN)) { - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - delimiter = tokens[1]; - } - return isPartial; - } - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - // remove ending ';' - return tokens[1].replace(";", ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - return dbCommand.startsWith(MYSQL_NESTING_TOKEN); - } - - @Override - public String getDelimiter() { - return delimiter; - } - - @Override - public boolean isNonExecCommand(String dbCommand) { - return super.isNonExecCommand(dbCommand) || - dbCommand.startsWith("/*") && dbCommand.endsWith("*/") || - dbCommand.startsWith(DELIMITER_TOKEN); - } - - @Override - public String cleanseCommand(String dbCommand) { - return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", ""); - } - - } - - // Postgres specific parser - public static class PostgresCommandParser extends AbstractCommandParser { - public static final String POSTGRES_STRING_COMMAND_FILTER = "SET standard_conforming_strings"; - public static final String POSTGRES_STRING_CLIENT_ENCODING = "SET client_encoding"; - public static final String POSTGRES_SKIP_STANDARD_STRING = "postgres.filter.81"; - private static final String POSTGRES_NESTING_TOKEN = "\\i"; - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - // remove ending ';' - return tokens[1].replace(";", ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - return dbCommand.startsWith(POSTGRES_NESTING_TOKEN); - } - - @Override - public boolean needsQuotedIdentifier() { - return true; - } - - @Override - public boolean isNonExecCommand(String dbCommand) { - // Skip "standard_conforming_strings" command which is not supported in older postgres - if (POSTGRES_SKIP_STANDARD_STRING.equalsIgnoreCase(getDbOpts()) - && (dbCommand.startsWith(POSTGRES_STRING_COMMAND_FILTER) || dbCommand.startsWith(POSTGRES_STRING_CLIENT_ENCODING))) { - return true; - } - return super.isNonExecCommand(dbCommand); - } - } - - //Oracle specific parser - public static class OracleCommandParser extends AbstractCommandParser { - private static final String ORACLE_NESTING_TOKEN = "@"; - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - if (!isNestedScript(dbCommand)) { - throw new IllegalArgumentException("Not a nested script format " + dbCommand); - } - // remove ending ';' and starting '@' - return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - return dbCommand.startsWith(ORACLE_NESTING_TOKEN); - } - } - - // DB2 commandline parser - public static class DB2CommandParser extends AbstractCommandParser { - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - //DB2 does not support nesting script - throw new IllegalArgumentException("DB2 does not support nesting script " + dbCommand); - } - - @Override - public boolean isNestedScript(String dbCommand) { - //DB2 does not support nesting script - return false; - } - } - - public static NestedScriptParser getDbCommandParser(String dbName) { - if (dbName.equalsIgnoreCase(DB_DERBY)) { - return new DerbyCommandParser(); - } else if (dbName.equalsIgnoreCase(DB_MYSQL)) { - return new MySqlCommandParser(); - } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) { - return new PostgresCommandParser(); - } else if (dbName.equalsIgnoreCase(DB_ORACLE)) { - return new OracleCommandParser(); - } else if (dbName.equalsIgnoreCase(DB_DB2)) { - return new DB2CommandParser(); - } else { - throw new IllegalArgumentException("Unknown dbType " + dbName); - } - } - - private SentrySchemaHelper() { - // Make constructor private to avoid instantiation - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaTool.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaTool.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaTool.java deleted file mode 100644 index d75e24b..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaTool.java +++ /dev/null @@ -1,595 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.provider.db.tools; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintStream; -import java.net.MalformedURLException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.IllegalFormatException; -import java.util.List; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.OptionGroup; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.io.output.NullOutputStream; -import org.apache.hadoop.conf.Configuration; -import org.apache.hive.beeline.BeeLine; -import org.apache.sentry.Command; -import org.apache.sentry.core.common.exception.SentryUserException; -import org.apache.sentry.core.common.exception.SentrySiteConfigurationException; -import org.apache.sentry.provider.db.service.persistent.SentryStoreSchemaInfo; -import org.apache.sentry.provider.db.tools.SentrySchemaHelper.NestedScriptParser; -import org.apache.sentry.service.thrift.SentryService; -import org.apache.sentry.service.thrift.ServiceConstants; - -public class SentrySchemaTool { - private static final String SENTRY_SCRIP_DIR = File.separatorChar + "scripts" - + File.separatorChar + "sentrystore" + File.separatorChar + "upgrade"; - private String userName = null; - private String passWord = null; - private String connectionURL = null; - private String driver = null; - private boolean dryRun = false; - private String dbOpts = null; - private boolean verbose = false; - private final Configuration sentryConf; - private final String dbType; - private final SentryStoreSchemaInfo sentryStoreSchemaInfo; - - public SentrySchemaTool(Configuration sentryConf, String dbType) - throws SentryUserException, IOException { - this(System.getenv("SENTRY_HOME") + SENTRY_SCRIP_DIR, sentryConf, dbType); - } - - public SentrySchemaTool(String sentryScripPath, Configuration sentryConf, - String dbType) throws SentryUserException, IOException { - if (sentryScripPath == null || sentryScripPath.isEmpty()) { - throw new SentryUserException("No Sentry script dir provided"); - } - this.sentryConf = sentryConf; - this.dbType = dbType; - this.sentryStoreSchemaInfo = new SentryStoreSchemaInfo(sentryScripPath, - dbType); - userName = sentryConf.get(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_USER, - ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_USER_DEFAULT); - //Password will be read from Credential provider specified using property - // CREDENTIAL_PROVIDER_PATH("hadoop.security.credential.provider.path" in sentry-site.xml - // it falls back to reading directly from sentry-site.xml - char[] passTmp = sentryConf.getPassword(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_PASS); - if(passTmp != null) { - passWord = new String(passTmp); - } else { - throw new SentrySiteConfigurationException("Error reading " + ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_PASS); - } - - try { - connectionURL = getValidConfVar(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_URL); - if(dbType.equalsIgnoreCase(SentrySchemaHelper.DB_DERBY)) { - driver = sentryConf.get(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_DRIVER, - ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_DRIVER_DEFAULT); - } else { - driver = getValidConfVar(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_DRIVER); - } - // load required JDBC driver - Class.forName(driver); - } catch (IOException e) { - throw new SentryUserException("Missing property: " + e.getMessage()); - } catch (ClassNotFoundException e) { - throw new SentryUserException("Failed to load driver", e); - } - } - - public Configuration getConfiguration() { - return sentryConf; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public void setPassWord(String passWord) { - this.passWord = passWord; - } - - public void setDryRun(boolean dryRun) { - this.dryRun = dryRun; - } - - public void setVerbose(boolean verbose) { - this.verbose = verbose; - } - - public String getDbOpts() { - return dbOpts; - } - - public void setDbOpts(String dbOpts) { - this.dbOpts = dbOpts; - } - - private static void printAndExit(Options cmdLineOptions) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("schemaTool", cmdLineOptions); - System.exit(1); - } - - /*** - * Print Hive version and schema version - * @throws SentryUserException - */ - public void showInfo() throws SentryUserException { - Connection sentryStoreConn = getConnectionToMetastore(true); - System.out.println("Sentry distribution version:\t " - + SentryStoreSchemaInfo.getSentryVersion()); - System.out.println("SentryStore schema version:\t " - + getMetaStoreSchemaVersion(sentryStoreConn)); - } - - // read schema version from sentry store - private String getMetaStoreSchemaVersion(Connection sentryStoreConn) - throws SentryUserException { - String versionQuery; - if (SentrySchemaHelper.getDbCommandParser(dbType).needsQuotedIdentifier()) { - versionQuery = "select t.\"SCHEMA_VERSION\" from \"SENTRY_VERSION\" t"; - } else { - versionQuery = "select t.SCHEMA_VERSION from SENTRY_VERSION t"; - } - try (Statement stmt = sentryStoreConn.createStatement(); - ResultSet res = stmt.executeQuery(versionQuery)) { - if (!res.next()) { - throw new SentryUserException("Didn't find version data in sentry store"); - } - String currentSchemaVersion = res.getString(1); - sentryStoreConn.close(); - return currentSchemaVersion; - } catch (SQLException e) { - throw new SentryUserException("Failed to get schema version.", e); - } - } - - // test the connection sentry store using the config property - private void testConnectionToMetastore() throws SentryUserException { - try (Connection conn = getConnectionToMetastore(true)) { - conn.close(); - } catch (SQLException e) { - throw new SentryUserException("Failed to close sentry store connection", e); - } - } - - /*** - * get JDBC connection to sentry store db - * - * @param printInfo print connection parameters - * @return - * @throws SentryUserException - */ - private Connection getConnectionToMetastore(boolean printInfo) - throws SentryUserException { - if (printInfo) { - System.out.println("Sentry store connection URL:\t " + connectionURL); - System.out.println("Sentry store Connection Driver :\t " + driver); - System.out.println("Sentry store connection User:\t " + userName); - } - if (userName == null || userName.isEmpty()) { - throw new SentryUserException("UserName empty "); - } - try { - // Connect using the JDBC URL and user/pass from conf - return DriverManager.getConnection(connectionURL, userName, passWord); - } catch (SQLException e) { - throw new SentryUserException("Failed to make connection to Sentry store.", e); - } - } - - /** - * check if the current schema version in sentry store matches the Hive version - * @throws SentryUserException - */ - public void verifySchemaVersion() throws SentryUserException { - // don't check version if its a dry run - if (dryRun) { - return; - } - String newSchemaVersion = - getMetaStoreSchemaVersion(getConnectionToMetastore(false)); - // verify that the new version is added to schema - if (!sentryStoreSchemaInfo.getSentrySchemaVersion().equalsIgnoreCase( - newSchemaVersion)) { - throw new SentryUserException("Found unexpected schema version " - + newSchemaVersion); - } - } - - /** - * Perform sentry store schema upgrade. extract the current schema version from sentry store - * @throws SentryUserException - */ - public void doUpgrade() throws SentryUserException { - String fromVersion = getMetaStoreSchemaVersion(getConnectionToMetastore(false)); - if (fromVersion == null || fromVersion.isEmpty()) { - throw new SentryUserException( - "Schema version not stored in the sentry store. " - + - "Metastore schema is too old or corrupt. Try specifying the version manually"); - } - doUpgrade(fromVersion); - } - - /** - * Perform sentry store schema upgrade - * - * @param fromSchemaVer - * Existing version of the sentry store. If null, then read from the sentry store - * @throws SentryUserException - */ - public void doUpgrade(String fromSchemaVer) throws SentryUserException { - if (sentryStoreSchemaInfo.getSentrySchemaVersion().equals(fromSchemaVer)) { - System.out.println("No schema upgrade required from version " + fromSchemaVer); - return; - } - // Find the list of scripts to execute for this upgrade - List upgradeScripts = - sentryStoreSchemaInfo.getUpgradeScripts(fromSchemaVer); - testConnectionToMetastore(); - System.out.println("Starting upgrade sentry store schema from version " + - fromSchemaVer + " to " - + sentryStoreSchemaInfo.getSentrySchemaVersion()); - String scriptDir = sentryStoreSchemaInfo.getSentryStoreScriptDir(); - try { - for (String scriptFile : upgradeScripts) { - System.out.println("Upgrade script " + scriptFile); - if (!dryRun) { - runBeeLine(scriptDir, scriptFile); - System.out.println("Completed " + scriptFile); - } - } - } catch (IOException eIO) { - throw new SentryUserException( - "Upgrade FAILED! Metastore state would be inconsistent !!", eIO); - } - - // Revalidated the new version after upgrade - verifySchemaVersion(); - } - - /** - * Initialize the sentry store schema to current version - * - * @throws SentryUserException - */ - public void doInit() throws SentryUserException { - doInit(sentryStoreSchemaInfo.getSentrySchemaVersion()); - - // Revalidated the new version after upgrade - verifySchemaVersion(); - } - - /** - * Initialize the sentry store schema - * - * @param toVersion - * If null then current hive version is used - * @throws SentryUserException - */ - public void doInit(String toVersion) throws SentryUserException { - testConnectionToMetastore(); - System.out.println("Starting sentry store schema initialization to " + toVersion); - - String initScriptDir = sentryStoreSchemaInfo.getSentryStoreScriptDir(); - String initScriptFile = sentryStoreSchemaInfo.generateInitFileName(toVersion); - - try { - System.out.println("Initialization script " + initScriptFile); - if (!dryRun) { - runBeeLine(initScriptDir, initScriptFile); - System.out.println("Initialization script completed"); - } - } catch (IOException e) { - throw new SentryUserException("Schema initialization FAILED!" - + " Metastore state would be inconsistent !!", e); - } - } - - // Flatten the nested upgrade script into a buffer - public static String buildCommand(NestedScriptParser dbCommandParser, - String scriptDir, String scriptFile) throws IllegalFormatException, IOException { - - BufferedReader bfReader = - new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile)); - String currLine; - StringBuilder sb = new StringBuilder(); - String currentCommand = null; - while ((currLine = bfReader.readLine()) != null) { - currLine = currLine.trim(); - if (currLine.isEmpty()) { - continue; // skip empty lines - } - - if (currentCommand == null) { - currentCommand = currLine; - } else { - currentCommand = currentCommand + " " + currLine; - } - if (dbCommandParser.isPartialCommand(currLine)) { - // if its a partial line, continue collecting the pieces - continue; - } - - // if this is a valid executable command then add it to the buffer - if (!dbCommandParser.isNonExecCommand(currentCommand)) { - currentCommand = dbCommandParser.cleanseCommand(currentCommand); - - if (dbCommandParser.isNestedScript(currentCommand)) { - // if this is a nested sql script then flatten it - String currScript = dbCommandParser.getScriptName(currentCommand); - sb.append(buildCommand(dbCommandParser, scriptDir, currScript)); - } else { - // Now we have a complete statement, process it - // write the line to buffer - sb.append(currentCommand); - sb.append(System.getProperty("line.separator")); - } - } - currentCommand = null; - } - bfReader.close(); - return sb.toString(); - } - - // run beeline on the given sentry store scrip, flatten the nested scripts into single file - private void runBeeLine(String scriptDir, String scriptFile) throws IOException { - NestedScriptParser dbCommandParser = - SentrySchemaHelper.getDbCommandParser(dbType); - dbCommandParser.setDbOpts(getDbOpts()); - // expand the nested script - String sqlCommands = buildCommand(dbCommandParser, scriptDir, scriptFile); - File tmpFile = File.createTempFile("schematool", ".sql"); - tmpFile.deleteOnExit(); - - // write out the buffer into a file. Add beeline commands for autocommit and close - try (FileWriter fstream = new FileWriter(tmpFile.getPath()); - BufferedWriter out = new BufferedWriter(fstream)) { - - out.write("!set Silent " + verbose + System.getProperty("line.separator")); - out.write("!autocommit on" + System.getProperty("line.separator")); - out.write("!set Isolation TRANSACTION_READ_COMMITTED" - + System.getProperty("line.separator")); - out.write("!set AllowMultiLineCommand false" - + System.getProperty("line.separator")); - out.write(sqlCommands); - out.write("!closeall" + System.getProperty("line.separator")); - out.close(); - } - runBeeLine(tmpFile.getPath()); - } - - // Generate the beeline args per hive conf and execute the given script - public void runBeeLine(String sqlScriptFile) throws IOException { - List argList = new ArrayList(); - argList.add("-u"); - argList.add(connectionURL); - argList.add("-d"); - argList - .add(driver); - argList.add("-n"); - argList.add(userName); - argList.add("-p"); - argList.add(passWord); - argList.add("-f"); - argList.add(sqlScriptFile); - - BeeLine beeLine = new BeeLine(); - if (!verbose) { - beeLine.setOutputStream(new PrintStream(new NullOutputStream())); - // beeLine.getOpts().setSilent(true); - } - // beeLine.getOpts().setAllowMultiLineCommand(false); - // beeLine.getOpts().setIsolation("TRANSACTION_READ_COMMITTED"); - int status = beeLine.begin(argList.toArray(new String[0]), null); - if (status != 0) { - throw new IOException("Schema script failed, errorcode " + status); - } - } - - private String getValidConfVar(String confVar) throws IOException { - String confVarKey = confVar; - String confVarValue = sentryConf.get(confVarKey); - if (confVarValue == null || confVarValue.isEmpty()) { - throw new IOException("Empty " + confVar); - } - return confVarValue; - } - - // Create the required command line options - @SuppressWarnings("static-access") - private static void initOptions(Options cmdLineOptions) { - Option help = new Option("help", "print this message"); - Option upgradeOpt = new Option("upgradeSchema", "Schema upgrade"); - Option upgradeFromOpt = OptionBuilder.withArgName("upgradeFrom").hasArg(). - withDescription("Schema upgrade from a version"). - create("upgradeSchemaFrom"); - Option initOpt = new Option("initSchema", "Schema initialization"); - Option initToOpt = OptionBuilder.withArgName("initTo").hasArg(). - withDescription("Schema initialization to a version"). - create("initSchemaTo"); - Option infoOpt = new Option("info", "Show config and schema details"); - - OptionGroup optGroup = new OptionGroup(); - optGroup.addOption(upgradeOpt).addOption(initOpt). - addOption(help).addOption(upgradeFromOpt). - addOption(initToOpt).addOption(infoOpt); - optGroup.setRequired(true); - - Option userNameOpt = OptionBuilder.withArgName("user") - .hasArg() - .withDescription("Override config file user name") - .create("userName"); - Option passwdOpt = OptionBuilder.withArgName("password") - .hasArg() - .withDescription("Override config file password") - .create("passWord"); - Option dbTypeOpt = OptionBuilder.withArgName("databaseType") - .hasArg().withDescription("Metastore database type [" + - SentrySchemaHelper.DB_DERBY + "," + - SentrySchemaHelper.DB_MYSQL + "," + - SentrySchemaHelper.DB_ORACLE + "," + - SentrySchemaHelper.DB_POSTGRACE + "," + - SentrySchemaHelper.DB_DB2 + "]") - .create("dbType"); - Option dbOpts = OptionBuilder.withArgName("databaseOpts") - .hasArgs().withDescription("Backend DB specific options") - .create("dbOpts"); - - Option dryRunOpt = new Option("dryRun", "list SQL scripts (no execute)"); - Option verboseOpt = new Option("verbose", "only print SQL statements"); - - Option configOpt = OptionBuilder.withArgName("confName").hasArgs() - .withDescription("Sentry Service configuration file").isRequired(true) - .create(ServiceConstants.ServiceArgs.CONFIG_FILE_LONG); - - cmdLineOptions.addOption(help); - cmdLineOptions.addOption(dryRunOpt); - cmdLineOptions.addOption(userNameOpt); - cmdLineOptions.addOption(passwdOpt); - cmdLineOptions.addOption(dbTypeOpt); - cmdLineOptions.addOption(verboseOpt); - cmdLineOptions.addOption(dbOpts); - cmdLineOptions.addOption(configOpt); - cmdLineOptions.addOptionGroup(optGroup); - } - - public static class CommandImpl implements Command { - @Override - public void run(String[] args) throws Exception { - CommandLineParser parser = new GnuParser(); - CommandLine line = null; - String dbType = null; - String schemaVer = null; - Options cmdLineOptions = new Options(); - String configFileName = null; - - // Argument handling - initOptions(cmdLineOptions); - try { - line = parser.parse(cmdLineOptions, args); - } catch (ParseException e) { - System.err.println("SentrySchemaTool:Parsing failed. Reason: " - + e.getLocalizedMessage()); - printAndExit(cmdLineOptions); - } - - if (line.hasOption("help")) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("schemaTool", cmdLineOptions); - return; - } - - if (line.hasOption("dbType")) { - dbType = line.getOptionValue("dbType"); - if (!dbType.equalsIgnoreCase(SentrySchemaHelper.DB_DERBY) - && !dbType.equalsIgnoreCase(SentrySchemaHelper.DB_MYSQL) - && !dbType.equalsIgnoreCase(SentrySchemaHelper.DB_POSTGRACE) - && !dbType.equalsIgnoreCase(SentrySchemaHelper.DB_ORACLE) - && !dbType.equalsIgnoreCase(SentrySchemaHelper.DB_DB2)) { - System.err.println("Unsupported dbType " + dbType); - printAndExit(cmdLineOptions); - } - } else { - System.err.println("no dbType supplied"); - printAndExit(cmdLineOptions); - } - if (line.hasOption(ServiceConstants.ServiceArgs.CONFIG_FILE_LONG)) { - configFileName = line - .getOptionValue(ServiceConstants.ServiceArgs.CONFIG_FILE_LONG); - } else { - System.err.println("no config file specified"); - printAndExit(cmdLineOptions); - } - try { - SentrySchemaTool schemaTool = new SentrySchemaTool( - SentryService.loadConfig(configFileName), dbType); - - if (line.hasOption("userName")) { - schemaTool.setUserName(line.getOptionValue("userName")); - } - if (line.hasOption("passWord")) { - schemaTool.setPassWord(line.getOptionValue("passWord")); - } - if (line.hasOption("dryRun")) { - schemaTool.setDryRun(true); - } - if (line.hasOption("verbose")) { - schemaTool.setVerbose(true); - } - if (line.hasOption("dbOpts")) { - schemaTool.setDbOpts(line.getOptionValue("dbOpts")); - } - - if (line.hasOption("info")) { - schemaTool.showInfo(); - } else if (line.hasOption("upgradeSchema")) { - schemaTool.doUpgrade(); - } else if (line.hasOption("upgradeSchemaFrom")) { - schemaVer = line.getOptionValue("upgradeSchemaFrom"); - schemaTool.doUpgrade(schemaVer); - } else if (line.hasOption("initSchema")) { - schemaTool.doInit(); - } else if (line.hasOption("initSchemaTo")) { - schemaVer = line.getOptionValue("initSchemaTo"); - schemaTool.doInit(schemaVer); - } else { - System.err.println("no valid option supplied"); - printAndExit(cmdLineOptions); - } - } catch (SentryUserException e) { - System.err.println(e); - if (line.hasOption("verbose")) { - e.printStackTrace(); - } - System.err.println("*** Sentry schemaTool failed ***"); - System.exit(1); - } catch (MalformedURLException e) { - System.err.println(e); - if (line.hasOption("verbose")) { - e.printStackTrace(); - } - System.err.println("*** Sentry schemaTool failed ***"); - System.exit(1); - } - System.out.println("Sentry schemaTool completed"); - } - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellCommon.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellCommon.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellCommon.java deleted file mode 100644 index c8b2eef..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellCommon.java +++ /dev/null @@ -1,284 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.provider.db.tools; - -import com.google.common.annotations.VisibleForTesting; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionGroup; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.Parser; -import org.apache.commons.lang.StringUtils; - -/** - * SentryShellCommon provides the function for parsing the argument. - * For hive model and generic model, child class should be implemented as a sentry admin tool. - */ -abstract public class SentryShellCommon { - - public enum TYPE { kafka, hive, solr, sqoop }; - - public static final String OPTION_DESC_HELP = "Shell usage"; - public static final String OPTION_DESC_CONF = "sentry-site file path"; - public static final String OPTION_DESC_ROLE_NAME = "Role name"; - public static final String OPTION_DESC_GROUP_NAME = "Group name"; - public static final String OPTION_DESC_PRIVILEGE = "Privilege string"; - public final static String OPTION_DESC_SERVICE = "Name of the service being managed"; - public static final String PREFIX_MESSAGE_MISSING_OPTION = "Missing required option: "; - - public static final String GROUP_SPLIT_CHAR = ","; - - protected String roleName; - protected String serviceName; - protected String groupName; - protected String privilegeStr; - protected String confPath; - // flag for the command - protected boolean isCreateRole; - protected boolean isDropRole; - protected boolean isAddRoleGroup; - protected boolean isDeleteRoleGroup; - protected boolean isGrantPrivilegeRole; - protected boolean isRevokePrivilegeRole; - protected boolean isListRole; - protected boolean isListPrivilege; - protected boolean isListGroup; - protected boolean isPrintHelp; - // flag for the parameter check - protected boolean roleNameRequired; - protected boolean groupNameRequired; - protected boolean privilegeStrRequired; - protected TYPE type; - - /** - * parse arguments - * - * 
-   *   -conf,--sentry_conf                              sentry config file path
-   *   -cr,--create_role            -r                  create role
-   *   -dr,--drop_role              -r                  drop role
-   *   -arg,--add_role_group        -r   -g  add role to group
-   *   -drg,--delete_role_group     -r   -g  delete role from group
-   *   -gpr,--grant_privilege_role  -r   -p  grant privilege to role
-   *   -rpr,--revoke_privilege_role -r   -p  revoke privilege from role
-   *   -lr,--list_role              -g                 list roles for group
-   *   -lp,--list_privilege         -r                  list privilege for role
-   *   -lg,--list_group                                           list all groups associated with all roles
-   *   -t,--type                                        the shell for hive model or generic model
-   *
- * - * @param args - */ - protected boolean parseArgs(String[] args) { - Options simpleShellOptions = new Options(); - - setupOptions(simpleShellOptions); - - - - // help option - Option helpOpt = new Option("h", "help", false, OPTION_DESC_HELP); - helpOpt.setRequired(false); - simpleShellOptions.addOption(helpOpt); - - // this Options is parsed first for help option - Options helpOptions = new Options(); - helpOptions.addOption(helpOpt); - - try { - Parser parser = new GnuParser(); - - // parse help option first - CommandLine cmd = parser.parse(helpOptions, args, true); - for (Option opt : cmd.getOptions()) { - if (opt.getOpt().equals("h")) { - // get the help option, print the usage and exit - usage(simpleShellOptions); - return false; - } - } - - // without help option - cmd = parser.parse(simpleShellOptions, args); - - parseOptions(cmd); - } catch (ParseException pe) { - System.out.println(pe.getMessage()); - usage(simpleShellOptions); - return false; - } - return true; - } - - protected void setupOptions(Options simpleShellOptions) { - OptionGroup simpleShellOptGroup = getMainOptions(); - simpleShellOptions.addOptionGroup(simpleShellOptGroup); - - Option sOpt = new Option("s", "service", true, OPTION_DESC_SERVICE); - sOpt.setRequired(false); - simpleShellOptions.addOption(sOpt); - - // optional args - Option pOpt = new Option("p", "privilege", true, OPTION_DESC_PRIVILEGE); - pOpt.setRequired(false); - simpleShellOptions.addOption(pOpt); - - Option gOpt = new Option("g", "groupname", true, OPTION_DESC_GROUP_NAME); - gOpt.setRequired(false); - simpleShellOptions.addOption(gOpt); - - Option rOpt = new Option("r", "rolename", true, OPTION_DESC_ROLE_NAME); - rOpt.setRequired(false); - simpleShellOptions.addOption(rOpt); - - // this argument should also be parsed in the bin/sentryShell - Option tOpt = new Option("t", "type", true, "[hive|solr|sqoop|.....]"); - tOpt.setRequired(false); - simpleShellOptions.addOption(tOpt); - - // file path of sentry-site - Option sentrySitePathOpt = new Option("conf", "sentry_conf", true, OPTION_DESC_CONF); - sentrySitePathOpt.setRequired(true); - simpleShellOptions.addOption(sentrySitePathOpt); - } - - protected OptionGroup getMainOptions() { - OptionGroup simpleShellOptGroup = new OptionGroup(); - Option crOpt = new Option("cr", "create_role", false, "Create role"); - crOpt.setRequired(false); - - Option drOpt = new Option("dr", "drop_role", false, "Drop role"); - drOpt.setRequired(false); - - Option argOpt = new Option("arg", "add_role_group", false, "Add role to group"); - argOpt.setRequired(false); - - Option drgOpt = new Option("drg", "delete_role_group", false, "Delete role from group"); - drgOpt.setRequired(false); - - Option gprOpt = new Option("gpr", "grant_privilege_role", false, "Grant privilege to role"); - gprOpt.setRequired(false); - - Option rprOpt = new Option("rpr", "revoke_privilege_role", false, "Revoke privilege from role"); - rprOpt.setRequired(false); - - Option lrOpt = new Option("lr", "list_role", false, "List role"); - lrOpt.setRequired(false); - - Option lpOpt = new Option("lp", "list_privilege", false, "List privilege"); - lpOpt.setRequired(false); - - Option lgOpt = new Option("lg", "list_group", false, "List groups"); - lgOpt.setRequired(false); - - - // required args group - simpleShellOptGroup.addOption(crOpt); - simpleShellOptGroup.addOption(drOpt); - simpleShellOptGroup.addOption(argOpt); - simpleShellOptGroup.addOption(drgOpt); - simpleShellOptGroup.addOption(gprOpt); - simpleShellOptGroup.addOption(rprOpt); - simpleShellOptGroup.addOption(lrOpt); - simpleShellOptGroup.addOption(lpOpt); - simpleShellOptGroup.addOption(lgOpt); - simpleShellOptGroup.setRequired(true); - return simpleShellOptGroup; - } - - protected void parseOptions(CommandLine cmd) throws ParseException { - for (Option opt : cmd.getOptions()) { - if (opt.getOpt().equals("p")) { - privilegeStr = opt.getValue(); - } else if (opt.getOpt().equals("g")) { - groupName = opt.getValue(); - } else if (opt.getOpt().equals("r")) { - roleName = opt.getValue(); - } else if (opt.getOpt().equals("s")) { - serviceName = opt.getValue(); - } else if (opt.getOpt().equals("cr")) { - isCreateRole = true; - roleNameRequired = true; - } else if (opt.getOpt().equals("dr")) { - isDropRole = true; - roleNameRequired = true; - } else if (opt.getOpt().equals("arg")) { - isAddRoleGroup = true; - roleNameRequired = true; - groupNameRequired = true; - } else if (opt.getOpt().equals("drg")) { - isDeleteRoleGroup = true; - roleNameRequired = true; - groupNameRequired = true; - } else if (opt.getOpt().equals("gpr")) { - isGrantPrivilegeRole = true; - roleNameRequired = true; - privilegeStrRequired = true; - } else if (opt.getOpt().equals("rpr")) { - isRevokePrivilegeRole = true; - roleNameRequired = true; - privilegeStrRequired = true; - } else if (opt.getOpt().equals("lr")) { - isListRole = true; - } else if (opt.getOpt().equals("lp")) { - isListPrivilege = true; - roleNameRequired = true; - } else if (opt.getOpt().equals("lg")) { - isListGroup = true; - } else if (opt.getOpt().equals("conf")) { - confPath = opt.getValue(); - } else if (opt.getOpt().equals("t")) { - type = TYPE.valueOf(opt.getValue()); - } - } - checkRequiredParameter(roleNameRequired, roleName, OPTION_DESC_ROLE_NAME); - checkRequiredParameter(groupNameRequired, groupName, OPTION_DESC_GROUP_NAME); - checkRequiredParameter(privilegeStrRequired, privilegeStr, OPTION_DESC_PRIVILEGE); - } - - protected void checkRequiredParameter(boolean isRequired, String paramValue, String paramName) throws ParseException { - if (isRequired && StringUtils.isEmpty(paramValue)) { - throw new ParseException(PREFIX_MESSAGE_MISSING_OPTION + paramName); - } - } - - // print usage - private void usage(Options sentryOptions) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("sentryShell", sentryOptions); - } - - // hive model and generic model should implement this method - public abstract void run() throws Exception; - - @VisibleForTesting - public boolean executeShell(String[] args) throws Exception { - boolean result = true; - if (parseArgs(args)) { - run(); - } else { - result = false; - } - return result; - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellHive.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellHive.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellHive.java deleted file mode 100644 index 785e27d..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellHive.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.provider.db.tools; - -import java.util.List; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient; -import org.apache.sentry.provider.db.tools.command.hive.*; -import org.apache.sentry.service.thrift.SentryServiceClientFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Sets; - -/** - * SentryShellHive is an admin tool, and responsible for the management of repository. - * The following function are supported: - * create role, drop role, add group to role, delete group from role, grant privilege to role, - * revoke privilege from role, list roles for group, list privilege for role. - */ -public class SentryShellHive extends SentryShellCommon { - - private static final Logger LOGGER = LoggerFactory.getLogger(SentryShellHive.class); - - public void run() throws Exception { - - try(SentryPolicyServiceClient client = - SentryServiceClientFactory.create(getSentryConf())) { - UserGroupInformation ugi = UserGroupInformation.getLoginUser(); - String requestorName = ugi.getShortUserName(); - ShellCommand command = new HiveShellCommand(client); - - // check the requestor name - if (StringUtils.isEmpty(requestorName)) { - // The exception message will be recorded in the log file. - throw new Exception("The requestor name is empty."); - } - - if (isCreateRole) { - command.createRole(requestorName, roleName); - } else if (isDropRole) { - command.dropRole(requestorName, roleName); - } else if (isAddRoleGroup) { - Set groups = Sets.newHashSet(groupName.split(SentryShellCommon.GROUP_SPLIT_CHAR)); - command.grantRoleToGroups(requestorName, roleName, groups); - } else if (isDeleteRoleGroup) { - Set groups = Sets.newHashSet(groupName.split(SentryShellCommon.GROUP_SPLIT_CHAR)); - command.revokeRoleFromGroups(requestorName, roleName, groups); - } else if (isGrantPrivilegeRole) { - command.grantPrivilegeToRole(requestorName, roleName, privilegeStr); - } else if (isRevokePrivilegeRole) { - command.revokePrivilegeFromRole(requestorName, roleName, privilegeStr); - } else if (isListRole) { - List roles = command.listRoles(requestorName, groupName); - for (String role : roles) { - System.out.println(role); - } - } else if (isListPrivilege) { - List privileges = command.listPrivileges(requestorName, roleName); - for (String privilege : privileges) { - System.out.println(privilege); - } - } else if (isListGroup) { - List groups = command.listGroupRoles(requestorName); - for (String group : groups) { - System.out.println(group); - } - } - } - } - - private Configuration getSentryConf() { - Configuration conf = new Configuration(); - conf.addResource(new Path(confPath), true); - return conf; - } - - public static void main(String[] args) throws Exception { - SentryShellHive sentryShell = new SentryShellHive(); - try { - sentryShell.executeShell(args); - } catch (Exception e) { - LOGGER.error(e.getMessage(), e); - Throwable current = e; - // find the first printable message; - while (current != null && current.getMessage() == null) { - current = current.getCause(); - } - - if (current != null) { - System.out.println("The operation failed." + - (current.getMessage() == null ? "" : " Message: " + current.getMessage())); - } - } - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/ShellCommand.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/ShellCommand.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/ShellCommand.java deleted file mode 100644 index eeb3a23..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/ShellCommand.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.provider.db.tools; - -import java.util.List; -import java.util.Set; - -import org.apache.sentry.core.common.exception.SentryUserException; - -/** - * The interface for all admin commands, eg, CreateRoleCmd. It is independent of the underlying mechanism (i.e. Generic or Hive) - */ -public interface ShellCommand { - - void createRole(String requestorName, String roleName) throws SentryUserException; - - void dropRole(String requestorName, String roleName) throws SentryUserException; - - void grantPrivilegeToRole(String requestorName, String roleName, String privilege) throws SentryUserException; - - void grantRoleToGroups(String requestorName, String roleName, Set groups) throws SentryUserException; - - void revokePrivilegeFromRole(String requestorName, String roleName, String privilege) throws SentryUserException; - - void revokeRoleFromGroups(String requestorName, String roleName, Set groups) throws SentryUserException; - - List listRoles(String requestorName, String group) throws SentryUserException; - - List listPrivileges(String requestorName, String roleName) throws SentryUserException; - - List listGroupRoles(String requestorName) throws SentryUserException; -} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/CommandUtil.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/CommandUtil.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/CommandUtil.java deleted file mode 100644 index 3f0b5fa..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/CommandUtil.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.provider.db.tools.command.hive; - -import org.apache.commons.lang.StringUtils; -import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; -import org.apache.sentry.service.thrift.ServiceConstants; - -public final class CommandUtil { - - public static final String SPLIT_CHAR = ","; - - private CommandUtil() { - // Make constructor private to avoid instantiation - } - - // check the privilege value for the specific privilege scope - // eg, for the table scope, server and database can't be empty - public static void validatePrivilegeHierarchy(TSentryPrivilege tSentryPrivilege) throws IllegalArgumentException { - String serverName = tSentryPrivilege.getServerName(); - String dbName = tSentryPrivilege.getDbName(); - String tableName = tSentryPrivilege.getTableName(); - String columnName = tSentryPrivilege.getColumnName(); - String uri = tSentryPrivilege.getURI(); - if (ServiceConstants.PrivilegeScope.SERVER.toString().equals(tSentryPrivilege.getPrivilegeScope())) { - if (StringUtils.isEmpty(serverName)) { - throw new IllegalArgumentException("The hierarchy of privilege is not correct."); - } - } else if (ServiceConstants.PrivilegeScope.URI.toString().equals(tSentryPrivilege.getPrivilegeScope())) { - if (StringUtils.isEmpty(serverName) || StringUtils.isEmpty(uri)) { - throw new IllegalArgumentException("The hierarchy of privilege is not correct."); - } - } else if (ServiceConstants.PrivilegeScope.DATABASE.toString().equals(tSentryPrivilege.getPrivilegeScope())) { - if (StringUtils.isEmpty(serverName) || StringUtils.isEmpty(dbName)) { - throw new IllegalArgumentException("The hierarchy of privilege is not correct."); - } - } else if (ServiceConstants.PrivilegeScope.TABLE.toString().equals(tSentryPrivilege.getPrivilegeScope())) { - if (StringUtils.isEmpty(serverName) || StringUtils.isEmpty(dbName) - || StringUtils.isEmpty(tableName)) { - throw new IllegalArgumentException("The hierarchy of privilege is not correct."); - } - } else if (ServiceConstants.PrivilegeScope.COLUMN.toString().equals(tSentryPrivilege.getPrivilegeScope()) - && (StringUtils.isEmpty(serverName) || StringUtils.isEmpty(dbName) - || StringUtils.isEmpty(tableName) || StringUtils.isEmpty(columnName))) { - throw new IllegalArgumentException("The hierarchy of privilege is not correct."); - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/HiveShellCommand.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/HiveShellCommand.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/HiveShellCommand.java deleted file mode 100644 index 3abba52..0000000 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/HiveShellCommand.java +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.provider.db.tools.command.hive; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.sentry.core.common.exception.SentryUserException; -import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient; -import org.apache.sentry.provider.db.service.thrift.TSentryGroup; -import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; -import org.apache.sentry.provider.db.service.thrift.TSentryRole; -import org.apache.sentry.provider.db.tools.ShellCommand; -import org.apache.sentry.service.thrift.SentryServiceUtil; - -/** - * The ShellCommand implementation for Hive. - */ -public class HiveShellCommand implements ShellCommand { - - private final SentryPolicyServiceClient client; - - public HiveShellCommand(SentryPolicyServiceClient client) { - this.client = client; - } - - public void createRole(String requestorName, String roleName) throws SentryUserException { - client.createRole(requestorName, roleName); - } - - public void dropRole(String requestorName, String roleName) throws SentryUserException { - client.dropRole(requestorName, roleName); - } - - public void grantPrivilegeToRole(String requestorName, String roleName, String privilege) throws SentryUserException { - TSentryPrivilege tSentryPrivilege = SentryServiceUtil.convertToTSentryPrivilege(privilege); - CommandUtil.validatePrivilegeHierarchy(tSentryPrivilege); - client.grantPrivilege(requestorName, roleName, tSentryPrivilege); - } - - public void grantRoleToGroups(String requestorName, String roleName, Set groups) throws SentryUserException { - client.grantRoleToGroups(requestorName, roleName, groups); - } - - public void revokePrivilegeFromRole(String requestorName, String roleName, String privilege) throws SentryUserException { - TSentryPrivilege tSentryPrivilege = SentryServiceUtil.convertToTSentryPrivilege(privilege); - CommandUtil.validatePrivilegeHierarchy(tSentryPrivilege); - client.revokePrivilege(requestorName, roleName, tSentryPrivilege); - } - - public void revokeRoleFromGroups(String requestorName, String roleName, Set groups) throws SentryUserException { - client.revokeRoleFromGroups(requestorName, roleName, groups); - } - - public List listRoles(String requestorName, String group) throws SentryUserException { - Set roles; - if (StringUtils.isEmpty(group)) { - roles = client.listAllRoles(requestorName); - } else { - roles = client.listRolesByGroupName(requestorName, group); - } - - List result = new ArrayList<>(); - if (roles != null) { - for (TSentryRole role : roles) { - result.add(role.getRoleName()); - } - } - - return result; - } - - public List listPrivileges(String requestorName, String roleName) throws SentryUserException { - Set privileges = client - .listAllPrivilegesByRoleName(requestorName, roleName); - - List result = new ArrayList<>(); - if (privileges != null) { - for (TSentryPrivilege privilege : privileges) { - String privilegeStr = SentryServiceUtil.convertTSentryPrivilegeToStr(privilege); - result.add(privilegeStr); - } - } - return result; - } - - public List listGroupRoles(String requestorName) throws SentryUserException { - Set roles = client.listAllRoles(requestorName); - if (roles == null || roles.isEmpty()) { - return Collections.emptyList(); - } - - // Set of all group names - Set groupNames = new HashSet<>(); - - // Map group to set of roles - Map> groupInfo = new HashMap<>(); - - // Get all group names - for (TSentryRole role: roles) { - for (TSentryGroup group : role.getGroups()) { - String groupName = group.getGroupName(); - groupNames.add(groupName); - Set groupRoles = groupInfo.get(groupName); - if (groupRoles != null) { - // Add a new or existing role - groupRoles.add(role.getRoleName()); - continue; - } - // Never seen this group before - groupRoles = new HashSet<>(); - groupRoles.add(role.getRoleName()); - groupInfo.put(groupName, groupRoles); - } - } - - List groups = new ArrayList<>(groupNames); - - // Produce printable result as - // group1 = role1, role2, ... - // group2 = ... - List result = new LinkedList<>(); - for (String groupName: groups) { - result.add(groupName + " = " + StringUtils.join(groupInfo.get(groupName), ", ")); - } - - return result; - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/generic/tools/TestPermissionsMigrationToolSolr.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/generic/tools/TestPermissionsMigrationToolSolr.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/generic/tools/TestPermissionsMigrationToolSolr.java deleted file mode 100644 index 69c067f..0000000 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/generic/tools/TestPermissionsMigrationToolSolr.java +++ /dev/null @@ -1,362 +0,0 @@ - /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.provider.db.generic.tools; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.io.FileUtils; -import org.apache.sentry.core.common.exception.SentryUserException; -import org.apache.sentry.provider.common.ProviderBackendContext; -import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceIntegrationBase; -import org.apache.sentry.provider.db.generic.service.thrift.TAuthorizable; -import org.apache.sentry.provider.db.generic.service.thrift.TSentryPrivilege; -import org.apache.sentry.provider.db.generic.service.thrift.TSentryRole; -import org.apache.sentry.provider.file.PolicyFile; -import org.apache.sentry.provider.file.SimpleFileProviderBackend; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.collect.Sets; -import com.google.common.collect.Table; -import com.google.common.io.Files; - -public class TestPermissionsMigrationToolSolr extends SentryGenericServiceIntegrationBase { - private File confDir; - private File confPath; - private String requestorName = ""; - private String service = "service1"; - - @Before - public void prepareForTest() throws Exception { - confDir = Files.createTempDir(); - confPath = new File(confDir, "sentry-site.xml"); - if (confPath.createNewFile()) { - FileOutputStream to = new FileOutputStream(confPath); - conf.writeXml(to); - to.close(); - } - requestorName = clientUgi.getShortUserName();//System.getProperty("user.name", ""); - Set requestorUserGroupNames = Sets.newHashSet(ADMIN_GROUP); - setLocalGroupMapping(requestorName, requestorUserGroupNames); - // add ADMIN_USER for the after() in SentryServiceIntegrationBase - setLocalGroupMapping(ADMIN_USER, requestorUserGroupNames); - setLocalGroupMapping("dev", Sets.newHashSet("dev_group")); - setLocalGroupMapping("user", Sets.newHashSet("user_group")); - writePolicyFile(); - } - - @After - public void clearTestData() throws Exception { - FileUtils.deleteQuietly(confDir); - - // clear roles and privileges - Set tRoles = client.listAllRoles(requestorName, SOLR); - for (TSentryRole tRole : tRoles) { - String role = tRole.getRoleName(); - Set privileges = client.listAllPrivilegesByRoleName( - requestorName, role, SOLR, service); - for (TSentryPrivilege privilege : privileges) { - client.revokePrivilege(requestorName, role, SOLR, privilege); - } - client.dropRole(requestorName, role, SOLR); - } - } - - @Test - public void testPermissionsMigrationFromSentrySvc_v1() throws Exception { - initializeSentryService(); - - String[] args = { "-s", "1.8.0", "-c", confPath.getAbsolutePath()}; - PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr(); - sentryTool.executeConfigTool(args); - - Map> groupMapping = new HashMap>(); - groupMapping.put("admin_role", Sets.newHashSet("admin_group")); - groupMapping.put("dev_role", Sets.newHashSet("dev_group")); - groupMapping.put("user_role", Sets.newHashSet("user_group")); - - Map> privilegeMapping = new HashMap>(); - privilegeMapping.put("admin_role", - Sets.newHashSet("admin=collections->action=*", "admin=cores->action=*")); - privilegeMapping.put("dev_role", - Sets.newHashSet("collection=*->action=*", "admin=collections->action=*", "admin=cores->action=*")); - privilegeMapping.put("user_role", - Sets.newHashSet("collection=foo->action=*")); - - verifySentryServiceState(groupMapping, privilegeMapping); - } - - @Test - public void testPermissionsMigrationFromSentryPolicyFile_v1() throws Exception { - Path policyFilePath = initializeSentryPolicyFile(); - Path outputFilePath = Paths.get(confDir.getAbsolutePath(), "sentry-provider_migrated.ini"); - - String[] args = { "-s", "1.8.0", "-p", policyFilePath.toFile().getAbsolutePath(), - "-o", outputFilePath.toFile().getAbsolutePath() }; - PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr(); - assertTrue(sentryTool.executeConfigTool(args)); - - Set groups = new HashSet<>(); - groups.add("admin_group"); - groups.add("dev_group"); - groups.add("user_group"); - - Map> privilegeMapping = new HashMap>(); - privilegeMapping.put("admin_role", - Sets.newHashSet("admin=collections->action=*", "admin=cores->action=*")); - privilegeMapping.put("dev_role", - Sets.newHashSet("collection=*->action=*", "admin=collections->action=*", "admin=cores->action=*")); - privilegeMapping.put("user_role", - Sets.newHashSet("collection=foo->action=*")); - - verifySentryPolicyFile(groups, privilegeMapping, outputFilePath); - } - - @Test - // For permissions created with Sentry 2.x, no migration necessary - public void testPermissionsMigrationFromSentrySvc_v2() throws Exception { - initializeSentryService(); - - String[] args = { "-s", "2.0.0", "-c", confPath.getAbsolutePath()}; - PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr(); - sentryTool.executeConfigTool(args); - - Map> groupMapping = new HashMap>(); - groupMapping.put("admin_role", Sets.newHashSet("admin_group")); - groupMapping.put("dev_role", Sets.newHashSet("dev_group")); - groupMapping.put("user_role", Sets.newHashSet("user_group")); - - Map> privilegeMapping = new HashMap>(); - privilegeMapping.put("admin_role", - Sets.newHashSet("collection=admin->action=*")); - privilegeMapping.put("dev_role", - Sets.newHashSet("collection=*->action=*")); - privilegeMapping.put("user_role", - Sets.newHashSet("collection=foo->action=*")); - - verifySentryServiceState(groupMapping, privilegeMapping); - } - - @Test - // For permissions created with Sentry 2.x, no migration necessary - public void testPermissionsMigrationFromSentryPolicyFile_v2() throws Exception { - Path policyFilePath = initializeSentryPolicyFile(); - Path outputFilePath = Paths.get(confDir.getAbsolutePath(), "sentry-provider_migrated.ini"); - - String[] args = { "-s", "2.0.0", "-p", policyFilePath.toFile().getAbsolutePath(), - "-o", outputFilePath.toFile().getAbsolutePath() }; - PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr(); - assertTrue(sentryTool.executeConfigTool(args)); - - Set groups = new HashSet<>(); - groups.add("admin_group"); - groups.add("dev_group"); - groups.add("user_group"); - - Map> privilegeMapping = new HashMap>(); - privilegeMapping.put("admin_role", - Sets.newHashSet("collection=admin->action=*")); - privilegeMapping.put("dev_role", - Sets.newHashSet("collection=*->action=*")); - privilegeMapping.put("user_role", - Sets.newHashSet("collection=foo->action=*")); - - verifySentryPolicyFile(groups, privilegeMapping, outputFilePath); - } - - @Test - public void testDryRunOption() throws Exception { - initializeSentryService(); - - String[] args = { "-s", "1.8.0", "-c", confPath.getAbsolutePath(), "--dry_run"}; - PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr(); - sentryTool.executeConfigTool(args); - - Map> groupMapping = new HashMap>(); - groupMapping.put("admin_role", Sets.newHashSet("admin_group")); - groupMapping.put("dev_role", Sets.newHashSet("dev_group")); - groupMapping.put("user_role", Sets.newHashSet("user_group")); - - // No change in the privileges - Map> privilegeMapping = new HashMap>(); - privilegeMapping.put("admin_role", - Sets.newHashSet("collection=admin->action=*")); - privilegeMapping.put("dev_role", - Sets.newHashSet("collection=*->action=*")); - privilegeMapping.put("user_role", - Sets.newHashSet("collection=foo->action=*")); - - verifySentryServiceState(groupMapping, privilegeMapping); - } - - @Test - public void testInvalidToolArguments() throws Exception { - PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr(); - - { - String[] args = { "-c", confPath.getAbsolutePath()}; - assertFalse("The execution should have failed due to missing source version", - sentryTool.executeConfigTool(args)); - } - - { - String[] args = { "-s", "1.8.0" }; - sentryTool.executeConfigTool(args); - assertFalse("The execution should have failed due to missing Sentry config file" - + " (or policy file) path", - sentryTool.executeConfigTool(args)); - } - - { - String[] args = { "-s", "1.8.0", "-p", "/test/path" }; - sentryTool.executeConfigTool(args); - assertFalse("The execution should have failed due to missing Sentry config output file path", - sentryTool.executeConfigTool(args)); - } - - { - String[] args = { "-s", "1.8.0", "-c", "/test/path1", "-p", "/test/path2" }; - sentryTool.executeConfigTool(args); - assertFalse("The execution should have failed due to providing both Sentry config file" - + " as well as policy file params", - sentryTool.executeConfigTool(args)); - } - } - - private void initializeSentryService() throws SentryUserException { - // Define an admin role - client.createRoleIfNotExist(requestorName, "admin_role", SOLR); - client.grantRoleToGroups(requestorName, "admin_role", SOLR, Sets.newHashSet("admin_group")); - - // Define a developer role - client.createRoleIfNotExist(requestorName, "dev_role", SOLR); - client.grantRoleToGroups(requestorName, "dev_role", SOLR, Sets.newHashSet("dev_group")); - - // Define a user role - client.createRoleIfNotExist(requestorName, "user_role", SOLR); - client.grantRoleToGroups(requestorName, "user_role", SOLR, Sets.newHashSet("user_group")); - - // Grant permissions - client.grantPrivilege(requestorName, "admin_role", SOLR, - new TSentryPrivilege(SOLR, "service1", - Arrays.asList(new TAuthorizable("collection", "admin")), "*")); - client.grantPrivilege(requestorName, "dev_role", SOLR, - new TSentryPrivilege(SOLR, "service1", - Arrays.asList(new TAuthorizable("collection", "*")), "*")); - client.grantPrivilege(requestorName, "user_role", SOLR, - new TSentryPrivilege(SOLR, "service1", - Arrays.asList(new TAuthorizable("collection", "foo")), "*")); - } - - private void verifySentryServiceState(Map> groupMapping, - Map> privilegeMapping) throws SentryUserException { - // check roles - Set tRoles = client.listAllRoles(requestorName, SOLR); - assertEquals("Unexpected number of roles", groupMapping.keySet().size(), tRoles.size()); - Set roles = new HashSet(); - for (TSentryRole tRole : tRoles) { - roles.add(tRole.getRoleName()); - } - - for (String expectedRole : groupMapping.keySet()) { - assertTrue("Didn't find expected role: " + expectedRole, roles.contains(expectedRole)); - } - - // check groups - for (TSentryRole tRole : tRoles) { - Set expectedGroups = groupMapping.get(tRole.getRoleName()); - assertEquals("Group size doesn't match for role: " + tRole.getRoleName(), - expectedGroups.size(), tRole.getGroups().size()); - assertTrue("Group does not contain all expected members for role: " + tRole.getRoleName(), - tRole.getGroups().containsAll(expectedGroups)); - } - - // check privileges - GenericPrivilegeConverter convert = new GenericPrivilegeConverter(SOLR, service); - for (String role : roles) { - Set privileges = client.listAllPrivilegesByRoleName( - requestorName, role, SOLR, service); - Set expectedPrivileges = privilegeMapping.get(role); - assertEquals("Privilege set size doesn't match for role: " + role + " Actual permissions : " + privileges, - expectedPrivileges.size(), privileges.size()); - - Set privilegeStrs = new HashSet(); - for (TSentryPrivilege privilege : privileges) { - privilegeStrs.add(convert.toString(privilege).toLowerCase()); - } - - for (String expectedPrivilege : expectedPrivileges) { - assertTrue("Did not find expected privilege: " + expectedPrivilege + " in " + privilegeStrs, - privilegeStrs.contains(expectedPrivilege)); - } - } - } - - private Path initializeSentryPolicyFile() throws Exception { - PolicyFile file = new PolicyFile(); - - file.addRolesToGroup("admin_group", "admin_role"); - file.addRolesToGroup("dev_group", "dev_role"); - file.addRolesToGroup("user_group", "user_role"); - - file.addPermissionsToRole("admin_role", "collection=admin->action=*"); - file.addPermissionsToRole("dev_role", "collection=*->action=*"); - file.addPermissionsToRole("user_role", "collection=foo->action=*"); - - Path policyFilePath = Paths.get(confDir.getAbsolutePath(), "sentry-provider.ini"); - file.write(policyFilePath.toFile()); - - return policyFilePath; - } - - private void verifySentryPolicyFile (Set groups, Map> privilegeMapping, - Path policyFilePath) throws IOException { - SimpleFileProviderBackend policyFileBackend = new SimpleFileProviderBackend(conf, - new org.apache.hadoop.fs.Path(policyFilePath.toUri())); - policyFileBackend.initialize(new ProviderBackendContext()); - Table> groupRolePrivilegeTable = - policyFileBackend.getGroupRolePrivilegeTable(); - - assertEquals(groups, groupRolePrivilegeTable.rowKeySet()); - assertEquals(privilegeMapping.keySet(), groupRolePrivilegeTable.columnKeySet()); - - for (String groupName : groupRolePrivilegeTable.rowKeySet()) { - for (String roleName : groupRolePrivilegeTable.columnKeySet()) { - if (groupRolePrivilegeTable.contains(groupName, roleName)) { - Set privileges = groupRolePrivilegeTable.get(groupName, roleName); - assertEquals(privilegeMapping.get(roleName), privileges); - } - } - } - } -}