sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cohei...@apache.org
Subject [8/9] sentry git commit: SENTRY-2207 Refactor out Sentry CLI from sentry-provider-db into own module. Steve Moist, reviewed by Colm O hEigeartaigh.
Date Mon, 30 Apr 2018 16:35:44 GMT
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaHelper.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaHelper.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaHelper.java
deleted file mode 100644
index cf1c725..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaHelper.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.provider.db.tools;
-
-import java.util.IllegalFormatException;
-
-public final class SentrySchemaHelper {
-  public static final String DB_DERBY = "derby";
-  public static final String DB_MYSQL = "mysql";
-  public static final String DB_POSTGRACE = "postgres";
-  public static final String DB_ORACLE = "oracle";
-  public static final String DB_DB2 = "db2";
-
-  public interface NestedScriptParser {
-
-    public enum CommandType {
-      PARTIAL_STATEMENT,
-      TERMINATED_STATEMENT,
-      COMMENT
-    }
-
-    String DEFAUTL_DELIMITER = ";";
-    /***
-     * Find the type of given command
-     * @param dbCommand
-     * @return
-     */
-    boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
-
-    /** Parse the DB specific nesting format and extract the inner script name if any
-     * @param dbCommand command from parent script
-     * @return
-     * @throws IllegalFormatException
-     */
-    String getScriptName(String dbCommand) throws IllegalArgumentException;
-
-    /***
-     * Find if the given command is a nested script execution
-     * @param dbCommand
-     * @return
-     */
-    boolean isNestedScript(String dbCommand);
-
-    /***
-     * Find if the given command is should be passed to DB
-     * @param dbCommand
-     * @return
-     */
-    boolean isNonExecCommand(String dbCommand);
-
-    /***
-     * Get the SQL statement delimiter
-     * @return
-     */
-    String getDelimiter();
-
-    /***
-     * Clear any client specific tags
-     * @return
-     */
-    String cleanseCommand(String dbCommand);
-
-    /***
-     * Does the DB required table/column names quoted
-     * @return
-     */
-    boolean needsQuotedIdentifier();
-
-    /***
-     * Set DB specific options if any
-     * @param dbOps
-     */
-    void setDbOpts(String dbOps);
-  }
-
-
-  /***
-   * Base implemenation of NestedScriptParser
-   * abstractCommandParser.
-   *
-   */
-  private static abstract class AbstractCommandParser implements NestedScriptParser {
-    private String dbOpts = null;
-
-    @Override
-    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
-      if (dbCommand == null || dbCommand.isEmpty()) {
-        throw new IllegalArgumentException("invalid command line " + dbCommand);
-      }
-      String trimmedDbCommand = dbCommand.trim();
-      return !(trimmedDbCommand.endsWith(getDelimiter()) || isNonExecCommand(trimmedDbCommand));
-    }
-
-    @Override
-    public boolean isNonExecCommand(String dbCommand) {
-      return dbCommand.startsWith("--") || dbCommand.startsWith("#");
-    }
-
-    @Override
-    public String getDelimiter() {
-      return DEFAUTL_DELIMITER;
-    }
-
-    @Override
-    public String cleanseCommand(String dbCommand) {
-      // strip off the delimiter
-      if (dbCommand.endsWith(getDelimiter())) {
-        dbCommand = dbCommand.substring(0,
-            dbCommand.length() - getDelimiter().length());
-      }
-      return dbCommand;
-    }
-
-    @Override
-    public boolean needsQuotedIdentifier() {
-      return false;
-    }
-
-    @Override
-    public void setDbOpts(String dbOpts) {
-      this.dbOpts = dbOpts;
-    }
-
-    protected String getDbOpts() {
-      return dbOpts;
-    }
-  }
-
-
-  // Derby commandline parser
-  public static class DerbyCommandParser extends AbstractCommandParser {
-    private static final String DERBY_NESTING_TOKEN = "RUN";
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
-      if (!isNestedScript(dbCommand)) {
-        throw new IllegalArgumentException("Not a script format " + dbCommand);
-      }
-      String[] tokens = dbCommand.split(" ");
-      if (tokens.length != 2) {
-        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-      }
-      return tokens[1].replace(";", "").replaceAll("'", "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      // Derby script format is RUN '<file>'
-     return dbCommand.startsWith(DERBY_NESTING_TOKEN);
-    }
-  }
-
-
-  // MySQL parser
-  public static class MySqlCommandParser extends AbstractCommandParser {
-    private static final String MYSQL_NESTING_TOKEN = "SOURCE";
-    private static final String DELIMITER_TOKEN = "DELIMITER";
-    private String delimiter = DEFAUTL_DELIMITER;
-
-    @Override
-    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
-      boolean isPartial = super.isPartialCommand(dbCommand);
-      // if this is a delimiter directive, reset our delimiter
-      if (dbCommand.startsWith(DELIMITER_TOKEN)) {
-        String[] tokens = dbCommand.split(" ");
-        if (tokens.length != 2) {
-          throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-        }
-        delimiter = tokens[1];
-      }
-      return isPartial;
-    }
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-      String[] tokens = dbCommand.split(" ");
-      if (tokens.length != 2) {
-        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-      }
-      // remove ending ';'
-      return tokens[1].replace(";", "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
-    }
-
-    @Override
-    public String getDelimiter() {
-      return delimiter;
-    }
-
-    @Override
-    public boolean isNonExecCommand(String dbCommand) {
-      return super.isNonExecCommand(dbCommand) ||
-          dbCommand.startsWith("/*") && dbCommand.endsWith("*/") ||
-          dbCommand.startsWith(DELIMITER_TOKEN);
-    }
-
-    @Override
-    public String cleanseCommand(String dbCommand) {
-      return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
-    }
-
-  }
-
-  // Postgres specific parser
-  public static class PostgresCommandParser extends AbstractCommandParser {
-    public static final String POSTGRES_STRING_COMMAND_FILTER = "SET standard_conforming_strings";
-    public static final String POSTGRES_STRING_CLIENT_ENCODING = "SET client_encoding";
-    public static final String POSTGRES_SKIP_STANDARD_STRING = "postgres.filter.81";
-    private static final String POSTGRES_NESTING_TOKEN = "\\i";
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-      String[] tokens = dbCommand.split(" ");
-      if (tokens.length != 2) {
-        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-      }
-      // remove ending ';'
-      return tokens[1].replace(";", "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
-    }
-
-    @Override
-    public boolean needsQuotedIdentifier() {
-      return true;
-    }
-
-    @Override
-    public boolean isNonExecCommand(String dbCommand) {
-      // Skip "standard_conforming_strings" command which is not supported in older postgres
-      if (POSTGRES_SKIP_STANDARD_STRING.equalsIgnoreCase(getDbOpts()) 
-        && (dbCommand.startsWith(POSTGRES_STRING_COMMAND_FILTER) || dbCommand.startsWith(POSTGRES_STRING_CLIENT_ENCODING))) {
-        return true;
-      }
-      return super.isNonExecCommand(dbCommand);
-    }
-  }
-
-  //Oracle specific parser
-  public static class OracleCommandParser extends AbstractCommandParser {
-    private static final String ORACLE_NESTING_TOKEN = "@";
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-      if (!isNestedScript(dbCommand)) {
-        throw new IllegalArgumentException("Not a nested script format " + dbCommand);
-      }
-      // remove ending ';' and starting '@'
-      return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
-    }
-  }
-
-  // DB2 commandline parser
-  public static class DB2CommandParser extends AbstractCommandParser {
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-        //DB2 does not support nesting script
-        throw new IllegalArgumentException("DB2 does not support nesting script " + dbCommand);
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-        //DB2 does not support nesting script
-     return false;
-    }
-  }
-
-  public static NestedScriptParser getDbCommandParser(String dbName) {
-    if (dbName.equalsIgnoreCase(DB_DERBY)) {
-      return new DerbyCommandParser();
-    } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
-      return new MySqlCommandParser();
-    } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
-      return new PostgresCommandParser();
-    } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
-        return new OracleCommandParser();
-    } else if (dbName.equalsIgnoreCase(DB_DB2)) {
-      return new DB2CommandParser();
-    } else {
-      throw new IllegalArgumentException("Unknown dbType " + dbName);
-    }
-  }
-  
-  private SentrySchemaHelper() {
-    // Make constructor private to avoid instantiation
-  }
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaTool.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaTool.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaTool.java
deleted file mode 100644
index d75e24b..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentrySchemaTool.java
+++ /dev/null
@@ -1,595 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.provider.db.tools;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.MalformedURLException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.IllegalFormatException;
-import java.util.List;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.io.output.NullOutputStream;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.beeline.BeeLine;
-import org.apache.sentry.Command;
-import org.apache.sentry.core.common.exception.SentryUserException;
-import org.apache.sentry.core.common.exception.SentrySiteConfigurationException;
-import org.apache.sentry.provider.db.service.persistent.SentryStoreSchemaInfo;
-import org.apache.sentry.provider.db.tools.SentrySchemaHelper.NestedScriptParser;
-import org.apache.sentry.service.thrift.SentryService;
-import org.apache.sentry.service.thrift.ServiceConstants;
-
-public class SentrySchemaTool {
-  private static final String SENTRY_SCRIP_DIR = File.separatorChar + "scripts"
-      + File.separatorChar + "sentrystore" + File.separatorChar + "upgrade";
-  private String userName = null;
-  private String passWord = null;
-  private String connectionURL = null;
-  private String driver = null;
-  private boolean dryRun = false;
-  private String dbOpts = null;
-  private boolean verbose = false;
-  private final Configuration sentryConf;
-  private final String dbType;
-  private final SentryStoreSchemaInfo sentryStoreSchemaInfo;
-
-  public SentrySchemaTool(Configuration sentryConf, String dbType)
-      throws SentryUserException, IOException {
-    this(System.getenv("SENTRY_HOME") + SENTRY_SCRIP_DIR, sentryConf, dbType);
-  }
-
-  public SentrySchemaTool(String sentryScripPath, Configuration sentryConf,
-      String dbType) throws SentryUserException, IOException {
-    if (sentryScripPath == null || sentryScripPath.isEmpty()) {
-      throw new SentryUserException("No Sentry script dir provided");
-    }
-    this.sentryConf = sentryConf;
-    this.dbType = dbType;
-    this.sentryStoreSchemaInfo = new SentryStoreSchemaInfo(sentryScripPath,
-        dbType);
-    userName = sentryConf.get(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_USER,
-        ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_USER_DEFAULT);
-    //Password will be read from Credential provider specified using property
-    // CREDENTIAL_PROVIDER_PATH("hadoop.security.credential.provider.path" in sentry-site.xml
-    // it falls back to reading directly from sentry-site.xml
-    char[] passTmp = sentryConf.getPassword(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_PASS);
-    if(passTmp != null) {
-      passWord = new String(passTmp);
-    } else {
-      throw new SentrySiteConfigurationException("Error reading " + ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_PASS);
-    }
-
-    try {
-      connectionURL = getValidConfVar(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_URL);
-      if(dbType.equalsIgnoreCase(SentrySchemaHelper.DB_DERBY)) {
-        driver = sentryConf.get(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_DRIVER,
-            ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_DRIVER_DEFAULT);
-      } else {
-        driver = getValidConfVar(ServiceConstants.ServerConfig.SENTRY_STORE_JDBC_DRIVER);
-      }
-      // load required JDBC driver
-      Class.forName(driver);
-    } catch (IOException e) {
-      throw new SentryUserException("Missing property: " + e.getMessage());
-    } catch (ClassNotFoundException e) {
-      throw new SentryUserException("Failed to load driver", e);
-    }
-  }
-
-  public Configuration getConfiguration() {
-    return sentryConf;
-  }
-
-  public void setUserName(String userName) {
-    this.userName = userName;
-  }
-
-  public void setPassWord(String passWord) {
-    this.passWord = passWord;
-  }
-
-  public void setDryRun(boolean dryRun) {
-    this.dryRun = dryRun;
-  }
-
-  public void setVerbose(boolean verbose) {
-    this.verbose = verbose;
-  }
-
-  public String getDbOpts() {
-    return dbOpts;
-  }
-
-  public void setDbOpts(String dbOpts) {
-    this.dbOpts = dbOpts;
-  }
-
-  private static void printAndExit(Options cmdLineOptions) {
-    HelpFormatter formatter = new HelpFormatter();
-    formatter.printHelp("schemaTool", cmdLineOptions);
-    System.exit(1);
-  }
-
-  /***
-   * Print Hive version and schema version
-   * @throws SentryUserException
-   */
-  public void showInfo() throws SentryUserException {
-    Connection sentryStoreConn = getConnectionToMetastore(true);
-    System.out.println("Sentry distribution version:\t "
-        + SentryStoreSchemaInfo.getSentryVersion());
-    System.out.println("SentryStore schema version:\t "
-        + getMetaStoreSchemaVersion(sentryStoreConn));
-  }
-
-  // read schema version from sentry store
-  private String getMetaStoreSchemaVersion(Connection sentryStoreConn)
-      throws SentryUserException {
-    String versionQuery;
-    if (SentrySchemaHelper.getDbCommandParser(dbType).needsQuotedIdentifier()) {
-      versionQuery = "select t.\"SCHEMA_VERSION\" from \"SENTRY_VERSION\" t";
-    } else {
-      versionQuery = "select t.SCHEMA_VERSION from SENTRY_VERSION t";
-    }
-    try (Statement stmt = sentryStoreConn.createStatement();
-      ResultSet res = stmt.executeQuery(versionQuery)) {
-      if (!res.next()) {
-        throw new SentryUserException("Didn't find version data in sentry store");
-      }
-      String currentSchemaVersion = res.getString(1);
-      sentryStoreConn.close();
-      return currentSchemaVersion;
-    } catch (SQLException e) {
-      throw new SentryUserException("Failed to get schema version.", e);
-    }
-  }
-
-  // test the connection sentry store using the config property
-  private void testConnectionToMetastore() throws SentryUserException {
-    try (Connection conn = getConnectionToMetastore(true)) {
-      conn.close();
-    } catch (SQLException e) {
-      throw new SentryUserException("Failed to close sentry store connection", e);
-    }
-  }
-
-  /***
-   * get JDBC connection to sentry store db
-   *
-   * @param printInfo print connection parameters
-   * @return
-   * @throws SentryUserException
-   */
-  private Connection getConnectionToMetastore(boolean printInfo)
-      throws SentryUserException {
-    if (printInfo) {
-      System.out.println("Sentry store connection URL:\t " + connectionURL);
-      System.out.println("Sentry store Connection Driver :\t " + driver);
-      System.out.println("Sentry store connection User:\t " + userName);
-    }
-    if (userName == null || userName.isEmpty()) {
-      throw new SentryUserException("UserName empty ");
-    }
-    try {
-      // Connect using the JDBC URL and user/pass from conf
-      return DriverManager.getConnection(connectionURL, userName, passWord);
-    } catch (SQLException e) {
-      throw new SentryUserException("Failed to make connection to Sentry store.", e);
-    }
-  }
-
-  /**
-   * check if the current schema version in sentry store matches the Hive version
-   * @throws SentryUserException
-   */
-  public void verifySchemaVersion() throws SentryUserException {
-    // don't check version if its a dry run
-    if (dryRun) {
-      return;
-    }
-    String newSchemaVersion =
-        getMetaStoreSchemaVersion(getConnectionToMetastore(false));
-    // verify that the new version is added to schema
-    if (!sentryStoreSchemaInfo.getSentrySchemaVersion().equalsIgnoreCase(
-        newSchemaVersion)) {
-      throw new SentryUserException("Found unexpected schema version "
-          + newSchemaVersion);
-    }
-  }
-
-  /**
-   * Perform sentry store schema upgrade. extract the current schema version from sentry store
-   * @throws SentryUserException
-   */
-  public void doUpgrade() throws SentryUserException {
-    String fromVersion = getMetaStoreSchemaVersion(getConnectionToMetastore(false));
-    if (fromVersion == null || fromVersion.isEmpty()) {
-      throw new SentryUserException(
-          "Schema version not stored in the sentry store. "
-              +
-          "Metastore schema is too old or corrupt. Try specifying the version manually");
-    }
-    doUpgrade(fromVersion);
-  }
-
-  /**
-   * Perform sentry store schema upgrade
-   *
-   * @param fromSchemaVer
-   *          Existing version of the sentry store. If null, then read from the sentry store
-   * @throws SentryUserException
-   */
-  public void doUpgrade(String fromSchemaVer) throws SentryUserException {
-    if (sentryStoreSchemaInfo.getSentrySchemaVersion().equals(fromSchemaVer)) {
-      System.out.println("No schema upgrade required from version " + fromSchemaVer);
-      return;
-    }
-    // Find the list of scripts to execute for this upgrade
-    List<String> upgradeScripts =
-        sentryStoreSchemaInfo.getUpgradeScripts(fromSchemaVer);
-    testConnectionToMetastore();
-    System.out.println("Starting upgrade sentry store schema from version " +
- fromSchemaVer + " to "
-        + sentryStoreSchemaInfo.getSentrySchemaVersion());
-    String scriptDir = sentryStoreSchemaInfo.getSentryStoreScriptDir();
-    try {
-      for (String scriptFile : upgradeScripts) {
-        System.out.println("Upgrade script " + scriptFile);
-        if (!dryRun) {
-          runBeeLine(scriptDir, scriptFile);
-          System.out.println("Completed " + scriptFile);
-        }
-      }
-    } catch (IOException eIO) {
-      throw new SentryUserException(
-          "Upgrade FAILED! Metastore state would be inconsistent !!", eIO);
-    }
-
-    // Revalidated the new version after upgrade
-    verifySchemaVersion();
-  }
-
-  /**
-   * Initialize the sentry store schema to current version
-   *
-   * @throws SentryUserException
-   */
-  public void doInit() throws SentryUserException {
-    doInit(sentryStoreSchemaInfo.getSentrySchemaVersion());
-
-    // Revalidated the new version after upgrade
-    verifySchemaVersion();
-  }
-
-  /**
-   * Initialize the sentry store schema
-   *
-   * @param toVersion
-   *          If null then current hive version is used
-   * @throws SentryUserException
-   */
-  public void doInit(String toVersion) throws SentryUserException {
-    testConnectionToMetastore();
-    System.out.println("Starting sentry store schema initialization to " + toVersion);
-
-    String initScriptDir = sentryStoreSchemaInfo.getSentryStoreScriptDir();
-    String initScriptFile = sentryStoreSchemaInfo.generateInitFileName(toVersion);
-
-    try {
-      System.out.println("Initialization script " + initScriptFile);
-      if (!dryRun) {
-        runBeeLine(initScriptDir, initScriptFile);
-        System.out.println("Initialization script completed");
-      }
-    } catch (IOException e) {
-      throw new SentryUserException("Schema initialization FAILED!"
-          + " Metastore state would be inconsistent !!", e);
-    }
-  }
-
-  // Flatten the nested upgrade script into a buffer
-  public static String buildCommand(NestedScriptParser dbCommandParser,
-        String scriptDir, String scriptFile) throws IllegalFormatException, IOException {
-
-    BufferedReader bfReader =
-        new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile));
-    String currLine;
-    StringBuilder sb = new StringBuilder();
-    String currentCommand = null;
-    while ((currLine = bfReader.readLine()) != null) {
-      currLine = currLine.trim();
-      if (currLine.isEmpty()) {
-        continue; // skip empty lines
-      }
-
-      if (currentCommand == null) {
-        currentCommand = currLine;
-      } else {
-        currentCommand = currentCommand + " " + currLine;
-      }
-      if (dbCommandParser.isPartialCommand(currLine)) {
-        // if its a partial line, continue collecting the pieces
-        continue;
-      }
-
-      // if this is a valid executable command then add it to the buffer
-      if (!dbCommandParser.isNonExecCommand(currentCommand)) {
-        currentCommand = dbCommandParser.cleanseCommand(currentCommand);
-
-        if (dbCommandParser.isNestedScript(currentCommand)) {
-          // if this is a nested sql script then flatten it
-          String currScript = dbCommandParser.getScriptName(currentCommand);
-          sb.append(buildCommand(dbCommandParser, scriptDir, currScript));
-        } else {
-          // Now we have a complete statement, process it
-          // write the line to buffer
-          sb.append(currentCommand);
-          sb.append(System.getProperty("line.separator"));
-        }
-      }
-      currentCommand = null;
-    }
-    bfReader.close();
-    return sb.toString();
-  }
-
-  // run beeline on the given sentry store scrip, flatten the nested scripts into single file
-  private void runBeeLine(String scriptDir, String scriptFile) throws IOException {
-    NestedScriptParser dbCommandParser =
-        SentrySchemaHelper.getDbCommandParser(dbType);
-    dbCommandParser.setDbOpts(getDbOpts());
-    // expand the nested script
-    String sqlCommands = buildCommand(dbCommandParser, scriptDir, scriptFile);
-    File tmpFile = File.createTempFile("schematool", ".sql");
-    tmpFile.deleteOnExit();
-
-    // write out the buffer into a file. Add beeline commands for autocommit and close
-    try (FileWriter fstream = new FileWriter(tmpFile.getPath());
-      BufferedWriter out = new BufferedWriter(fstream)) {
-
-      out.write("!set Silent " + verbose + System.getProperty("line.separator"));
-      out.write("!autocommit on" + System.getProperty("line.separator"));
-      out.write("!set Isolation TRANSACTION_READ_COMMITTED"
-          + System.getProperty("line.separator"));
-      out.write("!set AllowMultiLineCommand false"
-          + System.getProperty("line.separator"));
-      out.write(sqlCommands);
-      out.write("!closeall" + System.getProperty("line.separator"));
-      out.close();
-    }
-    runBeeLine(tmpFile.getPath());
-  }
-
-  // Generate the beeline args per hive conf and execute the given script
-  public void runBeeLine(String sqlScriptFile) throws IOException {
-    List<String> argList = new ArrayList<String>();
-    argList.add("-u");
-    argList.add(connectionURL);
-    argList.add("-d");
-    argList
-        .add(driver);
-    argList.add("-n");
-    argList.add(userName);
-    argList.add("-p");
-    argList.add(passWord);
-    argList.add("-f");
-    argList.add(sqlScriptFile);
-
-    BeeLine beeLine = new BeeLine();
-    if (!verbose) {
-      beeLine.setOutputStream(new PrintStream(new NullOutputStream()));
-      // beeLine.getOpts().setSilent(true);
-    }
-    // beeLine.getOpts().setAllowMultiLineCommand(false);
-    // beeLine.getOpts().setIsolation("TRANSACTION_READ_COMMITTED");
-    int status = beeLine.begin(argList.toArray(new String[0]), null);
-    if (status != 0) {
-      throw new IOException("Schema script failed, errorcode " + status);
-    }
-  }
-
-  private String getValidConfVar(String confVar) throws IOException {
-    String confVarKey = confVar;
-    String confVarValue = sentryConf.get(confVarKey);
-    if (confVarValue == null || confVarValue.isEmpty()) {
-      throw new IOException("Empty " + confVar);
-    }
-    return confVarValue;
-  }
-
-  // Create the required command line options
-  @SuppressWarnings("static-access")
-  private static void initOptions(Options cmdLineOptions) {
-    Option help = new Option("help", "print this message");
-    Option upgradeOpt = new Option("upgradeSchema", "Schema upgrade");
-    Option upgradeFromOpt = OptionBuilder.withArgName("upgradeFrom").hasArg().
-                withDescription("Schema upgrade from a version").
-                create("upgradeSchemaFrom");
-    Option initOpt = new Option("initSchema", "Schema initialization");
-    Option initToOpt = OptionBuilder.withArgName("initTo").hasArg().
-                withDescription("Schema initialization to a version").
-                create("initSchemaTo");
-    Option infoOpt = new Option("info", "Show config and schema details");
-
-    OptionGroup optGroup = new OptionGroup();
-    optGroup.addOption(upgradeOpt).addOption(initOpt).
-                addOption(help).addOption(upgradeFromOpt).
-                addOption(initToOpt).addOption(infoOpt);
-    optGroup.setRequired(true);
-
-    Option userNameOpt = OptionBuilder.withArgName("user")
-                .hasArg()
-                .withDescription("Override config file user name")
-                .create("userName");
-    Option passwdOpt = OptionBuilder.withArgName("password")
-                .hasArg()
-                 .withDescription("Override config file password")
-                 .create("passWord");
-    Option dbTypeOpt = OptionBuilder.withArgName("databaseType")
-                .hasArg().withDescription("Metastore database type [" +
-                SentrySchemaHelper.DB_DERBY + "," +
-                SentrySchemaHelper.DB_MYSQL + "," +
-                SentrySchemaHelper.DB_ORACLE + "," +
-                SentrySchemaHelper.DB_POSTGRACE + "," +
-                SentrySchemaHelper.DB_DB2 + "]")
-                .create("dbType");
-    Option dbOpts = OptionBuilder.withArgName("databaseOpts")
-                .hasArgs().withDescription("Backend DB specific options")
-                .create("dbOpts");
-
-    Option dryRunOpt = new Option("dryRun", "list SQL scripts (no execute)");
-    Option verboseOpt = new Option("verbose", "only print SQL statements");
-
-    Option configOpt = OptionBuilder.withArgName("confName").hasArgs()
-        .withDescription("Sentry Service configuration file").isRequired(true)
-        .create(ServiceConstants.ServiceArgs.CONFIG_FILE_LONG);
-
-    cmdLineOptions.addOption(help);
-    cmdLineOptions.addOption(dryRunOpt);
-    cmdLineOptions.addOption(userNameOpt);
-    cmdLineOptions.addOption(passwdOpt);
-    cmdLineOptions.addOption(dbTypeOpt);
-    cmdLineOptions.addOption(verboseOpt);
-    cmdLineOptions.addOption(dbOpts);
-    cmdLineOptions.addOption(configOpt);
-    cmdLineOptions.addOptionGroup(optGroup);
-  }
-
-  public static class CommandImpl implements Command {
-    @Override
-    public void run(String[] args) throws Exception {
-      CommandLineParser parser = new GnuParser();
-      CommandLine line = null;
-      String dbType = null;
-      String schemaVer = null;
-      Options cmdLineOptions = new Options();
-      String configFileName = null;
-
-      // Argument handling
-      initOptions(cmdLineOptions);
-      try {
-        line = parser.parse(cmdLineOptions, args);
-      } catch (ParseException e) {
-        System.err.println("SentrySchemaTool:Parsing failed.  Reason: "
-            + e.getLocalizedMessage());
-        printAndExit(cmdLineOptions);
-      }
-
-      if (line.hasOption("help")) {
-        HelpFormatter formatter = new HelpFormatter();
-        formatter.printHelp("schemaTool", cmdLineOptions);
-        return;
-      }
-
-      if (line.hasOption("dbType")) {
-        dbType = line.getOptionValue("dbType");
-        if (!dbType.equalsIgnoreCase(SentrySchemaHelper.DB_DERBY)
-            && !dbType.equalsIgnoreCase(SentrySchemaHelper.DB_MYSQL)
-            && !dbType.equalsIgnoreCase(SentrySchemaHelper.DB_POSTGRACE)
-            && !dbType.equalsIgnoreCase(SentrySchemaHelper.DB_ORACLE)
-            && !dbType.equalsIgnoreCase(SentrySchemaHelper.DB_DB2)) {
-          System.err.println("Unsupported dbType " + dbType);
-          printAndExit(cmdLineOptions);
-        }
-      } else {
-        System.err.println("no dbType supplied");
-        printAndExit(cmdLineOptions);
-      }
-      if (line.hasOption(ServiceConstants.ServiceArgs.CONFIG_FILE_LONG)) {
-        configFileName = line
-            .getOptionValue(ServiceConstants.ServiceArgs.CONFIG_FILE_LONG);
-      } else {
-        System.err.println("no config file specified");
-        printAndExit(cmdLineOptions);
-      }
-      try {
-        SentrySchemaTool schemaTool = new SentrySchemaTool(
-            SentryService.loadConfig(configFileName), dbType);
-
-        if (line.hasOption("userName")) {
-          schemaTool.setUserName(line.getOptionValue("userName"));
-        }
-        if (line.hasOption("passWord")) {
-          schemaTool.setPassWord(line.getOptionValue("passWord"));
-        }
-        if (line.hasOption("dryRun")) {
-          schemaTool.setDryRun(true);
-        }
-        if (line.hasOption("verbose")) {
-          schemaTool.setVerbose(true);
-        }
-        if (line.hasOption("dbOpts")) {
-          schemaTool.setDbOpts(line.getOptionValue("dbOpts"));
-        }
-
-        if (line.hasOption("info")) {
-          schemaTool.showInfo();
-        } else if (line.hasOption("upgradeSchema")) {
-          schemaTool.doUpgrade();
-        } else if (line.hasOption("upgradeSchemaFrom")) {
-          schemaVer = line.getOptionValue("upgradeSchemaFrom");
-          schemaTool.doUpgrade(schemaVer);
-        } else if (line.hasOption("initSchema")) {
-          schemaTool.doInit();
-        } else if (line.hasOption("initSchemaTo")) {
-          schemaVer = line.getOptionValue("initSchemaTo");
-          schemaTool.doInit(schemaVer);
-        } else {
-          System.err.println("no valid option supplied");
-          printAndExit(cmdLineOptions);
-        }
-      } catch (SentryUserException e) {
-        System.err.println(e);
-        if (line.hasOption("verbose")) {
-          e.printStackTrace();
-        }
-        System.err.println("*** Sentry schemaTool failed ***");
-        System.exit(1);
-      } catch (MalformedURLException e) {
-        System.err.println(e);
-        if (line.hasOption("verbose")) {
-          e.printStackTrace();
-        }
-        System.err.println("*** Sentry schemaTool failed ***");
-        System.exit(1);
-      }
-      System.out.println("Sentry schemaTool completed");
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellCommon.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellCommon.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellCommon.java
deleted file mode 100644
index c8b2eef..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellCommon.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.provider.db.tools;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
-import org.apache.commons.lang.StringUtils;
-
-/**
- * SentryShellCommon provides the function for parsing the argument.
- * For hive model and generic model, child class should be implemented as a sentry admin tool.
- */
-abstract public class SentryShellCommon {
-
-  public enum TYPE { kafka, hive, solr, sqoop };
-
-  public static final String OPTION_DESC_HELP = "Shell usage";
-  public static final String OPTION_DESC_CONF = "sentry-site file path";
-  public static final String OPTION_DESC_ROLE_NAME = "Role name";
-  public static final String OPTION_DESC_GROUP_NAME = "Group name";
-  public static final String OPTION_DESC_PRIVILEGE = "Privilege string";
-  public final static String OPTION_DESC_SERVICE = "Name of the service being managed";
-  public static final String PREFIX_MESSAGE_MISSING_OPTION = "Missing required option: ";
-
-  public static final String GROUP_SPLIT_CHAR = ",";
-
-  protected String roleName;
-  protected String serviceName;
-  protected String groupName;
-  protected String privilegeStr;
-  protected String confPath;
-  // flag for the command
-  protected boolean isCreateRole;
-  protected boolean isDropRole;
-  protected boolean isAddRoleGroup;
-  protected boolean isDeleteRoleGroup;
-  protected boolean isGrantPrivilegeRole;
-  protected boolean isRevokePrivilegeRole;
-  protected boolean isListRole;
-  protected boolean isListPrivilege;
-  protected boolean isListGroup;
-  protected boolean isPrintHelp;
-  // flag for the parameter check
-  protected boolean roleNameRequired;
-  protected boolean groupNameRequired;
-  protected boolean privilegeStrRequired;
-  protected TYPE type;
-
-  /**
-   * parse arguments
-   *
-   * <pre>
-   *   -conf,--sentry_conf             <filepath>                 sentry config file path
-   *   -cr,--create_role            -r <rolename>                 create role
-   *   -dr,--drop_role              -r <rolename>                 drop role
-   *   -arg,--add_role_group        -r <rolename>  -g <groupname> add role to group
-   *   -drg,--delete_role_group     -r <rolename>  -g <groupname> delete role from group
-   *   -gpr,--grant_privilege_role  -r <rolename>  -p <privilege> grant privilege to role
-   *   -rpr,--revoke_privilege_role -r <rolename>  -p <privilege> revoke privilege from role
-   *   -lr,--list_role              -g <groupname>                list roles for group
-   *   -lp,--list_privilege         -r <rolename>                 list privilege for role
-   *   -lg,--list_group                                           list all groups associated with all roles
-   *   -t,--type                    <typename>                    the shell for hive model or generic model
-   * </pre>
-   *
-   * @param args
-   */
-  protected boolean parseArgs(String[] args) {
-    Options simpleShellOptions = new Options();
-
-    setupOptions(simpleShellOptions);
-
-
-
-    // help option
-    Option helpOpt = new Option("h", "help", false, OPTION_DESC_HELP);
-    helpOpt.setRequired(false);
-    simpleShellOptions.addOption(helpOpt);
-
-    // this Options is parsed first for help option
-    Options helpOptions = new Options();
-    helpOptions.addOption(helpOpt);
-
-    try {
-      Parser parser = new GnuParser();
-
-      // parse help option first
-      CommandLine cmd = parser.parse(helpOptions, args, true);
-      for (Option opt : cmd.getOptions()) {
-        if (opt.getOpt().equals("h")) {
-          // get the help option, print the usage and exit
-          usage(simpleShellOptions);
-          return false;
-        }
-      }
-
-      // without help option
-      cmd = parser.parse(simpleShellOptions, args);
-
-      parseOptions(cmd);
-    } catch (ParseException pe) {
-      System.out.println(pe.getMessage());
-      usage(simpleShellOptions);
-      return false;
-    }
-    return true;
-  }
-
-  protected void setupOptions(Options simpleShellOptions) {
-    OptionGroup simpleShellOptGroup = getMainOptions();
-    simpleShellOptions.addOptionGroup(simpleShellOptGroup);
-
-    Option sOpt = new Option("s", "service", true, OPTION_DESC_SERVICE);
-    sOpt.setRequired(false);
-    simpleShellOptions.addOption(sOpt);
-
-    // optional args
-    Option pOpt = new Option("p", "privilege", true, OPTION_DESC_PRIVILEGE);
-    pOpt.setRequired(false);
-    simpleShellOptions.addOption(pOpt);
-
-    Option gOpt = new Option("g", "groupname", true, OPTION_DESC_GROUP_NAME);
-    gOpt.setRequired(false);
-    simpleShellOptions.addOption(gOpt);
-
-    Option rOpt = new Option("r", "rolename", true, OPTION_DESC_ROLE_NAME);
-    rOpt.setRequired(false);
-    simpleShellOptions.addOption(rOpt);
-
-    // this argument should also be parsed in the bin/sentryShell
-    Option tOpt = new Option("t", "type", true, "[hive|solr|sqoop|.....]");
-    tOpt.setRequired(false);
-    simpleShellOptions.addOption(tOpt);
-
-    // file path of sentry-site
-    Option sentrySitePathOpt = new Option("conf", "sentry_conf", true, OPTION_DESC_CONF);
-    sentrySitePathOpt.setRequired(true);
-    simpleShellOptions.addOption(sentrySitePathOpt);
-  }
-
-  protected OptionGroup getMainOptions() {
-    OptionGroup simpleShellOptGroup = new OptionGroup();
-    Option crOpt = new Option("cr", "create_role", false, "Create role");
-    crOpt.setRequired(false);
-
-    Option drOpt = new Option("dr", "drop_role", false, "Drop role");
-    drOpt.setRequired(false);
-
-    Option argOpt = new Option("arg", "add_role_group", false, "Add role to group");
-    argOpt.setRequired(false);
-
-    Option drgOpt = new Option("drg", "delete_role_group", false, "Delete role from group");
-    drgOpt.setRequired(false);
-
-    Option gprOpt = new Option("gpr", "grant_privilege_role", false, "Grant privilege to role");
-    gprOpt.setRequired(false);
-
-    Option rprOpt = new Option("rpr", "revoke_privilege_role", false, "Revoke privilege from role");
-    rprOpt.setRequired(false);
-
-    Option lrOpt = new Option("lr", "list_role", false, "List role");
-    lrOpt.setRequired(false);
-
-    Option lpOpt = new Option("lp", "list_privilege", false, "List privilege");
-    lpOpt.setRequired(false);
-
-    Option lgOpt = new Option("lg", "list_group", false, "List groups");
-    lgOpt.setRequired(false);
-
-
-    // required args group
-    simpleShellOptGroup.addOption(crOpt);
-    simpleShellOptGroup.addOption(drOpt);
-    simpleShellOptGroup.addOption(argOpt);
-    simpleShellOptGroup.addOption(drgOpt);
-    simpleShellOptGroup.addOption(gprOpt);
-    simpleShellOptGroup.addOption(rprOpt);
-    simpleShellOptGroup.addOption(lrOpt);
-    simpleShellOptGroup.addOption(lpOpt);
-    simpleShellOptGroup.addOption(lgOpt);
-    simpleShellOptGroup.setRequired(true);
-    return simpleShellOptGroup;
-  }
-
-  protected void parseOptions(CommandLine cmd) throws ParseException {
-    for (Option opt : cmd.getOptions()) {
-      if (opt.getOpt().equals("p")) {
-        privilegeStr = opt.getValue();
-      } else if (opt.getOpt().equals("g")) {
-        groupName = opt.getValue();
-      } else if (opt.getOpt().equals("r")) {
-        roleName = opt.getValue();
-      } else if (opt.getOpt().equals("s")) {
-        serviceName = opt.getValue();
-      } else if (opt.getOpt().equals("cr")) {
-        isCreateRole = true;
-        roleNameRequired = true;
-      } else if (opt.getOpt().equals("dr")) {
-        isDropRole = true;
-        roleNameRequired = true;
-      } else if (opt.getOpt().equals("arg")) {
-        isAddRoleGroup = true;
-        roleNameRequired = true;
-        groupNameRequired = true;
-      } else if (opt.getOpt().equals("drg")) {
-        isDeleteRoleGroup = true;
-        roleNameRequired = true;
-        groupNameRequired = true;
-      } else if (opt.getOpt().equals("gpr")) {
-        isGrantPrivilegeRole = true;
-        roleNameRequired = true;
-        privilegeStrRequired = true;
-      } else if (opt.getOpt().equals("rpr")) {
-        isRevokePrivilegeRole = true;
-        roleNameRequired = true;
-        privilegeStrRequired = true;
-      } else if (opt.getOpt().equals("lr")) {
-        isListRole = true;
-      } else if (opt.getOpt().equals("lp")) {
-        isListPrivilege = true;
-        roleNameRequired = true;
-      } else if (opt.getOpt().equals("lg")) {
-        isListGroup = true;
-      } else if (opt.getOpt().equals("conf")) {
-        confPath = opt.getValue();
-      } else if (opt.getOpt().equals("t")) {
-        type = TYPE.valueOf(opt.getValue());
-      }
-    }
-    checkRequiredParameter(roleNameRequired, roleName, OPTION_DESC_ROLE_NAME);
-    checkRequiredParameter(groupNameRequired, groupName, OPTION_DESC_GROUP_NAME);
-    checkRequiredParameter(privilegeStrRequired, privilegeStr, OPTION_DESC_PRIVILEGE);
-  }
-
-  protected void checkRequiredParameter(boolean isRequired, String paramValue, String paramName) throws ParseException {
-    if (isRequired && StringUtils.isEmpty(paramValue)) {
-      throw new ParseException(PREFIX_MESSAGE_MISSING_OPTION + paramName);
-    }
-  }
-
-  // print usage
-  private void usage(Options sentryOptions) {
-    HelpFormatter formatter = new HelpFormatter();
-    formatter.printHelp("sentryShell", sentryOptions);
-  }
-
-  // hive model and generic model should implement this method
-  public abstract void run() throws Exception;
-
-  @VisibleForTesting
-  public boolean executeShell(String[] args) throws Exception {
-    boolean result = true;
-    if (parseArgs(args)) {
-      run();
-    } else {
-      result = false;
-    }
-    return result;
-  }
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellHive.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellHive.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellHive.java
deleted file mode 100644
index 785e27d..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/SentryShellHive.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.provider.db.tools;
-
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
-import org.apache.sentry.provider.db.tools.command.hive.*;
-import org.apache.sentry.service.thrift.SentryServiceClientFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Sets;
-
-/**
- * SentryShellHive is an admin tool, and responsible for the management of repository.
- * The following function are supported:
- * create role, drop role, add group to role, delete group from role, grant privilege to role,
- * revoke privilege from role, list roles for group, list privilege for role.
- */
-public class SentryShellHive extends SentryShellCommon {
-
-  private static final Logger LOGGER = LoggerFactory.getLogger(SentryShellHive.class);
-
-  public void run() throws Exception {
-
-    try(SentryPolicyServiceClient client =
-                SentryServiceClientFactory.create(getSentryConf())) {
-      UserGroupInformation ugi = UserGroupInformation.getLoginUser();
-      String requestorName = ugi.getShortUserName();
-      ShellCommand command = new HiveShellCommand(client);
-
-      // check the requestor name
-      if (StringUtils.isEmpty(requestorName)) {
-        // The exception message will be recorded in the log file.
-        throw new Exception("The requestor name is empty.");
-      }
-
-      if (isCreateRole) {
-        command.createRole(requestorName, roleName);
-      } else if (isDropRole) {
-        command.dropRole(requestorName, roleName);
-      } else if (isAddRoleGroup) {
-        Set<String> groups = Sets.newHashSet(groupName.split(SentryShellCommon.GROUP_SPLIT_CHAR));
-        command.grantRoleToGroups(requestorName, roleName, groups);
-      } else if (isDeleteRoleGroup) {
-        Set<String> groups = Sets.newHashSet(groupName.split(SentryShellCommon.GROUP_SPLIT_CHAR));
-        command.revokeRoleFromGroups(requestorName, roleName, groups);
-      } else if (isGrantPrivilegeRole) {
-        command.grantPrivilegeToRole(requestorName, roleName, privilegeStr);
-      } else if (isRevokePrivilegeRole) {
-        command.revokePrivilegeFromRole(requestorName, roleName, privilegeStr);
-      } else if (isListRole) {
-        List<String> roles = command.listRoles(requestorName, groupName);
-        for (String role : roles) {
-          System.out.println(role);
-        }
-      } else if (isListPrivilege) {
-        List<String> privileges = command.listPrivileges(requestorName, roleName);
-        for (String privilege : privileges) {
-          System.out.println(privilege);
-        }
-      } else if (isListGroup) {
-        List<String> groups = command.listGroupRoles(requestorName);
-        for (String group : groups) {
-          System.out.println(group);
-        }
-      }
-    }
-  }
-
-  private Configuration getSentryConf() {
-    Configuration conf = new Configuration();
-    conf.addResource(new Path(confPath), true);
-    return conf;
-  }
-
-  public static void main(String[] args) throws Exception {
-    SentryShellHive sentryShell = new SentryShellHive();
-    try {
-      sentryShell.executeShell(args);
-    } catch (Exception e) {
-      LOGGER.error(e.getMessage(), e);
-      Throwable current =  e;
-      // find the first printable message;
-      while (current != null && current.getMessage() == null) {
-        current = current.getCause();
-      }
-
-      if (current != null) {
-        System.out.println("The operation failed." +
-           (current.getMessage() == null ? "" : "  Message: " + current.getMessage()));
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/ShellCommand.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/ShellCommand.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/ShellCommand.java
deleted file mode 100644
index eeb3a23..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/ShellCommand.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.provider.db.tools;
-
-import java.util.List;
-import java.util.Set;
-
-import org.apache.sentry.core.common.exception.SentryUserException;
-
-/**
- * The interface for all admin commands, eg, CreateRoleCmd. It is independent of the underlying mechanism (i.e. Generic or Hive)
- */
-public interface ShellCommand {
-
-  void createRole(String requestorName, String roleName) throws SentryUserException;
-
-  void dropRole(String requestorName, String roleName) throws SentryUserException;
-
-  void grantPrivilegeToRole(String requestorName, String roleName, String privilege) throws SentryUserException;
-
-  void grantRoleToGroups(String requestorName, String roleName, Set<String> groups) throws SentryUserException;
-
-  void revokePrivilegeFromRole(String requestorName, String roleName, String privilege) throws SentryUserException;
-
-  void revokeRoleFromGroups(String requestorName, String roleName, Set<String> groups) throws SentryUserException;
-
-  List<String> listRoles(String requestorName, String group) throws SentryUserException;
-
-  List<String> listPrivileges(String requestorName, String roleName) throws SentryUserException;
-
-  List<String> listGroupRoles(String requestorName) throws SentryUserException;
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/CommandUtil.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/CommandUtil.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/CommandUtil.java
deleted file mode 100644
index 3f0b5fa..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/CommandUtil.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.provider.db.tools.command.hive;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege;
-import org.apache.sentry.service.thrift.ServiceConstants;
-
-public final class CommandUtil {
-
-  public static final String SPLIT_CHAR = ",";
-
-  private CommandUtil() {
-    // Make constructor private to avoid instantiation
-  }
-
-  // check the privilege value for the specific privilege scope
-  // eg, for the table scope, server and database can't be empty
-  public static void validatePrivilegeHierarchy(TSentryPrivilege tSentryPrivilege) throws IllegalArgumentException {
-    String serverName = tSentryPrivilege.getServerName();
-    String dbName = tSentryPrivilege.getDbName();
-    String tableName = tSentryPrivilege.getTableName();
-    String columnName = tSentryPrivilege.getColumnName();
-    String uri = tSentryPrivilege.getURI();
-    if (ServiceConstants.PrivilegeScope.SERVER.toString().equals(tSentryPrivilege.getPrivilegeScope())) {
-      if (StringUtils.isEmpty(serverName)) {
-        throw new IllegalArgumentException("The hierarchy of privilege is not correct.");
-      }
-    } else if (ServiceConstants.PrivilegeScope.URI.toString().equals(tSentryPrivilege.getPrivilegeScope())) {
-      if (StringUtils.isEmpty(serverName) || StringUtils.isEmpty(uri)) {
-        throw new IllegalArgumentException("The hierarchy of privilege is not correct.");
-      }
-    } else if (ServiceConstants.PrivilegeScope.DATABASE.toString().equals(tSentryPrivilege.getPrivilegeScope())) {
-      if (StringUtils.isEmpty(serverName) || StringUtils.isEmpty(dbName)) {
-        throw new IllegalArgumentException("The hierarchy of privilege is not correct.");
-      }
-    } else if (ServiceConstants.PrivilegeScope.TABLE.toString().equals(tSentryPrivilege.getPrivilegeScope())) {
-      if (StringUtils.isEmpty(serverName) || StringUtils.isEmpty(dbName)
-              || StringUtils.isEmpty(tableName)) {
-        throw new IllegalArgumentException("The hierarchy of privilege is not correct.");
-      }
-    } else if (ServiceConstants.PrivilegeScope.COLUMN.toString().equals(tSentryPrivilege.getPrivilegeScope())
-      && (StringUtils.isEmpty(serverName) || StringUtils.isEmpty(dbName)
-              || StringUtils.isEmpty(tableName) || StringUtils.isEmpty(columnName))) {
-        throw new IllegalArgumentException("The hierarchy of privilege is not correct.");
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/HiveShellCommand.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/HiveShellCommand.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/HiveShellCommand.java
deleted file mode 100644
index 3abba52..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/tools/command/hive/HiveShellCommand.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.provider.db.tools.command.hive;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.sentry.core.common.exception.SentryUserException;
-import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
-import org.apache.sentry.provider.db.service.thrift.TSentryGroup;
-import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege;
-import org.apache.sentry.provider.db.service.thrift.TSentryRole;
-import org.apache.sentry.provider.db.tools.ShellCommand;
-import org.apache.sentry.service.thrift.SentryServiceUtil;
-
-/**
- * The ShellCommand implementation for Hive.
- */
-public class HiveShellCommand implements ShellCommand {
-
-  private final SentryPolicyServiceClient client;
-
-  public HiveShellCommand(SentryPolicyServiceClient client) {
-    this.client = client;
-  }
-
-  public void createRole(String requestorName, String roleName) throws SentryUserException {
-    client.createRole(requestorName, roleName);
-  }
-
-  public void dropRole(String requestorName, String roleName) throws SentryUserException {
-    client.dropRole(requestorName, roleName);
-  }
-
-  public void grantPrivilegeToRole(String requestorName, String roleName, String privilege) throws SentryUserException {
-    TSentryPrivilege tSentryPrivilege = SentryServiceUtil.convertToTSentryPrivilege(privilege);
-    CommandUtil.validatePrivilegeHierarchy(tSentryPrivilege);
-    client.grantPrivilege(requestorName, roleName, tSentryPrivilege);
-  }
-
-  public void grantRoleToGroups(String requestorName, String roleName, Set<String> groups) throws SentryUserException {
-    client.grantRoleToGroups(requestorName, roleName, groups);
-  }
-
-  public void revokePrivilegeFromRole(String requestorName, String roleName, String privilege) throws SentryUserException {
-    TSentryPrivilege tSentryPrivilege = SentryServiceUtil.convertToTSentryPrivilege(privilege);
-    CommandUtil.validatePrivilegeHierarchy(tSentryPrivilege);
-    client.revokePrivilege(requestorName, roleName, tSentryPrivilege);
-  }
-
-  public void revokeRoleFromGroups(String requestorName, String roleName, Set<String> groups) throws SentryUserException {
-    client.revokeRoleFromGroups(requestorName, roleName, groups);
-  }
-
-  public List<String> listRoles(String requestorName, String group) throws SentryUserException {
-    Set<TSentryRole> roles;
-    if (StringUtils.isEmpty(group)) {
-      roles = client.listAllRoles(requestorName);
-    } else {
-      roles = client.listRolesByGroupName(requestorName, group);
-    }
-
-    List<String> result = new ArrayList<>();
-    if (roles != null) {
-      for (TSentryRole role : roles) {
-        result.add(role.getRoleName());
-      }
-    }
-
-    return result;
-  }
-
-  public List<String> listPrivileges(String requestorName, String roleName) throws SentryUserException {
-    Set<TSentryPrivilege> privileges = client
-        .listAllPrivilegesByRoleName(requestorName, roleName);
-
-    List<String> result = new ArrayList<>();
-    if (privileges != null) {
-      for (TSentryPrivilege privilege : privileges) {
-        String privilegeStr = SentryServiceUtil.convertTSentryPrivilegeToStr(privilege);
-        result.add(privilegeStr);
-      }
-    }
-    return result;
-  }
-
-  public List<String> listGroupRoles(String requestorName) throws SentryUserException {
-    Set<TSentryRole> roles = client.listAllRoles(requestorName);
-    if (roles == null || roles.isEmpty()) {
-      return Collections.emptyList();
-    }
-
-    // Set of all group names
-    Set<String> groupNames = new HashSet<>();
-
-    // Map group to set of roles
-    Map<String, Set<String>> groupInfo = new HashMap<>();
-
-    // Get all group names
-    for (TSentryRole role: roles) {
-      for (TSentryGroup group : role.getGroups()) {
-        String groupName = group.getGroupName();
-        groupNames.add(groupName);
-        Set<String> groupRoles = groupInfo.get(groupName);
-        if (groupRoles != null) {
-          // Add a new or existing role
-          groupRoles.add(role.getRoleName());
-          continue;
-        }
-        // Never seen this group before
-        groupRoles = new HashSet<>();
-        groupRoles.add(role.getRoleName());
-        groupInfo.put(groupName, groupRoles);
-      }
-    }
-
-    List<String> groups = new ArrayList<>(groupNames);
-
-    // Produce printable result as
-    // group1 = role1, role2, ...
-    // group2 = ...
-    List<String> result = new LinkedList<>();
-    for (String groupName: groups) {
-      result.add(groupName + " = " + StringUtils.join(groupInfo.get(groupName), ", "));
-    }
-
-    return result;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/generic/tools/TestPermissionsMigrationToolSolr.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/generic/tools/TestPermissionsMigrationToolSolr.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/generic/tools/TestPermissionsMigrationToolSolr.java
deleted file mode 100644
index 69c067f..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/generic/tools/TestPermissionsMigrationToolSolr.java
+++ /dev/null
@@ -1,362 +0,0 @@
- /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.provider.db.generic.tools;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.sentry.core.common.exception.SentryUserException;
-import org.apache.sentry.provider.common.ProviderBackendContext;
-import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceIntegrationBase;
-import org.apache.sentry.provider.db.generic.service.thrift.TAuthorizable;
-import org.apache.sentry.provider.db.generic.service.thrift.TSentryPrivilege;
-import org.apache.sentry.provider.db.generic.service.thrift.TSentryRole;
-import org.apache.sentry.provider.file.PolicyFile;
-import org.apache.sentry.provider.file.SimpleFileProviderBackend;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Sets;
-import com.google.common.collect.Table;
-import com.google.common.io.Files;
-
-public class TestPermissionsMigrationToolSolr extends SentryGenericServiceIntegrationBase {
-  private File confDir;
-  private File confPath;
-  private String requestorName = "";
-  private String service = "service1";
-
-  @Before
-  public void prepareForTest() throws Exception {
-    confDir = Files.createTempDir();
-    confPath = new File(confDir, "sentry-site.xml");
-    if (confPath.createNewFile()) {
-      FileOutputStream to = new FileOutputStream(confPath);
-      conf.writeXml(to);
-      to.close();
-    }
-    requestorName = clientUgi.getShortUserName();//System.getProperty("user.name", "");
-    Set<String> requestorUserGroupNames = Sets.newHashSet(ADMIN_GROUP);
-    setLocalGroupMapping(requestorName, requestorUserGroupNames);
-    // add ADMIN_USER for the after() in SentryServiceIntegrationBase
-    setLocalGroupMapping(ADMIN_USER, requestorUserGroupNames);
-    setLocalGroupMapping("dev", Sets.newHashSet("dev_group"));
-    setLocalGroupMapping("user", Sets.newHashSet("user_group"));
-    writePolicyFile();
-  }
-
-  @After
-  public void clearTestData() throws Exception {
-    FileUtils.deleteQuietly(confDir);
-
-    // clear roles and privileges
-    Set<TSentryRole> tRoles = client.listAllRoles(requestorName, SOLR);
-    for (TSentryRole tRole : tRoles) {
-      String role = tRole.getRoleName();
-      Set<TSentryPrivilege> privileges = client.listAllPrivilegesByRoleName(
-          requestorName, role, SOLR, service);
-      for (TSentryPrivilege privilege : privileges) {
-        client.revokePrivilege(requestorName, role, SOLR, privilege);
-      }
-      client.dropRole(requestorName, role, SOLR);
-    }
-  }
-
-  @Test
-  public void testPermissionsMigrationFromSentrySvc_v1() throws Exception {
-    initializeSentryService();
-
-    String[] args = { "-s", "1.8.0", "-c", confPath.getAbsolutePath()};
-    PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr();
-    sentryTool.executeConfigTool(args);
-
-    Map<String, Set<String>> groupMapping = new HashMap<String, Set<String>>();
-    groupMapping.put("admin_role", Sets.newHashSet("admin_group"));
-    groupMapping.put("dev_role", Sets.newHashSet("dev_group"));
-    groupMapping.put("user_role", Sets.newHashSet("user_group"));
-
-    Map<String, Set<String>> privilegeMapping = new HashMap<String, Set<String>>();
-    privilegeMapping.put("admin_role",
-        Sets.newHashSet("admin=collections->action=*", "admin=cores->action=*"));
-    privilegeMapping.put("dev_role",
-        Sets.newHashSet("collection=*->action=*", "admin=collections->action=*", "admin=cores->action=*"));
-    privilegeMapping.put("user_role",
-        Sets.newHashSet("collection=foo->action=*"));
-
-    verifySentryServiceState(groupMapping, privilegeMapping);
-  }
-
-  @Test
-  public void testPermissionsMigrationFromSentryPolicyFile_v1() throws Exception {
-    Path policyFilePath = initializeSentryPolicyFile();
-    Path outputFilePath = Paths.get(confDir.getAbsolutePath(), "sentry-provider_migrated.ini");
-
-    String[] args = { "-s", "1.8.0", "-p", policyFilePath.toFile().getAbsolutePath(),
-                      "-o", outputFilePath.toFile().getAbsolutePath() };
-    PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr();
-    assertTrue(sentryTool.executeConfigTool(args));
-
-    Set<String> groups = new HashSet<>();
-    groups.add("admin_group");
-    groups.add("dev_group");
-    groups.add("user_group");
-
-    Map<String, Set<String>> privilegeMapping = new HashMap<String, Set<String>>();
-    privilegeMapping.put("admin_role",
-        Sets.newHashSet("admin=collections->action=*", "admin=cores->action=*"));
-    privilegeMapping.put("dev_role",
-        Sets.newHashSet("collection=*->action=*", "admin=collections->action=*", "admin=cores->action=*"));
-    privilegeMapping.put("user_role",
-        Sets.newHashSet("collection=foo->action=*"));
-
-    verifySentryPolicyFile(groups, privilegeMapping, outputFilePath);
-  }
-
-  @Test
-  // For permissions created with Sentry 2.x, no migration necessary
-  public void testPermissionsMigrationFromSentrySvc_v2() throws Exception {
-    initializeSentryService();
-
-    String[] args = { "-s", "2.0.0", "-c", confPath.getAbsolutePath()};
-    PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr();
-    sentryTool.executeConfigTool(args);
-
-    Map<String, Set<String>> groupMapping = new HashMap<String, Set<String>>();
-    groupMapping.put("admin_role", Sets.newHashSet("admin_group"));
-    groupMapping.put("dev_role", Sets.newHashSet("dev_group"));
-    groupMapping.put("user_role", Sets.newHashSet("user_group"));
-
-    Map<String, Set<String>> privilegeMapping = new HashMap<String, Set<String>>();
-    privilegeMapping.put("admin_role",
-        Sets.newHashSet("collection=admin->action=*"));
-    privilegeMapping.put("dev_role",
-        Sets.newHashSet("collection=*->action=*"));
-    privilegeMapping.put("user_role",
-        Sets.newHashSet("collection=foo->action=*"));
-
-    verifySentryServiceState(groupMapping, privilegeMapping);
-  }
-
-  @Test
-  // For permissions created with Sentry 2.x, no migration necessary
-  public void testPermissionsMigrationFromSentryPolicyFile_v2() throws Exception {
-    Path policyFilePath = initializeSentryPolicyFile();
-    Path outputFilePath = Paths.get(confDir.getAbsolutePath(), "sentry-provider_migrated.ini");
-
-    String[] args = { "-s", "2.0.0", "-p", policyFilePath.toFile().getAbsolutePath(),
-                      "-o", outputFilePath.toFile().getAbsolutePath() };
-    PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr();
-    assertTrue(sentryTool.executeConfigTool(args));
-
-    Set<String> groups = new HashSet<>();
-    groups.add("admin_group");
-    groups.add("dev_group");
-    groups.add("user_group");
-
-    Map<String, Set<String>> privilegeMapping = new HashMap<String, Set<String>>();
-    privilegeMapping.put("admin_role",
-        Sets.newHashSet("collection=admin->action=*"));
-    privilegeMapping.put("dev_role",
-        Sets.newHashSet("collection=*->action=*"));
-    privilegeMapping.put("user_role",
-        Sets.newHashSet("collection=foo->action=*"));
-
-    verifySentryPolicyFile(groups, privilegeMapping, outputFilePath);
-  }
-
-  @Test
-  public void testDryRunOption() throws Exception {
-    initializeSentryService();
-
-    String[] args = { "-s", "1.8.0", "-c", confPath.getAbsolutePath(), "--dry_run"};
-    PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr();
-    sentryTool.executeConfigTool(args);
-
-    Map<String, Set<String>> groupMapping = new HashMap<String, Set<String>>();
-    groupMapping.put("admin_role", Sets.newHashSet("admin_group"));
-    groupMapping.put("dev_role", Sets.newHashSet("dev_group"));
-    groupMapping.put("user_role", Sets.newHashSet("user_group"));
-
-    // No change in the privileges
-    Map<String, Set<String>> privilegeMapping = new HashMap<String, Set<String>>();
-    privilegeMapping.put("admin_role",
-        Sets.newHashSet("collection=admin->action=*"));
-    privilegeMapping.put("dev_role",
-        Sets.newHashSet("collection=*->action=*"));
-    privilegeMapping.put("user_role",
-        Sets.newHashSet("collection=foo->action=*"));
-
-    verifySentryServiceState(groupMapping, privilegeMapping);
-  }
-
-  @Test
-  public void testInvalidToolArguments() throws Exception {
-    PermissionsMigrationToolSolr sentryTool = new PermissionsMigrationToolSolr();
-
-    {
-      String[] args = { "-c", confPath.getAbsolutePath()};
-      assertFalse("The execution should have failed due to missing source version",
-          sentryTool.executeConfigTool(args));
-    }
-
-    {
-      String[] args = { "-s", "1.8.0" };
-      sentryTool.executeConfigTool(args);
-      assertFalse("The execution should have failed due to missing Sentry config file"
-          + " (or policy file) path",
-          sentryTool.executeConfigTool(args));
-    }
-
-    {
-      String[] args = { "-s", "1.8.0", "-p", "/test/path" };
-      sentryTool.executeConfigTool(args);
-      assertFalse("The execution should have failed due to missing Sentry config output file path",
-          sentryTool.executeConfigTool(args));
-    }
-
-    {
-      String[] args = { "-s", "1.8.0", "-c", "/test/path1", "-p", "/test/path2" };
-      sentryTool.executeConfigTool(args);
-      assertFalse("The execution should have failed due to providing both Sentry config file"
-          + " as well as policy file params",
-          sentryTool.executeConfigTool(args));
-    }
-  }
-
-  private void initializeSentryService() throws SentryUserException {
-    // Define an admin role
-    client.createRoleIfNotExist(requestorName, "admin_role", SOLR);
-    client.grantRoleToGroups(requestorName, "admin_role", SOLR, Sets.newHashSet("admin_group"));
-
-    // Define a developer role
-    client.createRoleIfNotExist(requestorName, "dev_role", SOLR);
-    client.grantRoleToGroups(requestorName, "dev_role", SOLR, Sets.newHashSet("dev_group"));
-
-    // Define a user role
-    client.createRoleIfNotExist(requestorName, "user_role", SOLR);
-    client.grantRoleToGroups(requestorName, "user_role", SOLR, Sets.newHashSet("user_group"));
-
-    // Grant permissions
-    client.grantPrivilege(requestorName, "admin_role", SOLR,
-        new TSentryPrivilege(SOLR, "service1",
-            Arrays.asList(new TAuthorizable("collection", "admin")), "*"));
-    client.grantPrivilege(requestorName, "dev_role", SOLR,
-        new TSentryPrivilege(SOLR, "service1",
-            Arrays.asList(new TAuthorizable("collection", "*")), "*"));
-    client.grantPrivilege(requestorName, "user_role", SOLR,
-        new TSentryPrivilege(SOLR, "service1",
-            Arrays.asList(new TAuthorizable("collection", "foo")), "*"));
-  }
-
-  private void verifySentryServiceState(Map<String, Set<String>> groupMapping,
-      Map<String, Set<String>> privilegeMapping) throws SentryUserException {
-    // check roles
-    Set<TSentryRole> tRoles = client.listAllRoles(requestorName, SOLR);
-    assertEquals("Unexpected number of roles", groupMapping.keySet().size(), tRoles.size());
-    Set<String> roles = new HashSet<String>();
-    for (TSentryRole tRole : tRoles) {
-      roles.add(tRole.getRoleName());
-    }
-
-    for (String expectedRole : groupMapping.keySet()) {
-      assertTrue("Didn't find expected role: " + expectedRole, roles.contains(expectedRole));
-    }
-
-    // check groups
-    for (TSentryRole tRole : tRoles) {
-      Set<String> expectedGroups = groupMapping.get(tRole.getRoleName());
-      assertEquals("Group size doesn't match for role: " + tRole.getRoleName(),
-          expectedGroups.size(), tRole.getGroups().size());
-      assertTrue("Group does not contain all expected members for role: " + tRole.getRoleName(),
-          tRole.getGroups().containsAll(expectedGroups));
-    }
-
-    // check privileges
-    GenericPrivilegeConverter convert = new GenericPrivilegeConverter(SOLR, service);
-    for (String role : roles) {
-      Set<TSentryPrivilege> privileges = client.listAllPrivilegesByRoleName(
-          requestorName, role, SOLR, service);
-      Set<String> expectedPrivileges = privilegeMapping.get(role);
-      assertEquals("Privilege set size doesn't match for role: " + role + " Actual permissions : " + privileges,
-          expectedPrivileges.size(), privileges.size());
-
-      Set<String> privilegeStrs = new HashSet<String>();
-      for (TSentryPrivilege privilege : privileges) {
-        privilegeStrs.add(convert.toString(privilege).toLowerCase());
-      }
-
-      for (String expectedPrivilege : expectedPrivileges) {
-        assertTrue("Did not find expected privilege: " + expectedPrivilege + " in " + privilegeStrs,
-            privilegeStrs.contains(expectedPrivilege));
-      }
-    }
-  }
-
-  private Path initializeSentryPolicyFile() throws Exception {
-    PolicyFile file = new PolicyFile();
-
-    file.addRolesToGroup("admin_group", "admin_role");
-    file.addRolesToGroup("dev_group", "dev_role");
-    file.addRolesToGroup("user_group", "user_role");
-
-    file.addPermissionsToRole("admin_role", "collection=admin->action=*");
-    file.addPermissionsToRole("dev_role", "collection=*->action=*");
-    file.addPermissionsToRole("user_role", "collection=foo->action=*");
-
-    Path policyFilePath = Paths.get(confDir.getAbsolutePath(), "sentry-provider.ini");
-    file.write(policyFilePath.toFile());
-
-    return policyFilePath;
-  }
-
-  private void verifySentryPolicyFile (Set<String> groups, Map<String, Set<String>> privilegeMapping,
-      Path policyFilePath) throws IOException {
-    SimpleFileProviderBackend policyFileBackend = new SimpleFileProviderBackend(conf,
-        new org.apache.hadoop.fs.Path(policyFilePath.toUri()));
-    policyFileBackend.initialize(new ProviderBackendContext());
-    Table<String, String, Set<String>> groupRolePrivilegeTable =
-        policyFileBackend.getGroupRolePrivilegeTable();
-
-    assertEquals(groups, groupRolePrivilegeTable.rowKeySet());
-    assertEquals(privilegeMapping.keySet(), groupRolePrivilegeTable.columnKeySet());
-
-    for (String groupName : groupRolePrivilegeTable.rowKeySet()) {
-      for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
-        if (groupRolePrivilegeTable.contains(groupName, roleName)) {
-          Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName);
-          assertEquals(privilegeMapping.get(roleName), privileges);
-        }
-      }
-    }
-  }
-}


Mime
View raw message