sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject [01/17] sentry git commit: SENTRY-2282: Remove hive-authzv2 binding and tests modules completely (Sergio Pena, reviewed by Na Li)
Date Wed, 27 Jun 2018 16:47:28 GMT
Repository: sentry
Updated Branches:
  refs/heads/master 0397fc5e3 -> e358fde7f


http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
deleted file mode 100644
index a9ff95d..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
+++ /dev/null
@@ -1,628 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.tests.e2e.metastore;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.sentry.provider.file.PolicyFile;
-import org.apache.sentry.tests.e2e.hive.Context;
-import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
-import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.Resources;
-
-public class TestMetastoreEndToEnd extends
-    AbstractMetastoreTestWithStaticConfiguration {
-
-  private PolicyFile policyFile;
-  private File dataFile;
-  private static final String dbName = "db_1";
-  private static final String db_all_role = "all_db1";
-  private static final String uri_role = "uri_role";
-  private static final String tab1_all_role = "tab1_all_role";
-  private static final String tab1_read_role = "tab1_read_role";
-  private static final String tab2_all_role = "tab2_all_role";
-  private static final String tab2_read_role = "tab2_read_role";
-  private static final String tabName1 = "tab1";
-  private static final String tabName2 = "tab2";
-  private static final String tabName3 = "tab3";
-
-  @BeforeClass
-  public static void setupTestStaticConfiguration() throws Exception {
-    setMetastoreListener = false;
-    AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration();
-  }
-
-  @Override
-  @Before
-  public void setup() throws Exception {
-    policyFile = setAdminOnServer1(ADMINGROUP);
-    policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping());
-    writePolicyFile(policyFile);
-    super.setup();
-
-    dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME);
-    FileOutputStream to = new FileOutputStream(dataFile);
-    Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to);
-    to.close();
-
-    HiveMetaStoreClient client = null;
-    for (int i=0; i < 10; i++) {
-      try {
-        client = context.getMetaStoreClient(ADMIN1);
-        break;
-      } catch (Throwable e) {
-        // ignore
-      }
-      Thread.sleep(6000);
-    }
-    client.dropDatabase(dbName, true, true, true);
-    createMetastoreDB(client, dbName);
-    client.close();
-
-    policyFile
-            .addRolesToGroup(USERGROUP1, db_all_role)
-            .addRolesToGroup(USERGROUP2, "read_db_role")
-            .addRolesToGroup(USERGROUP2, tab1_all_role)
-            .addRolesToGroup(USERGROUP2, tab2_all_role)
-            .addRolesToGroup(USERGROUP3, tab1_read_role)
-            .addRolesToGroup(USERGROUP3, tab2_read_role)
-            .addPermissionsToRole(db_all_role, "server=server1->db=" + dbName)
-            .addPermissionsToRole("read_db_role",
-                    "server=server1->db=" + dbName + "->action=SELECT")
-            .addPermissionsToRole(tab1_all_role,
-                    "server=server1->db=" + dbName + "->table=" + tabName1)
-            .addPermissionsToRole(tab2_all_role,
-                    "server=server1->db=" + dbName + "->table=" + tabName2)
-            .addPermissionsToRole(tab1_read_role,
-                    "server=server1->db=" + dbName + "->table=" + tabName1 + "->action=SELECT")
-            .addPermissionsToRole(tab2_read_role,
-                    "server=server1->db=" + dbName + "->table=" + tabName2 + "->action=SELECT")
-            .setUserGroupMapping(StaticUserGroup.getStaticMapping());
-    writePolicyFile(policyFile);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    if (context != null) {
-      context.close();
-    }
-  }
-
-  /**
-   * Setup admin privileges for user ADMIN1 verify user can create DB and tables
-   * @throws Exception
-   */
-//  @Test
-//  public void testServerPrivileges() throws Exception {
-//    String tabName = "tab1";
-//    HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
-//    client.dropDatabase(dbName, true, true, true);
-//
-//    createMetastoreDB(client, dbName);
-//    createMetastoreTable(client, dbName, tabName,
-//        Lists.newArrayList(new FieldSchema("col1", "int", "")));
-//    assertEquals(1, client.getTables(dbName, tabName).size());
-//
-//    AuthzPathsCache authzPathCache = new AuthzPathsCache(null, new String[]{"/"}, 0);
-//    SentryPolicyServiceClient sentryClient = new SentryServiceClientFactory().create(sentryConf);
-//    waitToCommit(authzPathCache, sentryClient);
-//    assertEquals("/%PREFIX[data%DIR[db_1.db%AUTHZ_OBJECT#db_1[tab1%AUTHZ_OBJECT#db_1.tab1[]]]]", authzPathCache.serializeAllPaths());
-//    client.dropTable(dbName, tabName);
-//    client.dropDatabase(dbName, true, true, true);
-//    waitToCommit(authzPathCache, sentryClient);
-//    assertEquals("/%PREFIX[]", authzPathCache.serializeAllPaths());
-//  }
-//
-//  private void waitToCommit(AuthzPathsCache authzPathCache, SentryPolicyServiceClient sentryClient)
-//      throws Exception {
-//    SentryAuthzUpdate allUpdates = sentryClient.getAllUpdatesFrom(0, 0);
-//    for (HMSUpdate update : allUpdates.pathUpdates) {
-//      authzPathCache.handleUpdateNotification(update);
-//    }
-//    int counter = 0;
-//    while(!authzPathCache.areAllUpdatesCommited()) {
-//      Thread.sleep(200);
-//      counter++;
-//      if (counter > 10000) {
-//        fail("Updates taking too long to commit !!");
-//      }
-//    }
-//  }
-
-  /**
-   * verify non-admin user can not create or drop DB
-   * @throws Exception
-   */
-  @Test
-  public void testNegativeServerPrivileges() throws Exception {
-    HiveMetaStoreClient client = context.getMetaStoreClient(USER1_1);
-    try {
-      createMetastoreDB(client, "fooDb");
-      fail("Creat db should have failed for non-admin user");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    try {
-      createMetastoreDB(client, "barDb");
-      fail("create db should have failed for non-admin user");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-  }
-
-  /**
-   * Verify the user with DB permission can create table in that db Verify the
-   * user can't create table in DB where he doesn't have ALL permissions
-   * @throws Exception
-   */
-  @Test
-  public void testTablePrivileges() throws Exception {
-    HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
-    createMetastoreTable(client, dbName, tabName1,
-        Lists.newArrayList(new FieldSchema("col1", "int", "")));
-    client.close();
-
-    client = context.getMetaStoreClient(USER1_1);
-    createMetastoreTable(client, dbName, tabName2,
-        Lists.newArrayList(new FieldSchema("col1", "int", "")));
-    assertEquals(1, client.getTables(dbName, tabName2).size());
-    client.dropTable(dbName, tabName1);
-    createMetastoreTable(client, dbName, tabName1,
-        Lists.newArrayList(new FieldSchema("col1", "int", "")));
-    client.close();
-
-    // group2 users can't create the table, but can drop it
-    client = context.getMetaStoreClient(USER2_1);
-    try {
-      createMetastoreTable(client, dbName, "barTab",
-          Lists.newArrayList(new FieldSchema("col1", "int", "")));
-      fail("Create table should have failed for non-privilege user");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    client.dropTable(dbName, tabName2);
-    client.close();
-
-    // group3 users can't create or drop it
-    client = context.getMetaStoreClient(USER3_1);
-    try {
-      createMetastoreTable(client, dbName, "barTab",
-          Lists.newArrayList(new FieldSchema("col1", "int", "")));
-      fail("Create table should have failed for non-privilege user");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-
-    try {
-      client.dropTable(dbName, tabName1);
-      fail("drop table should have failed for non-privilege user");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    client.close();
-  }
-
-  /**
-   * Verify alter table privileges
-   * @throws Exception
-   */
-  @Test
-  public void testAlterTablePrivileges() throws Exception {
-
-    HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
-    createMetastoreTable(client, dbName, tabName1,
-        Lists.newArrayList(new FieldSchema("col1", "int", "")));
-    client.close();
-
-    // verify group1 users with DDL privileges can alter tables in db_1
-    client = context.getMetaStoreClient(USER1_1);
-    Table metaTable2 = client.getTable(dbName, tabName1);
-    metaTable2.getSd().setCols(
-        Lists.newArrayList(new FieldSchema("col2", "double", "")));
-    client.alter_table(dbName, tabName1, metaTable2);
-    Table metaTable3 = client.getTable(dbName, tabName1);
-    assertEquals(metaTable2.getSd().getCols(), metaTable3.getSd().getCols());
-
-    // verify group1 users with DDL privileges can alter tables in db_1
-    client = context.getMetaStoreClient(USER2_1);
-    metaTable2 = client.getTable(dbName, tabName1);
-    metaTable2.getSd().setCols(
-        Lists.newArrayList(new FieldSchema("col3", "string", "")));
-    client.alter_table(dbName, tabName1, metaTable2);
-    metaTable3 = client.getTable(dbName, tabName1);
-    assertEquals(metaTable2.getSd().getCols(), metaTable3.getSd().getCols());
-
-    // verify group3 users can't alter tables in db_1
-    client = context.getMetaStoreClient(USER3_1);
-    metaTable2 = client.getTable(dbName, tabName1);
-    metaTable2.getSd().setCols(
-        Lists.newArrayList(new FieldSchema("col3", "string", "")));
-    try {
-      client.alter_table(dbName, tabName1, metaTable2);
-      fail("alter table should have failed for non-privilege user");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    client.close();
-  }
-
-  /**
-   * Verify add partition privileges
-   * @throws Exception
-   */
-  @Test
-  public void testAddPartitionPrivileges() throws Exception {
-    ArrayList<String> partVals1 = Lists.newArrayList("part1");
-    ArrayList<String> partVals2 = Lists.newArrayList("part2");
-    ArrayList<String> partVals3 = Lists.newArrayList("part3");
-    ArrayList<String> partVals4 = Lists.newArrayList("part4");
-
-    // user with ALL on DB should be able to add partition
-    HiveMetaStoreClient client = context.getMetaStoreClient(USER1_1);
-    Table tbl1 = createMetastoreTableWithPartition(client, dbName, tabName1,
-        Lists.newArrayList(new FieldSchema("col1", "int", "")),
-        Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
-    assertEquals(1, client.getTables(dbName, tabName1).size());
-    addPartition(client, dbName, tabName1, partVals1, tbl1);
-    addPartition(client, dbName, tabName1, partVals2, tbl1);
-    client.close();
-
-    // user with ALL on Table should be able to add partition
-    client = context.getMetaStoreClient(USER2_1);
-    tbl1 = client.getTable(dbName, tabName1);
-    addPartition(client, dbName, tabName1, partVals3, tbl1);
-    client.close();
-
-    // user without ALL on DB or Table should NOT be able to add partition
-    client = context.getMetaStoreClient(USER3_1);
-    tbl1 = client.getTable(dbName, tabName1);
-    try {
-      addPartition(client, dbName, tabName1, partVals4, tbl1);
-      fail("Add partition should have failed for non-admin user");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    client.close();
-
-    // user with ALL on DB should be able to drop partition
-    client = context.getMetaStoreClient(USER1_1);
-    tbl1 = client.getTable(dbName, tabName1);
-    client.dropPartition(dbName, tabName1, partVals1, true);
-    client.close();
-
-    // user with ALL on Table should be able to drop partition
-    client = context.getMetaStoreClient(USER2_1);
-    tbl1 = client.getTable(dbName, tabName1);
-    client.dropPartition(dbName, tabName1, partVals2, true);
-    client.close();
-
-    // user without ALL on DB or Table should NOT be able to drop partition
-    client = context.getMetaStoreClient(USER3_1);
-    try {
-      addPartition(client, dbName, tabName1, partVals3, tbl1);
-      fail("Drop partition should have failed for non-admin user");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-  }
-
-  /**
-   * Verify URI privileges for alter table table
-   * @throws Exception
-   */
-  @Test
-  public void testUriTablePrivileges() throws Exception {
-    String newPath1 = "fooTab1";
-    String newPath2 = "fooTab2";
-
-    String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
-        + File.separator + newPath1;
-    String tabDir2 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
-        + File.separator + newPath2;
-    policyFile.addRolesToGroup(USERGROUP1, uri_role)
-        .addRolesToGroup(USERGROUP2, uri_role)
-        .addRolesToGroup(USERGROUP3, db_all_role)
-        .addPermissionsToRole(uri_role, "server=server1->URI=" + tabDir1)
-        .addPermissionsToRole(uri_role, "server=server1->URI=" + tabDir2);
-    writePolicyFile(policyFile);
-
-    // user with URI privileges should be able to create table with that
-    // specific location
-    HiveMetaStoreClient client = context.getMetaStoreClient(USER1_1);
-    createMetastoreTableWithLocation(client, dbName, tabName1,
-        Lists.newArrayList(new FieldSchema("col1", "int", "")), tabDir1);
-
-    createMetastoreTableWithLocation(client, dbName, tabName2,
-        Lists.newArrayList(new FieldSchema("col1", "int", "")), tabDir2);
-    client.close();
-
-    // user without URI privileges should be NOT able to create table with that specific location
-    client = context.getMetaStoreClient(USER3_1);
-    try {
-      createMetastoreTableWithLocation(client, dbName, "fooTab",
-          Lists.newArrayList(new FieldSchema("col1", "int", "")), tabDir2);
-      fail("Create table with location should fail without URI privilege");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    client.close();
-
-    // user with URI privileges should be able to alter table to set that specific location
-    client = context.getMetaStoreClient(USER1_1);
-    Table metaTable1 = client.getTable(dbName, tabName1);
-    metaTable1.getSd().setLocation(tabDir2);
-    client.alter_table(dbName, tabName1, metaTable1);
-    client.close();
-
-    // user with URI privileges and table all should be able to alter table to
-    // set that specific location
-    client = context.getMetaStoreClient(USER2_1);
-    metaTable1 = client.getTable(dbName, tabName2);
-    metaTable1.getSd().setLocation(tabDir1);
-    client.alter_table(dbName, tabName2, metaTable1);
-    client.close();
-
-    // user without URI privileges should be NOT able to alter table to set that
-    // specific location
-    client = context.getMetaStoreClient(USER3_1);
-    Table metaTable2 = client.getTable(dbName, tabName2);
-    metaTable2.getSd().setLocation(tabDir2);
-    try {
-      client.alter_table(dbName, tabName2, metaTable2);
-      fail("Alter table with location should fail without URI privilege");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    client.close();
-  }
-
-  /**
-   * Verify URI privileges for alter table table
-   * @throws Exception
-   */
-  @Test
-  public void testUriPartitionPrivileges() throws Exception {
-    String tabName1 = "tab1";
-    String newPath1 = "fooTab1";
-    String newPath2 = "fooTab2";
-    ArrayList<String> partVals1 = Lists.newArrayList("part1");
-    ArrayList<String> partVals2 = Lists.newArrayList("part2");
-    ArrayList<String> partVals3 = Lists.newArrayList("part2");
-
-    String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
-        + File.separator + newPath1;
-    String tabDir2 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
-        + File.separator + newPath2;
-    policyFile.addRolesToGroup(USERGROUP1, uri_role)
-        .addRolesToGroup(USERGROUP2, db_all_role)
-        .addPermissionsToRole(uri_role, "server=server1->URI=" + tabDir1)
-        .addPermissionsToRole(uri_role, "server=server1->URI=" + tabDir2);
-    writePolicyFile(policyFile);
-
-    // user with URI privileges should be able to alter partition to set that specific location
-    HiveMetaStoreClient client = context.getMetaStoreClient(USER1_1);
-    Table tbl1 = createMetastoreTableWithPartition(client, dbName,
-        tabName1, Lists.newArrayList(new FieldSchema("col1", "int", "")),
-        Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
-    addPartition(client, dbName, tabName1, partVals1, tbl1);
-    addPartitionWithLocation(client, dbName, tabName1, partVals2, tbl1,
-        tabDir1);
-    client.close();
-
-    // user without URI privileges should be NOT able to alter partition to set
-    // that specific location
-    client = context.getMetaStoreClient(USER2_1);
-    try {
-      tbl1 = client.getTable(dbName, tabName1);
-      addPartitionWithLocation(client, dbName, tabName1, partVals3,
-          tbl1, tabDir2);
-      fail("Add partition with location should have failed");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    client.close();
-  }
-
-  /**
-   * Verify alter partion privileges
-   * TODO: We seem to have a bit inconsistency with Alter partition. It's only
-   * allowed with SERVER privilege. If we allow add/drop partition with DB
-   * level privilege, then this should also be at the same level.
-   * @throws Exception
-   */
-  @Test
-  public void testAlterSetLocationPrivileges() throws Exception {
-    String newPath1 = "fooTab1";
-    ArrayList<String> partVals1 = Lists.newArrayList("part1");
-    ArrayList<String> partVals2 = Lists.newArrayList("part2");
-    String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
-        + File.separator + newPath1;
-
-    policyFile.addRolesToGroup(USERGROUP1, uri_role)
-        .addRolesToGroup(USERGROUP2, uri_role)
-        .addPermissionsToRole(uri_role, "server=server1->URI=" + tabDir1);
-    writePolicyFile(policyFile);
-
-    HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
-    Table tbl1 = createMetastoreTableWithPartition(client, dbName,
-        tabName1, Lists.newArrayList(new FieldSchema("col1", "int", "")),
-        Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
-    addPartition(client, dbName, tabName1, partVals1, tbl1);
-    tbl1 = client.getTable(dbName, tabName1);
-    addPartition(client, dbName, tabName1, partVals2, tbl1);
-    client.close();
-
-    // user with DB and URI privileges should be able to alter partition set location
-    client = context.getMetaStoreClient(USER1_1);
-    Partition newPartition = client.getPartition(dbName, tabName1, partVals1);
-    newPartition.getSd().setLocation(tabDir1);
-    client.alter_partition(dbName, tabName1, newPartition);
-    client.close();
-
-    // user with Table and URI privileges should be able to alter partition set location
-    client = context.getMetaStoreClient(USER2_1);
-    newPartition = client.getPartition(dbName, tabName1, partVals2);
-    newPartition.getSd().setLocation(tabDir1);
-    client.alter_partition(dbName, tabName1, newPartition);
-    client.close();
-
-    policyFile.addRolesToGroup(USERGROUP3, db_all_role);
-    writePolicyFile(policyFile);
-    // user without URI privileges should not be able to alter partition set location
-    client = context.getMetaStoreClient(USER3_1);
-    newPartition = client.getPartition(dbName, tabName1, partVals2);
-    newPartition.getSd().setLocation(tabDir1);
-    try {
-      client.alter_partition(dbName, tabName1, newPartition);
-      fail("alter partition with location should have failed");
-    } catch (MetaException e) {
-      Context.verifyMetastoreAuthException(e);
-    }
-    client.close();
-
-  }
-
-  /**
-   * Verify data load into new partition using INSERT .. PARTITION statement
-   */
-  @Test
-  public void testPartionInsert() throws Exception {
-    String partVal1 = "part1", partVal2 = "part2";
-
-    policyFile.addRolesToGroup(USERGROUP1, uri_role).addPermissionsToRole(
-        uri_role, "server=server1->uri=file://" + dataFile.getPath());
-    writePolicyFile(policyFile);
-
-    execHiveSQL("CREATE TABLE " + dbName + "." + tabName1
-        + " (id int) PARTITIONED BY (part_col string)", USER1_1);
-    execHiveSQL("CREATE TABLE " + dbName + "." + tabName2 + " (id int)",
-        USER1_1);
-    execHiveSQL("LOAD DATA LOCAL INPATH '" + dataFile.getPath()
-        + "' INTO TABLE " + dbName + "." + tabName2, USER1_1);
-
-    // verify that user with DB all can add partition using INSERT .. PARTITION
-    execHiveSQL("INSERT OVERWRITE TABLE " + dbName + "." + tabName1
-        + " PARTITION (part_col='" + partVal1 + "') SELECT * FROM " + dbName
-        + "." + tabName2, USER1_1);
-    verifyPartitionExists(dbName, tabName1, partVal1);
-
-    // verify that user with Table all can add partition using INSERT
-    execHiveSQL("INSERT OVERWRITE TABLE " + dbName + "." + tabName1
-        + " PARTITION (part_col='" + partVal2 + "') SELECT * FROM " + dbName
-        + "." + tabName2, USER2_1);
-    verifyPartitionExists(dbName, tabName1, partVal2);
-
-    // verify that user with Table all can add dynamic partition using INSERT
-    Map<String, String> dynamicInsertProperties = Maps.newHashMap();
-    dynamicInsertProperties.put(ConfVars.DYNAMICPARTITIONING.varname, "true");
-    dynamicInsertProperties.put(ConfVars.DYNAMICPARTITIONINGMODE.varname,
-        "nonstrict");
-
-    execHiveSQL("CREATE TABLE " + dbName + "." + tabName3
-        + " (id int) PARTITIONED BY (part_col string)", USER1_1);
-    execHiveSQLwithOverlay("INSERT OVERWRITE TABLE " + dbName + "." + tabName3
-        + " partition (part_col) SELECT id, part_col FROM "
-        + dbName + "." + tabName1, USER1_1, dynamicInsertProperties);
-  }
-
-  @Test
-  public void testAddPartion() throws Exception {
-    String partVal1 = "part1", partVal2 = "part2";
-    String newPath1 = "fooTab1";
-    String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
-        + File.separator + newPath1;
-
-    //URI privilege required when "using location"
-    policyFile.addRolesToGroup(USERGROUP1, uri_role).addPermissionsToRole(
-        uri_role, "server=server1->URI=" + tabDir1);
-    writePolicyFile(policyFile);
-
-    execHiveSQL("DROP TABLE IF EXISTS " + dbName + "." + tabName1, USER1_1);
-    execHiveSQL("CREATE TABLE " + dbName + "." + tabName1
-        + " (id int) PARTITIONED BY (part_col string)", USER1_1);
-
-    //User with all on table
-    execHiveSQL("ALTER TABLE " + dbName + "." + tabName1
-        + " ADD PARTITION (part_col ='" + partVal1 + "')", USER1_1);
-    verifyPartitionExists(dbName, tabName1, partVal1);
-
-    execHiveSQL("ALTER TABLE " + dbName + "." + tabName1
-        + " ADD PARTITION (part_col ='" + partVal2 +  "') location '"
-        + tabDir1 + "'", USER1_1);
-    verifyPartitionExists(dbName, tabName1, partVal2);
-
-    try {
-      execHiveSQL("ALTER TABLE " + dbName + "." + tabName1
-          + " ADD PARTITION (part_col ='" + partVal2 + "') location '"
-          + tabDir1 + "'", USER2_1);
-      fail("alter table should have failed due to URI privilege missed");
-    } catch (IOException e) {
-      // Expected error
-    }
-
-  }
-
-
-  @Test
-  public void testInsertInto() throws Exception {
-    String partVal1 = "part1";
-
-    writePolicyFile(policyFile);
-
-    execHiveSQL("DROP TABLE IF EXISTS " + dbName + "." + tabName1, USER1_1);
-    execHiveSQL("CREATE TABLE " + dbName + "." + tabName1
-        + " (id int) PARTITIONED BY (part_col string)", USER1_1);
-
-    execHiveSQL("INSERT INTO " + dbName + "." + tabName1 +
-        " PARTITION(part_col ='" + partVal1 + "') select 1 from " + dbName + "." + tabName1, USER2_1);
-    verifyPartitionExists(dbName, tabName1, partVal1);
-
-  }
-
-  private void verifyPartitionExists(String dbName, String tabName,
-      String partVal) throws Exception {
-    HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
-    Partition newPartition = client.getPartition(dbName, tabName,
-        Lists.newArrayList(partVal));
-    Assert.assertNotNull(newPartition);
-    client.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java
deleted file mode 100644
index 1596daa..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.tests.e2e.minisentry;
-
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.service.thrift.SentryService;
-import org.apache.sentry.service.thrift.SentryServiceFactory;
-import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.server.ServerContext;
-import org.apache.thrift.server.TServerEventHandler;
-import org.apache.thrift.transport.TTransport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-public class InternalSentrySrv implements SentrySrv {
-
-  public static class SentryServerContext implements ServerContext {
-    private long contextId;
-
-    public SentryServerContext(long contextId) {
-      this.contextId = contextId;
-    }
-
-    public long getContextId() {
-      return contextId;
-    }
-  }
-
-  /**
-   * Thrift even handler class to track client connections to Sentry service
-   */
-  public static class SentryThriftEvenHandler implements TServerEventHandler {
-    // unique id for each client connection. We could see multiple simultaneous
-    // client connections, some make it thread safe.
-    private AtomicLong clientId = new AtomicLong();
-    // Lists of clientId currently connected
-    private List<Long> clientList = Lists.newArrayList();
-
-    /**
-     * Thrift callback when a new client is connecting
-     */
-    @Override
-    public ServerContext createContext(TProtocol inputProto,
-        TProtocol outputProto) {
-      clientList.add(clientId.incrementAndGet());
-      LOGGER.info("Client Connected: " + clientId.get());
-      return new SentryServerContext(clientId.get());
-    }
-
-    /**
-     * Thrift callback when a client is disconnecting
-     */
-    @Override
-    public void deleteContext(ServerContext arg0, TProtocol arg1, TProtocol arg2) {
-      clientList.remove(((SentryServerContext) arg0).getContextId());
-      LOGGER.info("Client Disonnected: "
-          + ((SentryServerContext) arg0).getContextId());
-    }
-
-    @Override
-    public void preServe() {
-    }
-
-    @Override
-    public void processContext(ServerContext arg0, TTransport arg1,
-        TTransport arg2) {
-    }
-
-    public long getClientCount() {
-      return clientList.size();
-    }
-
-    public List<Long> getClienList() {
-      return clientList;
-    }
-
-    public long getClientId() {
-      return clientId.get();
-    }
-  }
-
-  private List<SentryService> sentryServers = Lists.newArrayList();
-  private static final Logger LOGGER = LoggerFactory
-      .getLogger(InternalSentrySrv.class);
-  private boolean isActive = false;
-
-  public InternalSentrySrv(Configuration sentryConf, int numServers)
-      throws Exception {
-    for (int count = 0; count < numServers; count++) {
-      Configuration servConf = new Configuration(sentryConf);
-      SentryService sentryServer = new SentryServiceFactory().create(servConf);
-      servConf.set(ClientConfig.SERVER_RPC_ADDRESS, sentryServer.getAddress()
-          .getHostName());
-      servConf.setInt(ClientConfig.SERVER_RPC_PORT, sentryServer.getAddress()
-          .getPort());
-      sentryServers.add(sentryServer);
-    }
-    isActive = true;
-  }
-
-  @Override
-  public void startAll() throws Exception {
-    if (!isActive) {
-      throw new IllegalStateException("SentrySrv is no longer active");
-    }
-    for (int sentryServerNum = 0; sentryServerNum < sentryServers.size(); sentryServerNum++) {
-      start(sentryServerNum);
-    }
-  }
-
-  @Override
-  public void start(int serverNum) throws Exception {
-    if (!isActive) {
-      throw new IllegalStateException("SentrySrv is no longer active");
-    }
-    SentryService sentryServer = sentryServers.get(serverNum);
-    sentryServer.start();
-
-    // wait for startup
-    final long start = System.currentTimeMillis();
-    while (!sentryServer.isRunning()) {
-      Thread.sleep(1000);
-      if (System.currentTimeMillis() - start > 60000L) {
-        throw new TimeoutException("Server did not start after 60 seconds");
-      }
-    }
-    sentryServer.setThriftEventHandler(new SentryThriftEvenHandler());
-  }
-
-  @Override
-  public void stopAll() throws Exception {
-    boolean cleanStop = true;
-    if (!isActive) {
-      throw new IllegalStateException("SentrySrv is no longer active");
-    }
-    for (int sentryServerNum = 0; sentryServerNum < sentryServers.size(); sentryServerNum++) {
-      try {
-        stop(sentryServerNum);
-      } catch (Exception e) {
-        LOGGER.error("Sentry Server " + sentryServerNum + " failed to stop");
-        cleanStop = false;
-      }
-    }
-    if (!cleanStop) {
-      throw new IllegalStateException(
-          "At least one of the servers failed to stop cleanly");
-    }
-  }
-
-  @Override
-  public void stop(int serverNum) throws Exception {
-    if (!isActive) {
-      throw new IllegalStateException("SentrySrv is no longer active");
-    }
-    SentryService sentryServer = sentryServers.get(serverNum);
-    sentryServer.stop();
-  }
-
-  @Override
-  public void close() {
-    for (SentryService sentryServer : sentryServers) {
-      try {
-        sentryServer.stop();
-      } catch (Exception e) {
-        LOGGER.error("Error stoping Sentry service ", e);
-      }
-    }
-    sentryServers.clear();
-    isActive = false;
-  }
-
-  @Override
-  public SentryService get(int serverNum) {
-    return sentryServers.get(serverNum);
-  }
-
-  @Override
-  public long getNumActiveClients(int serverNum) {
-    SentryThriftEvenHandler thriftHandler = (SentryThriftEvenHandler) get(
-        serverNum).getThriftEventHandler();
-    LOGGER.warn("Total clients: " + thriftHandler.getClientId());
-    for (Long clientId: thriftHandler.getClienList()) {
-      LOGGER.warn("Got clients: " + clientId);
-    }
-    return thriftHandler.getClientCount();
-  }
-
-  @Override
-  public long getNumActiveClients() {
-    long numClients = 0;
-    for (int sentryServerNum = 0; sentryServerNum < sentryServers.size(); sentryServerNum++) {
-      numClients += getNumActiveClients(sentryServerNum);
-    }
-    return numClients;
-
-  }
-
-  @Override
-  public long getTotalClients() {
-    long totalClients = 0;
-    for (int sentryServerNum = 0; sentryServerNum < sentryServers.size(); sentryServerNum++) {
-      totalClients += getTotalClients(sentryServerNum);
-    }
-    return totalClients;
-  }
-
-  @Override
-  public long getTotalClients(int serverNum) {
-    SentryThriftEvenHandler thriftHandler = (SentryThriftEvenHandler) get(
-        serverNum).getThriftEventHandler();
-    return thriftHandler.getClientId();
-  }
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrv.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrv.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrv.java
deleted file mode 100644
index 9139706..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrv.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.tests.e2e.minisentry;
-
-import org.apache.sentry.service.thrift.SentryService;
-
-public interface SentrySrv {
-
-  /**
-   * Start all the sentry services
-   * @throws Exception
-   */
-  void startAll() throws Exception;
-
-  /**
-   * Start the given server
-   * @param serverNum
-   *          - Server number (0 to N-1)
-   * @throws Exception
-   */
-  void start(int serverNum) throws Exception ;
-
-  /**
-   * Stop all the Sentry servers
-   * @throws Exception
-   */
-  void stopAll() throws Exception;
-
-  /**
-   * Stop the specified Sentry server
-   * @param serverNum
-   *          - Server number (0 to N-1)
-   * @throws Exception
-   */
-  void stop(int serverNum) throws Exception ;
-
-  /**
-   * Get the underlying Sentry service object
-   * @param serverNum
-   *          - Server number (0 to N-1)
-   * @return
-   */
-  SentryService get(int serverNum);
-
-  /**
-   * Stop all the nodes and ZK if started. The SentrySrv can't be reused once
-   * closed.
-   */
-  void close();
-
-  /**
-   * Get the number of active clients connections across servers
-   */
-  long getNumActiveClients();
-
-  /**
-   * Get the number of active clients connections for the given server
-   */
-  long getNumActiveClients(int serverNum);
-
-  /**
-   * Get the total number of clients connected so far
-   */
-  long getTotalClients();
-
-  /**
-   * Get the total number of clients connected so far
-   */
-  long getTotalClients(int serverNum);
-
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrvFactory.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrvFactory.java b/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrvFactory.java
deleted file mode 100644
index 9381e88..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/java/org/apache/sentry/tests/e2e/minisentry/SentrySrvFactory.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.tests.e2e.minisentry;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class SentrySrvFactory {
-
-  public static enum SentrySrvType {
-    INTERNAL_SERVER, EXTERNAL_SERVER
-  }
-
-  public static SentrySrv create(SentrySrvType srvType,
-      Configuration sentryConf)
-          throws Exception {
-    return create(srvType, sentryConf, 1);
-  }
-
-  public static SentrySrv create(SentrySrvType srvType,
-      Configuration sentryConf,
-      int numServers) throws Exception {
-    if (!srvType.equals(SentrySrvType.INTERNAL_SERVER)) {
-      throw new IOException("Server type " + srvType.name()
-          + " is not supported");
-    }
-    return new InternalSentrySrv(sentryConf, numServers);
-  }
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/core-site-for-sentry-test.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/core-site-for-sentry-test.xml b/sentry-tests/sentry-tests-hive-v2/src/test/resources/core-site-for-sentry-test.xml
deleted file mode 100644
index 01b8576..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/core-site-for-sentry-test.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<configuration>
-  <property>
-    <name>hadoop.security.group.mapping</name>
-    <value>org.apache.sentry.tests.e2e.hive.fs.MiniDFS$PseudoGroupMappingService</value>
-  </property>
-  <property>
-    <name>fs.permissions</name>
-    <value>false</value>
-  </property>
-  <property>
-    <name>fs.permissions.umask-mode</name>
-    <value>000</value>
-  </property>
-</configuration>
-

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/emp.dat
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/emp.dat b/sentry-tests/sentry-tests-hive-v2/src/test/resources/emp.dat
deleted file mode 100644
index 5922b20..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/emp.dat
+++ /dev/null
@@ -1,12 +0,0 @@
-16|john
-17|robert
-18|andrew
-19|katty
-21|tom
-22|tim
-23|james
-24|paul
-27|edward
-29|alan
-31|kerry
-34|terri
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/hadoop
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/hadoop b/sentry-tests/sentry-tests-hive-v2/src/test/resources/hadoop
deleted file mode 100755
index 914d3db..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/hadoop
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script runs the hadoop core commands. 
-
-bin=`which $0`
-bin=`dirname ${bin}`
-bin=`cd "$bin"; pwd`
- 
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m
-
-# check envvars which might override default args
-if [ "$HADOOP_HEAPSIZE" != "" ]; then
-  JAVA_HEAP_MAX="-Xmx""$HADOOP_HEAPSIZE""m"
-fi
-
-if [ "$SENTRY_HADOOP_TEST_CLASSPATH" != "" ]; then
-  CLASSPATH=${CLASSPATH}:${SENTRY_HADOOP_TEST_CLASSPATH}
-  echo "Got Sentry classpath ${SENTRY_HADOOP_TEST_CLASSPATH}"
-else    
-  echo "Error: SENTRY_HADOOP_TEST_CLASSPATH not defined."
-  exit 1  
-fi
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-
-function print_usage(){
-  echo "Usage: hadoop [--config confdir] COMMAND"
-  echo "       where COMMAND is one of:"
-  echo "  fs                   run a generic filesystem user client"
-  echo "  version              print the version"
-  echo "  jar <jar>            run a jar file"
-  echo ""
-  echo "Most commands print help when invoked w/o parameters."
-}
-
-if [ $# = 0 ]; then
-  print_usage
-  exit
-fi
-
-COMMAND=$1
-case $COMMAND in
-  # usage flags
-  --help|-help|-h)
-    print_usage
-    exit
-    ;;
-
-
-  classpath)
-    echo $CLASSPATH
-    exit
-    ;;
-
-  #core commands  
-  *)
-    # the core commands
-    if [ "$COMMAND" = "fs" ] ; then
-      CLASS=org.apache.hadoop.fs.FsShell
-    elif [ "$COMMAND" = "version" ] ; then
-      CLASS=org.apache.hadoop.util.VersionInfo
-    elif [ "$COMMAND" = "jar" ] ; then
-      CLASS=org.apache.hadoop.util.RunJar
-    elif [ "$COMMAND" = "checknative" ] ; then
-      CLASS=org.apache.hadoop.util.NativeLibraryChecker
-    elif [ "$COMMAND" = "distcp" ] ; then
-      CLASS=org.apache.hadoop.tools.DistCp
-      CLASSPATH=${CLASSPATH}:${TOOL_PATH}
-    elif [ "$COMMAND" = "archive" ] ; then
-      CLASS=org.apache.hadoop.tools.HadoopArchives
-      CLASSPATH=${CLASSPATH}:${TOOL_PATH}
-    elif [[ "$COMMAND" = -*  ]] ; then
-        # class and package names cannot begin with a -
-        echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'"
-        exit 1
-    else
-      CLASS=$COMMAND
-    fi
-    shift
-    
-    # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
-    HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-
-    #make sure security appender is turned off
-    HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
-
-    export CLASSPATH=$CLASSPATH
-    exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
-    ;;
-
-esac

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/kv1.dat
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/kv1.dat b/sentry-tests/sentry-tests-hive-v2/src/test/resources/kv1.dat
deleted file mode 100644
index 20fb0dc..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/kv1.dat
+++ /dev/null
@@ -1,500 +0,0 @@
-238
-86
-311
-27
-165
-409
-255
-278
-98
-484
-265
-193
-401
-150
-273
-224
-369
-66
-128
-213
-146
-406
-429
-374
-152
-469
-145
-495
-37
-327
-281
-277
-209
-15
-82
-403
-166
-417
-430
-252
-292
-219
-287
-153
-193
-338
-446
-459
-394
-237
-482
-174
-413
-494
-207
-199
-466
-208
-174
-399
-396
-247
-417
-489
-162
-377
-397
-309
-365
-266
-439
-342
-367
-325
-167
-195
-475
-17
-113
-155
-203
-339
-0
-455
-128
-311
-316
-57
-302
-205
-149
-438
-345
-129
-170
-20
-489
-157
-378
-221
-92
-111
-47
-72
-4
-280
-35
-427
-277
-208
-356
-399
-169
-382
-498
-125
-386
-437
-469
-192
-286
-187
-176
-54
-459
-51
-138
-103
-239
-213
-216
-430
-278
-176
-289
-221
-65
-318
-332
-311
-275
-137
-241
-83
-333
-180
-284
-12
-230
-181
-67
-260
-404
-384
-489
-353
-373
-272
-138
-217
-84
-348
-466
-58
-8
-411
-230
-208
-348
-24
-463
-431
-179
-172
-42
-129
-158
-119
-496
-0
-322
-197
-468
-393
-454
-100
-298
-199
-191
-418
-96
-26
-165
-327
-230
-205
-120
-131
-51
-404
-43
-436
-156
-469
-468
-308
-95
-196
-288
-481
-457
-98
-282
-197
-187
-318
-318
-409
-470
-137
-369
-316
-169
-413
-85
-77
-0
-490
-87
-364
-179
-118
-134
-395
-282
-138
-238
-419
-15
-118
-72
-90
-307
-19
-435
-10
-277
-273
-306
-224
-309
-389
-327
-242
-369
-392
-272
-331
-401
-242
-452
-177
-226
-5
-497
-402
-396
-317
-395
-58
-35
-336
-95
-11
-168
-34
-229
-233
-143
-472
-322
-498
-160
-195
-42
-321
-430
-119
-489
-458
-78
-76
-41
-223
-492
-149
-449
-218
-228
-138
-453
-30
-209
-64
-468
-76
-74
-342
-69
-230
-33
-368
-103
-296
-113
-216
-367
-344
-167
-274
-219
-239
-485
-116
-223
-256
-263
-70
-487
-480
-401
-288
-191
-5
-244
-438
-128
-467
-432
-202
-316
-229
-469
-463
-280
-2
-35
-283
-331
-235
-80
-44
-193
-321
-335
-104
-466
-366
-175
-403
-483
-53
-105
-257
-406
-409
-190
-406
-401
-114
-258
-90
-203
-262
-348
-424
-12
-396
-201
-217
-164
-431
-454
-478
-298
-125
-431
-164
-424
-187
-382
-5
-70
-397
-480
-291
-24
-351
-255
-104
-70
-163
-438
-119
-414
-200
-491
-237
-439
-360
-248
-479
-305
-417
-199
-444
-120
-429
-169
-443
-323
-325
-277
-230
-478
-178
-468
-310
-317
-333
-493
-460
-207
-249
-265
-480
-83
-136
-353
-172
-214
-462
-233
-406
-133
-175
-189
-454
-375
-401
-421
-407
-384
-256
-26
-134
-67
-384
-379
-18
-462
-492
-100
-298
-9
-341
-498
-146
-458
-362
-186
-285
-348
-167
-18
-273
-183
-281
-344
-97
-469
-315
-84
-28
-37
-448
-152
-348
-307
-194
-414
-477
-222
-126
-90
-169
-403
-400
-200
-97

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/log4j.properties b/sentry-tests/sentry-tests-hive-v2/src/test/resources/log4j.properties
deleted file mode 100644
index d941816..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Define some default values that can be overridden by system properties.
-#
-# For testing, it may also be convenient to specify
-
-sentry.root.logger=INFO,console
-log4j.rootLogger=${sentry.root.logger}
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d (%t) [%p - %l] %m%n
-
-log4j.logger.org.apache.hadoop.conf.Configuration=ERROR
-log4j.logger.org.apache.sentry=DEBUG
-
-log4j.category.DataNucleus=ERROR

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/log4j2.properties
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/log4j2.properties b/sentry-tests/sentry-tests-hive-v2/src/test/resources/log4j2.properties
deleted file mode 100644
index a6ddc29..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Define some default values that can be overridden by system properties.
-#
-# For testing, it may also be convenient to specify
-
-# list of properties
-property.sentry.root.logger = INFO
-
-# list of all appenders
-appenders = console
-
-# console appender
-appender.console.name = console
-appender.console.type = Console
-appender.console.target = SYSTEM_ERR
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d (%t) [%p - %l] %m%n
-
-# list of all loggers
-loggers = Sentry, HadoopConf, Datanucleus
-
-logger.Sentry.name = org.apache.sentry
-logger.Sentry.level = DEBUG
-
-logger.HadoopConf.name = org.apache.hadoop.conf.Configuration
-logger.HadoopConf.level = ERROR
-
-logger.Datanucleus.name = org.datanucleus
-logger.Datanucleus.level = ERROR
-
-# root logger
-rootLogger.level = ${sys:sentry.root.logger}
-rootLogger.appenderRefs = console
-rootLogger.appenderRef.console.ref = console
-rootLogger.appenderRef.console.level = ${sys:sentry.root.logger}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/sentry-provider.ini
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/sentry-provider.ini b/sentry-tests/sentry-tests-hive-v2/src/test/resources/sentry-provider.ini
deleted file mode 100644
index 014d827..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/sentry-provider.ini
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-foo = all_default
-
-[roles]
-all_default = server=server1->db=default
-
-[users]
-foo = foo

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/sentry-site.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/sentry-site.xml b/sentry-tests/sentry-tests-hive-v2/src/test/resources/sentry-site.xml
deleted file mode 100644
index d068011..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/sentry-site.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<configuration>
-  <property>
-    <name>sentry.provider</name>
-    <value>invalid</value>
-  </property>
-  <property>
-    <name>sentry.hive.provider.resource</name>
-    <value>invalid</value>
-  </property>
-  <property>
-    <name>sentry.hive.server</name>
-    <value>myHS2</value>
-  </property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImport.ini
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImport.ini b/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImport.ini
deleted file mode 100644
index 15fc5bf..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImport.ini
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-group1=roleImport1,roleImport2
-group2=roleImport1,roleImport2,roleImport3
-group3=roleImport2,roleImport3
-[roles]
-roleImport1=server=server1,server=server1->action=select->grantoption=false,server=server1->db=db2->action=insert->grantoption=true,server=server1->db=db1->table=tbl1->action=insert
-roleImport2=server=server1->db=db2->action=insert->grantoption=true,server=server1->db=db1->table=tbl1->action=insert,server=server1->db=db1->table=tbl2->column=col1->action=insert,server=server1->db=db1->table=tbl3->column=col1->action=*->grantoption=true
-roleImport3=server=server1->db=db1->table=tbl2->column=col1->action=insert,server=server1->db=db1->table=tbl3->column=col1->action=*->grantoption=true,server=server1->db=db1->table=tbl4->column=col1->action=all->grantoption=true,server=server1->uri=hdfs://testserver:9999/path2->action=insert

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImportAdmin.ini
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImportAdmin.ini b/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImportAdmin.ini
deleted file mode 100644
index c778d05..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImportAdmin.ini
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-admin=adminRole
-
-[roles]
-adminRole=server=server1

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImportError.ini
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImportError.ini b/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImportError.ini
deleted file mode 100644
index 4d53f2b..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/resources/testPolicyImportError.ini
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-group1=roleImport1
-[roles]
-roleImport1=server->db=db_1

http://git-wip-us.apache.org/repos/asf/sentry/blob/e358fde7/sentry-tests/sentry-tests-hive-v2/src/test/scripts/scale-test/create-many-dbs-tables.sh
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive-v2/src/test/scripts/scale-test/create-many-dbs-tables.sh b/sentry-tests/sentry-tests-hive-v2/src/test/scripts/scale-test/create-many-dbs-tables.sh
deleted file mode 100755
index dcdddeb..0000000
--- a/sentry-tests/sentry-tests-hive-v2/src/test/scripts/scale-test/create-many-dbs-tables.sh
+++ /dev/null
@@ -1,277 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This script means to create many testing objects (database, tables,
-# partitions and a wide table with many partitions). The way to run it:
-# !/usr/bin/env bash
-# export HS2="HOSTNAME"
-# export REALM="REALM.NAME"
-# bash /root/tests/create-many-dbs-tables.sh &
-# bash /root/tests/create-many-dbs-tables.sh &
-
-if [[ ${HS2} == "" ]]; then
-  echo "error: need to export HS2=hostname"
-  exit 1
-fi
-
-if [[ ${REALM} == "" ]]; then
-  echo "error: need to export REALM"
-  exit 1
-fi
-
-# Define default test scale
-NUM_OF_DATABASES=60
-NUM_OF_TABLES_PER_DATABASE=20
-NUM_OF_ROLES_FOR_DATABASES=60 # <= NUM_OF_DATABASES
-NUM_OF_ROLES_FOR_TABLES_PER_DATABASE=5 # <= NUM_OF_TABLES_PER_DATABASE
-NUM_OF_GROUPS=60 # >= NUM_OF_DATABASES
-
-# Number of partitions varies between max and min
-MAX_NUM_OF_PARTITIONS_PER_TABLE=10
-MIN_NUM_OF_PARTITIONS_PER_TABLE=2
-
-BASE_EXTERNAL_DIR="/data"
-LOCAL_OUTPUT_DIR="/tmp"
-BL="beeline -n hive -p hive --silent=true -u 'jdbc:hive2://${HS2}:10000/default;principal=hive/_HOST@${REALM}'"
-
-# Number of external partitions wide tables have
-declare -a NUM_OF_WIDE_TABLE_PARTITIONS=(10 100 1000)
-wLen=${#NUM_OF_WIDE_TABLE_PARTITIONS[@]}
-
-process_id=$$
-
-while getopts "d:t:g:b:l" OPTION
-do	case "${OPTION}" in
-  b)      BASE_EXTERNAL_DIR="$OPTARG";;
-  d)	  NUM_OF_DATABASES="$OPTARG";;
-  l)      LOCAL_OUTPUT_DIR="$OPTARG";;
-  t)      NUM_OF_TABLES_PER_DATABASE="$OPTARG";;
-  g)	  NUM_OF_GROUPS="$OPTARG";;
-  [?])	  print >&2 "Usage: $0 [-b BASE_EXTERNAL_DIR] [-d NUM_OF_DATABASES] [-l LOCAL_OUTPUT_DIR] [-t NUM_OF_TABLES_PER_DATABASE] [-g NUM_OF_GROUPS]"
-	  exit 1;;
-	esac
-done
-
-NUM_OF_PERMISSIONS=$(( NUM_OF_ROLES_FOR_DATABASES + NUM_OF_ROLES_FOR_TABLES_PER_DATABASE * NUM_OF_DATABASES))
-AVG_NUM_OF_PARTITIONS_PER_TABLE=$((( MAX_NUM_OF_PARTITIONS_PER_TABLE + MIN_NUM_OF_PARTITIONS_PER_TABLE) / 2 ))
-
-echo "[${process_id}]	Scale numbers:"
-echo "[${process_id}]	number of databases: ${NUM_OF_DATABASES}"
-echo "[${process_id}]	number of tables: $((NUM_OF_TABLES_PER_DATABASE * NUM_OF_DATABASES))"
-echo "[${process_id}]	number of wide tables: ${wLen}"
-echo "[${process_id}]	number of partitions per table: ${AVG_NUM_OF_PARTITIONS_PER_TABLE}"
-echo "[${process_id}]	number of min partitions per wide table: ${NUM_OF_WIDE_TABLE_PARTITIONS[0]}"
-echo "[${process_id}]	number of max partitions per wide table: ${NUM_OF_WIDE_TABLE_PARTITIONS[${wLen}-1]}"
-echo "[${process_id}]	number of permissions: ${NUM_OF_PERMISSIONS}"
-echo "[${process_id}]	number of groups: ${NUM_OF_GROUPS}"
-
-# Random string as prefix for test databases and tables
-prefix_string=$(cat /dev/urandom | tr -dc 'a-z' | fold -w 4 | head -n 1)
-prefix_string=${prefix_string}$(date +%s | cut -c1-4)
-
-DB_NAME=${prefix_string}_db
-
-function validate_ret () {
-  ret=$1
-  if [[ ${ret} != "" && ${ret} -ne 0 ]]; then
-    echo "ERROR!! when running query in bulk mode"
-    exit $ret
-  fi
-}
-
-function get_group () {
-  count=$1
-  group_name=group_$((count % NUM_OF_GROUPS))
-  echo "$group_name"
-}
-
-# Create groups
-function create_groups () {
-  for g in $(seq ${NUM_OF_GROUPS}); do
-    group_name=$(get_group $g)
-    getent passwd ${group_name} | grep "${group_name}" 1>&2>/dev/null
-    if [[ $? -ne 0 ]]; then
-      sudo groupadd ${group_name}
-      sudo useradd -g ${group_name} ${group_name}
-    fi
-  done
-}
-
-# Convenience function to create one table with many external partitons
-function create_wide_table () {
-  db_name=$1
-  tbl_name=$2
-  num_of_pars=$3
-  file_name=$4
-  dir_file_name=$5
-  echo "-- [${process_id}]	Create ${tbl_name} in ${db_name} with ${num_of_pars} external partitions; " >> ${file_name}
-  echo "CREATE DATABASE IF NOT EXISTS ${db_name}; " >> ${file_name}
-  echo "USE ${db_name};" >> ${file_name}
-  table_dir=${BASE_EXTERNAL_DIR}/${db_name}/${tbl_name}
-  echo "sudo -u hdfs hdfs dfs -rm -R -skipTrash ${table_dir} 2>/dev/null" >> ${dir_file_name}
-  echo "DROP TABLE IF EXISTS ${tbl_name}; " >> ${file_name}
-  echo "CREATE TABLE ${tbl_name} (s STRING, i INT) PARTITIONED BY (par INT);" >> ${file_name}
-  echo "-- create ${num_of_pars} partitions on table ${tbl_name}" >> ${file_name}
-  for p in $(seq ${num_of_pars}); do
-    dir=${table_dir}/$p
-    echo "sudo -u hdfs hdfs dfs -mkdir -p ${dir}" >> ${dir_file_name}
-    echo "ALTER TABLE ${tbl_name} ADD PARTITION (par=$p) LOCATION '${dir}';" >> ${file_name}
-  done
-}
-
-# Convenience function to create wide tables with many external partitions
-function create_external_par_dirs_bulk_file () {
-  file_name=$1
-  dir_file_name=$2
-  echo "-- [${process_id}]	Start bulk process to create wide tables" > ${file_name}
-  echo "# [${process_id}]	Start to create external dirs for partitions" > ${dir_file_name}
-  db_id=$(awk -v n="${NUM_OF_DATABASES}" 'BEGIN{srand();print int(rand()*n+1)}')
-  db_name=${DB_NAME}_${db_id}
-  for p in "${!NUM_OF_WIDE_TABLE_PARTITIONS[@]}"; do
-    tbl_name=${db_name}_wide_tbl_$p
-	  create_wide_table ${db_name} ${tbl_name} ${NUM_OF_WIDE_TABLE_PARTITIONS[p]} ${file_name} ${dir_file_name}
-  done
-  chmod a+x ${file_name}
-  chmod a+x ${dir_file_name}
-}
-
-# Create internal databases and their tables in one bulk file
-function create_dbs_tbls_bulk_file () {
-  file_name=$1
-  echo "-- [${process_id}]	start bulk load " > ${file_name}
-  for d in $(seq ${NUM_OF_DATABASES}); do
-    db_name=${DB_NAME}_${d}
-    echo "drop database if exists ${db_name}; " >> ${file_name}
-    echo "create database ${db_name}; " >> ${file_name}
-    echo "use ${db_name};" >> ${file_name}
-    NUM_OF_COLS=$(awk -v mn="${MIN_NUM_OF_PARTITIONS_PER_TABLE}" -v mx="${MAX_NUM_OF_PARTITIONS_PER_TABLE}" 'BEGIN{srand();print int(rand()*(mx-mn)+1)}')
-    NUM_OF_PARS=$(awk -v mn="${MIN_NUM_OF_PARTITIONS_PER_TABLE}" -v mx="${MAX_NUM_OF_PARTITIONS_PER_TABLE}" 'BEGIN{srand();print int(rand()*(mx-mn)+1)}')
-
-    for t in $(seq ${NUM_OF_TABLES_PER_DATABASE}); do
-      tbl_name=${db_name}_tbl_${t}
-      # create table
-      echo "create table ${tbl_name} (col_start INT, " >> ${file_name}
-      for c in $(seq ${NUM_OF_COLS}); do
-        echo "col_${c} STRING, " >> ${file_name}
-      done
-      echo "col_end INT) partitioned by (par_start STRING, " >> ${file_name}
-      # create many partitions
-      for p in $(seq ${NUM_OF_PARS}); do
-        echo "par_${p} INT, " >> ${file_name}
-      done
-      echo "par_end STRING); " >> ${file_name}
-    done
-  done
-  chmod a+x ${file_name}
-}
-
-# Create database roles
-function create_dbs_roles () {
-  db_file_name=$1
-  total_db_permissions=0
-  echo "-- [${process_id}] Start to create database roles" > ${db_file_name}
-  for d in $(seq ${NUM_OF_ROLES_FOR_DATABASES}); do
-    db_name=${DB_NAME}_${d}
-    role_name=${db_name}_db_role_${d}
-    group_name=$(get_group $d)
-    echo "create role ${role_name}; " >> ${db_file_name}
-    echo "grant all on database ${db_name} to role ${role_name}; " >> ${db_file_name}
-    echo "grant ${role_name} to group ${group_name};" >> ${db_file_name}
-  done
-  chmod a+x ${db_file_name}
-}
-
-# Create table roles
-function create_tbls_roles () {
-  tbl_file_name=$1
-  echo "-- [${process_id}] Start to create table roles;" > ${tbl_file_name}
-  # create table roles
-  for d in $(seq ${NUM_OF_DATABASES}); do
-    db_name=${DB_NAME}_${d}
-    echo "USE ${db_name};" >> ${tbl_file_name}
-    for t in $(seq ${NUM_OF_ROLES_FOR_TABLES_PER_DATABASE}); do
-      tbl_name=${db_name}_tbl_${t}
-      role_name=${tbl_name}_role_${t}
-      echo "CREATE ROLE ${role_name};" >> ${tbl_file_name}
-      rand_number=$(awk 'BEGIN{srand();print int(rand()*3)}')
-      case "$((rand_number % 3))" in
-          0) echo "grant all on table ${tbl_name} to role ${role_name}; " >> ${tbl_file_name}
-             ;;
-          1) echo "grant insert on table ${tbl_name} to role ${role_name}; "  >> ${tbl_file_name}
-             ;;
-          *) echo "grant select on table ${tbl_name} to role ${role_name}; " >> ${tbl_file_name}
-             ;;
-      esac
-      group_name=$(get_group $d)
-      echo "grant role ${role_name} to group ${group_name}; "  >> ${tbl_file_name}
-    done
-  done
-  chmod a+x ${tbl_file_name}
-}
-
-###########################
-# Start from here!
-###########################
-create_groups
-echo "# [${process_id}]	Created ${NUM_OF_GROUPS} groups"
-
-# Use Hive to create the partitions because it supports bulk adding of partitions.
-# Hive doesn't allow fully qualified table names in ALTER statements, so start with a
-# USE <db>.
-create_tables_file_name=${LOCAL_OUTPUT_DIR}/hive_${prefix_string}_bulk_tables.q
-create_dbs_tbls_bulk_file ${create_tables_file_name}
-echo "# [${process_id}]	Created ${create_tables_file_name} to create databases and tables in bulk mode"
-
-create_wide_tables_file_name=${LOCAL_OUTPUT_DIR}/hive_${prefix_string}_bulk_wide_tables.q
-create_wide_tables_dir_file_name=${LOCAL_OUTPUT_DIR}/hive_${prefix_string}_bulk_wide_tables_dirs.sh
-create_external_par_dirs_bulk_file ${create_wide_tables_file_name} ${create_wide_tables_dir_file_name}
-echo "# [${process_id}]	Created ${create_wide_tables_file_name} to create wide tables with external partitions in bulk mode"
-echo "# [${process_id}]	Created ${create_wide_tables_dir_file_name} to create external dirs for external partitions in bulk mode"
-
-create_db_role_file_name=${LOCAL_OUTPUT_DIR}/hive_${prefix_string}_bulk_db_roles.q
-create_dbs_roles ${create_db_role_file_name}
-echo "# [${process_id}]	Created ${create_db_role_file_name} to create database roles"
-
-create_tbl_role_file_name=${LOCAL_OUTPUT_DIR}/hive_${prefix_string}_bulk_tbl_roles.q
-create_tbls_roles ${create_tbl_role_file_name}
-echo "# [${process_id}]	Created ${create_tbl_role_file_name} to create table roles"
-
-sudo -u hive hive -S -f ${create_tables_file_name}
-validate_ret $?
-echo "# [${process_id}]	Succeessfully ran bulk file ${create_tables_file_name} to create databases and tables"
-
-. ${create_wide_tables_dir_file_name}
-echo "# [${process_id}]	Successfully ran ${create_wide_tables_dir_file_name} to create dirs for external partitions"
-
-sudo -u hive hive -S -f ${create_wide_tables_file_name}
-validate_ret $?
-echo "# [${process_id}]	Successfully ran bulk file ${create_wide_tables_file_name} to create wide tables with external partitions"
-
-sudo -u hive ${BL} -f ${create_db_role_file_name} 1>/dev/null # to remove white lines after execution
-validate_ret $?
-echo "# [${process_id}]	Successfully created database level roles and privileges"
-
-sudo -u hive ${BL} -f ${create_tbl_role_file_name} 1>/dev/null # to remove white lines after execution
-validate_ret $?
-echo "# [${process_id}]	Successfully created table level roles and privileges"
-
-res_file=${LOCAL_OUTPUT_DIR}/hive_${prefix_string}.res
-echo "-- [${process_id}]	List all databases and roles in ${res_file}" > ${res_file}
-sudo -u hive  ${BL} -e "show databases" 2>/dev/null 1>>${res_file}
-sudo -u hive  ${BL} -e "show roles" 2>/dev/null 1>>${res_file}
-echo "[${process_id}]	Successfully listed all databases and roles in ${res_file}"


Mime
View raw message