sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kal...@apache.org
Subject sentry git commit: SENTRY-2034: Add e2e tests for testing HMS notification processing.(Kalyan Kumar Kalvagadda, reviewed by Sergio Pena, Lina Li)
Date Tue, 16 Jan 2018 20:52:52 GMT
Repository: sentry
Updated Branches:
  refs/heads/master 5a7b07643 -> b87651cf9


SENTRY-2034: Add e2e tests for testing HMS notification processing.(Kalyan Kumar Kalvagadda,
reviewed by Sergio Pena, Lina Li)


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/b87651cf
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/b87651cf
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/b87651cf

Branch: refs/heads/master
Commit: b87651cf9b140f03d44297129fbe68b82052da8e
Parents: 5a7b076
Author: Kalyan Kumar Kalvagadda <kkalyan@cloudera.com>
Authored: Tue Jan 16 14:08:54 2018 -0600
Committer: Kalyan Kumar Kalvagadda <kkalyan@cloudera.com>
Committed: Tue Jan 16 14:48:55 2018 -0600

----------------------------------------------------------------------
 .../TestDbPrivilegeCleanupOnDrop.java           |  40 ++------
 .../TestHmsNotificationProcessing.java          |  92 +++++++++++++++++
 .../TestHmsNotificationProcessingBase.java      |  79 +++++++++++++++
 ...msNotificationProcessingWithOutHdfsSync.java | 101 +++++++++++++++++++
 ...tificationProcessingWithOutSyncOnCreate.java |  96 ++++++++++++++++++
 ...NotificationProcessingWithOutSyncOnDrop.java |  92 +++++++++++++++++
 .../e2e/hdfs/TestHDFSIntegrationAdvanced.java   |   6 ++
 .../tests/e2e/hdfs/TestHDFSIntegrationBase.java | 100 +++++++++++++-----
 .../e2e/hdfs/TestHDFSIntegrationEnd2End.java    |   7 +-
 .../e2e/hdfs/TestHDFSIntegrationWithHA.java     |   1 +
 10 files changed, 551 insertions(+), 63 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
index 2138304..5fe6625 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
@@ -18,7 +18,6 @@
 package org.apache.sentry.tests.e2e.dbprovider;
 
 import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
@@ -26,7 +25,6 @@ import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase;
@@ -59,8 +57,6 @@ public class TestDbPrivilegeCleanupOnDrop extends TestHDFSIntegrationBase
{
           DB1 = "db_1",
           DB2 = "db_2";
 
-  static final long WAIT_FOR_NOTIFICATION_PROCESSING = 5000;
-
   private Connection connection;
   private Statement statement;
 
@@ -326,13 +322,13 @@ public class TestDbPrivilegeCleanupOnDrop extends TestHDFSIntegrationBase
{
   private void verifyTablePrivilegesDropped(Statement statement)
       throws Exception {
     List<String> roles = getRoles(statement);
-    verifyPrivilegeDropped(statement, roles, tableName1,
+    verifyIfAllPrivilegeAreDropped(statement, roles, tableName1,
         SHOW_GRANT_TABLE_POSITION);
-    verifyPrivilegeDropped(statement, roles, tableName2,
+    verifyIfAllPrivilegeAreDropped(statement, roles, tableName2,
         SHOW_GRANT_TABLE_POSITION);
-    verifyPrivilegeDropped(statement, roles, tableName3,
+    verifyIfAllPrivilegeAreDropped(statement, roles, tableName3,
         SHOW_GRANT_TABLE_POSITION);
-    verifyPrivilegeDropped(statement, roles, tableName4,
+    verifyIfAllPrivilegeAreDropped(statement, roles, tableName4,
         SHOW_GRANT_TABLE_POSITION);
 
   }
@@ -340,24 +336,9 @@ public class TestDbPrivilegeCleanupOnDrop extends TestHDFSIntegrationBase
{
   // verify all the test privileges are dropped as we drop the objects
   private void verifyDbPrivilegesDropped(Statement statement) throws Exception {
     List<String> roles = getRoles(statement);
-    verifyPrivilegeDropped(statement, roles, DB2, SHOW_GRANT_DB_POSITION);
-    verifyPrivilegeDropped(statement, roles, DB1, SHOW_GRANT_DB_POSITION);
-
-  }
+    verifyIfAllPrivilegeAreDropped(statement, roles, DB2, SHOW_GRANT_DB_POSITION);
+    verifyIfAllPrivilegeAreDropped(statement, roles, DB1, SHOW_GRANT_DB_POSITION);
 
-  // verify given table/DB has no longer permissions
-  private void verifyPrivilegeDropped(Statement statement, List<String> roles,
-      String objectName, int resultPos) throws Exception {
-    for (String roleName : roles) {
-      ResultSet resultSet = statement.executeQuery("SHOW GRANT ROLE "
-          + roleName);
-      while (resultSet.next()) {
-        String returned = resultSet.getString(resultPos);
-        assertFalse("value " + objectName + " shouldn't be detected, but actually " + returned
+ " is found from resultSet",
-                objectName.equalsIgnoreCase(returned));
-          }
-      resultSet.close();
-    }
   }
 
   // verify given table is part of the role
@@ -370,13 +351,4 @@ public class TestDbPrivilegeCleanupOnDrop extends TestHDFSIntegrationBase
{
       resultSet.close();
     }
   }
-
-  private List<String> getRoles(Statement statement) throws Exception {
-    ArrayList<String> roleList = Lists.newArrayList();
-    ResultSet resultSet = statement.executeQuery("SHOW ROLES ");
-    while (resultSet.next()) {
-      roleList.add(resultSet.getString(1));
-    }
-    return roleList;
-  }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessing.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessing.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessing.java
new file mode 100644
index 0000000..e730dd5
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessing.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.dbprovider;
+
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.Statement;
+
+public class TestHmsNotificationProcessing  extends TestHmsNotificationProcessingBase {
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    hiveSyncOnDrop = true;
+    hiveSyncOnCreate = true;
+    TestHDFSIntegrationBase.setup();
+  }
+  /*
+  Tests basic sanity of Hms notification processing by verifying below when new Hive objects
are created
+  1. Making sure that HDFS ACL rules for an new Hive objects created.
+  2. Making sue that stale permissions are deleted for the new Hive object that are created.
+  3. Making sure that permissions are deleted when hive objects are deleted
+  */
+
+  @Test
+  public void testHmsNotificationProcessingSanity() throws Throwable {
+    dbNames = new String[]{DB1};
+    roles = new String[]{"admin_role", "read_db1", "select_tbl1"};
+    admin = "hive";
+
+    Connection connection = hiveServer2.createConnection(admin, admin);
+    Statement statement = connection.createStatement();
+    statement.execute("create role admin_role");
+    statement.execute("grant role admin_role to group hive");
+    statement.execute("grant all on server server1 to role admin_role");
+
+    // Add privileges for an objects that do not exist yet
+    statement.execute("create role read_db1");
+    statement.execute("create role select_tbl1");
+    statement.execute("grant role read_db1 to group hbase");
+    statement.execute("grant role select_tbl1 to group hbase");
+    statement.execute("grant select on database " + DB1 + " to role read_db1");
+    String str = "grant select on table " + DB1 + "." + tableName1
+            + " TO ROLE select_tbl1";
+    statement.execute(str);
+
+    //Add object
+    statement.execute("CREATE DATABASE " + DB1);
+    statement.execute("use " + DB1);
+    statement.execute("create table " + DB1 + "." + tableName1
+            + " (under_col int comment 'the under column', value string)");
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    //Make sure that the privileges for that object are removed.
+    verifyPrivilegesDropped(statement);
+
+    //add "select" sentry permission for the object's
+    statement.execute("GRANT select ON DATABASE " + DB1 + " TO ROLE read_db1");
+    statement.execute("USE " + DB1);
+    statement.execute("GRANT SELECT ON TABLE " + tableName1
+            + " TO ROLE select_tbl1");
+
+    // Make sure that an ACL is added for that
+    verifyOnAllSubDirs("/user/hive/warehouse/db_1.db", FsAction.READ_EXECUTE, "hbase", true);
+    verifyOnAllSubDirs("/user/hive/warehouse/db_1.db/tb_1", FsAction.READ_EXECUTE, "hbase",
true);
+
+    //Drop the object
+    statement.execute("DROP DATABASE " + DB1 + " CASCADE");
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    //Make sure that the privileges added for that object are removed.
+    verifyPrivilegesDropped(statement);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingBase.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingBase.java
new file mode 100644
index 0000000..ed80e88
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingBase.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.dbprovider;
+
+import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestHmsNotificationProcessingBase  extends TestHDFSIntegrationBase{
+  static final Logger LOGGER = LoggerFactory.getLogger(TestHmsNotificationProcessingBase.class);
+  protected final static int SHOW_GRANT_TABLE_POSITION = 2;
+  protected final static int SHOW_GRANT_DB_POSITION = 1;
+  protected static final String DB1 = "db_1",
+          DB2 = "db_2",
+          tableName1 = "tb_1";
+
+  // verify all the test privileges are dropped as we drop the objects
+  protected void verifyPrivilegesDropped(Statement statement)
+          throws Exception {
+    verifyDbPrivilegesDropped(statement);
+    verifyTablePrivilegesDropped(statement);
+  }
+
+  // verify all the test privileges are dropped as we drop the objects
+  protected void verifyTablePrivilegesDropped(Statement statement)
+          throws Exception {
+    List<String> roles = getRoles(statement);
+    verifyIfAllPrivilegeAreDropped(statement, roles, tableName1,
+            SHOW_GRANT_TABLE_POSITION);
+  }
+
+  // verify all the test privileges are dropped as we drop the objects
+  protected void verifyDbPrivilegesDropped(Statement statement) throws Exception {
+    List<String> roles = getRoles(statement);
+    verifyIfAllPrivilegeAreDropped(statement, roles, DB2, SHOW_GRANT_DB_POSITION);
+    verifyIfAllPrivilegeAreDropped(statement, roles, DB1, SHOW_GRANT_DB_POSITION);
+
+  }
+
+  // verify all the test privileges are not dropped as we drop the objects
+  protected void verifyPrivilegesCount(Statement statement, int count)
+          throws Exception {
+    int privilegeCount = 0;
+    List<String> roles = getRoles(statement);
+    for (String roleName : roles) {
+      if(roleName.compareTo("admin_role") == 0) {
+        continue;
+      }
+      ResultSet resultSet = statement.executeQuery("SHOW GRANT ROLE "
+              + roleName);
+      while (resultSet.next()) {
+        privilegeCount++;
+      }
+      resultSet.close();
+    }
+    assertEquals("Privilege count do not match", count, privilegeCount);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutHdfsSync.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutHdfsSync.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutHdfsSync.java
new file mode 100644
index 0000000..9634ea1
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutHdfsSync.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.dbprovider;
+
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.Statement;
+
+
+public class TestHmsNotificationProcessingWithOutHdfsSync extends TestHmsNotificationProcessingBase
{
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    hdfsSyncEnabled = false;
+    hiveSyncOnDrop = true;
+    hiveSyncOnCreate = true;
+    TestHDFSIntegrationBase.setup();
+  }
+
+  /*
+  Tests basic sanity of Hms notification processing by verifying below when new Hive objects
are created
+  1. Making sure that HDFS ACL rules for an new Hive objects are not added.
+  2. Making sure that stale permissions are deleted for the new Hive object that are created.
+  3. Making sure that permissions are deleted when hive objects are deleted
+  */
+
+  @Test
+  public void testHmsNotificationProcessingSanity() throws Throwable {
+    dbNames = new String[]{DB1};
+    roles = new String[]{"admin_role", "read_db1", "select_tbl1"};
+    admin = "hive";
+
+    Connection connection = hiveServer2.createConnection(admin, admin);
+    Statement statement = connection.createStatement();
+    statement.execute("create role admin_role");
+    statement.execute("grant role admin_role to group hive");
+    statement.execute("grant all on server server1 to role admin_role");
+
+    // Add privileges for an objects that do not exist yet
+    statement.execute("create role read_db1");
+    statement.execute("create role select_tbl1");
+    statement.execute("grant role read_db1 to group hbase");
+    statement.execute("grant role select_tbl1 to group hbase");
+
+    //add "select" sentry permission for the object's
+    statement.execute("grant select on database " + DB1 + " to role read_db1");
+    String str = "grant select on table " + DB1 + "." + tableName1
+            + " TO ROLE select_tbl1";
+    statement.execute(str);
+
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    verifyPrivilegesCount(statement, 2);
+
+    //Add object
+    statement.execute("CREATE DATABASE " + DB1);
+    statement.execute("use " + DB1);
+    statement.execute("create table " + DB1 + "." + tableName1
+            + " (under_col int comment 'the under column', value string)");
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    //Make sure that the privileges for that object are removed.
+    verifyPrivilegesCount(statement, 0);
+
+    //add "select" sentry permission for the object's
+    statement.execute("GRANT select ON DATABASE " + DB1 + " TO ROLE read_db1");
+    statement.execute("USE " + DB1);
+    statement.execute("GRANT SELECT ON TABLE " + tableName1
+            + " TO ROLE select_tbl1");
+
+    // Make sure that an ACL is added for that
+    verifyOnAllSubDirs("/user/hive/warehouse/db_1.db", FsAction.READ_EXECUTE, "hbase", false);
+    verifyOnAllSubDirs("/user/hive/warehouse/db_1.db/tb_1", FsAction.READ_EXECUTE, "hbase",
false);
+
+    //Drop the object
+    statement.execute("DROP DATABASE " + DB1 + " CASCADE");
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    //Make sure that the privileges added for that object are removed.
+    verifyPrivilegesCount(statement, 0);
+  }
+ }

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutSyncOnCreate.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutSyncOnCreate.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutSyncOnCreate.java
new file mode 100644
index 0000000..0339c7b
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutSyncOnCreate.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.dbprovider;
+
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.Statement;
+
+public class TestHmsNotificationProcessingWithOutSyncOnCreate extends TestHmsNotificationProcessingBase
{
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    hiveSyncOnCreate = false;
+    hiveSyncOnDrop = true;
+    TestHDFSIntegrationBase.setup();
+  }
+
+  /*
+  Tests basic sanity of Hms notification processing by verifying below when new Hive objects
are created
+  1. Making sure that HDFS ACL rules for an new Hive objects created.
+  2. Making sure that stale permissions are not deleted for the new Hive object that are
created.
+  3. Making sure that permissions are deleted when hive objects are deleted
+  */
+  @Test
+  public void testHmsNotificationProcessingSanity() throws Throwable {
+    dbNames = new String[]{DB1};
+    roles = new String[]{"admin_role", "read_db1", "select_tbl1"};
+    admin = "hive";
+
+    Connection connection = hiveServer2.createConnection(admin, admin);
+    Statement statement = connection.createStatement();
+    statement.execute("create role admin_role");
+    statement.execute("grant role admin_role to group hive");
+    statement.execute("grant all on server server1 to role admin_role");
+
+    // Add privileges for an objects that do not exist yet
+    statement.execute("create role read_db1");
+    statement.execute("create role select_tbl1");
+    statement.execute("grant role read_db1 to group hbase");
+    statement.execute("grant role select_tbl1 to group hbase");
+
+    //add "select" sentry permission for the object's
+    statement.execute("grant select on database " + DB1 + " to role read_db1");
+    String str = "grant select on table " + DB1 + "." + tableName1
+            + " TO ROLE select_tbl1";
+    statement.execute(str);
+
+    //Add object
+    statement.execute("CREATE DATABASE " + DB1);
+    statement.execute("use " + DB1);
+    statement.execute("create table " + DB1 + "." + tableName1
+            + " (under_col int comment 'the under column', value string)");
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    //Make sure that the privileges for that object are removed.
+    verifyPrivilegesCount(statement, 2);
+
+    //add "select" sentry permission for the object's
+    statement.execute("GRANT select ON DATABASE " + DB1 + " TO ROLE read_db1");
+    statement.execute("USE " + DB1);
+    statement.execute("GRANT SELECT ON TABLE " + tableName1
+            + " TO ROLE select_tbl1");
+
+
+    // Make sure that an ACL is added for that
+    verifyOnAllSubDirs("/user/hive/warehouse/db_1.db", FsAction.READ_EXECUTE, "hbase", true);
+    verifyOnAllSubDirs("/user/hive/warehouse/db_1.db/tb_1", FsAction.READ_EXECUTE, "hbase",
true);
+
+    //Drop the object
+    statement.execute("DROP DATABASE " + DB1 + " CASCADE");
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    //Make sure that the privileges added for that object are removed.
+    verifyPrivilegesCount(statement, 0);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutSyncOnDrop.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutSyncOnDrop.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutSyncOnDrop.java
new file mode 100644
index 0000000..f70b6ab
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestHmsNotificationProcessingWithOutSyncOnDrop.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.dbprovider;
+
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.Statement;
+
+public class TestHmsNotificationProcessingWithOutSyncOnDrop extends TestHmsNotificationProcessingBase
{
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    hiveSyncOnDrop = false;
+    hiveSyncOnCreate = true;
+    hdfsSyncEnabled = true;
+    TestHDFSIntegrationBase.setup();
+  }
+
+  /*
+  Tests basic sanity of Hms notification processing by verifying below when new Hive objects
are created
+  1. Making sure that HDFS ACL rules for an new Hive objects created.
+  2. Making sure that stale permissions are deleted for the new Hive object that are created.
+  3. Making sure that permissions are not deleted when hive objects are deleted
+  */
+
+  @Test
+  public void testHmsNotificationProcessingSanity() throws Throwable {
+    dbNames = new String[]{DB1};
+    roles = new String[]{"admin_role", "read_db1", "select_tbl1"};
+    admin = "hive";
+
+    Connection connection = hiveServer2.createConnection(admin, admin);
+    Statement statement = connection.createStatement();
+    statement.execute("create role admin_role");
+    statement.execute("grant role admin_role to group hive");
+    statement.execute("grant all on server server1 to role admin_role");
+
+    // Add privileges for an objects that do not exist yet
+    statement.execute("create role read_db1");
+    statement.execute("create role select_tbl1");
+    statement.execute("grant role read_db1 to group hbase");
+    statement.execute("grant role select_tbl1 to group hbase");
+
+    //Add object
+    statement.execute("CREATE DATABASE " + DB1);
+    statement.execute("use " + DB1);
+    statement.execute("create table " + DB1 + "." + tableName1
+            + " (under_col int comment 'the under column', value string)");
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    //Make sure that the privileges for that object are removed.
+    verifyPrivilegesCount(statement, 0);
+
+    //add "select" sentry permission for the object's
+    statement.execute("GRANT select ON DATABASE " + DB1 + " TO ROLE read_db1");
+    statement.execute("USE " + DB1);
+    statement.execute("GRANT SELECT ON TABLE " + tableName1
+            + " TO ROLE select_tbl1");
+
+    verifyPrivilegesCount(statement, 2);
+
+    // Make sure that an ACL is added for that
+    verifyOnAllSubDirs("/user/hive/warehouse/db_1.db", FsAction.READ_EXECUTE, "hbase", true);
+    verifyOnAllSubDirs("/user/hive/warehouse/db_1.db/tb_1", FsAction.READ_EXECUTE, "hbase",
true);
+
+    //Drop the object
+    statement.execute("DROP DATABASE " + DB1 + " CASCADE");
+
+    Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING);
+    //Make sure that the privileges added for that object are removed.
+    verifyPrivilegesCount(statement, 2);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
index 33ace57..c8fc019 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
@@ -31,6 +31,7 @@ import org.apache.sentry.hdfs.PathsUpdate;
 import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
 
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.slf4j.Logger;
@@ -46,6 +47,11 @@ public class TestHDFSIntegrationAdvanced extends TestHDFSIntegrationBase
{
   private static final Logger LOGGER = LoggerFactory
       .getLogger(TestHDFSIntegrationAdvanced.class);
 
+  @BeforeClass
+  public static void setup() throws Exception{
+    hdfsSyncEnabled = true;
+    TestHDFSIntegrationBase.setup();
+  }
   @Test
   public void testNoPartitionInsert() throws Throwable {
     dbNames = new String[]{"db1"};

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
index c8eef09..4cd00e6 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
@@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.base.Preconditions;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -96,6 +97,7 @@ import com.google.common.io.Files;
 import com.google.common.io.Resources;
 
 import static org.apache.sentry.hdfs.ServiceConstants.ServerConfig.SENTRY_HDFS_INTEGRATION_PATH_PREFIXES;
+import static org.junit.Assert.assertFalse;
 
 /**
  * Base abstract class for HDFS Sync integration
@@ -170,6 +172,9 @@ public abstract class TestHDFSIntegrationBase {
           ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT +
               ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT * 2 + CACHE_REFRESH
* 2;
 
+  protected static final long HMSFOLLOWER_INTERVAL_MILLS = 50;
+  protected static final long WAIT_FOR_NOTIFICATION_PROCESSING = HMSFOLLOWER_INTERVAL_MILLS
* 3;
+
   // Time to wait before running next tests. The unit is milliseconds.
   // Deleting HDFS may finish, but HDFS may not be ready for creating the same file again.
   // We need to to make sure that creating the same file in the next test will succeed
@@ -194,7 +199,11 @@ public abstract class TestHDFSIntegrationBase {
   protected String[] dbNames;
   protected String[] roles;
   protected String admin;
+  protected static Boolean hdfsSyncEnabled = true;
+  protected static Boolean hiveSyncOnCreate = false;
+  protected static Boolean hiveSyncOnDrop = true;
   protected static Configuration hadoopConf;
+  protected static final Map<String, String> sentryProperties = Maps.newHashMap();
 
   protected static File assertCreateDir(File dir) {
     if(!dir.isDirectory()) {
@@ -251,7 +260,7 @@ public abstract class TestHDFSIntegrationBase {
       if (groupShouldExist) {
         Assert.assertEquals("Error at verifying Path action : " + p + " ;", fsAction, getAcls(p).get(group));
       } else {
-        Assert.assertFalse("Error at verifying Path : " + p + " ," +
+        assertFalse("Error at verifying Path : " + p + " ," +
             " group : " + group + " ;", getAcls(p).containsKey(group));
       }
       LOGGER.info("Successfully found acls for path = " + p.getName());
@@ -347,7 +356,7 @@ public abstract class TestHDFSIntegrationBase {
     try {
       fs.listFiles(p, true);
       if(!hasPermission) {
-        Assert.assertFalse("Expected listing files to fail", false);
+        assertFalse("Expected listing files to fail", false);
       }
     } catch (Exception e) {
       if(hasPermission) {
@@ -415,6 +424,30 @@ public abstract class TestHDFSIntegrationBase {
 
   }
 
+  // verify given table/DB has no longer permissions
+  protected void verifyIfAllPrivilegeAreDropped(Statement statement, List<String> roles,
+                                                String objectName, int resultPos) throws
Exception {
+    for (String roleName : roles) {
+      ResultSet resultSet = statement.executeQuery("SHOW GRANT ROLE "
+              + roleName);
+      while (resultSet.next()) {
+        String returned = resultSet.getString(resultPos);
+        assertFalse("value " + objectName + " shouldn't be detected, but actually " + returned
+ " is found from resultSet",
+                objectName.equalsIgnoreCase(returned));
+      }
+      resultSet.close();
+    }
+  }
+
+  protected List<String> getRoles(Statement statement) throws Exception {
+    ArrayList<String> roleList = Lists.newArrayList();
+    ResultSet resultSet = statement.executeQuery("SHOW ROLES ");
+    while (resultSet.next()) {
+      roleList.add(resultSet.getString(1));
+    }
+    return roleList;
+  }
+
   protected void loadDataTwoCols(Statement stmt) throws IOException, SQLException {
     FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt"));
     f1.writeChars("m1d1_t1, m1d1_t2\n");
@@ -800,37 +833,48 @@ public abstract class TestHDFSIntegrationBase {
         public Void run() throws Exception {
           Configuration sentryConf = new Configuration(false);
           sentryConf.set(SENTRY_HDFS_INTEGRATION_PATH_PREFIXES, MANAGED_PREFIXES);
-          Map<String, String> properties = Maps.newHashMap();
-          properties.put(HiveServerFactory.AUTHZ_PROVIDER_BACKEND,
+          sentryProperties.put(HiveServerFactory.AUTHZ_PROVIDER_BACKEND,
               SimpleDBProviderBackend.class.getName());
-          properties.put(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname,
+          sentryProperties.put(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname,
               SentryHiveAuthorizationTaskFactoryImpl.class.getName());
-          properties
+          sentryProperties
               .put(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS.varname, "2");
-          properties.put("hive.exec.local.scratchdir", Files.createTempDir().getAbsolutePath());
-          properties.put(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_NONE);
-//        properties.put("sentry.service.server.compact.transport", "true");
-          properties.put("sentry.hive.testing.mode", "true");
-          properties.put("sentry.service.reporting", "JMX");
-          properties.put(ServerConfig.ADMIN_GROUPS, "hive,admin");
-          properties.put(ServerConfig.RPC_ADDRESS, "localhost");
-          properties.put(ServerConfig.RPC_PORT, String.valueOf(sentryPort > 0 ? sentryPort
: 0));
-          properties.put(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false");
-          properties.put("sentry.hive.server", "server1");
-
-          properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING);
-          properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath());
-          properties.put(ServerConfig.SENTRY_STORE_JDBC_URL,
+          sentryProperties.put("hive.exec.local.scratchdir", Files.createTempDir().getAbsolutePath());
+          sentryProperties.put(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_NONE);
+//        sentryProperties.put("sentry.service.server.compact.transport", "true");
+          sentryProperties.put("sentry.hive.testing.mode", "true");
+          sentryProperties.put("sentry.service.reporting", "JMX");
+          sentryProperties.put(ServerConfig.ADMIN_GROUPS, "hive,admin");
+          sentryProperties.put(ServerConfig.RPC_ADDRESS, "localhost");
+          sentryProperties.put(ServerConfig.RPC_PORT, String.valueOf(sentryPort > 0 ?
sentryPort : 0));
+          sentryProperties.put(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false");
+          sentryProperties.put("sentry.hive.server", "server1");
+
+          sentryProperties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING);
+          sentryProperties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath());
+          sentryProperties.put(ServerConfig.SENTRY_STORE_JDBC_URL,
               "jdbc:derby:;databaseName=" + baseDir.getPath()
                   + "/sentrystore_db;create=true");
-          properties.put(ServerConfig.SENTRY_STORE_JDBC_PASS, "dummy");
-          properties.put("sentry.service.processor.factories",
-              "org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory");
-          properties.put("sentry.policy.store.plugins", "org.apache.sentry.hdfs.SentryPlugin");
-          properties.put(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS, "10000");
-          properties.put(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS, "50");
-          properties.put(ServerConfig.RPC_MIN_THREADS, "3");
-          for (Map.Entry<String, String> entry : properties.entrySet()) {
+          sentryProperties.put(ServerConfig.SENTRY_STORE_JDBC_PASS, "dummy");
+          sentryProperties.put(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS, "10000");
+          sentryProperties.put(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS, String.valueOf(HMSFOLLOWER_INTERVAL_MILLS));
+          sentryProperties.put(ServerConfig.RPC_MIN_THREADS, "3");
+          if(hiveSyncOnCreate) {
+            sentryProperties.put("sentry.hive.sync.create", "true");
+          } else {
+            sentryProperties.put("sentry.hive.sync.create", "false");
+          }
+          if(hiveSyncOnDrop) {
+            sentryProperties.put("sentry.hive.sync.drop", "true");
+          } else {
+            sentryProperties.put("sentry.hive.sync.drop", "false");
+          }
+          if(hdfsSyncEnabled) {
+            sentryProperties.put("sentry.service.processor.factories",
+                    "org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory");
+            sentryProperties.put("sentry.policy.store.plugins", "org.apache.sentry.hdfs.SentryPlugin");
+          }
+            for (Map.Entry<String, String> entry : sentryProperties.entrySet()) {
             sentryConf.set(entry.getKey(), entry.getValue());
           }
           sentryServer = SentrySrvFactory.create(SentrySrvFactory.SentrySrvType.INTERNAL_SERVER,

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
index a909b91..061900a 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
 
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -46,7 +47,11 @@ public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase
{
 
   private static String adminRole = "admin_role";
 
-
+  @BeforeClass
+  public static void setup() throws Exception{
+    hdfsSyncEnabled = true;
+    TestHDFSIntegrationBase.setup();
+  }
   @Test
   public void testEnd2EndManagedPaths() throws Throwable {
     tmpHDFSDir = new Path("/tmp/external");

http://git-wip-us.apache.org/repos/asf/sentry/blob/b87651cf/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationWithHA.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationWithHA.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationWithHA.java
index cc0449b..96a2f90 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationWithHA.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationWithHA.java
@@ -22,6 +22,7 @@ import org.junit.BeforeClass;
 public class TestHDFSIntegrationWithHA extends TestHDFSIntegrationEnd2End {
   @BeforeClass
   public static void setup() throws Exception {
+    hdfsSyncEnabled = true;
     TestHDFSIntegrationBase.testSentryHA = true;
     TestHDFSIntegrationBase.setup();
   }


Mime
View raw message