sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject sentry git commit: SENTRY-1577: Support "create function using jar" for hive when Sentry is enabled (Aihua Xu, Reviewed by: Hao Hao, Alexander Kolbasov)
Date Thu, 12 Jan 2017 19:06:52 GMT
Repository: sentry
Updated Branches:
  refs/heads/master aba45daf7 -> 58d76b93a


SENTRY-1577: Support "create function using jar" for hive when Sentry is enabled (Aihua Xu,
Reviewed by: Hao Hao, Alexander Kolbasov)

Change-Id: I1862b5b6ac919d2d3ccf8179b7710a8df24a09e7


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/58d76b93
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/58d76b93
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/58d76b93

Branch: refs/heads/master
Commit: 58d76b93a0765f906ac4246d0a252baa8c807bae
Parents: aba45da
Author: hahao <hao.hao@cloudera.com>
Authored: Wed Jan 11 14:25:37 2017 -0800
Committer: hahao <hao.hao@cloudera.com>
Committed: Thu Jan 12 11:06:35 2017 -0800

----------------------------------------------------------------------
 .../binding/hive/HiveAuthzBindingHookBase.java  |  38 ++++++++++--
 .../hive/SentryOnFailureHookContext.java        |   5 +-
 .../hive/SentryOnFailureHookContextImpl.java    |  13 ++--
 .../binding/hive/v2/HiveAuthzBindingHookV2.java |  14 ++++-
 .../binding/hive/HiveAuthzBindingHook.java      |  14 ++++-
 sentry-tests/data/xudf.jar                      | Bin 0 -> 2430 bytes
 .../e2e/hive/TestPrivilegesAtFunctionScope.java |  60 +++++++++++++++++--
 7 files changed, 124 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
index d706b83..c4daea1 100644
--- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
+++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
 import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.sentry.binding.hive.authz.HiveAuthzBinding;
 import org.apache.sentry.binding.hive.authz.HiveAuthzPrivileges;
@@ -76,6 +77,7 @@ import org.slf4j.LoggerFactory;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerHook {
@@ -85,7 +87,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
   protected final HiveAuthzConf authzConf;
   protected Database currDB = Database.ALL;
   protected Table currTab;
-  protected AccessURI udfURI;
+  protected List<AccessURI> udfURIs;
   protected AccessURI serdeURI;
   protected AccessURI partitionURI;
   protected Table currOutTab = null;
@@ -120,6 +122,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
       throw new IllegalStateException("Session HiveConf is null");
     }
     authzConf = loadAuthzConf(hiveConf);
+    udfURIs = Lists.newArrayList();
     hiveAuthzBinding = new HiveAuthzBinding(hiveConf, authzConf);
     String serdeWhiteLists =
         authzConf.get(HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST,
@@ -174,7 +177,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
       HiveOperation hiveOp, AuthorizationException e) {
     SentryOnFailureHookContext hookCtx = new SentryOnFailureHookContextImpl(
         context.getCommand(), context.getInputs(), context.getOutputs(),
-        hiveOp, currDB, currTab, udfURI, null, context.getUserName(),
+        hiveOp, currDB, currTab, udfURIs, null, context.getUserName(),
         context.getIpAddress(), e, context.getConf());
     String csHooks = authzConf.get(
         HiveAuthzConf.AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), "").trim();
@@ -188,6 +191,33 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
     }
   }
 
+  /**
+   * The command 'create function ... using jar <jar resources>' can create a function
+   * with the supplied jar resources in the command, which is translated into ASTNode being
+   * [functionName functionClass resourceList] and resourceList being [resourceType resourcePath].
+   * This function collects all the jar paths for the supplied jar resources.
+   *
+   * @param ast the AST node for the command
+   * @return    the jar path list if any or an empty list
+   */
+  protected List<String> getFunctionJars(ASTNode ast) {
+    ASTNode resourcesNode = (ASTNode) ast.getFirstChildWithType(HiveParser.TOK_RESOURCE_LIST);
+
+    List<String> resources = new ArrayList<String>();
+    if (resourcesNode != null) {
+      for (int idx = 0; idx < resourcesNode.getChildCount(); ++idx) {
+        ASTNode resNode = (ASTNode) resourcesNode.getChild(idx);
+        ASTNode resTypeNode = (ASTNode) resNode.getChild(0);
+        ASTNode resUriNode = (ASTNode) resNode.getChild(1);
+        if (resTypeNode.getType() == HiveParser.TOK_JAR) {
+          resources.add(PlanUtils.stripQuotes(resUriNode.getText()));
+        }
+      }
+    }
+
+    return resources;
+  }
+
   @VisibleForTesting
   protected static AccessURI extractPartition(ASTNode ast) throws SemanticException {
     for (int i = 0; i < ast.getChildCount(); i++) {
@@ -386,10 +416,10 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
        *  - CREATE TEMP FUNCTION
        *  - DROP TEMP FUNCTION.
        */
-      if (udfURI != null) {
+      if (!udfURIs.isEmpty()) {
         List<DBModelAuthorizable> udfUriHierarchy = new ArrayList<DBModelAuthorizable>();
         udfUriHierarchy.add(hiveAuthzBinding.getAuthServer());
-        udfUriHierarchy.add(udfURI);
+        udfUriHierarchy.addAll(udfURIs);
         inputHierarchy.add(udfUriHierarchy);
         for (WriteEntity writeEntity : outputs) {
           List<DBModelAuthorizable> entityHierarchy = new ArrayList<DBModelAuthorizable>();

http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
index c101a4f..ac14fb1 100644
--- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
+++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
@@ -18,6 +18,7 @@
 
 package org.apache.sentry.binding.hive;
 
+import java.util.List;
 import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
@@ -76,9 +77,9 @@ public interface SentryOnFailureHookContext  {
   Table getTable();
 
   /**
-   * @return the udf URI
+   * @return the udf URIs
    */
-  AccessURI getUdfURI();
+  List<AccessURI> getUdfURIs();
 
   /**
    * @return the partition URI

http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
index f97d7f3..15bfbea 100644
--- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
+++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
@@ -18,6 +18,7 @@
 
 package org.apache.sentry.binding.hive;
 
+import java.util.List;
 import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
@@ -39,14 +40,14 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex
   private final String ipAddress;
   private final Database database;
   private final Table table;
-  private final AccessURI udfURI;
+  private final List<AccessURI> udfURIs;
   private final AccessURI partitionURI;
   private final AuthorizationException authException;
   private final Configuration conf;
 
   public SentryOnFailureHookContextImpl(String command,
       Set<ReadEntity> inputs, Set<WriteEntity> outputs, HiveOperation hiveOp,
-      Database db, Table tab, AccessURI udfURI, AccessURI partitionURI,
+      Database db, Table tab, List<AccessURI> udfURIs, AccessURI partitionURI,
       String userName, String ipAddress, AuthorizationException e,
       Configuration conf) {
     this.command = command;
@@ -57,7 +58,7 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex
     this.ipAddress = ipAddress;
     this.database = db;
     this.table = tab;
-    this.udfURI = udfURI;
+    this.udfURIs = udfURIs;
     this.partitionURI = partitionURI;
     this.authException = e;
     this.conf = conf;
@@ -104,8 +105,8 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex
   }
 
   @Override
-  public AccessURI getUdfURI() {
-    return udfURI;
+  public List<AccessURI> getUdfURIs() {
+    return udfURIs;
   }
 
   @Override
@@ -122,4 +123,4 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex
   public Configuration getConf() {
     return conf;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
b/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
index 91a2507..018ebf3 100644
--- a/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
+++ b/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
@@ -69,10 +69,20 @@ public class HiveAuthzBindingHookV2 extends HiveAuthzBindingHookBase {
             throw new SemanticException("Could not find the jar for UDF class " + udfClassName
+
                 "to validate privileges");
           }
-          udfURI = parseURI(udfSrc.getLocation().toString(), true);
+          udfURIs.add(parseURI(udfSrc.getLocation().toString(), true));
         } catch (ClassNotFoundException e) {
-          throw new SemanticException("Error retrieving udf class", e);
+          List<String> functionJars = getFunctionJars(ast);
+          if (functionJars.isEmpty()) {
+            throw new SemanticException("Error retrieving udf class", e);
+          } else {
+            // Add the jars from the command "Create function using jar" to the access list
+            // Defer to hive to check if the class is in the jars
+            for(String jar : functionJars) {
+              udfURIs.add(parseURI(jar, false));
+            }
+          }
         }
+
         // create/drop function is allowed with any database
         currDB = Database.ALL;
         break;

http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
index 98c7eb0..f41960f 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
@@ -208,10 +208,20 @@ public class HiveAuthzBindingHook extends HiveAuthzBindingHookBase {
             throw new SemanticException("Could not find the jar for UDF class " + udfClassName
+
                 "to validate privileges");
           }
-          udfURI = parseURI(udfSrc.getLocation().toString(), true);
+          udfURIs.add(parseURI(udfSrc.getLocation().toString(), true));
         } catch (ClassNotFoundException e) {
-          throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e);
+          List<String> functionJars = getFunctionJars(ast);
+          if (functionJars.isEmpty()) {
+            throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e);
+          } else {
+            // Add the jars from the command "Create function using jar" to the access list
+            // Defer to hive to check if the class is in the jars
+            for(String jar : functionJars) {
+              udfURIs.add(parseURI(jar, false));
+            }
+          }
         }
+
         // create/drop function is allowed with any database
         currDB = Database.ALL;
         break;

http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-tests/data/xudf.jar
----------------------------------------------------------------------
diff --git a/sentry-tests/data/xudf.jar b/sentry-tests/data/xudf.jar
new file mode 100644
index 0000000..5bc4cee
Binary files /dev/null and b/sentry-tests/data/xudf.jar differ

http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
index 103a057..e440fb2 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
@@ -17,9 +17,7 @@ printf_test_3 * Licensed to the Apache Software Foundation (ASF) under one
or mo
 
 package org.apache.sentry.tests.e2e.hive;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -86,6 +84,8 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu
     String udf1ClassName = "org.apache.sentry.tests.e2e.hive.TestUDF";
     CodeSource udf1Src = Class.forName(udf1ClassName).getProtectionDomain().getCodeSource();
     String udf1Location = udf1Src.getLocation().getPath();
+    String udf2Location = new File("../data/xudf.jar").getCanonicalPath();
+
     Connection connection = context.createConnection(ADMIN1);
     Statement statement = context.createStatement(connection);
     statement.execute("DROP DATABASE IF EXISTS " + DB1 + " CASCADE");
@@ -100,13 +100,14 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu
     context.close();
 
     policyFile
-        .addRolesToGroup(USERGROUP1, "db1_all", "UDF1_JAR", "UDF_JAR", "data_read")
+        .addRolesToGroup(USERGROUP1, "db1_all", "UDF2_JAR", "UDF1_JAR", "UDF_JAR", "data_read")
         .addRolesToGroup(USERGROUP2, "db1_tab1", "UDF_JAR")
         .addRolesToGroup(USERGROUP3, "db1_tab1")
         .addPermissionsToRole("db1_all", "server=server1->db=" + DB1)
         .addPermissionsToRole("db1_tab1", "server=server1->db=" + DB1 + "->table="
+ tableName1)
         .addPermissionsToRole("UDF_JAR", "server=server1->uri=file://" + udfLocation)
         .addPermissionsToRole("UDF1_JAR", "server=server1->uri=file://" + udf1Location)
+        .addPermissionsToRole("UDF2_JAR", "server=server1->uri=file://" + udf2Location)
         .addPermissionsToRole("data_read", "server=server1->URI=" + "file:///tmp");
     writePolicyFile(policyFile);
   }
@@ -343,6 +344,57 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu
     context.close();
   }
 
+  /**
+   * Test create function using jar functionality
+   * @throws Exception
+   */
+  @Test
+  public void testAndVerifyFuncPrivilegesPart4() throws Exception {
+    setUpContext();
+    Connection connection = context.createConnection(USER1_1);
+    Statement statement = context.createStatement(connection);
+    statement.execute("USE " + DB1);
+
+    // USER1 has URI perm on jarFiles
+    try {
+      String udfLocation = new File("../data/xudf.jar").getCanonicalPath();
+
+      statement
+          .execute("CREATE FUNCTION "
+              + DB1
+              + ".xadd AS 'xudf.XAdd'"
+              + " using jar 'file://" + udfLocation + "'");
+      ResultSet rs = statement.executeQuery("select xadd(1)");
+      assertTrue(rs.next());
+      assertTrue(rs.getInt(1) == 1);
+    } catch (SQLException e) {
+      assertFalse("CREATE FUNCTION should succeed for user1:" + e, true);
+    } finally {
+      connection.close();
+    }
+
+    connection = context.createConnection(USER2_1);
+    statement = context.createStatement(connection);
+    statement.execute("USE " + DB1);
+
+    // USER2 doesn't have URI perm on jarFiles
+    try {
+      String udfLocation = new File("../data/xudf.jar").getCanonicalPath();
+
+      statement
+          .execute("CREATE FUNCTION "
+              + DB1
+              + ".xadd AS 'xudf.XAdd'"
+              + " using jar 'file://" + udfLocation + "'");
+      assertFalse("CREATE FUNCTION should fail for user2", true);
+    } catch (SQLException e) {
+      context.verifyAuthzException(e);
+    } finally {
+      connection.close();
+    }
+    context.close();
+  }
+
   @Test
   public void testUdfWhiteList () throws Exception {
     String tableName1 = "tab1";


Mime
View raw message