Return-Path: X-Original-To: apmail-drill-commits-archive@www.apache.org Delivered-To: apmail-drill-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A4CFD18C54 for ; Thu, 25 Jun 2015 16:51:57 +0000 (UTC) Received: (qmail 87839 invoked by uid 500); 25 Jun 2015 16:51:57 -0000 Delivered-To: apmail-drill-commits-archive@drill.apache.org Received: (qmail 87808 invoked by uid 500); 25 Jun 2015 16:51:57 -0000 Mailing-List: contact commits-help@drill.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@drill.apache.org Delivered-To: mailing list commits@drill.apache.org Received: (qmail 87798 invoked by uid 99); 25 Jun 2015 16:51:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 25 Jun 2015 16:51:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5317FE3690; Thu, 25 Jun 2015 16:51:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: venki@apache.org To: commits@drill.apache.org Date: Thu, 25 Jun 2015 16:51:58 -0000 Message-Id: <3f5879f591ec46fd852c53fac8f759aa@git.apache.org> In-Reply-To: <9b68f8316b0745e9914294a227d5eb5b@git.apache.org> References: <9b68f8316b0745e9914294a227d5eb5b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] drill git commit: DRILL-3203: Add support for impersonation in Hive storage plugin DRILL-3203: Add support for impersonation in Hive storage plugin Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/c1b847ac Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/c1b847ac Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/c1b847ac Branch: refs/heads/master Commit: c1b847acdc8cb90a1498b236b3bb5c81ca75c044 Parents: 58c3c4c Author: vkorukanti Authored: Sat Jun 20 19:29:23 2015 -0700 Committer: vkorukanti Committed: Thu Jun 25 07:53:36 2015 -0700 ---------------------------------------------------------------------- contrib/storage-hive/core/pom.xml | 12 + .../store/hive/DrillHiveMetaStoreClient.java | 375 +++++++++++++++++++ .../store/hive/HiveAuthorizationHelper.java | 153 ++++++++ .../apache/drill/exec/store/hive/HiveScan.java | 31 +- .../exec/store/hive/schema/DrillHiveTable.java | 4 +- .../store/hive/schema/DrillHiveViewTable.java | 5 +- .../store/hive/schema/HiveSchemaFactory.java | 207 +++------- .../drill/exec/hive/HiveTestUtilities.java | 50 +++ .../hive/BaseTestHiveImpersonation.java | 140 +++++++ .../hive/TestSqlStdBasedAuthorization.java | 296 +++++++++++++++ .../hive/TestStorageBasedHiveAuthorization.java | 372 ++++++++++++++++++ .../exec/store/hive/HiveTestDataGenerator.java | 51 +-- .../core/src/test/resources/student.txt | 10 + .../core/src/test/resources/voter.txt | 10 + .../org/apache/drill/exec/ops/QueryContext.java | 38 +- .../apache/drill/exec/store/AbstractSchema.java | 7 +- .../drill/exec/util/ImpersonationUtil.java | 16 +- .../java/org/apache/drill/BaseTestQuery.java | 31 ++ .../impersonation/BaseTestImpersonation.java | 66 +++- .../TestImpersonationDisabledWithMiniDFS.java | 37 +- .../TestImpersonationMetadata.java | 47 +-- .../impersonation/TestImpersonationQueries.java | 66 +--- 22 files changed, 1686 insertions(+), 338 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index 546fd7b..f0330f1 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -87,6 +87,18 @@ hadoop-yarn-api test + + org.apache.hadoop + hadoop-common + tests + test + + + org.apache.hadoop + hadoop-hdfs + tests + test + http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java new file mode 100644 index 0000000..ef70b2e --- /dev/null +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/DrillHiveMetaStoreClient.java @@ -0,0 +1,375 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.hive; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Lists; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.util.ImpersonationUtil; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.thrift.TException; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * Override HiveMetaStoreClient to provide additional capabilities such as caching, reconnecting with user + * credentials and higher level APIs to get the metadata in form that Drill needs directly. + */ +public abstract class DrillHiveMetaStoreClient extends HiveMetaStoreClient { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHiveMetaStoreClient.class); + + protected final Map hiveConfigOverride; + + /** + * Create a DrillHiveMetaStoreClient for cases where: + * 1. Drill impersonation is enabled and + * 2. either storage (in remote HiveMetaStore server) or SQL standard based authorization (in Hive storage plugin) + * is enabled + * @param hiveConf Conf including authorization configuration + * @param hiveConfigOverride + * @param userName User who is trying to access the Hive metadata + * @param ignoreAuthzErrors When browsing info schema, we want to ignore permission denied errors. If a permission + * denied error occurs while accessing metadata for an object, it will not be shown in the + * info schema. + * @return + * @throws MetaException + */ + public static DrillHiveMetaStoreClient createClientWithAuthz(final HiveConf hiveConf, + final Map hiveConfigOverride, final String userName, final boolean ignoreAuthzErrors) + throws MetaException { + try { + final UserGroupInformation ugiForRpc; // UGI credentials to use for RPC communication with Hive MetaStore server + if (!hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS)) { + // If the user impersonation is disabled in Hive storage plugin (not Drill impersonation), use the process + // user UGI credentials. + ugiForRpc = ImpersonationUtil.getProcessUserUGI(); + } else { + ugiForRpc = ImpersonationUtil.createProxyUgi(userName); + } + return ugiForRpc.doAs(new PrivilegedExceptionAction() { + @Override + public DrillHiveMetaStoreClient run() throws Exception { + return new HiveClientWithAuthz(hiveConf, hiveConfigOverride, ugiForRpc, userName, ignoreAuthzErrors); + } + }); + } catch (final Exception e) { + throw new DrillRuntimeException("Failure setting up HiveMetaStore client.", e); + } + } + + /** + * Create a DrillMetaStoreClient that can be shared across multiple users. This is created when impersonation is + * disabled. + * @param hiveConf + * @param hiveConfigOverride + * @return + * @throws MetaException + */ + public static DrillHiveMetaStoreClient createNonCloseableClientWithCaching(final HiveConf hiveConf, + final Map hiveConfigOverride) throws MetaException { + return new NonCloseableHiveClientWithCaching(hiveConf, hiveConfigOverride); + } + + private DrillHiveMetaStoreClient(final HiveConf hiveConf, final Map hiveConfigOverride) + throws MetaException { + super(hiveConf); + this.hiveConfigOverride = hiveConfigOverride; + } + + + /** + * Higher level API that returns the databases in Hive. + * @return + * @throws TException + */ + public abstract List getDatabases() throws TException; + + /** + * Higher level API that returns the tables in given database. + * @param dbName + * @return + * @throws TException + */ + public abstract List getTableNames(final String dbName) throws TException; + + /** + * Higher level API that returns the {@link HiveReadEntry} for given database and table. + * @param dbName + * @param tableName + * @return + * @throws TException + */ + public abstract HiveReadEntry getHiveReadEntry(final String dbName, final String tableName) throws TException; + + /** Helper method which gets database. Retries once if the first call to fetch the metadata fails */ + protected static List getDatabasesHelper(final IMetaStoreClient mClient) throws TException { + try { + return mClient.getAllDatabases(); + } catch (TException e) { + logger.warn("Failure while attempting to get hive databases", e); + mClient.reconnect(); + return mClient.getAllDatabases(); + } + } + + /** Helper method which gets tables in a database. Retries once if the first call to fetch the metadata fails */ + protected static List getTableNamesHelper(final IMetaStoreClient mClient, final String dbName) + throws TException { + try { + return mClient.getAllTables(dbName); + } catch (TException e) { + logger.warn("Failure while attempting to get hive tables", e); + mClient.reconnect(); + return mClient.getAllTables(dbName); + } + } + + /** Helper method which gets table metadata. Retries once if the first call to fetch the metadata fails */ + protected static HiveReadEntry getHiveReadEntryHelper(final IMetaStoreClient mClient, final String dbName, + final String tableName, final Map hiveConfigOverride) throws TException { + Table t = null; + try { + t = mClient.getTable(dbName, tableName); + } catch (TException e) { + mClient.reconnect(); + t = mClient.getTable(dbName, tableName); + } + + if (t == null) { + throw new UnknownTableException(String.format("Unable to find table '%s'.", tableName)); + } + + List partitions; + try { + partitions = mClient.listPartitions(dbName, tableName, (short) -1); + } catch (TException e) { + mClient.reconnect(); + partitions = mClient.listPartitions(dbName, tableName, (short) -1); + } + + List hivePartitions = Lists.newArrayList(); + for (Partition part : partitions) { + hivePartitions.add(new HiveTable.HivePartition(part)); + } + + if (hivePartitions.size() == 0) { + hivePartitions = null; + } + + return new HiveReadEntry(new HiveTable(t), hivePartitions, hiveConfigOverride); + } + + /** + * HiveMetaStoreClient to create and maintain (reconnection cases) connection to Hive metastore with given user + * credentials and check authorization privileges if set. + */ + private static class HiveClientWithAuthz extends DrillHiveMetaStoreClient { + private final UserGroupInformation ugiForRpc; + private final boolean ignoreAuthzErrors; + private HiveAuthorizationHelper authorizer; + + private HiveClientWithAuthz(final HiveConf hiveConf, final Map hiveConfigOverride, + final UserGroupInformation ugiForRpc, final String userName, final boolean ignoreAuthzErrors) + throws TException { + super(hiveConf, hiveConfigOverride); + this.ugiForRpc = ugiForRpc; + this.ignoreAuthzErrors = ignoreAuthzErrors; + this.authorizer = new HiveAuthorizationHelper(this, hiveConf, userName); + } + + @Override + public void reconnect() throws MetaException { + try { + ugiForRpc.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + reconnectSuper(); + return null; + } + }); + } catch (final InterruptedException | IOException e) { + throw new DrillRuntimeException("Failed to reconnect to HiveMetaStore: " + e.getMessage(), e); + } + } + + private void reconnectSuper() throws MetaException { + super.reconnect(); + } + + public List getDatabases() throws TException { + try { + authorizer.authorizeShowDatabases(); + } catch (final HiveAccessControlException e) { + if (ignoreAuthzErrors) { + return Collections.emptyList(); + } + throw UserException.permissionError(e).build(logger); + } + return getDatabasesHelper(this); + } + + public List getTableNames(final String dbName) throws TException { + try { + authorizer.authorizeShowTables(dbName); + } catch (final HiveAccessControlException e) { + if (ignoreAuthzErrors) { + return Collections.emptyList(); + } + throw UserException.permissionError(e).build(logger); + } + return getTableNamesHelper(this, dbName); + } + + public HiveReadEntry getHiveReadEntry(final String dbName, final String tableName) throws TException { + try { + authorizer.authorizeReadTable(dbName, tableName); + } catch (final HiveAccessControlException e) { + if (!ignoreAuthzErrors) { + throw UserException.permissionError(e).build(logger); + } + } + return getHiveReadEntryHelper(this, dbName, tableName, hiveConfigOverride); + } + } + + /** + * HiveMetaStoreClient that provides a shared MetaStoreClient implementation with caching. + */ + private static class NonCloseableHiveClientWithCaching extends DrillHiveMetaStoreClient { + private final LoadingCache> databases; + private final LoadingCache> tableNameLoader; + private final LoadingCache> tableLoaders; + + private NonCloseableHiveClientWithCaching(final HiveConf hiveConf, + final Map hiveConfigOverride) throws MetaException { + super(hiveConf, hiveConfigOverride); + + databases = CacheBuilder // + .newBuilder() // + .expireAfterAccess(1, TimeUnit.MINUTES) // + .build(new DatabaseLoader()); + + tableNameLoader = CacheBuilder // + .newBuilder() // + .expireAfterAccess(1, TimeUnit.MINUTES) // + .build(new TableNameLoader()); + + tableLoaders = CacheBuilder // + .newBuilder() // + .expireAfterAccess(4, TimeUnit.HOURS) // + .maximumSize(20) // + .build(new TableLoaderLoader()); + } + + @Override + public List getDatabases() throws TException { + try { + return databases.get("databases"); + } catch (final ExecutionException e) { + throw new TException(e); + } + } + + @Override + public List getTableNames(final String dbName) throws TException { + try { + return tableNameLoader.get(dbName); + } catch (final ExecutionException e) { + throw new TException(e); + } + } + + @Override + public HiveReadEntry getHiveReadEntry(final String dbName, final String tableName) throws TException { + try { + return tableLoaders.get(dbName).get(tableName); + } catch (final ExecutionException e) { + throw new TException(e); + } + } + + @Override + public void close() { + // No-op. + } + + private class DatabaseLoader extends CacheLoader> { + @Override + public List load(String key) throws Exception { + if (!"databases".equals(key)) { + throw new UnsupportedOperationException(); + } + synchronized (NonCloseableHiveClientWithCaching.this) { + return getDatabasesHelper(NonCloseableHiveClientWithCaching.this); + } + } + } + + private class TableNameLoader extends CacheLoader> { + @Override + public List load(String dbName) throws Exception { + synchronized (NonCloseableHiveClientWithCaching.this) { + return getTableNamesHelper(NonCloseableHiveClientWithCaching.this, dbName); + } + } + } + + private class TableLoaderLoader extends CacheLoader> { + @Override + public LoadingCache load(String key) throws Exception { + return CacheBuilder + .newBuilder() + .expireAfterAccess(1, TimeUnit.MINUTES) + .build(new TableLoader(key)); + } + } + + private class TableLoader extends CacheLoader { + private final String dbName; + + public TableLoader(final String dbName) { + this.dbName = dbName; + } + + @Override + public HiveReadEntry load(String key) throws Exception { + synchronized (NonCloseableHiveClientWithCaching.this) { + return getHiveReadEntryHelper(NonCloseableHiveClientWithCaching.this, dbName, key, hiveConfigOverride); + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveAuthorizationHelper.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveAuthorizationHelper.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveAuthorizationHelper.java new file mode 100644 index 0000000..643b121 --- /dev/null +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveAuthorizationHelper.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.hive; + +import com.google.common.collect.ImmutableList; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.UserException; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; +import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizerFactory; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext.CLIENT_TYPE; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactory; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; +import org.apache.hadoop.hive.ql.session.SessionState; + +import java.util.Collections; +import java.util.List; + +/** + * Helper class for initializing and checking privileges according to authorization configuration set in Hive storage + * plugin config. + */ +public class HiveAuthorizationHelper { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveAuthorizationHelper.class); + + final boolean authzEnabled; + final HiveAuthorizer authorizerV2; + + public HiveAuthorizationHelper(final IMetaStoreClient mClient, final HiveConf hiveConf, final String user) { + authzEnabled = hiveConf.getBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED); + if (!authzEnabled) { + authorizerV2 = null; + return; + } + + try { + final HiveConf hiveConfCopy = new HiveConf(hiveConf); + hiveConfCopy.set("user.name", user); + + final HiveAuthenticationProvider authenticator = HiveUtils.getAuthenticator(hiveConfCopy, + HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER); + SessionState ss = new SessionState(hiveConfCopy, user); + SessionState.start(ss); + + authenticator.setSessionState(ss); + + HiveAuthorizerFactory authorizerFactory = + HiveUtils.getAuthorizerFactory(hiveConfCopy, HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER); + + HiveAuthzSessionContext.Builder authzContextBuilder = new HiveAuthzSessionContext.Builder(); + authzContextBuilder.setClientType(CLIENT_TYPE.HIVESERVER2); // Drill is emulating HS2 here + + authorizerV2 = authorizerFactory.createHiveAuthorizer( + new HiveMetastoreClientFactory() { + @Override + public IMetaStoreClient getHiveMetastoreClient() throws HiveAuthzPluginException { + return mClient; + } + }, + hiveConf, authenticator, authzContextBuilder.build()); + + authorizerV2.applyAuthorizationConfigPolicy(hiveConfCopy); + } catch (final HiveException e) { + throw new DrillRuntimeException("Failed to initialize Hive authorization components: " + e.getMessage(), e); + } + + logger.trace("Hive authorization enabled"); + } + + /** + * Check authorization for "SHOW DATABASES" command. A {@link HiveAccessControlException} is thrown + * for illegal access. + */ + public void authorizeShowDatabases() throws HiveAccessControlException { + if (!authzEnabled) { + return; + } + + authorize(HiveOperationType.SHOWDATABASES, Collections.EMPTY_LIST, Collections.EMPTY_LIST, "SHOW DATABASES"); + } + + /** + * Check authorization for "SHOW TABLES" command in given Hive db. A {@link HiveAccessControlException} is thrown + * for illegal access. + * @param dbName + */ + public void authorizeShowTables(final String dbName) throws HiveAccessControlException { + if (!authzEnabled) { + return; + } + + final HivePrivilegeObject toRead = new HivePrivilegeObject(HivePrivilegeObjectType.DATABASE, dbName, null); + + authorize(HiveOperationType.SHOWTABLES, ImmutableList.of(toRead), Collections.EMPTY_LIST, "SHOW TABLES"); + } + + /** + * Check authorization for "READ TABLE" for given db.table. A {@link HiveAccessControlException} is thrown + * for illegal access. + * @param dbName + * @param tableName + */ + public void authorizeReadTable(final String dbName, final String tableName) throws HiveAccessControlException { + if (!authzEnabled) { + return; + } + + HivePrivilegeObject toRead = new HivePrivilegeObject(HivePrivilegeObjectType.TABLE_OR_VIEW, dbName, tableName); + authorize(HiveOperationType.QUERY, ImmutableList.of(toRead), Collections.EMPTY_LIST, "READ TABLE"); + } + + /* Helper method to check privileges */ + private void authorize(final HiveOperationType hiveOpType, final List toRead, + final List toWrite, final String cmd) throws HiveAccessControlException { + try { + HiveAuthzContext.Builder authzContextBuilder = new HiveAuthzContext.Builder(); + authzContextBuilder.setUserIpAddress("Not available"); + authzContextBuilder.setCommandString(cmd); + + authorizerV2.checkPrivileges(hiveOpType, toRead, toWrite, authzContextBuilder.build()); + } catch (final HiveAccessControlException e) { + throw e; + } catch (final Exception e) { + throw new DrillRuntimeException("Failed to use the Hive authorization components: " + e.getMessage(), e); + } + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java index 8a2e498..9ada569 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.store.hive; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -40,6 +41,7 @@ import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.hive.HiveTable.HivePartition; +import org.apache.drill.exec.util.ImpersonationUtil; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.MetaStoreUtils; @@ -61,6 +63,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.io.ByteArrayDataOutput; import com.google.common.io.ByteStreams; +import org.apache.hadoop.security.UserGroupInformation; @JsonTypeName("hive-scan") public class HiveScan extends AbstractGroupScan { @@ -104,7 +107,7 @@ public class HiveScan extends AbstractGroupScan { this.storagePluginName = storagePluginName; this.storagePlugin = (HiveStoragePlugin) pluginRegistry.getPlugin(storagePluginName); this.columns = columns; - getSplits(); + getSplitsWithUGI(); endpoints = storagePlugin.getContext().getBits(); } @@ -113,7 +116,7 @@ public class HiveScan extends AbstractGroupScan { this.hiveReadEntry = hiveReadEntry; this.columns = columns; this.storagePlugin = storagePlugin; - getSplits(); + getSplitsWithUGI(); endpoints = storagePlugin.getContext().getBits(); this.storagePluginName = storagePlugin.getName(); } @@ -135,6 +138,22 @@ public class HiveScan extends AbstractGroupScan { return columns; } + private void getSplitsWithUGI() throws ExecutionSetupException { + final UserGroupInformation ugi = ImpersonationUtil.createProxyUgi(getUserName()); + try { + ugi.doAs(new PrivilegedExceptionAction() { + public Void run() throws Exception { + getSplits(); + return null; + } + }); + } catch (final InterruptedException | IOException e) { + final String errMsg = String.format("Failed to create input splits: %s", e.getMessage()); + logger.error(errMsg, e); + throw new DrillRuntimeException(errMsg, e); + } + } + private void getSplits() throws ExecutionSetupException { try { final List partitions = hiveReadEntry.getPartitions(); @@ -169,12 +188,10 @@ public class HiveScan extends AbstractGroupScan { final Path path = new Path(sd.getLocation()); final FileSystem fs = path.getFileSystem(job); - // Use new JobConf that has FS configuration - final JobConf jobWithFsConf = new JobConf(fs.getConf()); if (fs.exists(path)) { - FileInputFormat.addInputPath(jobWithFsConf, path); - format = jobWithFsConf.getInputFormat(); - for (final InputSplit split : format.getSplits(jobWithFsConf, 1)) { + FileInputFormat.addInputPath(job, path); + format = job.getInputFormat(); + for (final InputSplit split : format.getSplits(job, 1)) { inputSplits.add(split); partitionMap.put(split, partition); } http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java index d0ea143..b459ee4 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java @@ -45,8 +45,8 @@ public class DrillHiveTable extends DrillTable{ protected final Table hiveTable; - public DrillHiveTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) { - super(storageEngineName, plugin, readEntry); + public DrillHiveTable(String storageEngineName, HiveStoragePlugin plugin, String userName, HiveReadEntry readEntry) { + super(storageEngineName, plugin, userName, readEntry); this.hiveTable = new Table(readEntry.getTable()); } http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java index 1e02301..1a08433 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java @@ -25,8 +25,9 @@ import org.apache.drill.exec.store.hive.HiveStoragePlugin; public class DrillHiveViewTable extends DrillHiveTable implements DrillViewInfoProvider { - public DrillHiveViewTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) { - super(storageEngineName, plugin, readEntry); + public DrillHiveViewTable(String storageEngineName, HiveStoragePlugin plugin, String userName, + HiveReadEntry readEntry) { + super(storageEngineName, plugin, userName, readEntry); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java index 83f250b..c8f2490 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java @@ -18,187 +18,105 @@ package org.apache.drill.exec.store.hive.schema; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.planner.logical.DrillTable; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.SchemaConfig; import org.apache.drill.exec.store.SchemaFactory; +import org.apache.drill.exec.store.hive.DrillHiveMetaStoreClient; import org.apache.drill.exec.store.hive.HiveReadEntry; import org.apache.drill.exec.store.hive.HiveStoragePlugin; import org.apache.drill.exec.store.hive.HiveStoragePluginConfig; -import org.apache.drill.exec.store.hive.HiveTable; +import org.apache.drill.exec.util.ImpersonationUtil; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.thrift.TException; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; public class HiveSchemaFactory implements SchemaFactory { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveSchemaFactory.class); - private static final String DATABASES = "databases"; - - private final HiveMetaStoreClient mClient; - private LoadingCache> databases; - private LoadingCache> tableNameLoader; - private LoadingCache> tableLoaders; - private HiveStoragePlugin plugin; - private final String schemaName; + private final DrillHiveMetaStoreClient globalMetastoreClient; + private final HiveStoragePlugin plugin; private final Map hiveConfigOverride; + private final String schemaName; + private final HiveConf hiveConf; + private final boolean isDrillImpersonationEnabled; + private final boolean isHS2DoAsSet; public HiveSchemaFactory(HiveStoragePlugin plugin, String name, Map hiveConfigOverride) throws ExecutionSetupException { this.schemaName = name; this.plugin = plugin; this.hiveConfigOverride = hiveConfigOverride; - HiveConf hiveConf = new HiveConf(); + hiveConf = new HiveConf(); if (hiveConfigOverride != null) { for (Map.Entry entry : hiveConfigOverride.entrySet()) { - hiveConf.set(entry.getKey(), entry.getValue()); + final String property = entry.getKey(); + final String value = entry.getValue(); + hiveConf.set(property, value); + logger.trace("HiveConfig Override {}={}", property, value); } } - try { - this.mClient = new HiveMetaStoreClient(hiveConf); - } catch (MetaException e) { - throw new ExecutionSetupException("Failure setting up Hive metastore client.", e); - } - - databases = CacheBuilder // - .newBuilder() // - .expireAfterAccess(1, TimeUnit.MINUTES) // - .build(new DatabaseLoader()); - - tableNameLoader = CacheBuilder // - .newBuilder() // - .expireAfterAccess(1, TimeUnit.MINUTES) // - .build(new TableNameLoader()); - - tableLoaders = CacheBuilder // - .newBuilder() // - .expireAfterAccess(4, TimeUnit.HOURS) // - .maximumSize(20) // - .build(new TableLoaderLoader()); - } - - private class TableNameLoader extends CacheLoader> { + isHS2DoAsSet = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS); + isDrillImpersonationEnabled = plugin.getContext().getConfig().getBoolean(ExecConstants.IMPERSONATION_ENABLED); - @Override - public List load(String dbName) throws Exception { + if (!isDrillImpersonationEnabled) { try { - return mClient.getAllTables(dbName); - } catch (TException e) { - logger.warn("Failure while attempting to get hive tables", e); - mClient.reconnect(); - return mClient.getAllTables(dbName); + globalMetastoreClient = DrillHiveMetaStoreClient.createNonCloseableClientWithCaching(hiveConf, hiveConfigOverride); + } catch (MetaException e) { + throw new ExecutionSetupException("Failure setting up Hive metastore client.", e); } + } else { + globalMetastoreClient = null; } - } - private class DatabaseLoader extends CacheLoader> { - - @Override - public List load(String key) throws Exception { - if (!DATABASES.equals(key)) { - throw new UnsupportedOperationException(); - } - try { - return mClient.getAllDatabases(); - } catch (TException e) { - logger.warn("Failure while attempting to get hive tables", e); - mClient.reconnect(); - return mClient.getAllDatabases(); - } - } + /** + * Does Drill needs to impersonate as user connected to Drill when reading data from Hive warehouse location? + * @return True when both Drill impersonation and Hive impersonation are enabled. + */ + private boolean needToImpersonateReadingData() { + return isDrillImpersonationEnabled && isHS2DoAsSet; } - private class TableLoaderLoader extends CacheLoader> { - - @Override - public LoadingCache load(String key) throws Exception { - return CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.MINUTES).build(new TableLoader(key)); - } - - } - - private class TableLoader extends CacheLoader { - - private final String dbName; - - public TableLoader(String dbName) { - super(); - this.dbName = dbName; - } - - @Override - public HiveReadEntry load(String key) throws Exception { - Table t = null; - try { - t = mClient.getTable(dbName, key); - } catch (TException e) { - mClient.reconnect(); - t = mClient.getTable(dbName, key); - } - - if (t == null) { - throw new UnknownTableException(String.format("Unable to find table '%s'.", key)); - } - - List partitions = null; + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { + DrillHiveMetaStoreClient mClientForSchemaTree = globalMetastoreClient; + if (isDrillImpersonationEnabled) { try { - partitions = mClient.listPartitions(dbName, key, Short.MAX_VALUE); - } catch (TException e) { - mClient.reconnect(); - partitions = mClient.listPartitions(dbName, key, Short.MAX_VALUE); - } - - List hivePartitions = Lists.newArrayList(); - for (Partition part : partitions) { - hivePartitions.add(new HiveTable.HivePartition(part)); + mClientForSchemaTree = DrillHiveMetaStoreClient.createClientWithAuthz(hiveConf, hiveConfigOverride, + schemaConfig.getUserName(), schemaConfig.getIgnoreAuthErrors()); + } catch (final TException e) { + throw new IOException("Failure setting up Hive metastore client.", e); } - - if (hivePartitions.size() == 0) { - hivePartitions = null; - } - return new HiveReadEntry(new HiveTable(t), hivePartitions, hiveConfigOverride); - } - - } - - @Override - public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { - HiveSchema schema = new HiveSchema(schemaName); + HiveSchema schema = new HiveSchema(schemaConfig, mClientForSchemaTree, schemaName); SchemaPlus hPlus = parent.add(schemaName, schema); schema.setHolder(hPlus); } class HiveSchema extends AbstractSchema { + private final SchemaConfig schemaConfig; + private final DrillHiveMetaStoreClient mClient; private HiveDatabaseSchema defaultSchema; - public HiveSchema(String name) { + public HiveSchema(final SchemaConfig schemaConfig, final DrillHiveMetaStoreClient mClient, final String name) { super(ImmutableList.of(), name); + this.schemaConfig = schemaConfig; + this.mClient = mClient; getSubSchema("default"); } @@ -206,25 +124,24 @@ public class HiveSchemaFactory implements SchemaFactory { public AbstractSchema getSubSchema(String name) { List tables; try { - List dbs = databases.get(DATABASES); + List dbs = mClient.getDatabases(); if (!dbs.contains(name)) { - logger.debug(String.format("Database '%s' doesn't exists in Hive storage '%s'", name, schemaName)); + logger.debug("Database '{}' doesn't exists in Hive storage '{}'", name, schemaName); return null; } - tables = tableNameLoader.get(name); + tables = mClient.getTableNames(name); HiveDatabaseSchema schema = new HiveDatabaseSchema(tables, this, name); if (name.equals("default")) { this.defaultSchema = schema; } return schema; - } catch (ExecutionException e) { + } catch (final TException e) { logger.warn("Failure while attempting to access HiveDatabase '{}'.", name, e.getCause()); return null; } } - void setHolder(SchemaPlus plusOfThis) { for (String s : getSubSchemaNames()) { plusOfThis.add(s, getSubSchema(s)); @@ -239,9 +156,9 @@ public class HiveSchemaFactory implements SchemaFactory { @Override public Set getSubSchemaNames() { try { - List dbs = databases.get(DATABASES); + List dbs = mClient.getDatabases(); return Sets.newHashSet(dbs); - } catch (ExecutionException e) { + } catch (final TException e) { logger.warn("Failure while getting Hive database list.", e); } return super.getSubSchemaNames(); @@ -263,25 +180,19 @@ public class HiveSchemaFactory implements SchemaFactory { return defaultSchema.getTableNames(); } - List getTableNames(String dbName) { - try{ - return tableNameLoader.get(dbName); - } catch (ExecutionException e) { - logger.warn("Failure while loading table names for database '{}'.", dbName, e.getCause()); - return Collections.emptyList(); - } - } - DrillTable getDrillTable(String dbName, String t) { HiveReadEntry entry = getSelectionBaseOnName(dbName, t); if (entry == null) { return null; } + final String userToImpersonate = needToImpersonateReadingData() ? schemaConfig.getUserName() : + ImpersonationUtil.getProcessUserName(); + if (entry.getJdbcTableType() == TableType.VIEW) { - return new DrillHiveViewTable(schemaName, plugin, entry); + return new DrillHiveViewTable(schemaName, plugin, userToImpersonate, entry); } else { - return new DrillHiveTable(schemaName, plugin, entry); + return new DrillHiveTable(schemaName, plugin, userToImpersonate, entry); } } @@ -290,8 +201,8 @@ public class HiveSchemaFactory implements SchemaFactory { dbName = "default"; } try{ - return tableLoaders.get(dbName).get(t); - }catch(ExecutionException e) { + return mClient.getHiveReadEntry(dbName, t); + }catch(final TException e) { logger.warn("Exception occurred while trying to read table. {}.{}", dbName, t, e.getCause()); return null; } @@ -307,6 +218,12 @@ public class HiveSchemaFactory implements SchemaFactory { return HiveStoragePluginConfig.NAME; } + @Override + public void close() throws Exception { + if (mClient != null) { + mClient.close(); + } + } } } http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestUtilities.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestUtilities.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestUtilities.java new file mode 100644 index 0000000..49738aa --- /dev/null +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestUtilities.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.hive; + +import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; + +public class HiveTestUtilities { + + /** + * Execute the give query on given hiveDriver instance. If a {@link CommandNeedRetryException} + * exception is thrown, it tries upto 3 times before returning failure. + * @param hiveDriver + * @param query + */ + public static void executeQuery(Driver hiveDriver, String query) { + CommandProcessorResponse response = null; + boolean failed = false; + int retryCount = 3; + + try { + response = hiveDriver.run(query); + } catch(CommandNeedRetryException ex) { + if (--retryCount == 0) { + failed = true; + } + } + + if (failed || response.getResponseCode() != 0 ) { + throw new RuntimeException(String.format("Failed to execute command '%s', errorMsg = '%s'", + query, (response != null ? response.getErrorMessage() : ""))); + } + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java new file mode 100644 index 0000000..8004155 --- /dev/null +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.impersonation.hive; + +import org.apache.drill.TestBuilder; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.dotdrill.DotDrillType; +import org.apache.drill.exec.impersonation.BaseTestImpersonation; +import org.apache.drill.exec.store.hive.HiveStoragePluginConfig; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.shims.ShimLoader; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; + +public class BaseTestHiveImpersonation extends BaseTestImpersonation { + protected static final String hivePluginName = "hive"; + + protected static HiveConf hiveConf; + protected static String whDir; + + protected static String studentData; + protected static String voterData; + + protected static final String studentDef = "CREATE TABLE %s.%s" + + "(rownum int, name string, age int, gpa float, studentnum bigint) " + + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE"; + protected static final String voterDef = "CREATE TABLE %s.%s" + + "(voter_id int,name varchar(30), age tinyint, registration string, " + + "contributions double,voterzone smallint,create_time timestamp) " + + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE"; + + protected static void prepHiveConfAndData() throws Exception { + hiveConf = new HiveConf(); + + // Configure metastore persistence db location on local filesystem + final String dbUrl = String.format("jdbc:derby:;databaseName=%s;create=true", getTempDir("metastore_db")); + hiveConf.set(ConfVars.METASTORECONNECTURLKEY.varname, dbUrl); + + hiveConf.set(ConfVars.SCRATCHDIR.varname, getTempDir("scratch_dir")); + hiveConf.set(ConfVars.LOCALSCRATCHDIR.varname, getTempDir("local_scratch_dir")); + + // Set MiniDFS conf in HiveConf + hiveConf.set(FS_DEFAULT_NAME_KEY, dfsConf.get(FS_DEFAULT_NAME_KEY)); + + whDir = hiveConf.get(ConfVars.METASTOREWAREHOUSE.varname); + FileSystem.mkdirs(fs, new Path(whDir), new FsPermission((short) 0777)); + + studentData = getPhysicalFileFromResource("student.txt"); + voterData = getPhysicalFileFromResource("voter.txt"); + } + + protected static void startHiveMetaStore() throws Exception { + final int port = MetaStoreUtils.findFreePort(); + + hiveConf.set(METASTOREURIS.varname, "thrift://localhost:" + port); + + MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf); + } + + protected static HiveStoragePluginConfig createHiveStoragePlugin(final Map hiveConfig) throws Exception { + HiveStoragePluginConfig pluginConfig = new HiveStoragePluginConfig(hiveConfig); + pluginConfig.setEnabled(true); + return pluginConfig; + } + + protected static Path getWhPathForHiveObject(final String dbName, final String tableName) { + if (dbName == null) { + return new Path(whDir); + } + + if (tableName == null) { + return new Path(whDir, dbName + ".db"); + } + + return new Path(new Path(whDir, dbName + ".db"), tableName); + } + + protected static void addHiveStoragePlugin(final Map hiveConfig) throws Exception { + getDrillbitContext().getStorage().createOrUpdate(hivePluginName, createHiveStoragePlugin(hiveConfig), true); + } + + protected void showTablesHelper(final String db, List expectedTables) throws Exception { + final String dbQualified = hivePluginName + "." + db; + final TestBuilder testBuilder = testBuilder() + .sqlQuery("SHOW TABLES IN " + dbQualified) + .unOrdered() + .baselineColumns("TABLE_SCHEMA", "TABLE_NAME"); + + if (expectedTables.size() == 0) { + testBuilder.expectsEmptyResultSet(); + } else { + for (String tbl : expectedTables) { + testBuilder.baselineValues(dbQualified, tbl); + } + } + + testBuilder.go(); + } + + protected static void createView(final String viewOwner, final String viewGroup, final String viewName, + final String viewDef) throws Exception { + updateClient(viewOwner); + test(String.format("ALTER SESSION SET `%s`='%o';", ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY, (short) 0750)); + test("CREATE VIEW %s.%s.%s AS %s", MINIDFS_STORAGE_PLUGIN_NAME, "tmp", viewName, viewDef); + final Path viewFilePath = new Path("/tmp/", viewName + DotDrillType.VIEW.getEnding()); + fs.setOwner(viewFilePath, viewOwner, viewGroup); + } + + public static void stopHiveMetaStore() throws Exception { + // Unfortunately Hive metastore doesn't provide an API to shut it down. It will be exited as part of the test JVM + // exit. As each metastore server instance is using its own resources and not sharing it with other metastore + // server instances this should be ok. + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestSqlStdBasedAuthorization.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestSqlStdBasedAuthorization.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestSqlStdBasedAuthorization.java new file mode 100644 index 0000000..8dc292d --- /dev/null +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestSqlStdBasedAuthorization.java @@ -0,0 +1,296 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.impersonation.hive; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import org.apache.drill.exec.store.dfs.WorkspaceConfig; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +import org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator; +import org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory; +import org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Map; + +import static org.apache.drill.exec.hive.HiveTestUtilities.executeQuery; +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI; + +public class TestSqlStdBasedAuthorization extends BaseTestHiveImpersonation { + + private static final String db_general = "db_general"; + + // Tables in "db_general" + private static final String g_student_user0 = "student_user0"; + private static final String g_voter_role0 = "voter_role0"; + private static final String g_student_user2 = "student_user2"; + + + // Create a view on "g_student_user0". View is owned by user0:group0 and has permissions 750 + private static final String v_student_u0g0_750 = "v_student_u0g0_750"; + + // Create a view on "v_student_u0g0_750". View is owned by user1:group1 and has permissions 750 + private static final String v_student_u1g1_750 = "v_student_u1g1_750"; + + private static final String query_v_student_u0g0_750 = String.format( + "SELECT rownum FROM %s.%s.%s ORDER BY rownum LIMIT 1", MINIDFS_STORAGE_PLUGIN_NAME, "tmp", v_student_u0g0_750); + + private static final String query_v_student_u1g1_750 = String.format( + "SELECT rownum FROM %s.%s.%s ORDER BY rownum LIMIT 1", MINIDFS_STORAGE_PLUGIN_NAME, "tmp", v_student_u1g1_750); + + // Role for testing purpose + private static final String test_role0 = "role0"; + + @BeforeClass + public static void setup() throws Exception { + startMiniDfsCluster(TestSqlStdBasedAuthorization.class.getSimpleName()); + prepHiveConfAndData(); + setSqlStdBasedAuthorizationInHiveConf(); + startHiveMetaStore(); + startDrillCluster(true); + addHiveStoragePlugin(getHivePluginConfig()); + addMiniDfsBasedStorage(Maps.newHashMap()); + generateTestData(); + } + + private static void setSqlStdBasedAuthorizationInHiveConf() { + hiveConf.set(ConfVars.HIVE_AUTHORIZATION_ENABLED.varname, "true"); + hiveConf.set(HIVE_AUTHENTICATOR_MANAGER.varname, SessionStateConfigUserAuthenticator.class.getName()); + hiveConf.set(HIVE_AUTHORIZATION_MANAGER.varname, SQLStdConfOnlyAuthorizerFactory.class.getName()); + hiveConf.set(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false"); + hiveConf.set(ConfVars.METASTORE_EXECUTE_SET_UGI.varname, "false"); + hiveConf.set(ConfVars.USERS_IN_ADMIN_ROLE.varname, processUser); + } + + private static Map getHivePluginConfig() { + final Map hiveConfig = Maps.newHashMap(); + hiveConfig.put(METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)); + hiveConfig.put(FS_DEFAULT_NAME_KEY, dfsConf.get(FS_DEFAULT_NAME_KEY)); + hiveConfig.put(HIVE_SERVER2_ENABLE_DOAS.varname, hiveConf.get(HIVE_SERVER2_ENABLE_DOAS.varname)); + hiveConfig.put(METASTORE_EXECUTE_SET_UGI.varname, hiveConf.get(METASTORE_EXECUTE_SET_UGI.varname)); + hiveConfig.put(HIVE_AUTHORIZATION_ENABLED.varname, hiveConf.get(HIVE_AUTHORIZATION_ENABLED.varname)); + hiveConfig.put(HIVE_AUTHENTICATOR_MANAGER.varname, SessionStateUserAuthenticator.class.getName()); + hiveConfig.put(HIVE_AUTHORIZATION_MANAGER.varname, SQLStdHiveAuthorizerFactory.class.getName()); + return hiveConfig; + } + + private static void generateTestData() throws Exception { + final SessionState ss = new SessionState(hiveConf); + SessionState.start(ss); + final Driver driver = new Driver(hiveConf); + + executeQuery(driver, "CREATE DATABASE " + db_general); + createTbl(driver, db_general, g_student_user0, studentDef, studentData); + createTbl(driver, db_general, g_voter_role0, voterDef, voterData); + createTbl(driver, db_general, g_student_user2, studentDef, studentData); + + executeQuery(driver, "SET ROLE admin"); + executeQuery(driver, "CREATE ROLE " + test_role0); + executeQuery(driver, "GRANT ROLE " + test_role0 + " TO USER " + org1Users[1]); + executeQuery(driver, "GRANT ROLE " + test_role0 + " TO USER " + org1Users[2]); + + executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO USER %s", db_general, g_student_user0, org1Users[0])); + executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO ROLE %s", db_general, g_voter_role0, test_role0)); + executeQuery(driver, String.format("GRANT SELECT ON %s.%s TO USER %s", db_general, g_student_user2, org1Users[2])); + + createView(org1Users[0], org1Groups[0], v_student_u0g0_750, + String.format("SELECT rownum, name, age, studentnum FROM %s.%s.%s", + hivePluginName, db_general, g_student_user0)); + + createView(org1Users[1], org1Groups[1], v_student_u1g1_750, + String.format("SELECT rownum, name, age FROM %s.%s.%s", + MINIDFS_STORAGE_PLUGIN_NAME, "tmp", v_student_u0g0_750)); + } + + private static void createTbl(final Driver driver, final String db, final String tbl, final String tblDef, + final String data) throws Exception { + executeQuery(driver, String.format(tblDef, db, tbl)); + executeQuery(driver, String.format("LOAD DATA LOCAL INPATH '%s' INTO TABLE %s.%s", data, db, tbl)); + } + + // Irrespective of each db permissions, all dbs show up in "SHOW SCHEMAS" + @Test + public void showSchemas() throws Exception { + testBuilder() + .sqlQuery("SHOW SCHEMAS LIKE 'hive.%'") + .unOrdered() + .baselineColumns("SCHEMA_NAME") + .baselineValues("hive.db_general") + .baselineValues("hive.default") + .go(); + } + + @Test + public void showTables_user0() throws Exception { + updateClient(org1Users[0]); + showTablesHelper(db_general, + // Users are expected to see all tables in a database even if they don't have permissions to read from tables. + ImmutableList.of( + g_student_user0, + g_student_user2, + g_voter_role0 + )); + } + + @Test + public void showTables_user1() throws Exception { + updateClient(org1Users[1]); + showTablesHelper(db_general, + // Users are expected to see all tables in a database even if they don't have permissions to read from tables. + ImmutableList.of( + g_student_user0, + g_student_user2, + g_voter_role0 + )); + } + + @Test + public void select_user0_1() throws Exception { + // SELECT on "student_user0" table is granted to user "user0" + updateClient(org1Users[0]); + test("USE " + hivePluginName + "." + db_general); + test(String.format("SELECT * FROM %s ORDER BY name LIMIT 2", g_student_user0)); + } + + @Test + public void select_user0_2() throws Exception { + // SELECT on table "student_user0" is NOT granted to user "user0" directly or indirectly through role "role0" as + // user "user0" is not part of role "role0" + updateClient(org1Users[0]); + test("USE " + hivePluginName + "." + db_general); + final String query = String.format("SELECT * FROM %s ORDER BY name LIMIT 2", g_voter_role0); + errorMsgTestHelper(query, "Principal [name=user0_1, type=USER] does not have following privileges for " + + "operation QUERY [[SELECT] on Object [type=TABLE_OR_VIEW, name=db_general.voter_role0]]\n"); + } + + @Test + public void select_user1_1() throws Exception { + // SELECT on table "student_user0" is NOT granted to user "user1" + updateClient(org1Users[1]); + test("USE " + hivePluginName + "." + db_general); + final String query = String.format("SELECT * FROM %s ORDER BY name LIMIT 2", g_student_user0); + errorMsgTestHelper(query, "Principal [name=user1_1, type=USER] does not have following privileges for " + + "operation QUERY [[SELECT] on Object [type=TABLE_OR_VIEW, name=db_general.student_user0]]\n"); + } + + @Test + public void select_user1_2() throws Exception { + // SELECT on "voter_role0" table is granted to role "role0" and user "user1" is part the role "role0" + updateClient(org1Users[1]); + test("USE " + hivePluginName + "." + db_general); + test(String.format("SELECT * FROM %s ORDER BY name LIMIT 2", g_voter_role0)); + } + + @Test + public void select_user1_3() throws Exception { + // SELECT on "voter_role0" table is granted to role "role0" and user "user1" is part the role "role0" + // SELECT on "student_user2" table is NOT granted to either role "role0" or user "user1" + updateClient(org1Users[1]); + test("USE " + hivePluginName + "." + db_general); + final String query = + String.format("SELECT * FROM %s v JOIN %s s on v.name = s.name limit 2;", g_voter_role0, g_student_user2); + errorMsgTestHelper(query, "Principal [name=user1_1, type=USER] does not have following privileges for " + + "operation QUERY [[SELECT] on Object [type=TABLE_OR_VIEW, name=db_general.student_user2]]"); + } + + @Test + public void select_user2_1() throws Exception { + // SELECT on "voter_role0" table is granted to role "role0" and user "user2" is part the role "role0" + updateClient(org1Users[2]); + test("USE " + hivePluginName + "." + db_general); + test(String.format("SELECT * FROM %s ORDER BY name LIMIT 2", g_voter_role0)); + } + + @Test + public void select_user2_2() throws Exception { + // SELECT on "student_user2" table is granted to user "user2" + updateClient(org1Users[2]); + test("USE " + hivePluginName + "." + db_general); + test(String.format("SELECT * FROM %s ORDER BY name LIMIT 2", g_student_user2)); + } + + @Test + public void select_user2_3() throws Exception { + // SELECT on "voter_role0" table is granted to role "role0" and user "user2" is part the role "role0" + // SELECT on "student_user2" table is granted to user "user2" + updateClient(org1Users[2]); + test("USE " + hivePluginName + "." + db_general); + test(String.format("SELECT * FROM %s v JOIN %s s on v.name = s.name limit 2;", g_voter_role0, g_student_user2)); + } + + private static void queryViewHelper(final String queryUser, final String query) throws Exception { + updateClient(queryUser); + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("rownum") + .baselineValues(1) + .go(); + } + + @Test + public void selectUser0_v_student_u0g0_750() throws Exception { + queryViewHelper(org1Users[0], query_v_student_u0g0_750); + } + + @Test + public void selectUser1_v_student_u0g0_750() throws Exception { + queryViewHelper(org1Users[1], query_v_student_u0g0_750); + } + + @Test + public void selectUser2_v_student_u0g0_750() throws Exception { + updateClient(org1Users[2]); + errorMsgTestHelper(query_v_student_u0g0_750, + "Not authorized to read view [v_student_u0g0_750] in schema [miniDfsPlugin.tmp]"); + } + + @Test + public void selectUser0_v_student_u1g1_750() throws Exception { + updateClient(org1Users[0]); + errorMsgTestHelper(query_v_student_u1g1_750, + "Not authorized to read view [v_student_u1g1_750] in schema [miniDfsPlugin.tmp]"); + } + + @Test + public void selectUser1_v_student_u1g1_750() throws Exception { + queryViewHelper(org1Users[1], query_v_student_u1g1_750); + } + + @Test + public void selectUser2_v_student_u1g1_750() throws Exception { + queryViewHelper(org1Users[2], query_v_student_u1g1_750); + } + + @AfterClass + public static void shutdown() throws Exception { + stopMiniDfsCluster(); + stopHiveMetaStore(); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java new file mode 100644 index 0000000..69e4f8d --- /dev/null +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java @@ -0,0 +1,372 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.impersonation.hive; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.dotdrill.DotDrillType; +import org.apache.drill.exec.store.dfs.WorkspaceConfig; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator; +import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener; +import org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; + +import static org.apache.drill.exec.hive.HiveTestUtilities.executeQuery; +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_AUTH_READS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS; + +public class TestStorageBasedHiveAuthorization extends BaseTestHiveImpersonation { + + // DB whose warehouse directory has permissions 755, available everyone to read + private static final String db_general = "db_general"; + + // Tables in "db_general" + private static final String g_student_u0_700 = "student_u0_700"; + private static final String g_student_u0g0_750 = "student_u0g0_750"; + private static final String g_student_all_755 = "student_all_755"; + private static final String g_voter_u1_700 = "voter_u1_700"; + private static final String g_voter_u2g1_750 = "voter_u2g1_750"; + private static final String g_voter_all_755 = "voter_all_755"; + + // DB whose warehouse directory has permissions 700 and owned by user0 + private static final String db_u0_only = "db_u0_only"; + + // Tables in "db_u0_only" + private static final String u0_student_all_755 = "student_all_755"; + private static final String u0_voter_all_755 = "voter_all_755"; + + // DB whose warehouse directory has permissions 750 and owned by user1 and group1 + private static final String db_u1g1_only = "db_u1g1_only"; + + // Tables in "db_u1g1_only" + private static final String u1g1_student_all_755 = "student_all_755"; + private static final String u1g1_student_u1_700 = "student_u1_700"; + private static final String u1g1_voter_all_755 = "voter_all_755"; + private static final String u1g1_voter_u1_700 = "voter_u1_700"; + + // Create a view on "student_u0_700". View is owned by user0:group0 and has permissions 750 + private static final String v_student_u0g0_750 = "v_student_u0g0_750"; + + // Create a view on "v_student_u0g0_750". View is owned by user1:group1 and has permissions 750 + private static final String v_student_u1g1_750 = "v_student_u1g1_750"; + + private static final String query_v_student_u0g0_750 = String.format( + "SELECT rownum FROM %s.%s.%s ORDER BY rownum LIMIT 1", MINIDFS_STORAGE_PLUGIN_NAME, "tmp", v_student_u0g0_750); + + private static final String query_v_student_u1g1_750 = String.format( + "SELECT rownum FROM %s.%s.%s ORDER BY rownum LIMIT 1", MINIDFS_STORAGE_PLUGIN_NAME, "tmp", v_student_u1g1_750); + + @BeforeClass + public static void setup() throws Exception { + startMiniDfsCluster(TestStorageBasedHiveAuthorization.class.getName()); + prepHiveConfAndData(); + setStorabaseBasedAuthorizationInHiveConf(); + startHiveMetaStore(); + startDrillCluster(true); + addHiveStoragePlugin(getHivePluginConfig()); + addMiniDfsBasedStorage(Maps.newHashMap()); + generateTestData(); + } + + private static void setStorabaseBasedAuthorizationInHiveConf() { + // Turn on metastore-side authorization + hiveConf.set(METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); + hiveConf.set(HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); + hiveConf.set(HIVE_METASTORE_AUTHORIZATION_MANAGER.varname, StorageBasedAuthorizationProvider.class.getName()); + hiveConf.set(HIVE_METASTORE_AUTHORIZATION_AUTH_READS.varname, "true"); + hiveConf.set(METASTORE_EXECUTE_SET_UGI.varname, "true"); + } + + private static Map getHivePluginConfig() { + final Map hiveConfig = Maps.newHashMap(); + hiveConfig.put(METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)); + hiveConfig.put(FS_DEFAULT_NAME_KEY, dfsConf.get(FS_DEFAULT_NAME_KEY)); + hiveConfig.put(HIVE_SERVER2_ENABLE_DOAS.varname, hiveConf.get(HIVE_SERVER2_ENABLE_DOAS.varname)); + hiveConfig.put(METASTORE_EXECUTE_SET_UGI.varname, hiveConf.get(METASTORE_EXECUTE_SET_UGI.varname)); + return hiveConfig; + } + + private static void generateTestData() throws Exception { + + // Generate Hive test tables + final SessionState ss = new SessionState(hiveConf); + SessionState.start(ss); + final Driver driver = new Driver(hiveConf); + + executeQuery(driver, "CREATE DATABASE " + db_general); + + createTable(driver, + db_general, g_student_u0_700, studentDef, studentData, org1Users[0], org1Groups[0], (short) 0700); + createTable(driver, + db_general, g_student_u0g0_750, studentDef, studentData, org1Users[0], org1Groups[0], (short) 0750); + createTable(driver, + db_general, g_student_all_755, studentDef, studentData, org1Users[2], org1Groups[2], (short) 0755); + createTable(driver, + db_general, g_voter_u1_700, voterDef, voterData, org1Users[1], org1Groups[1], (short) 0700); + createTable(driver, + db_general, g_voter_u2g1_750, voterDef, voterData, org1Users[2], org1Groups[1], (short) 0750); + createTable(driver, + db_general, g_voter_all_755, voterDef, voterData, org1Users[1], org1Groups[1], (short) 0755); + + changeDBPermissions(db_general, (short) 0755, org1Users[0], org1Groups[0]); + + executeQuery(driver, "CREATE DATABASE " + db_u1g1_only); + + createTable(driver, + db_u1g1_only, u1g1_student_all_755, studentDef, studentData, org1Users[1], org1Groups[1], (short) 0755); + createTable(driver, + db_u1g1_only, u1g1_student_u1_700, studentDef, studentData, org1Users[1], org1Groups[1], (short) 0700); + createTable(driver, + db_u1g1_only, u1g1_voter_all_755, voterDef, voterData, org1Users[1], org1Groups[1], (short) 0755); + createTable(driver, + db_u1g1_only, u1g1_voter_u1_700, voterDef, voterData, org1Users[1], org1Groups[1], (short) 0700); + + changeDBPermissions(db_u1g1_only, (short) 0750, org1Users[1], org1Groups[1]); + + executeQuery(driver, "CREATE DATABASE " + db_u0_only); + + createTable(driver, db_u0_only, u0_student_all_755, studentDef, studentData, org1Users[0], org1Groups[0], (short) 0755); + createTable(driver, db_u0_only, u0_voter_all_755, voterDef, voterData, org1Users[0], org1Groups[0], (short) 0755); + + changeDBPermissions(db_u0_only, (short) 0700, org1Users[0], org1Groups[0]); + + createView(org1Users[0], org1Groups[0], v_student_u0g0_750, + String.format("SELECT rownum, name, age, studentnum FROM %s.%s.%s", + hivePluginName, db_general, g_student_u0_700)); + + createView(org1Users[1], org1Groups[1], v_student_u1g1_750, + String.format("SELECT rownum, name, age FROM %s.%s.%s", + MINIDFS_STORAGE_PLUGIN_NAME, "tmp", v_student_u0g0_750)); + } + + private static void createTable(final Driver hiveDriver, final String db, final String tbl, final String tblDef, + final String tblData, final String user, final String group, final short permissions) throws Exception { + executeQuery(hiveDriver, String.format(tblDef, db, tbl)); + executeQuery(hiveDriver, String.format("LOAD DATA LOCAL INPATH '%s' INTO TABLE %s.%s", tblData, db, tbl)); + final Path p = getWhPathForHiveObject(db, tbl); + fs.setPermission(p, new FsPermission(permissions)); + fs.setOwner(p, user, group); + } + + private static void changeDBPermissions(final String db, final short perm, final String u, final String g) + throws Exception { + Path p = getWhPathForHiveObject(db, null); + fs.setPermission(p, new FsPermission(perm)); + fs.setOwner(p, u, g); + } + + // Irrespective of each db permissions, all dbs show up in "SHOW SCHEMAS" + @Test + public void showSchemas() throws Exception { + testBuilder() + .sqlQuery("SHOW SCHEMAS LIKE 'hive.%'") + .unOrdered() + .baselineColumns("SCHEMA_NAME") + .baselineValues("hive.db_general") + .baselineValues("hive.db_u0_only") + .baselineValues("hive.db_u1g1_only") + .baselineValues("hive.default") + .go(); + } + + /** + * "SHOW TABLE" output for a db, should only contain the tables that the user + * has access to read. If the user has no read access to the db, the list will be always empty even if the user has + * read access to the tables inside the db. + * @throws Exception + */ + @Test + public void showTablesUser0() throws Exception { + updateClient(org1Users[0]); + + showTablesHelper(db_general, + ImmutableList.of( + g_student_u0_700, + g_student_u0g0_750, + g_student_all_755, + g_voter_all_755 + )); + + showTablesHelper(db_u0_only, + ImmutableList.of( + u0_student_all_755, + u0_voter_all_755 + )); + + showTablesHelper(db_u1g1_only, Collections.EMPTY_LIST); + } + + @Test + public void showTablesUser1() throws Exception { + updateClient(org1Users[1]); + + showTablesHelper(db_general, + ImmutableList.of( + g_student_u0g0_750, + g_student_all_755, + g_voter_u1_700, + g_voter_u2g1_750, + g_voter_all_755 + )); + + showTablesHelper(db_u1g1_only, + ImmutableList.of( + u1g1_student_all_755, + u1g1_student_u1_700, + u1g1_voter_all_755, + u1g1_voter_u1_700 + )); + + showTablesHelper(db_u0_only, Collections.EMPTY_LIST); + } + + @Test + public void showTablesUser2() throws Exception { + updateClient(org1Users[2]); + + showTablesHelper(db_general, + ImmutableList.of( + g_student_all_755, + g_voter_u2g1_750, + g_voter_all_755 + )); + + showTablesHelper(db_u1g1_only, + ImmutableList.of( + u1g1_student_all_755, + u1g1_voter_all_755 + )); + + showTablesHelper(db_u0_only, Collections.EMPTY_LIST); + } + + // Try to read the tables "user0" has access to read in db_general. + @Test + public void selectUser0_db_general() throws Exception { + updateClient(org1Users[0]); + + test(String.format("SELECT * FROM hive.%s.%s ORDER BY gpa DESC LIMIT 2", db_general, g_student_u0_700)); + test(String.format("SELECT * FROM hive.%s.%s ORDER BY gpa DESC LIMIT 2", db_general, g_student_all_755)); + test(String.format("SELECT * FROM hive.%s.%s ORDER BY name DESC LIMIT 2", db_general, g_voter_all_755)); + } + + // Try to read the table that "user0" has access to read in db_u0_only + @Test + public void selectUser0_db_u0_only() throws Exception { + updateClient(org1Users[0]); + + test(String.format("SELECT * FROM hive.%s.%s ORDER BY gpa DESC LIMIT 2", db_u0_only, u0_student_all_755)); + test(String.format("SELECT * FROM hive.%s.%s ORDER BY name DESC LIMIT 2", db_u0_only, u0_voter_all_755)); + } + + // Try to read the tables "user0" has no access to read in db_u1g1_only + @Test + public void selectUser0_db_u1g1_only() throws Exception { + updateClient(org1Users[0]); + + errorMsgTestHelper( + String.format("SELECT * FROM hive.%s.%s ORDER BY gpa DESC LIMIT 2", db_u1g1_only, u1g1_student_all_755), + String.format("Table 'hive.%s.%s' not found", db_u1g1_only, u1g1_student_all_755)); + } + + // Try to read the tables "user1" has access to read in db_general. + @Test + public void selectUser1_db_general() throws Exception { + updateClient(org1Users[1]); + + test(String.format("SELECT * FROM hive.%s.%s ORDER BY gpa DESC LIMIT 2", db_general, g_student_u0g0_750)); + test(String.format("SELECT * FROM hive.%s.%s ORDER BY gpa DESC LIMIT 2", db_general, g_student_all_755)); + test(String.format("SELECT * FROM hive.%s.%s ORDER BY name DESC LIMIT 2", db_general, g_voter_u2g1_750)); + } + + // Try to read the tables "user1" has no access to read in db_u0_only + @Test + public void selectUser1_db_u0_only() throws Exception { + updateClient(org1Users[1]); + + errorMsgTestHelper( + String.format("SELECT * FROM hive.%s.%s ORDER BY gpa DESC LIMIT 2", db_u0_only, u0_student_all_755), + String.format("Table 'hive.%s.%s' not found", db_u0_only, u0_student_all_755)); + } + + private static void queryViewHelper(final String queryUser, final String query) throws Exception { + updateClient(queryUser); + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("rownum") + .baselineValues(1) + .go(); + } + + @Test + public void selectUser0_v_student_u0g0_750() throws Exception { + queryViewHelper(org1Users[0], query_v_student_u0g0_750); + } + + @Test + public void selectUser1_v_student_u0g0_750() throws Exception { + queryViewHelper(org1Users[1], query_v_student_u0g0_750); + } + + @Test + public void selectUser2_v_student_u0g0_750() throws Exception { + updateClient(org1Users[2]); + errorMsgTestHelper(query_v_student_u0g0_750, + "Not authorized to read view [v_student_u0g0_750] in schema [miniDfsPlugin.tmp]"); + } + + @Test + public void selectUser0_v_student_u1g1_750() throws Exception { + updateClient(org1Users[0]); + errorMsgTestHelper(query_v_student_u1g1_750, + "Not authorized to read view [v_student_u1g1_750] in schema [miniDfsPlugin.tmp]"); + } + + @Test + public void selectUser1_v_student_u1g1_750() throws Exception { + queryViewHelper(org1Users[1], query_v_student_u1g1_750); + } + + @Test + public void selectUser2_v_student_u1g1_750() throws Exception { + queryViewHelper(org1Users[2], query_v_student_u1g1_750); + } + + @AfterClass + public static void shutdown() throws Exception { + stopMiniDfsCluster(); + stopHiveMetaStore(); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/c1b847ac/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index ea8d90f..965a863 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -24,7 +24,6 @@ import java.sql.Date; import java.sql.Timestamp; import java.util.Map; -import com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.drill.BaseTestQuery; import org.apache.drill.common.exceptions.DrillException; @@ -32,16 +31,16 @@ import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import com.google.common.collect.Maps; +import static org.apache.drill.BaseTestQuery.getTempDir; +import static org.apache.drill.exec.hive.HiveTestUtilities.executeQuery; + public class HiveTestDataGenerator { private static final String HIVE_TEST_PLUGIN_NAME = "hive"; - private static final int RETRIES = 5; private static HiveTestDataGenerator instance; private final String dbDir; @@ -50,13 +49,8 @@ public class HiveTestDataGenerator { public static synchronized HiveTestDataGenerator getInstance() throws Exception { if (instance == null) { - final File db = Files.createTempDir(); - db.deleteOnExit(); - final String dbDir = db.getAbsolutePath() + File.separator + "metastore_db"; - - final File wh = Files.createTempDir(); - wh.deleteOnExit(); - final String whDir = wh.getAbsolutePath(); + final String dbDir = getTempDir("metastore_db"); + final String whDir = getTempDir("warehouse"); instance = new HiveTestDataGenerator(dbDir, whDir); instance.generateTestData(); @@ -121,7 +115,8 @@ public class HiveTestDataGenerator { conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); conf.set("hive.metastore.warehouse.dir", whDir); conf.set("mapred.job.tracker", "local"); - conf.set("hive.exec.scratchdir", Files.createTempDir().getAbsolutePath() + File.separator + "scratch_dir"); + conf.set(ConfVars.SCRATCHDIR.varname, getTempDir("scratch_dir")); + conf.set(ConfVars.LOCALSCRATCHDIR.varname, getTempDir("local_scratch_dir")); SessionState ss = new SessionState(conf); SessionState.start(ss); @@ -151,7 +146,8 @@ public class HiveTestDataGenerator { "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' " + "STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' " + "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' " + - "TBLPROPERTIES ('avro.schema.url'='file:///%s')", getSchemaFile("avro_test_schema.json")); + "TBLPROPERTIES ('avro.schema.url'='file:///%s')", + BaseTestQuery.getPhysicalFileFromResource("avro_test_schema.json")); executeQuery(hiveDriver, avroCreateQuery); executeQuery(hiveDriver, "INSERT INTO TABLE db1.avro SELECT * FROM default.kv"); @@ -320,15 +316,6 @@ public class HiveTestDataGenerator { return file.getPath(); } - private String getSchemaFile(final String resource) throws Exception { - final File file = getTempFile(); - PrintWriter printWriter = new PrintWriter(file); - printWriter.write(BaseTestQuery.getFile(resource)); - printWriter.close(); - - return file.getPath(); - } - private String generateTestDataFileWithDate() throws Exception { final File file = getTempFile(); @@ -355,24 +342,4 @@ public class HiveTestDataGenerator { return file.getPath(); } - - private void executeQuery(Driver hiveDriver, String query) { - CommandProcessorResponse response = null; - boolean failed = false; - int retryCount = RETRIES; - - try { - response = hiveDriver.run(query); - } catch(CommandNeedRetryException ex) { - if (--retryCount == 0) { - failed = true; - } - } - - if (failed || response.getResponseCode() != 0 ) { - throw new RuntimeException(String.format("Failed to execute command '%s', errorMsg = '%s'", - query, (response != null ? response.getErrorMessage() : ""))); - } - } - }