Return-Path: X-Original-To: apmail-falcon-commits-archive@minotaur.apache.org Delivered-To: apmail-falcon-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6B0BD109C9 for ; Fri, 14 Nov 2014 02:57:39 +0000 (UTC) Received: (qmail 79282 invoked by uid 500); 14 Nov 2014 02:57:39 -0000 Delivered-To: apmail-falcon-commits-archive@falcon.apache.org Received: (qmail 79231 invoked by uid 500); 14 Nov 2014 02:57:39 -0000 Mailing-List: contact commits-help@falcon.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.incubator.apache.org Delivered-To: mailing list commits@falcon.incubator.apache.org Received: (qmail 79222 invoked by uid 99); 14 Nov 2014 02:57:39 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 14 Nov 2014 02:57:39 +0000 X-ASF-Spam-Status: No, hits=-2000.6 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Fri, 14 Nov 2014 02:57:37 +0000 Received: (qmail 75010 invoked by uid 99); 14 Nov 2014 02:54:47 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 14 Nov 2014 02:54:47 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 20E4D93A9FA; Fri, 14 Nov 2014 02:54:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: venkatesh@apache.org To: commits@falcon.incubator.apache.org Date: Fri, 14 Nov 2014 02:54:53 -0000 Message-Id: <615e5c5a7c364d5cb9e30b037904e3ef@git.apache.org> In-Reply-To: <12e02d50c86c428aad37eec156c9f27f@git.apache.org> References: <12e02d50c86c428aad37eec156c9f27f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [8/8] incubator-falcon git commit: FALCON-894 Cluster submission with hive registry fails in secure setup. Contributed by Venkatesh Seetharam X-Virus-Checked: Checked by ClamAV on apache.org FALCON-894 Cluster submission with hive registry fails in secure setup. Contributed by Venkatesh Seetharam Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/c4dd440d Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/c4dd440d Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/c4dd440d Branch: refs/heads/master Commit: c4dd440d9ebe5052226028ebbab66bfe2c981ae0 Parents: e8b1d11 Author: Venkatesh Seetharam Authored: Thu Nov 13 18:30:54 2014 -0800 Committer: Venkatesh Seetharam Committed: Thu Nov 13 18:55:24 2014 -0800 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../falcon/catalog/AbstractCatalogService.java | 11 +++---- .../falcon/catalog/HiveCatalogService.java | 34 +++++++++----------- .../entity/parser/ClusterEntityParser.java | 6 ++-- .../falcon/entity/parser/FeedEntityParser.java | 10 +++--- .../falcon/catalog/HiveCatalogServiceIT.java | 8 ++--- 6 files changed, 35 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index fda0338..7804f79 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -144,6 +144,9 @@ Trunk (Unreleased) OPTIMIZATIONS BUG FIXES + FALCON-894 Cluster submission with hive registry fails in secure setup + (Venkatesh Seetharam) + FALCON-892 HCatReplication fails in secure setup (Venkatesh Seetharam) FALCON-889 Windows azure replication fails with "wasb" as the scheme to an http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java index e64a5be..348fac0 100644 --- a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java +++ b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java @@ -33,26 +33,25 @@ public abstract class AbstractCatalogService { /** * This method checks if the catalog service is alive. * + * @param conf conf * @param catalogUrl url for the catalog service - * @param metaStorePrincipal kerberos principal for hive metastore as this is executed in falcon on behalf of user * @return if the service was reachable * @throws FalconException exception */ - public abstract boolean isAlive(String catalogUrl, - String metaStorePrincipal) throws FalconException; + public abstract boolean isAlive(Configuration conf, String catalogUrl) throws FalconException; /** * This method checks if the given table exists in the catalog. * + * @param conf conf * @param catalogUrl url for the catalog service * @param database database the table belongs to * @param tableName tableName to check if it exists - * @param metaStorePrincipal kerberos principal for hive metastore as this is executed in falcon on behalf of user * @return if the table exists * @throws FalconException exception */ - public abstract boolean tableExists(String catalogUrl, String database, String tableName, - String metaStorePrincipal) throws FalconException; + public abstract boolean tableExists(Configuration conf, String catalogUrl, + String database, String tableName) throws FalconException; /** * Returns if the table is external or not. Executed in the workflow engine. http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java index 51fb6b7..3216f1e 100644 --- a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java +++ b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java @@ -20,6 +20,7 @@ package org.apache.falcon.catalog; import org.apache.falcon.FalconException; import org.apache.falcon.security.CurrentUser; +import org.apache.falcon.security.SecurityUtil; import org.apache.falcon.workflow.util.OozieActionConfigurationHelper; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -63,7 +64,7 @@ public class HiveCatalogService extends AbstractCatalogService { */ public static HCatClient getHCatClient(String metastoreUrl) throws FalconException { try { - HiveConf hcatConf = createHiveConf(metastoreUrl); + HiveConf hcatConf = createHiveConf(new Configuration(false), metastoreUrl); return HCatClient.create(hcatConf); } catch (HCatException e) { throw new FalconException("Exception creating HCatClient: " + e.getMessage(), e); @@ -72,10 +73,6 @@ public class HiveCatalogService extends AbstractCatalogService { } } - private static HiveConf createHiveConf(String metastoreUrl) throws IOException { - return createHiveConf(new Configuration(false), metastoreUrl); - } - private static HiveConf createHiveConf(Configuration conf, String metastoreUrl) throws IOException { HiveConf hcatConf = new HiveConf(conf, HiveConf.class); @@ -156,19 +153,18 @@ public class HiveCatalogService extends AbstractCatalogService { /** * This is used from with in falcon namespace. * - * @param catalogUrl metastore uri - * @param metaStoreServicePrincipal metastore principal + * @param conf conf + * @param catalogUrl metastore uri * @return hive metastore client handle * @throws FalconException */ - private static synchronized HCatClient createProxiedHCatClient(String catalogUrl, - String metaStoreServicePrincipal) - throws FalconException { + private static HCatClient createProxiedHCatClient(Configuration conf, + String catalogUrl) throws FalconException { try { - final HiveConf hcatConf = createHiveConf(catalogUrl); + final HiveConf hcatConf = createHiveConf(conf, catalogUrl); UserGroupInformation proxyUGI = CurrentUser.getProxyUGI(); - addSecureCredentialsAndToken(metaStoreServicePrincipal, hcatConf, proxyUGI); + addSecureCredentialsAndToken(conf, hcatConf, proxyUGI); LOG.info("Creating HCatalog client object for {}", catalogUrl); return proxyUGI.doAs(new PrivilegedExceptionAction() { @@ -183,10 +179,11 @@ public class HiveCatalogService extends AbstractCatalogService { } } - private static void addSecureCredentialsAndToken(String metaStoreServicePrincipal, + private static void addSecureCredentialsAndToken(Configuration conf, HiveConf hcatConf, UserGroupInformation proxyUGI) throws IOException { if (UserGroupInformation.isSecurityEnabled()) { + String metaStoreServicePrincipal = conf.get(SecurityUtil.HIVE_METASTORE_PRINCIPAL); hcatConf.set(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, metaStoreServicePrincipal); hcatConf.set(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, "true"); @@ -215,12 +212,11 @@ public class HiveCatalogService extends AbstractCatalogService { } @Override - public boolean isAlive(final String catalogUrl, - final String metaStorePrincipal) throws FalconException { + public boolean isAlive(Configuration conf, final String catalogUrl) throws FalconException { LOG.info("Checking if the service is alive for: {}", catalogUrl); try { - HCatClient client = createProxiedHCatClient(catalogUrl, metaStorePrincipal); + HCatClient client = createProxiedHCatClient(conf, catalogUrl); HCatDatabase database = client.getDatabase("default"); return database != null; } catch (HCatException e) { @@ -229,12 +225,12 @@ public class HiveCatalogService extends AbstractCatalogService { } @Override - public boolean tableExists(final String catalogUrl, final String database, final String tableName, - final String metaStorePrincipal) throws FalconException { + public boolean tableExists(Configuration conf, final String catalogUrl, final String database, + final String tableName) throws FalconException { LOG.info("Checking if the table exists: {}", tableName); try { - HCatClient client = createProxiedHCatClient(catalogUrl, metaStorePrincipal); + HCatClient client = createProxiedHCatClient(conf, catalogUrl); HCatTable table = client.getTable(database, tableName); return table != null; } catch (HCatException e) { http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java index b3496c4..cd51804 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java @@ -189,15 +189,15 @@ public class ClusterEntityParser extends EntityParser { LOG.info("Validating catalog registry interface: {}", catalogUrl); try { - String metaStorePrincipal = null; + Configuration clusterConf = ClusterHelper.getConfiguration(cluster); if (UserGroupInformation.isSecurityEnabled()) { - metaStorePrincipal = ClusterHelper.getPropertyValue(cluster, SecurityUtil.HIVE_METASTORE_PRINCIPAL); + String metaStorePrincipal = clusterConf.get(SecurityUtil.HIVE_METASTORE_PRINCIPAL); Validate.notEmpty(metaStorePrincipal, "Cluster definition missing required metastore credential property: " + SecurityUtil.HIVE_METASTORE_PRINCIPAL); } - if (!CatalogServiceFactory.getCatalogService().isAlive(catalogUrl, metaStorePrincipal)) { + if (!CatalogServiceFactory.getCatalogService().isAlive(clusterConf, catalogUrl)) { throw new ValidationException("Unable to reach Catalog server:" + catalogUrl); } } catch (FalconException e) { http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java index 63f9202..448fd70 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java @@ -40,7 +40,7 @@ import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.expression.ExpressionHelper; import org.apache.falcon.group.FeedGroup; import org.apache.falcon.group.FeedGroupMap; -import org.apache.falcon.security.SecurityUtil; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authorize.AuthorizationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -418,10 +418,10 @@ public class FeedEntityParser extends EntityParser { } CatalogStorage catalogStorage = (CatalogStorage) storage; - String metaStorePrincipal = ClusterHelper.getPropertyValue(clusterEntity, - SecurityUtil.HIVE_METASTORE_PRINCIPAL); - if (!CatalogServiceFactory.getCatalogService().tableExists(catalogStorage.getCatalogUrl(), - catalogStorage.getDatabase(), catalogStorage.getTable(), metaStorePrincipal)) { + Configuration clusterConf = ClusterHelper.getConfiguration(clusterEntity); + if (!CatalogServiceFactory.getCatalogService().tableExists( + clusterConf, catalogStorage.getCatalogUrl(), + catalogStorage.getDatabase(), catalogStorage.getTable())) { buffer.append("Table [") .append(catalogStorage.getTable()) .append("] does not exist for feed: ") http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java index b422119..87101c5 100644 --- a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java +++ b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java @@ -176,23 +176,23 @@ public class HiveCatalogServiceIT { @Test public void testIsAlive() throws Exception { - Assert.assertTrue(hiveCatalogService.isAlive(METASTORE_URL, "metaStorePrincipal")); + Assert.assertTrue(hiveCatalogService.isAlive(conf, METASTORE_URL)); } @Test (expectedExceptions = Exception.class) public void testIsAliveNegative() throws Exception { - hiveCatalogService.isAlive("thrift://localhost:9999", "metaStorePrincipal"); + hiveCatalogService.isAlive(conf, "thrift://localhost:9999"); } @Test (expectedExceptions = FalconException.class) public void testTableExistsNegative() throws Exception { - hiveCatalogService.tableExists(METASTORE_URL, DATABASE_NAME, "blah", "metaStorePrincipal"); + hiveCatalogService.tableExists(conf, METASTORE_URL, DATABASE_NAME, "blah"); } @Test public void testTableExists() throws Exception { Assert.assertTrue(hiveCatalogService.tableExists( - METASTORE_URL, DATABASE_NAME, TABLE_NAME, "metaStorePrincipal")); + conf, METASTORE_URL, DATABASE_NAME, TABLE_NAME)); } @Test