falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [8/8] incubator-falcon git commit: FALCON-894 Cluster submission with hive registry fails in secure setup. Contributed by Venkatesh Seetharam
Date Fri, 14 Nov 2014 02:54:53 GMT
FALCON-894 Cluster submission with hive registry fails in secure setup. Contributed by Venkatesh
Seetharam


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/c4dd440d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/c4dd440d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/c4dd440d

Branch: refs/heads/master
Commit: c4dd440d9ebe5052226028ebbab66bfe2c981ae0
Parents: e8b1d11
Author: Venkatesh Seetharam <venkatesh@apache.org>
Authored: Thu Nov 13 18:30:54 2014 -0800
Committer: Venkatesh Seetharam <venkatesh@apache.org>
Committed: Thu Nov 13 18:55:24 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 .../falcon/catalog/AbstractCatalogService.java  | 11 +++----
 .../falcon/catalog/HiveCatalogService.java      | 34 +++++++++-----------
 .../entity/parser/ClusterEntityParser.java      |  6 ++--
 .../falcon/entity/parser/FeedEntityParser.java  | 10 +++---
 .../falcon/catalog/HiveCatalogServiceIT.java    |  8 ++---
 6 files changed, 35 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fda0338..7804f79 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -144,6 +144,9 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
+   FALCON-894 Cluster submission with hive registry fails in secure setup
+   (Venkatesh Seetharam)
+
    FALCON-892 HCatReplication fails in secure setup (Venkatesh Seetharam)
 
    FALCON-889 Windows azure replication fails with "wasb" as the scheme to an

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
index e64a5be..348fac0 100644
--- a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
+++ b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
@@ -33,26 +33,25 @@ public abstract class AbstractCatalogService {
     /**
      * This method checks if the catalog service is alive.
      *
+     * @param conf conf
      * @param catalogUrl url for the catalog service
-     * @param metaStorePrincipal kerberos principal for hive metastore as this is executed
in falcon on behalf of user
      * @return if the service was reachable
      * @throws FalconException exception
      */
-    public abstract boolean isAlive(String catalogUrl,
-                                    String metaStorePrincipal) throws FalconException;
+    public abstract boolean isAlive(Configuration conf, String catalogUrl) throws FalconException;
 
     /**
      * This method checks if the given table exists in the catalog.
      *
+     * @param conf  conf
      * @param catalogUrl url for the catalog service
      * @param database database the table belongs to
      * @param tableName tableName to check if it exists
-     * @param metaStorePrincipal kerberos principal for hive metastore as this is executed
in falcon on behalf of user
      * @return if the table exists
      * @throws FalconException exception
      */
-    public abstract boolean tableExists(String catalogUrl, String database, String tableName,
-                                        String metaStorePrincipal) throws FalconException;
+    public abstract boolean tableExists(Configuration conf, String catalogUrl,
+                                        String database, String tableName) throws FalconException;
 
     /**
      * Returns if the table is external or not. Executed in the workflow engine.

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
index 51fb6b7..3216f1e 100644
--- a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
+++ b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
@@ -20,6 +20,7 @@ package org.apache.falcon.catalog;
 
 import org.apache.falcon.FalconException;
 import org.apache.falcon.security.CurrentUser;
+import org.apache.falcon.security.SecurityUtil;
 import org.apache.falcon.workflow.util.OozieActionConfigurationHelper;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -63,7 +64,7 @@ public class HiveCatalogService extends AbstractCatalogService {
      */
     public static HCatClient getHCatClient(String metastoreUrl) throws FalconException {
         try {
-            HiveConf hcatConf = createHiveConf(metastoreUrl);
+            HiveConf hcatConf = createHiveConf(new Configuration(false), metastoreUrl);
             return HCatClient.create(hcatConf);
         } catch (HCatException e) {
             throw new FalconException("Exception creating HCatClient: " + e.getMessage(),
e);
@@ -72,10 +73,6 @@ public class HiveCatalogService extends AbstractCatalogService {
         }
     }
 
-    private static HiveConf createHiveConf(String metastoreUrl) throws IOException {
-        return createHiveConf(new Configuration(false), metastoreUrl);
-    }
-
     private static HiveConf createHiveConf(Configuration conf,
                                            String metastoreUrl) throws IOException {
         HiveConf hcatConf = new HiveConf(conf, HiveConf.class);
@@ -156,19 +153,18 @@ public class HiveCatalogService extends AbstractCatalogService {
     /**
      * This is used from with in falcon namespace.
      *
-     * @param catalogUrl metastore uri
-     * @param metaStoreServicePrincipal metastore principal
+     * @param conf                      conf
+     * @param catalogUrl                metastore uri
      * @return hive metastore client handle
      * @throws FalconException
      */
-    private static synchronized HCatClient createProxiedHCatClient(String catalogUrl,
-                                                                   String metaStoreServicePrincipal)
-        throws FalconException {
+    private static HCatClient createProxiedHCatClient(Configuration conf,
+                                                      String catalogUrl) throws FalconException
{
 
         try {
-            final HiveConf hcatConf = createHiveConf(catalogUrl);
+            final HiveConf hcatConf = createHiveConf(conf, catalogUrl);
             UserGroupInformation proxyUGI = CurrentUser.getProxyUGI();
-            addSecureCredentialsAndToken(metaStoreServicePrincipal, hcatConf, proxyUGI);
+            addSecureCredentialsAndToken(conf, hcatConf, proxyUGI);
 
             LOG.info("Creating HCatalog client object for {}", catalogUrl);
             return proxyUGI.doAs(new PrivilegedExceptionAction<HCatClient>() {
@@ -183,10 +179,11 @@ public class HiveCatalogService extends AbstractCatalogService {
         }
     }
 
-    private static void addSecureCredentialsAndToken(String metaStoreServicePrincipal,
+    private static void addSecureCredentialsAndToken(Configuration conf,
                                                      HiveConf hcatConf,
                                                      UserGroupInformation proxyUGI) throws
IOException {
         if (UserGroupInformation.isSecurityEnabled()) {
+            String metaStoreServicePrincipal = conf.get(SecurityUtil.HIVE_METASTORE_PRINCIPAL);
             hcatConf.set(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname,
                 metaStoreServicePrincipal);
             hcatConf.set(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, "true");
@@ -215,12 +212,11 @@ public class HiveCatalogService extends AbstractCatalogService {
     }
 
     @Override
-    public boolean isAlive(final String catalogUrl,
-                           final String metaStorePrincipal) throws FalconException {
+    public boolean isAlive(Configuration conf, final String catalogUrl) throws FalconException
{
         LOG.info("Checking if the service is alive for: {}", catalogUrl);
 
         try {
-            HCatClient client = createProxiedHCatClient(catalogUrl, metaStorePrincipal);
+            HCatClient client = createProxiedHCatClient(conf, catalogUrl);
             HCatDatabase database = client.getDatabase("default");
             return database != null;
         } catch (HCatException e) {
@@ -229,12 +225,12 @@ public class HiveCatalogService extends AbstractCatalogService {
     }
 
     @Override
-    public boolean tableExists(final String catalogUrl, final String database, final String
tableName,
-                               final String metaStorePrincipal) throws FalconException {
+    public boolean tableExists(Configuration conf, final String catalogUrl, final String
database,
+                               final String tableName) throws FalconException {
         LOG.info("Checking if the table exists: {}", tableName);
 
         try {
-            HCatClient client = createProxiedHCatClient(catalogUrl, metaStorePrincipal);
+            HCatClient client = createProxiedHCatClient(conf, catalogUrl);
             HCatTable table = client.getTable(database, tableName);
             return table != null;
         } catch (HCatException e) {

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
index b3496c4..cd51804 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
@@ -189,15 +189,15 @@ public class ClusterEntityParser extends EntityParser<Cluster>
{
         LOG.info("Validating catalog registry interface: {}", catalogUrl);
 
         try {
-            String metaStorePrincipal = null;
+            Configuration clusterConf = ClusterHelper.getConfiguration(cluster);
             if (UserGroupInformation.isSecurityEnabled()) {
-                metaStorePrincipal = ClusterHelper.getPropertyValue(cluster, SecurityUtil.HIVE_METASTORE_PRINCIPAL);
+                String metaStorePrincipal = clusterConf.get(SecurityUtil.HIVE_METASTORE_PRINCIPAL);
                 Validate.notEmpty(metaStorePrincipal,
                         "Cluster definition missing required metastore credential property:
"
                                 + SecurityUtil.HIVE_METASTORE_PRINCIPAL);
             }
 
-            if (!CatalogServiceFactory.getCatalogService().isAlive(catalogUrl, metaStorePrincipal))
{
+            if (!CatalogServiceFactory.getCatalogService().isAlive(clusterConf, catalogUrl))
{
                 throw new ValidationException("Unable to reach Catalog server:" + catalogUrl);
             }
         } catch (FalconException e) {

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
index 63f9202..448fd70 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
@@ -40,7 +40,7 @@ import org.apache.falcon.entity.v0.process.Process;
 import org.apache.falcon.expression.ExpressionHelper;
 import org.apache.falcon.group.FeedGroup;
 import org.apache.falcon.group.FeedGroupMap;
-import org.apache.falcon.security.SecurityUtil;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.authorize.AuthorizationException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -418,10 +418,10 @@ public class FeedEntityParser extends EntityParser<Feed> {
             }
 
             CatalogStorage catalogStorage = (CatalogStorage) storage;
-            String metaStorePrincipal = ClusterHelper.getPropertyValue(clusterEntity,
-                    SecurityUtil.HIVE_METASTORE_PRINCIPAL);
-            if (!CatalogServiceFactory.getCatalogService().tableExists(catalogStorage.getCatalogUrl(),
-                    catalogStorage.getDatabase(), catalogStorage.getTable(), metaStorePrincipal))
{
+            Configuration clusterConf = ClusterHelper.getConfiguration(clusterEntity);
+            if (!CatalogServiceFactory.getCatalogService().tableExists(
+                    clusterConf, catalogStorage.getCatalogUrl(),
+                    catalogStorage.getDatabase(), catalogStorage.getTable())) {
                 buffer.append("Table [")
                         .append(catalogStorage.getTable())
                         .append("] does not exist for feed: ")

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/c4dd440d/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
index b422119..87101c5 100644
--- a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
+++ b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
@@ -176,23 +176,23 @@ public class HiveCatalogServiceIT {
 
     @Test
     public void testIsAlive() throws Exception {
-        Assert.assertTrue(hiveCatalogService.isAlive(METASTORE_URL, "metaStorePrincipal"));
+        Assert.assertTrue(hiveCatalogService.isAlive(conf, METASTORE_URL));
     }
 
     @Test (expectedExceptions = Exception.class)
     public void testIsAliveNegative() throws Exception {
-        hiveCatalogService.isAlive("thrift://localhost:9999", "metaStorePrincipal");
+        hiveCatalogService.isAlive(conf, "thrift://localhost:9999");
     }
 
     @Test (expectedExceptions = FalconException.class)
     public void testTableExistsNegative() throws Exception {
-        hiveCatalogService.tableExists(METASTORE_URL, DATABASE_NAME, "blah", "metaStorePrincipal");
+        hiveCatalogService.tableExists(conf, METASTORE_URL, DATABASE_NAME, "blah");
     }
 
     @Test
     public void testTableExists() throws Exception {
         Assert.assertTrue(hiveCatalogService.tableExists(
-                METASTORE_URL, DATABASE_NAME, TABLE_NAME, "metaStorePrincipal"));
+                conf, METASTORE_URL, DATABASE_NAME, TABLE_NAME));
     }
 
     @Test


Mime
View raw message