hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject hive git commit: HIVE-8696 : HCatClientHMSImpl doesn't use a Retrying-HiveMetastoreClient (Thiruvel Thirumoolan via Sushanth Sowmyan)
Date Thu, 07 May 2015 08:13:29 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1.2 7e0e5cf0e -> d78f19d59


HIVE-8696 : HCatClientHMSImpl doesn't use a Retrying-HiveMetastoreClient (Thiruvel Thirumoolan
via Sushanth Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d78f19d5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d78f19d5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d78f19d5

Branch: refs/heads/branch-1.2
Commit: d78f19d59f2288ed9de089273d18b943b27aa6fe
Parents: 7e0e5cf
Author: Sushanth Sowmyan <khorgath@gmail.com>
Authored: Thu May 7 01:12:11 2015 -0700
Committer: Sushanth Sowmyan <khorgath@gmail.com>
Committed: Thu May 7 01:13:26 2015 -0700

----------------------------------------------------------------------
 .../hive/hcatalog/common/HiveClientCache.java   |  9 +++-
 .../hcatalog/mapreduce/TestPassProperties.java  |  5 +-
 .../hive/metastore/RetryingMetaStoreClient.java | 53 ++++++++++++++++----
 3 files changed, 55 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/d78f19d5/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
index 578b6ea..0966581 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.shims.Utils;
@@ -81,7 +82,7 @@ class HiveClientCache {
   }
 
   public static IMetaStoreClient getNonCachedHiveMetastoreClient(HiveConf hiveConf) throws
MetaException {
-    return new HiveMetaStoreClient(hiveConf);
+    return RetryingMetaStoreClient.getProxy(hiveConf);
   }
 
   public HiveClientCache(HiveConf hiveConf) {
@@ -226,7 +227,11 @@ class HiveClientCache {
       return hiveCache.get(cacheKey, new Callable<ICacheableMetaStoreClient>() {
         @Override
         public ICacheableMetaStoreClient call() throws MetaException {
-          return new CacheableHiveMetaStoreClient(cacheKey.getHiveConf(), timeout);
+          return
+              (ICacheableMetaStoreClient) RetryingMetaStoreClient.getProxy(cacheKey.getHiveConf(),
+                  new Class<?>[]{HiveConf.class, Integer.class},
+                  new Object[]{cacheKey.getHiveConf(), timeout},
+                  CacheableHiveMetaStoreClient.class.getName());
         }
       });
     } catch (ExecutionException e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/d78f19d5/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
index 735ab5f..8673b48 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 
 import org.apache.hadoop.conf.Configuration;
@@ -109,8 +110,10 @@ public class TestPassProperties {
       new FileOutputCommitterContainer(job, null).cleanupJob(job);
     } catch (Exception e) {
       caughtException = true;
-      assertTrue(e.getCause().getMessage().contains(
+      assertTrue(((InvocationTargetException)e.getCause().getCause().getCause()).getTargetException().getMessage().contains(
           "Could not connect to meta store using any of the URIs provided"));
+      assertTrue(e.getCause().getMessage().contains(
+          "Unable to instantiate org.apache.hive.hcatalog.common.HiveClientCache$CacheableHiveMetaStoreClient"));
     }
     assertTrue(caughtException);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/d78f19d5/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
index 1b6487a..fb44484 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
@@ -60,6 +60,18 @@ public class RetryingMetaStoreClient implements InvocationHandler {
 
   protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
       Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass)
throws MetaException {
+
+    this(hiveConf,
+        new Class[] {HiveConf.class, HiveMetaHookLoader.class},
+        new Object[] {hiveConf, hookLoader},
+        metaCallTimeMap,
+        msClientClass);
+  }
+
+  protected RetryingMetaStoreClient(HiveConf hiveConf, Class<?>[] constructorArgTypes,
+      Object[] constructorArgs, Map<String, Long> metaCallTimeMap, Class<? extends
IMetaStoreClient> msClientClass)
+      throws MetaException {
+
     this.retryLimit = hiveConf.getIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES);
     this.retryDelaySeconds = hiveConf.getTimeVar(
         HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS);
@@ -71,12 +83,14 @@ public class RetryingMetaStoreClient implements InvocationHandler {
     localMetaStore = (msUri == null) || msUri.trim().isEmpty();
 
     reloginExpiringKeytabUser();
-    this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] {
-        HiveConf.class, HiveMetaHookLoader.class}, new Object[] {hiveConf, hookLoader});
+    this.base = (IMetaStoreClient) MetaStoreUtils.newInstance(msClientClass, constructorArgTypes,
constructorArgs);
   }
 
-  public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader)
throws MetaException {
-    return getProxy(hiveConf, hookLoader, null, HiveMetaStoreClient.class.getName());
+  public static IMetaStoreClient getProxy(HiveConf hiveConf) throws MetaException {
+
+    return getProxy(hiveConf, new Class[]{HiveConf.class}, new Object[]{hiveConf}, null,
+        HiveMetaStoreClient.class.getName()
+    );
   }
 
   public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
@@ -84,19 +98,40 @@ public class RetryingMetaStoreClient implements InvocationHandler {
     return getProxy(hiveConf, hookLoader, null, mscClassName);
   }
 
+  public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
+      Map<String, Long> metaCallTimeMap, String mscClassName) throws MetaException
{
+
+    return getProxy(hiveConf,
+        new Class[] {HiveConf.class, HiveMetaHookLoader.class},
+        new Object[] {hiveConf, hookLoader},
+        metaCallTimeMap,
+        mscClassName
+    );
+  }
+
   /**
    * This constructor is meant for Hive internal use only.
    * Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose.
    */
-  public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
-      Map<String, Long> metaCallTimeMap, String mscClassName) throws MetaException
{
+  public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes,
+      Object[] constructorArgs, String mscClassName) throws MetaException {
+    return getProxy(hiveConf, constructorArgTypes, constructorArgs, null, mscClassName);
+  }
+
+  /**
+   * This constructor is meant for Hive internal use only.
+   * Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose.
+   */
+  public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes,
+      Object[] constructorArgs, Map<String, Long> metaCallTimeMap,
+      String mscClassName) throws MetaException {
 
     Class<? extends IMetaStoreClient> baseClass = (Class<? extends IMetaStoreClient>)
MetaStoreUtils
         .getClass(mscClassName);
 
-    RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, hookLoader,
-        metaCallTimeMap, baseClass);
-
+    RetryingMetaStoreClient handler =
+        new RetryingMetaStoreClient(hiveConf, constructorArgTypes, constructorArgs,
+            metaCallTimeMap, baseClass);
     return (IMetaStoreClient) Proxy.newProxyInstance(
         RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler);
   }


Mime
View raw message