hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vgumas...@apache.org
Subject hive git commit: HIVE-9508: MetaStore client socket connection should have a lifetime (Thiruvel Thirumoolan reviewed by Vaibhav Gumashta)
Date Tue, 05 May 2015 17:44:35 GMT
Repository: hive
Updated Branches:
  refs/heads/master 652febcda -> e2a12c9a6


HIVE-9508: MetaStore client socket connection should have a lifetime (Thiruvel Thirumoolan
reviewed by Vaibhav Gumashta)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e2a12c9a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e2a12c9a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e2a12c9a

Branch: refs/heads/master
Commit: e2a12c9a630d037f3e3aaf42acc873eac86bc9f3
Parents: 652febc
Author: Vaibhav Gumashta <vgumashta@apache.org>
Authored: Tue May 5 10:44:16 2015 -0700
Committer: Vaibhav Gumashta <vgumashta@apache.org>
Committed: Tue May 5 10:44:16 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  6 ++++
 .../hive/metastore/TestHiveMetaStore.java       | 30 ++++++++++++++++++++
 .../hive/metastore/RetryingMetaStoreClient.java | 27 +++++++++++++++---
 3 files changed, 59 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5d4dbea..69fda45 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -119,6 +119,7 @@ public class HiveConf extends Configuration {
       HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES,
       HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY,
       HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
+      HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME,
       HiveConf.ConfVars.METASTOREPWD,
       HiveConf.ConfVars.METASTORECONNECTURLHOOK,
       HiveConf.ConfVars.METASTORECONNECTURLKEY,
@@ -398,6 +399,11 @@ public class HiveConf extends Configuration {
     METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", "600s",
         new TimeValidator(TimeUnit.SECONDS),
         "MetaStore Client socket timeout in seconds"),
+    METASTORE_CLIENT_SOCKET_LIFETIME("hive.metastore.client.socket.lifetime", "0s",
+        new TimeValidator(TimeUnit.SECONDS),
+        "MetaStore Client socket lifetime in seconds. After this time is exceeded, client\n"
+
+        "reconnects on the next MetaStore operation. A value of 0s means the connection\n"
+
+        "has an infinite lifetime."),
     METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine",
         "password to use against metastore database"),
     METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", "",

http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 130fd67..dffeb34 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -2867,4 +2867,34 @@ public abstract class TestHiveMetaStore extends TestCase {
         ownerName, ownerType, createTime, functionType, resources);
     client.createFunction(func);
   }
+
+  public void testRetriableClientWithConnLifetime() throws Exception {
+
+    HiveConf conf = new HiveConf(hiveConf);
+    conf.setLong(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME.name(), 60);
+    long timeout = 65 * 1000; // Lets use a timeout more than the socket lifetime to simulate
a reconnect
+
+    // Test a normal retriable client
+    IMetaStoreClient client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName());
+    client.getAllDatabases();
+    client.close();
+
+    // Connect after the lifetime, there should not be any failures
+    client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName());
+    Thread.sleep(timeout);
+    client.getAllDatabases();
+    client.close();
+  }
+
+  private HiveMetaHookLoader getHookLoader() {
+    HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
+      @Override
+      public HiveMetaHook getHook(
+          org.apache.hadoop.hive.metastore.api.Table tbl)
+          throws MetaException {
+        return null;
+      }
+    };
+    return hookLoader;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
index 77da6f7..1b6487a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
@@ -54,9 +54,9 @@ public class RetryingMetaStoreClient implements InvocationHandler {
   private final int retryLimit;
   private final long retryDelaySeconds;
   private final Map<String, Long> metaCallTimeMap;
-
-
-
+  private final long connectionLifeTimeInMillis;
+  private long lastConnectionTime;
+  private boolean localMetaStore;
 
   protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
       Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass)
throws MetaException {
@@ -64,6 +64,11 @@ public class RetryingMetaStoreClient implements InvocationHandler {
     this.retryDelaySeconds = hiveConf.getTimeVar(
         HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS);
     this.metaCallTimeMap = metaCallTimeMap;
+    this.connectionLifeTimeInMillis =
+        hiveConf.getTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, TimeUnit.SECONDS)
* 1000;
+    this.lastConnectionTime = System.currentTimeMillis();
+    String msUri = hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS);
+    localMetaStore = (msUri == null) || msUri.trim().isEmpty();
 
     reloginExpiringKeytabUser();
     this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] {
@@ -104,8 +109,9 @@ public class RetryingMetaStoreClient implements InvocationHandler {
     while (true) {
       try {
         reloginExpiringKeytabUser();
-        if(retriesMade > 0){
+        if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) {
           base.reconnect();
+          lastConnectionTime = System.currentTimeMillis();
         }
         if (metaCallTimeMap == null) {
           ret = method.invoke(base, args);
@@ -171,6 +177,19 @@ public class RetryingMetaStoreClient implements InvocationHandler {
     return methodSb.toString();
   }
 
+  private boolean hasConnectionLifeTimeReached(Method method) {
+    if (connectionLifeTimeInMillis <= 0 || localMetaStore ||
+        method.getName().equalsIgnoreCase("close")) {
+      return false;
+    }
+    boolean shouldReconnect =
+        (System.currentTimeMillis() - lastConnectionTime) >= connectionLifeTimeInMillis;
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Reconnection status for Method: " + method.getName() + " is " + shouldReconnect);
+    }
+    return shouldReconnect;
+  }
+
   /**
    * Relogin if login user is logged in using keytab
    * Relogin is actually done by ugi code only if sufficient time has passed


Mime
View raw message