hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aihu...@apache.org
Subject hive git commit: HIVE-12481: Occasionally "Request is a replay" will be thrown from HS2 (Aihua Xu, reviewed by Yongzhi Chen)
Date Mon, 11 Jan 2016 20:31:10 GMT
Repository: hive
Updated Branches:
  refs/heads/master c278860a9 -> 361d72d96


HIVE-12481: Occasionally "Request is a replay" will be thrown from HS2 (Aihua Xu, reviewed
by Yongzhi Chen)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/361d72d9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/361d72d9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/361d72d9

Branch: refs/heads/master
Commit: 361d72d963053eafe0ee9f6c6e24d127708def02
Parents: c278860
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Fri Dec 4 14:33:35 2015 -0500
Committer: Aihua Xu <aihuaxu@apache.org>
Committed: Mon Jan 11 15:24:45 2016 -0500

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  4 ++
 .../hive/thrift/TestHadoopAuthBridge23.java     | 54 +++++++++++++++++++-
 .../hadoop/hive/metastore/HiveMetaStore.java    |  3 +-
 .../hive/service/auth/HiveAuthFactory.java      |  3 +-
 .../hive/thrift/HadoopThriftAuthBridge.java     | 33 ++++++++++--
 5 files changed, 88 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/361d72d9/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index eedd205..ab7e3c6 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1951,6 +1951,10 @@ public class HiveConf extends Configuration {
         "HttpOnly attribute of the HS2 generated cookie."),
 
     // binary transport settings
+    HIVE_SERVER2_THRIFT_AUTH_MAX_RETRIES("hive.server2.thrift.auth.max.retries", 1,
+        "Number of maximum retries to authenticate HS2 server or HMS server against Kerberos
service.\n"
+        + "This is to mitigate some false alarm auth issues, such that concurrent query executions\n"
+        + "against single HS2 server may fail to authenticate due to 'Request is a replay'."),
     HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000,
         "Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is
'binary'."),
     HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", "auth",

http://git-wip-us.apache.org/repos/asf/hive/blob/361d72d9/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
index 6d0776a..64fa0ec 100644
--- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
+++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.thrift;
 
 
 import junit.framework.TestCase;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -41,8 +42,12 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecret
 import org.apache.hadoop.security.token.delegation.DelegationKey;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.thrift.transport.TSaslServerTransport;
+import org.apache.thrift.transport.TTransport;
 import org.apache.thrift.transport.TTransportException;
 import org.apache.thrift.transport.TTransportFactory;
+import org.junit.Test;
+import org.mockito.Mockito;
+import static org.mockito.Mockito.*;
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
@@ -78,7 +83,7 @@ public class TestHadoopAuthBridge23 extends TestCase {
         super();
       }
       @Override
-      public TTransportFactory createTransportFactory(Map<String, String> saslProps)
+      public TTransportFactory createTransportFactory(Map<String, String> saslProps,
int authMaxRetries)
       throws TTransportException {
         TSaslServerTransport.Factory transFactory =
           new TSaslServerTransport.Factory();
@@ -87,7 +92,7 @@ public class TestHadoopAuthBridge23 extends TestCase {
             saslProps,
             new SaslDigestCallbackHandler(secretManager));
 
-        return new TUGIAssumingTransportFactory(transFactory, realUgi);
+        return new TUGIAssumingTransportFactory(transFactory, realUgi, authMaxRetries);
       }
       static DelegationTokenStore TOKEN_STORE = new MemoryTokenStore();
 
@@ -230,6 +235,51 @@ public class TestHadoopAuthBridge23 extends TestCase {
     obtainTokenAndAddIntoUGI(clientUgi, "tokenForFooTablePartition");
   }
 
+  /**
+   * Verifies that the expected result returned after 2 unsuccessful retries
+   * @throws Exception
+   */
+  @Test
+  public void testRetryGetTransport() throws Exception {
+    TTransport inputTransport = Mockito.mock(TTransport.class);
+    TTransport expectedTransport = Mockito.mock(TTransport.class);
+    TTransportFactory mockWrapped = Mockito.mock(TTransportFactory.class);
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    when(mockWrapped.getTransport(any(TTransport.class)))
+    .thenThrow(new RuntimeException(new TTransportException()))
+    .thenThrow(new RuntimeException(new TTransportException()))
+    .thenReturn(expectedTransport);
+
+    TTransportFactory factory = new HadoopThriftAuthBridge.Server.TUGIAssumingTransportFactory(mockWrapped,
ugi, 3);
+    TTransport transport = factory.getTransport(inputTransport);
+
+    assertEquals(expectedTransport, transport);
+    verify(mockWrapped, times(3)).getTransport(any(TTransport.class));
+  }
+
+  /**
+   * Verifies exception is thrown after 3 unsuccessful retries
+   * @throws Exception
+   */
+  @Test
+  public void testRetryGetTransport2() throws Exception {
+    Exception expectedException = new RuntimeException(new TTransportException());
+    TTransport inputTransport = Mockito.mock(TTransport.class);
+    TTransportFactory mockWrapped = Mockito.mock(TTransportFactory.class);
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    when(mockWrapped.getTransport(any(TTransport.class)))
+    .thenThrow(expectedException);
+
+    try {
+      TTransportFactory factory = new HadoopThriftAuthBridge.Server.TUGIAssumingTransportFactory(mockWrapped,
ugi, 3);
+      factory.getTransport(inputTransport);
+    } catch(Exception e) {
+      assertEquals(expectedException, e);
+    } finally {
+      verify(mockWrapped, times(3)).getTransport(any(TTransport.class));
+    }
+  }
+
   public void testMetastoreProxyUser() throws Exception {
     setup();
 

http://git-wip-us.apache.org/repos/asf/hive/blob/361d72d9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index ace644b..9ea9375 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -6189,8 +6189,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
             conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL));
         // start delegation token manager
         saslServer.startDelegationTokenSecretManager(conf, baseHandler, ServerMode.METASTORE);
+        int authMaxRetries = conf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_AUTH_MAX_RETRIES);
         transFactory = saslServer.createTransportFactory(
-                MetaStoreUtils.getMetaStoreSaslProperties(conf));
+                MetaStoreUtils.getMetaStoreSaslProperties(conf), authMaxRetries);
         processor = saslServer.wrapProcessor(
           new ThriftHiveMetastore.Processor<IHMSHandler>(handler));
         LOG.info("Starting DB backed MetaStore Server in Secure Mode");

http://git-wip-us.apache.org/repos/asf/hive/blob/361d72d9/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
index 3471f12..0c7c9ee 100644
--- a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
+++ b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
@@ -140,7 +140,8 @@ public class HiveAuthFactory {
     TTransportFactory transportFactory;
     if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) {
       try {
-        transportFactory = saslServer.createTransportFactory(getSaslProperties());
+        int authMaxRetries = conf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_AUTH_MAX_RETRIES);
+        transportFactory = saslServer.createTransportFactory(getSaslProperties(), authMaxRetries);
       } catch (TTransportException e) {
         throw new LoginException(e.getMessage());
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/361d72d9/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
index 6fe5969..731555c 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
@@ -26,6 +26,7 @@ import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Random;
 
 import javax.security.auth.callback.Callback;
 import javax.security.auth.callback.CallbackHandler;
@@ -44,7 +45,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.shims.Utils;
 import org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport;
 import org.apache.hadoop.security.SaslRpcServer;
@@ -367,7 +367,7 @@ public abstract class HadoopThriftAuthBridge {
      * @param saslProps Map of SASL properties
      */
 
-    public TTransportFactory createTransportFactory(Map<String, String> saslProps)
+    public TTransportFactory createTransportFactory(Map<String, String> saslProps,
int authMaxRetries)
         throws TTransportException {
       // Parse out the kerberos principal, host, realm.
       String kerberosName = realUgi.getUserName();
@@ -386,7 +386,7 @@ public abstract class HadoopThriftAuthBridge {
           null, SaslRpcServer.SASL_DEFAULT_REALM,
           saslProps, new SaslDigestCallbackHandler(secretManager));
 
-      return new TUGIAssumingTransportFactory(transFactory, realUgi);
+      return new TUGIAssumingTransportFactory(transFactory, realUgi, authMaxRetries);
     }
 
     /**
@@ -721,12 +721,14 @@ public abstract class HadoopThriftAuthBridge {
     static class TUGIAssumingTransportFactory extends TTransportFactory {
       private final UserGroupInformation ugi;
       private final TTransportFactory wrapped;
+      private final int authMaxRetries;
 
-      public TUGIAssumingTransportFactory(TTransportFactory wrapped, UserGroupInformation
ugi) {
+      public TUGIAssumingTransportFactory(TTransportFactory wrapped, UserGroupInformation
ugi, int authMaxRetries) {
         assert wrapped != null;
         assert ugi != null;
         this.wrapped = wrapped;
         this.ugi = ugi;
+        this.authMaxRetries = authMaxRetries;
       }
 
 
@@ -735,7 +737,28 @@ public abstract class HadoopThriftAuthBridge {
         return ugi.doAs(new PrivilegedAction<TTransport>() {
           @Override
           public TTransport run() {
-            return wrapped.getTransport(trans);
+            // Retry the authentication after sleeping for random microseconds
+            short numRetries = 0;
+            Random rand = new Random();
+
+            while (true) {
+              try {
+                return wrapped.getTransport(trans);
+              } catch(RuntimeException e) {
+                if (e.getCause() instanceof TTransportException) {
+                  if (++numRetries < authMaxRetries) {
+                    LOG.warn(e.getMessage());
+                    try {
+                      Thread.sleep(rand.nextInt(1000));
+                    } catch (InterruptedException ie) {
+                    }
+                    continue;
+                  }
+                }
+
+                throw e;
+              }
+            }
           }
         });
       }


Mime
View raw message