hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [1/2] git commit: HADOOP-11187 NameNode - KMS communication fails after a long period of inactivity. Contributed by Arun Suresh. (cherry picked from commit d593035d50e9997f31ddd67275b6e68504f9ca3c)
Date Thu, 06 Nov 2014 02:18:23 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 0f9d0d9d8 -> d698ed1d6
  refs/heads/trunk 86eb27ba1 -> ef5af4f8d


HADOOP-11187 NameNode - KMS communication fails after a long period of inactivity. Contributed
by Arun Suresh.
(cherry picked from commit d593035d50e9997f31ddd67275b6e68504f9ca3c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d698ed1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d698ed1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d698ed1d

Branch: refs/heads/branch-2
Commit: d698ed1d6f641a7377c1600ebe7a33757936a2b1
Parents: 0f9d0d9
Author: Aaron T. Myers <atm@apache.org>
Authored: Wed Nov 5 18:15:12 2014 -0800
Committer: Aaron T. Myers <atm@apache.org>
Committed: Wed Nov 5 18:17:43 2014 -0800

----------------------------------------------------------------------
 .../server/AuthenticationFilter.java            |  9 ++++++
 .../server/AuthenticationHandler.java           |  3 ++
 .../server/KerberosAuthenticationHandler.java   |  2 +-
 .../server/PseudoAuthenticationHandler.java     |  9 ++++--
 .../client/TestPseudoAuthenticator.java         |  5 +--
 .../server/TestAuthenticationFilter.java        |  4 ++-
 .../server/TestPseudoAuthenticationHandler.java |  9 ++----
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 ++
 .../crypto/key/kms/KMSClientProvider.java       |  5 ++-
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 32 ++++++++------------
 10 files changed, 49 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 58d97ca..0ac352b 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -17,6 +17,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
 import org.apache.hadoop.security.authentication.util.Signer;
 import org.apache.hadoop.security.authentication.util.SignerException;
 import org.apache.hadoop.security.authentication.util.RandomSignerSecretProvider;
@@ -36,6 +37,7 @@ import javax.servlet.http.Cookie;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletRequestWrapper;
 import javax.servlet.http.HttpServletResponse;
+
 import java.io.IOException;
 import java.security.Principal;
 import java.text.SimpleDateFormat;
@@ -565,6 +567,13 @@ public class AuthenticationFilter implements Filter {
       if (!httpResponse.isCommitted()) {
         createAuthCookie(httpResponse, "", getCookieDomain(),
                 getCookiePath(), 0, isHttps);
+        // If response code is 401. Then WWW-Authenticate Header should be
+        // present.. reset to 403 if not found..
+        if ((errCode == HttpServletResponse.SC_UNAUTHORIZED)
+            && (!httpResponse.containsHeader(
+                KerberosAuthenticator.WWW_AUTHENTICATE))) {
+          errCode = HttpServletResponse.SC_FORBIDDEN;
+        }
         if (authenticationEx == null) {
           httpResponse.sendError(errCode, "Authentication required");
         } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
index 7cafe8b..04984be 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
@@ -18,6 +18,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+
 import java.io.IOException;
 import java.util.Properties;
 
@@ -30,6 +31,8 @@ import java.util.Properties;
  */
 public interface AuthenticationHandler {
 
+  public static final String WWW_AUTHENTICATE = "WWW-Authenticate";
+
   /**
    * Returns the authentication type of the authentication handler.
    * <p/>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
index 9852460..92bc57c 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
@@ -331,7 +331,7 @@ public class KerberosAuthenticationHandler implements AuthenticationHandler
{
     String authorization = request.getHeader(KerberosAuthenticator.AUTHORIZATION);
 
     if (authorization == null || !authorization.startsWith(KerberosAuthenticator.NEGOTIATE))
{
-      response.setHeader(KerberosAuthenticator.WWW_AUTHENTICATE, KerberosAuthenticator.NEGOTIATE);
+      response.setHeader(WWW_AUTHENTICATE, KerberosAuthenticator.NEGOTIATE);
       response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
       if (authorization == null) {
         LOG.trace("SPNEGO starting");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
index 0b329e0..2c7db88 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
@@ -15,13 +15,13 @@ package org.apache.hadoop.security.authentication.server;
 
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
-
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.apache.http.NameValuePair;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.List;
@@ -54,6 +54,9 @@ public class PseudoAuthenticationHandler implements AuthenticationHandler
{
   public static final String ANONYMOUS_ALLOWED = TYPE + ".anonymous.allowed";
 
   private static final Charset UTF8_CHARSET = Charset.forName("UTF-8");
+
+  private static final String PSEUDO_AUTH = "PseudoAuth";
+
   private boolean acceptAnonymous;
   private String type;
 
@@ -181,7 +184,9 @@ public class PseudoAuthenticationHandler implements AuthenticationHandler
{
       if (getAcceptAnonymous()) {
         token = AuthenticationToken.ANONYMOUS;
       } else {
-        throw new AuthenticationException("Anonymous requests are disallowed");
+        response.setStatus(HttpServletResponse.SC_FORBIDDEN);
+        response.setHeader(WWW_AUTHENTICATE, PSEUDO_AUTH);
+        token = null;
       }
     } else {
       token = new AuthenticationToken(userName, userName, getType());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestPseudoAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestPseudoAuthenticator.java
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestPseudoAuthenticator.java
index 4a33fa9..3a5764b 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestPseudoAuthenticator.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestPseudoAuthenticator.java
@@ -63,8 +63,9 @@ public class TestPseudoAuthenticator {
       URL url = new URL(auth.getBaseURL());
       HttpURLConnection conn = (HttpURLConnection) url.openConnection();
       conn.connect();
-      Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN, conn.getResponseCode());
-      Assert.assertEquals("Anonymous requests are disallowed", conn.getResponseMessage());
+      Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, conn.getResponseCode());
+      Assert.assertTrue(conn.getHeaderFields().containsKey("WWW-Authenticate"));
+      Assert.assertEquals("Authentication required", conn.getResponseMessage());
     } finally {
       auth.stop();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
index 3b6b958..c01c182 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
@@ -537,11 +537,11 @@ public class TestAuthenticationFilter {
         }
       ).when(chain).doFilter(Mockito.<ServletRequest>anyObject(), Mockito.<ServletResponse>anyObject());
 
+      Mockito.when(response.containsHeader("WWW-Authenticate")).thenReturn(true);
       filter.doFilter(request, response, chain);
 
       Mockito.verify(response).sendError(
           HttpServletResponse.SC_UNAUTHORIZED, "Authentication required");
-      Mockito.verify(response).setHeader("WWW-Authenticate", "dummyauth");
     } finally {
       filter.destroy();
     }
@@ -852,6 +852,7 @@ public class TestAuthenticationFilter {
       Mockito.when(request.getCookies()).thenReturn(new Cookie[]{cookie});
 
       HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+      Mockito.when(response.containsHeader("WWW-Authenticate")).thenReturn(true);
       FilterChain chain = Mockito.mock(FilterChain.class);
 
       verifyUnauthorized(filter, request, response, chain);
@@ -930,6 +931,7 @@ public class TestAuthenticationFilter {
       Mockito.when(request.getCookies()).thenReturn(new Cookie[]{cookie});
 
       HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+      Mockito.when(response.containsHeader("WWW-Authenticate")).thenReturn(true);
       FilterChain chain = Mockito.mock(FilterChain.class);
 
       verifyUnauthorized(filter, request, response, chain);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
index 91c1103..b52915d 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
@@ -21,6 +21,7 @@ import org.mockito.Mockito;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+
 import java.util.Properties;
 
 public class TestPseudoAuthenticationHandler {
@@ -74,12 +75,8 @@ public class TestPseudoAuthenticationHandler {
       HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
       HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
 
-      handler.authenticate(request, response);
-      Assert.fail();
-    } catch (AuthenticationException ex) {
-      // Expected
-    } catch (Exception ex) {
-      Assert.fail();
+      AuthenticationToken token = handler.authenticate(request, response);
+      Assert.assertNull(token);
     } finally {
       handler.destroy();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 735962f..8e21669 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -69,6 +69,9 @@ Release 2.7.0 - UNRELEASED
     HADOOP-11272. Allow ZKSignerSecretProvider and
     ZKDelegationTokenSecretManager to use the same curator client. (Arun Suresh via atm)
 
+    HADOOP-11187 NameNode - KMS communication fails after a long period of
+    inactivity. (Arun Suresh via atm)
+
 Release 2.6.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 5c332a8..e9e8af4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -81,6 +81,8 @@ import com.google.common.base.Preconditions;
 public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     KeyProviderDelegationTokenExtension.DelegationTokenExtension {
 
+  private static final String INVALID_SIGNATURE = "Invalid signature";
+
   private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous requests are disallowed";
 
   public static final String TOKEN_KIND = "kms-dt";
@@ -453,7 +455,8 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
       throw ex;
     }
     if ((conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN
-        && conn.getResponseMessage().equals(ANONYMOUS_REQUESTS_DISALLOWED))
+        && (conn.getResponseMessage().equals(ANONYMOUS_REQUESTS_DISALLOWED) ||
+            conn.getResponseMessage().contains(INVALID_SIGNATURE)))
         || conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
       // Ideally, this should happen only when there is an Authentication
       // failure. Unfortunately, the AuthenticationFilter returns 403 when it

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d698ed1d/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 9e76178..86e6484 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -900,6 +900,7 @@ public class TestKMS {
         keytab.getAbsolutePath());
     conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
     conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
+    conf.set("hadoop.kms.authentication.token.validity", "1");
 
     for (KMSACLs.Type type : KMSACLs.Type.values()) {
       conf.set(type.getAclConfigKey(), type.toString());
@@ -930,11 +931,16 @@ public class TestKMS {
                   @Override
                   public Void run() throws Exception {
                     KMSClientProvider kp = new KMSClientProvider(uri, conf);
+
+                    kp.createKey("k0", new byte[16],
+                        new KeyProvider.Options(conf));
+                    // This happens before rollover
                     kp.createKey("k1", new byte[16],
                         new KeyProvider.Options(conf));
-                    makeAuthTokenStale(kp);
+                    // Atleast 2 rollovers.. so should induce signer Exception
+                    Thread.sleep(3500);
                     kp.createKey("k2", new byte[16],
-                        new KeyProvider.Options(conf));
+                      new KeyProvider.Options(conf));
                     return null;
                   }
                 });
@@ -958,15 +964,16 @@ public class TestKMS {
                     KMSClientProvider kp = new KMSClientProvider(uri, conf);
                     kp.createKey("k3", new byte[16],
                         new KeyProvider.Options(conf));
-                    makeAuthTokenStale(kp);
+                    // Atleast 2 rollovers.. so should induce signer Exception
+                    Thread.sleep(3500);
                     try {
                       kp.createKey("k4", new byte[16],
                           new KeyProvider.Options(conf));
-                      Assert.fail("Shoud fail since retry count == 0");
+                      Assert.fail("This should not succeed..");
                     } catch (IOException e) {
                       Assert.assertTrue(
-                          "HTTP exception must be a 403 : " + e.getMessage(), e
-                              .getMessage().contains("403"));
+                          "HTTP exception must be a 401 : " + e.getMessage(), e
+                              .getMessage().contains("401"));
                     }
                     return null;
                   }
@@ -976,19 +983,6 @@ public class TestKMS {
         });
   }
 
-  private void makeAuthTokenStale(KMSClientProvider kp) throws Exception {
-    Field tokF = KMSClientProvider.class.getDeclaredField("authToken");
-    tokF.setAccessible(true);
-    DelegationTokenAuthenticatedURL.Token delToken =
-        (DelegationTokenAuthenticatedURL.Token) tokF.get(kp);
-    String oldTokStr = delToken.toString();
-    Method setM =
-        AuthenticatedURL.Token.class.getDeclaredMethod("set", String.class);
-    setM.setAccessible(true);
-    String newTokStr = oldTokStr.replaceAll("e=[^&]*", "e=1000");
-    setM.invoke(((AuthenticatedURL.Token)delToken), newTokStr);
-  }
-
   @Test
   public void testACLs() throws Exception {
     Configuration conf = new Configuration();


Mime
View raw message