Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CB6F5113EE for ; Thu, 28 Aug 2014 21:46:15 +0000 (UTC) Received: (qmail 31833 invoked by uid 500); 28 Aug 2014 21:46:15 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 31762 invoked by uid 500); 28 Aug 2014 21:46:15 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 31753 invoked by uid 99); 28 Aug 2014 21:46:15 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Aug 2014 21:46:15 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id CF6BE82CA37; Thu, 28 Aug 2014 21:46:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tucu@apache.org To: common-commits@hadoop.apache.org Message-Id: <34f06472265249af84c327139fba3637@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: HADOOP-10880. Move HTTP delegation tokens out of URL querystring to a header. (tucu) Date: Thu, 28 Aug 2014 21:46:14 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk c4c9a7841 -> d1ae479aa HADOOP-10880. Move HTTP delegation tokens out of URL querystring to a header. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1ae479a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1ae479a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1ae479a Branch: refs/heads/trunk Commit: d1ae479aa5ae4d3e7ec80e35892e1699c378f813 Parents: c4c9a78 Author: Alejandro Abdelnur Authored: Thu Aug 28 14:45:40 2014 -0700 Committer: Alejandro Abdelnur Committed: Thu Aug 28 14:45:40 2014 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../web/DelegationTokenAuthenticatedURL.java | 81 ++++++++++++++++---- .../DelegationTokenAuthenticationHandler.java | 14 +++- .../web/DelegationTokenAuthenticator.java | 19 ++++- ...tionTokenAuthenticationHandlerWithMocks.java | 46 ++++++++++- .../delegation/web/TestWebDelegationToken.java | 50 +++++++++++- 6 files changed, 187 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ecbaaab..641635b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -518,6 +518,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10998. Fix bash tab completion code to work (Jim Hester via aw) + HADOOP-10880. Move HTTP delegation tokens out of URL querystring to + a header. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java index d955ada..5aeb177 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java @@ -125,6 +125,8 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL { } } + private boolean useQueryStringforDelegationToken = false; + /** * Creates an DelegationTokenAuthenticatedURL. *

@@ -171,6 +173,34 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL { } /** + * Sets if delegation token should be transmitted in the URL query string. + * By default it is transmitted using the + * {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header. + *

+ * This method is provided to enable WebHDFS backwards compatibility. + * + * @param useQueryString TRUE if the token is transmitted in the + * URL query string, FALSE if the delegation token is transmitted + * using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP + * header. + */ + @Deprecated + protected void setUseQueryStringForDelegationToken(boolean useQueryString) { + useQueryStringforDelegationToken = useQueryString; + } + + /** + * Returns if delegation token is transmitted as a HTTP header. + * + * @return TRUE if the token is transmitted in the URL query + * string, FALSE if the delegation token is transmitted using the + * {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header. + */ + public boolean useQueryStringForDelegationToken() { + return useQueryStringforDelegationToken; + } + + /** * Returns an authenticated {@link HttpURLConnection}, it uses a Delegation * Token only if the given auth token is an instance of {@link Token} and * it contains a Delegation Token, otherwise use the configured @@ -235,23 +265,41 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL { * @throws IOException if an IO error occurred. * @throws AuthenticationException if an authentication exception occurred. */ + @SuppressWarnings("unchecked") public HttpURLConnection openConnection(URL url, Token token, String doAs) throws IOException, AuthenticationException { Preconditions.checkNotNull(url, "url"); Preconditions.checkNotNull(token, "token"); Map extraParams = new HashMap(); - - // delegation token - Credentials creds = UserGroupInformation.getCurrentUser().getCredentials(); - if (!creds.getAllTokens().isEmpty()) { - InetSocketAddress serviceAddr = new InetSocketAddress(url.getHost(), - url.getPort()); - Text service = SecurityUtil.buildTokenService(serviceAddr); - org.apache.hadoop.security.token.Token dt = - creds.getToken(service); - if (dt != null) { - extraParams.put(KerberosDelegationTokenAuthenticator.DELEGATION_PARAM, - dt.encodeToUrlString()); + org.apache.hadoop.security.token.Token dToken + = null; + // if we have valid auth token, it takes precedence over a delegation token + // and we don't even look for one. + if (!token.isSet()) { + // delegation token + Credentials creds = UserGroupInformation.getCurrentUser(). + getCredentials(); + if (!creds.getAllTokens().isEmpty()) { + InetSocketAddress serviceAddr = new InetSocketAddress(url.getHost(), + url.getPort()); + Text service = SecurityUtil.buildTokenService(serviceAddr); + dToken = creds.getToken(service); + if (dToken != null) { + if (useQueryStringForDelegationToken()) { + // delegation token will go in the query string, injecting it + extraParams.put( + KerberosDelegationTokenAuthenticator.DELEGATION_PARAM, + dToken.encodeToUrlString()); + } else { + // delegation token will go as request header, setting it in the + // auth-token to ensure no authentication handshake is triggered + // (if we have a delegation token, we are authenticated) + // the delegation token header is injected in the connection request + // at the end of this method. + token.delegationToken = (org.apache.hadoop.security.token.Token + ) dToken; + } + } } } @@ -261,7 +309,14 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL { } url = augmentURL(url, extraParams); - return super.openConnection(url, token); + HttpURLConnection conn = super.openConnection(url, token); + if (!token.isSet() && !useQueryStringForDelegationToken() && dToken != null) { + // injecting the delegation token header in the connection request + conn.setRequestProperty( + DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER, + dToken.encodeToUrlString()); + } + return conn; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java index 670ec55..e4d9424 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java @@ -331,8 +331,7 @@ public abstract class DelegationTokenAuthenticationHandler HttpServletResponse response) throws IOException, AuthenticationException { AuthenticationToken token; - String delegationParam = ServletUtils.getParameter(request, - KerberosDelegationTokenAuthenticator.DELEGATION_PARAM); + String delegationParam = getDelegationToken(request); if (delegationParam != null) { try { Token dt = @@ -356,4 +355,15 @@ public abstract class DelegationTokenAuthenticationHandler return token; } + private String getDelegationToken(HttpServletRequest request) + throws IOException { + String dToken = request.getHeader( + DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER); + if (dToken == null) { + dToken = ServletUtils.getParameter(request, + KerberosDelegationTokenAuthenticator.DELEGATION_PARAM); + } + return dToken; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java index ec192da..18df56c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java @@ -56,6 +56,9 @@ public abstract class DelegationTokenAuthenticator implements Authenticator { public static final String OP_PARAM = "op"; + public static final String DELEGATION_TOKEN_HEADER = + "X-Hadoop-Delegation-Token"; + public static final String DELEGATION_PARAM = "delegation"; public static final String TOKEN_PARAM = "token"; public static final String RENEWER_PARAM = "renewer"; @@ -101,15 +104,23 @@ public abstract class DelegationTokenAuthenticator implements Authenticator { authenticator.setConnectionConfigurator(configurator); } - private boolean hasDelegationToken(URL url) { - String queryStr = url.getQuery(); - return (queryStr != null) && queryStr.contains(DELEGATION_PARAM + "="); + private boolean hasDelegationToken(URL url, AuthenticatedURL.Token token) { + boolean hasDt = false; + if (token instanceof DelegationTokenAuthenticatedURL.Token) { + hasDt = ((DelegationTokenAuthenticatedURL.Token) token). + getDelegationToken() != null; + } + if (!hasDt) { + String queryStr = url.getQuery(); + hasDt = (queryStr != null) && queryStr.contains(DELEGATION_PARAM + "="); + } + return hasDt; } @Override public void authenticate(URL url, AuthenticatedURL.Token token) throws IOException, AuthenticationException { - if (!hasDelegationToken(url)) { + if (!hasDelegationToken(url, token)) { authenticator.authenticate(url, token); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java index c9d255d..7880fa1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java @@ -284,11 +284,13 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { @Test public void testAuthenticate() throws Exception { - testValidDelegationToken(); - testInvalidDelegationToken(); + testValidDelegationTokenQueryString(); + testValidDelegationTokenHeader(); + testInvalidDelegationTokenQueryString(); + testInvalidDelegationTokenHeader(); } - private void testValidDelegationToken() throws Exception { + private void testValidDelegationTokenQueryString() throws Exception { HttpServletRequest request = Mockito.mock(HttpServletRequest.class); HttpServletResponse response = Mockito.mock(HttpServletResponse.class); Token dToken = @@ -307,7 +309,26 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { Assert.assertTrue(token.isExpired()); } - private void testInvalidDelegationToken() throws Exception { + private void testValidDelegationTokenHeader() throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Token dToken = + handler.getTokenManager().createToken( + UserGroupInformation.getCurrentUser(), "user"); + Mockito.when(request.getHeader(Mockito.eq( + DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER))).thenReturn( + dToken.encodeToUrlString()); + + AuthenticationToken token = handler.authenticate(request, response); + Assert.assertEquals(UserGroupInformation.getCurrentUser(). + getShortUserName(), token.getUserName()); + Assert.assertEquals(0, token.getExpires()); + Assert.assertEquals(handler.getType(), + token.getType()); + Assert.assertTrue(token.isExpired()); + } + + private void testInvalidDelegationTokenQueryString() throws Exception { HttpServletRequest request = Mockito.mock(HttpServletRequest.class); HttpServletResponse response = Mockito.mock(HttpServletResponse.class); Mockito.when(request.getQueryString()).thenReturn( @@ -323,4 +344,21 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks { } } + private void testInvalidDelegationTokenHeader() throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.when(request.getHeader(Mockito.eq( + DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER))).thenReturn( + "invalid"); + + try { + handler.authenticate(request, response); + Assert.fail(); + } catch (AuthenticationException ex) { + //NOP + } catch (Exception ex) { + Assert.fail(); + } + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java index 1b452f1..118abff 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java @@ -149,6 +149,15 @@ public class TestWebDelegationToken { throws ServletException, IOException { resp.setStatus(HttpServletResponse.SC_OK); resp.getWriter().write("ping"); + if (req.getHeader(DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER) + != null) { + resp.setHeader("UsingHeader", "true"); + } + if (req.getQueryString() != null && + req.getQueryString().contains( + DelegationTokenAuthenticator.DELEGATION_PARAM + "=")) { + resp.setHeader("UsingQueryString", "true"); + } } @Override @@ -314,7 +323,20 @@ public class TestWebDelegationToken { } @Test - public void testDelegationTokenAuthenticatorCalls() throws Exception { + public void testDelegationTokenAuthenticatorCallsWithHeader() + throws Exception { + testDelegationTokenAuthenticatorCalls(false); + } + + @Test + public void testDelegationTokenAuthenticatorCallsWithQueryString() + throws Exception { + testDelegationTokenAuthenticatorCalls(true); + } + + + private void testDelegationTokenAuthenticatorCalls(final boolean useQS) + throws Exception { final Server jetty = createJettyServer(); Context context = new Context(); context.setContextPath("/foo"); @@ -324,14 +346,15 @@ public class TestWebDelegationToken { try { jetty.start(); - URL nonAuthURL = new URL(getJettyURL() + "/foo/bar"); + final URL nonAuthURL = new URL(getJettyURL() + "/foo/bar"); URL authURL = new URL(getJettyURL() + "/foo/bar?authenticated=foo"); URL authURL2 = new URL(getJettyURL() + "/foo/bar?authenticated=bar"); DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token(); - DelegationTokenAuthenticatedURL aUrl = + final DelegationTokenAuthenticatedURL aUrl = new DelegationTokenAuthenticatedURL(); + aUrl.setUseQueryStringForDelegationToken(useQS); try { aUrl.getDelegationToken(nonAuthURL, token, FOO_USER); @@ -379,6 +402,27 @@ public class TestWebDelegationToken { Assert.assertTrue(ex.getMessage().contains("401")); } + aUrl.getDelegationToken(authURL, token, "foo"); + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + ugi.addToken(token.getDelegationToken()); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + HttpURLConnection conn = aUrl.openConnection(nonAuthURL, new DelegationTokenAuthenticatedURL.Token()); + Assert.assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + if (useQS) { + Assert.assertNull(conn.getHeaderField("UsingHeader")); + Assert.assertNotNull(conn.getHeaderField("UsingQueryString")); + } else { + Assert.assertNotNull(conn.getHeaderField("UsingHeader")); + Assert.assertNull(conn.getHeaderField("UsingQueryString")); + } + return null; + } + }); + + } finally { jetty.stop(); }