Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C39FB17D9C for ; Mon, 9 Mar 2015 20:56:27 +0000 (UTC) Received: (qmail 99944 invoked by uid 500); 9 Mar 2015 20:56:22 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 99877 invoked by uid 500); 9 Mar 2015 20:56:22 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 99868 invoked by uid 99); 9 Mar 2015 20:56:22 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 09 Mar 2015 20:56:22 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4CE70E1827; Mon, 9 Mar 2015 20:56:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zjshen@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-3287. Made TimelineClient put methods do as the correct login context. Contributed by Daryn Sharp and Jonathan Eagles. Date: Mon, 9 Mar 2015 20:56:22 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2 45cc7514f -> a94d23762 YARN-3287. Made TimelineClient put methods do as the correct login context. Contributed by Daryn Sharp and Jonathan Eagles. (cherry picked from commit d6e05c5ee26feefc17267b7c9db1e2a3dbdef117) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a94d2376 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a94d2376 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a94d2376 Branch: refs/heads/branch-2 Commit: a94d23762e2cf4211fe84661eb67504c7072db49 Parents: 45cc751 Author: Zhijie Shen Authored: Mon Mar 9 13:54:36 2015 -0700 Committer: Zhijie Shen Committed: Mon Mar 9 13:55:51 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../client/api/impl/TimelineClientImpl.java | 82 +++---- .../TestTimelineAuthenticationFilter.java | 221 +++++++++---------- 3 files changed, 135 insertions(+), 171 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a94d2376/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 026cfbb..702d4d6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -695,6 +695,9 @@ Release 2.7.0 - UNRELEASED YARN-3275. CapacityScheduler: Preemption happening on non-preemptable queues (Eric Payne via jlowe) + YARN-3287. Made TimelineClient put methods do as the correct login context. + (Daryn Sharp and Jonathan Eagles via zjshen) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/a94d2376/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index c05d65b..df6c7a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -47,6 +47,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.token.Token; @@ -108,6 +109,8 @@ public class TimelineClientImpl extends TimelineClient { private DelegationTokenAuthenticator authenticator; private DelegationTokenAuthenticatedURL.Token token; private URI resURI; + private UserGroupInformation authUgi; + private String doAsUser; @Private @VisibleForTesting @@ -252,6 +255,15 @@ public class TimelineClientImpl extends TimelineClient { } protected void serviceInit(Configuration conf) throws Exception { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation realUgi = ugi.getRealUser(); + if (realUgi != null) { + authUgi = realUgi; + doAsUser = ugi.getShortUserName(); + } else { + authUgi = ugi; + doAsUser = null; + } ClientConfig cc = new DefaultClientConfig(); cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); connConfigurator = newConnConfigurator(conf); @@ -301,16 +313,20 @@ public class TimelineClientImpl extends TimelineClient { doPosting(domain, "domain"); } - private ClientResponse doPosting(Object obj, String path) throws IOException, YarnException { + private ClientResponse doPosting(final Object obj, final String path) + throws IOException, YarnException { ClientResponse resp; try { - resp = doPostingObject(obj, path); - } catch (RuntimeException re) { - // runtime exception is expected if the client cannot connect the server - String msg = - "Failed to get the response from the timeline server."; - LOG.error(msg, re); - throw re; + resp = authUgi.doAs(new PrivilegedExceptionAction() { + @Override + public ClientResponse run() throws Exception { + return doPostingObject(obj, path); + } + }); + } catch (UndeclaredThrowableException e) { + throw new IOException(e.getCause()); + } catch (InterruptedException ie) { + throw new IOException(ie); } if (resp == null || resp.getClientResponseStatus() != ClientResponse.Status.OK) { @@ -331,11 +347,6 @@ public class TimelineClientImpl extends TimelineClient { @Override public Token getDelegationToken( final String renewer) throws IOException, YarnException { - boolean isProxyAccess = - UserGroupInformation.getCurrentUser().getAuthenticationMethod() - == UserGroupInformation.AuthenticationMethod.PROXY; - final String doAsUser = isProxyAccess ? - UserGroupInformation.getCurrentUser().getShortUserName() : null; PrivilegedExceptionAction> getDTAction = new PrivilegedExceptionAction>() { @@ -357,11 +368,6 @@ public class TimelineClientImpl extends TimelineClient { public long renewDelegationToken( final Token timelineDT) throws IOException, YarnException { - boolean isProxyAccess = - UserGroupInformation.getCurrentUser().getAuthenticationMethod() - == UserGroupInformation.AuthenticationMethod.PROXY; - final String doAsUser = isProxyAccess ? - UserGroupInformation.getCurrentUser().getShortUserName() : null; boolean useHttps = YarnConfiguration.useHttps(this.getConfig()); final String scheme = useHttps ? "https" : "http"; final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT); @@ -393,11 +399,6 @@ public class TimelineClientImpl extends TimelineClient { public void cancelDelegationToken( final Token timelineDT) throws IOException, YarnException { - boolean isProxyAccess = - UserGroupInformation.getCurrentUser().getAuthenticationMethod() - == UserGroupInformation.AuthenticationMethod.PROXY; - final String doAsUser = isProxyAccess ? - UserGroupInformation.getCurrentUser().getShortUserName() : null; boolean useHttps = YarnConfiguration.useHttps(this.getConfig()); final String scheme = useHttps ? "https" : "http"; final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT); @@ -433,15 +434,9 @@ public class TimelineClientImpl extends TimelineClient { @Override public Object run() throws IOException { // Try pass the request, if fail, keep retrying - boolean isProxyAccess = - UserGroupInformation.getCurrentUser().getAuthenticationMethod() - == UserGroupInformation.AuthenticationMethod.PROXY; - UserGroupInformation callerUGI = isProxyAccess ? - UserGroupInformation.getCurrentUser().getRealUser() - : UserGroupInformation.getCurrentUser(); - callerUGI.checkTGTAndReloginFromKeytab(); + authUgi.checkTGTAndReloginFromKeytab(); try { - return callerUGI.doAs(action); + return authUgi.doAs(action); } catch (UndeclaredThrowableException e) { throw new IOException(e.getCause()); } catch (InterruptedException e) { @@ -481,28 +476,15 @@ public class TimelineClientImpl extends TimelineClient { @Override public HttpURLConnection getHttpURLConnection(final URL url) throws IOException { - boolean isProxyAccess = - UserGroupInformation.getCurrentUser().getAuthenticationMethod() - == UserGroupInformation.AuthenticationMethod.PROXY; - UserGroupInformation callerUGI = isProxyAccess ? - UserGroupInformation.getCurrentUser().getRealUser() - : UserGroupInformation.getCurrentUser(); - final String doAsUser = isProxyAccess ? - UserGroupInformation.getCurrentUser().getShortUserName() : null; - callerUGI.checkTGTAndReloginFromKeytab(); + authUgi.checkTGTAndReloginFromKeytab(); try { - return callerUGI.doAs(new PrivilegedExceptionAction() { - @Override - public HttpURLConnection run() throws Exception { - return new DelegationTokenAuthenticatedURL( - authenticator, connConfigurator).openConnection(url, token, - doAsUser); - } - }); + return new DelegationTokenAuthenticatedURL( + authenticator, connConfigurator).openConnection(url, token, + doAsUser); } catch (UndeclaredThrowableException e) { throw new IOException(e.getCause()); - } catch (InterruptedException e) { - throw new IOException(e); + } catch (AuthenticationException ae) { + throw new IOException(ae); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a94d2376/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java index 53d8c81..c93e8f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java @@ -34,6 +34,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.KerberosTestUtils; +import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; @@ -47,9 +48,9 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -77,20 +78,19 @@ public class TestTimelineAuthenticationFilter { return Arrays.asList(new Object[][] { { false }, { true } }); } - private MiniKdc testMiniKDC; - private String keystoresDir; - private String sslConfDir; - private ApplicationHistoryServer testTimelineServer; - private Configuration conf; - private TimelineClient client; - private boolean withSsl; + private static MiniKdc testMiniKDC; + private static String keystoresDir; + private static String sslConfDir; + private static ApplicationHistoryServer testTimelineServer; + private static Configuration conf; + private static boolean withSsl; public TestTimelineAuthenticationFilter(boolean withSsl) { - this.withSsl = withSsl; + TestTimelineAuthenticationFilter.withSsl = withSsl; } - @Before - public void setup() { + @BeforeClass + public static void setup() { try { testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir); testMiniKDC.start(); @@ -127,6 +127,7 @@ public class TestTimelineAuthenticationFilter { "localhost:8190"); conf.set("hadoop.proxyuser.HTTP.hosts", "*"); conf.set("hadoop.proxyuser.HTTP.users", FOO_USER); + conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 1); if (withSsl) { conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, @@ -146,14 +147,17 @@ public class TestTimelineAuthenticationFilter { } catch (Exception e) { assertTrue("Couldn't setup TimelineServer", false); } + } - client = TimelineClient.createTimelineClient(); + private TimelineClient createTimelineClientForUGI() { + TimelineClient client = TimelineClient.createTimelineClient(); client.init(conf); client.start(); + return client; } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { if (testMiniKDC != null) { testMiniKDC.stop(); } @@ -162,10 +166,6 @@ public class TestTimelineAuthenticationFilter { testTimelineServer.stop(); } - if (client != null) { - client.stop(); - } - if (withSsl) { KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); File base = new File(BASEDIR); @@ -178,6 +178,7 @@ public class TestTimelineAuthenticationFilter { KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable() { @Override public Void call() throws Exception { + TimelineClient client = createTimelineClientForUGI(); TimelineEntity entityToStore = new TimelineEntity(); entityToStore.setEntityType( TestTimelineAuthenticationFilter.class.getName()); @@ -199,6 +200,7 @@ public class TestTimelineAuthenticationFilter { KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable() { @Override public Void call() throws Exception { + TimelineClient client = createTimelineClientForUGI(); TimelineDomain domainToStore = new TimelineDomain(); domainToStore.setId(TestTimelineAuthenticationFilter.class.getName()); domainToStore.setReaders("*"); @@ -215,119 +217,96 @@ public class TestTimelineAuthenticationFilter { @Test public void testDelegationTokenOperations() throws Exception { - KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable() { - @Override - public Void call() throws Exception { - // Let HTTP user to get the delegation for itself - Token token = - client.getDelegationToken( - UserGroupInformation.getCurrentUser().getShortUserName()); - Assert.assertNotNull(token); - TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier(); - Assert.assertNotNull(tDT); - Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner()); + TimelineClient httpUserClient = + KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable() { + @Override + public TimelineClient call() throws Exception { + return createTimelineClientForUGI(); + } + }); + UserGroupInformation httpUser = + KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable() { + @Override + public UserGroupInformation call() throws Exception { + return UserGroupInformation.getCurrentUser(); + } + }); + // Let HTTP user to get the delegation for itself + Token token = + httpUserClient.getDelegationToken(httpUser.getShortUserName()); + Assert.assertNotNull(token); + TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier(); + Assert.assertNotNull(tDT); + Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner()); - // Renew token - long renewTime1 = client.renewDelegationToken(token); - Thread.sleep(100); - long renewTime2 = client.renewDelegationToken(token); - Assert.assertTrue(renewTime1 < renewTime2); + // Renew token + long renewTime1 = httpUserClient.renewDelegationToken(token); + Thread.sleep(100); + long renewTime2 = httpUserClient.renewDelegationToken(token); + Assert.assertTrue(renewTime1 < renewTime2); - // Cancel token - client.cancelDelegationToken(token); - // Renew should not be successful because the token is canceled - try { - client.renewDelegationToken(token); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains( - "Renewal request for unknown token")); - } + // Cancel token + httpUserClient.cancelDelegationToken(token); + // Renew should not be successful because the token is canceled + try { + httpUserClient.renewDelegationToken(token); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains( + "Renewal request for unknown token")); + } - // Let HTTP user to get the delegation token for FOO user - UserGroupInformation fooUgi = UserGroupInformation.createProxyUser( - FOO_USER, UserGroupInformation.getCurrentUser()); - token = fooUgi.doAs( - new PrivilegedExceptionAction>() { + // Let HTTP user to get the delegation token for FOO user + UserGroupInformation fooUgi = UserGroupInformation.createProxyUser( + FOO_USER, httpUser); + TimelineClient fooUserClient = fooUgi.doAs( + new PrivilegedExceptionAction() { @Override - public Token run() - throws Exception { - return client.getDelegationToken( - UserGroupInformation.getCurrentUser().getShortUserName()); + public TimelineClient run() throws Exception { + return createTimelineClientForUGI(); } }); - Assert.assertNotNull(token); - tDT = token.decodeIdentifier(); - Assert.assertNotNull(tDT); - Assert.assertEquals(new Text(FOO_USER), tDT.getOwner()); - Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser()); + token = fooUserClient.getDelegationToken(httpUser.getShortUserName()); + Assert.assertNotNull(token); + tDT = token.decodeIdentifier(); + Assert.assertNotNull(tDT); + Assert.assertEquals(new Text(FOO_USER), tDT.getOwner()); + Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser()); - // Renew token - final Token tokenToRenew = token; - renewTime1 = fooUgi.doAs( - new PrivilegedExceptionAction() { - @Override - public Long run() throws Exception { - return client.renewDelegationToken(tokenToRenew); - } - }); - renewTime2 = fooUgi.doAs( - new PrivilegedExceptionAction() { - @Override - public Long run() throws Exception { - return client.renewDelegationToken(tokenToRenew); - } - }); - Assert.assertTrue(renewTime1 < renewTime2); + // Renew token as the renewer + final Token tokenToRenew = token; + renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew); + renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew); + Assert.assertTrue(renewTime1 < renewTime2); - // Cancel token - fooUgi.doAs( - new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - client.cancelDelegationToken(tokenToRenew); - return null; - } - }); - // Renew should not be successful because the token is canceled - try { - fooUgi.doAs( - new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - client.renewDelegationToken(tokenToRenew); - return null; - } - }); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains( - "Renewal request for unknown token")); - } + // Cancel token + fooUserClient.cancelDelegationToken(tokenToRenew); + + // Renew should not be successful because the token is canceled + try { + httpUserClient.renewDelegationToken(tokenToRenew); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage().contains("Renewal request for unknown token")); + } - // Let HTTP user to get the delegation token for BAR user - UserGroupInformation barUgi = UserGroupInformation.createProxyUser( - BAR_USER, UserGroupInformation.getCurrentUser()); - token = barUgi.doAs( - new PrivilegedExceptionAction>() { + // Let HTTP user to get the delegation token for BAR user + UserGroupInformation barUgi = UserGroupInformation.createProxyUser( + BAR_USER, httpUser); + TimelineClient barUserClient = barUgi.doAs( + new PrivilegedExceptionAction() { @Override - public Token run() - throws Exception { - try { - Token token = - client.getDelegationToken( - UserGroupInformation.getCurrentUser().getShortUserName()); - Assert.fail(); - return token; - } catch (Exception e) { - Assert.assertTrue(e instanceof AuthorizationException); - return null; - } + public TimelineClient run() { + return createTimelineClientForUGI(); } }); - return null; - } - }); - } + try { + barUserClient.getDelegationToken(httpUser.getShortUserName()); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getCause() instanceof AuthorizationException || e.getCause() instanceof AuthenticationException); + } + } }