Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 436C5200B36 for ; Wed, 22 Jun 2016 02:26:08 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 42040160A6A; Wed, 22 Jun 2016 00:26:08 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 48184160A60 for ; Wed, 22 Jun 2016 02:26:07 +0200 (CEST) Received: (qmail 63496 invoked by uid 500); 22 Jun 2016 00:26:00 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 61916 invoked by uid 99); 22 Jun 2016 00:25:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jun 2016 00:25:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6E7A6E9438; Wed, 22 Jun 2016 00:25:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: subru@apache.org To: common-commits@hadoop.apache.org Date: Wed, 22 Jun 2016 00:26:19 -0000 Message-Id: <6e2c4c7968a5490f81623a7ef363dae4@git.apache.org> In-Reply-To: <85a9aecfdfad441186f9f7f2137dd235@git.apache.org> References: <85a9aecfdfad441186f9f7f2137dd235@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [21/51] [abbrv] hadoop git commit: HADOOP-13255. KMSClientProvider should check and renew tgt when doing delegation token operations. Contributed by Xiao Chen. archived-at: Wed, 22 Jun 2016 00:26:08 -0000 HADOOP-13255. KMSClientProvider should check and renew tgt when doing delegation token operations. Contributed by Xiao Chen. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1674caa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1674caa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1674caa Branch: refs/heads/YARN-2915 Commit: b1674caa409ca2c616207acb72aeb2767d28b10c Parents: 127d2c7 Author: Xiaoyu Yao Authored: Thu Jun 16 15:22:00 2016 -0700 Committer: Xiaoyu Yao Committed: Thu Jun 16 15:22:00 2016 -0700 ---------------------------------------------------------------------- .../crypto/key/kms/KMSClientProvider.java | 2 - .../hadoop/security/UserGroupInformation.java | 2 +- .../web/DelegationTokenAuthenticator.java | 3 + .../hadoop/crypto/key/kms/server/TestKMS.java | 91 +++++++++++++++++--- .../src/test/resources/log4j.properties | 2 +- .../java/org/apache/hadoop/minikdc/MiniKdc.java | 11 ++- 6 files changed, 95 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1674caa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index f4103b4..7e06ddd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -536,8 +536,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, UserGroupInformation.AuthenticationMethod.PROXY) ? currentUgi.getShortUserName() : null; - // check and renew TGT to handle potential expiration - actualUgi.checkTGTAndReloginFromKeytab(); // creating the HTTP connection using the current UGI at constructor time conn = actualUgi.doAs(new PrivilegedExceptionAction() { @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1674caa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 798aa01..93822a1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -103,7 +103,7 @@ public class UserGroupInformation { * @param immediate true if we should login without waiting for ticket window */ @VisibleForTesting - static void setShouldRenewImmediatelyForTests(boolean immediate) { + public static void setShouldRenewImmediatelyForTests(boolean immediate) { shouldRenewImmediatelyForTests = immediate; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1674caa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java index 46a0b1f..53978a6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java @@ -20,6 +20,7 @@ package org.apache.hadoop.security.token.delegation.web; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.Authenticator; @@ -143,6 +144,8 @@ public abstract class DelegationTokenAuthenticator implements Authenticator { public void authenticate(URL url, AuthenticatedURL.Token token) throws IOException, AuthenticationException { if (!hasDelegationToken(url, token)) { + // check and renew TGT to handle potential expiration + UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab(); authenticator.authenticate(url, token); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1674caa/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index db34aa9..94b9d06 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -42,12 +42,9 @@ import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.log4j.Level; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -88,11 +85,11 @@ public class TestKMS { public final Timeout testTimeout = new Timeout(180000); @Before - public void cleanUp() { + public void setUp() throws Exception { + setUpMiniKdc(); // resetting kerberos security Configuration conf = new Configuration(); UserGroupInformation.setConfiguration(conf); - GenericTestUtils.setLogLevel(LOG, Level.INFO); } public static File getTestDir() throws Exception { @@ -232,10 +229,8 @@ public class TestKMS { private static MiniKdc kdc; private static File keytab; - @BeforeClass - public static void setUpMiniKdc() throws Exception { + private static void setUpMiniKdc(Properties kdcConf) throws Exception { File kdcDir = getTestDir(); - Properties kdcConf = MiniKdc.createConf(); kdc = new MiniKdc(kdcConf, kdcDir); kdc.start(); keytab = new File(kdcDir, "keytab"); @@ -255,11 +250,18 @@ public class TestKMS { principals.toArray(new String[principals.size()])); } - @AfterClass - public static void tearDownMiniKdc() throws Exception { + private void setUpMiniKdc() throws Exception { + Properties kdcConf = MiniKdc.createConf(); + setUpMiniKdc(kdcConf); + } + + @After + public void tearDownMiniKdc() throws Exception { if (kdc != null) { kdc.stop(); + kdc = null; } + UserGroupInformation.setShouldRenewImmediatelyForTests(false); } private T doAs(String user, final PrivilegedExceptionAction action) @@ -2053,6 +2055,73 @@ public class TestKMS { doWebHDFSProxyUserTest(false); } + @Test + public void testTGTRenewal() throws Exception { + tearDownMiniKdc(); + Properties kdcConf = MiniKdc.createConf(); + kdcConf.setProperty(MiniKdc.MAX_TICKET_LIFETIME, "3"); + kdcConf.setProperty(MiniKdc.MIN_TICKET_LIFETIME, "3"); + setUpMiniKdc(kdcConf); + + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(conf); + final File testDir = getTestDir(); + conf = createBaseKMSConf(testDir); + conf.set("hadoop.kms.authentication.type", "kerberos"); + conf.set("hadoop.kms.authentication.kerberos.keytab", + keytab.getAbsolutePath()); + conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost"); + conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT"); + conf.set("hadoop.kms.proxyuser.client.users", "*"); + conf.set("hadoop.kms.proxyuser.client.hosts", "*"); + writeConf(testDir, conf); + + runServer(null, null, testDir, new KMSCallable() { + @Override + public Void call() throws Exception { + final Configuration conf = new Configuration(); + final URI uri = createKMSUri(getKMSUrl()); + UserGroupInformation.setShouldRenewImmediatelyForTests(true); + UserGroupInformation + .loginUserFromKeytab("client", keytab.getAbsolutePath()); + final UserGroupInformation clientUgi = + UserGroupInformation.getCurrentUser(); + clientUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + // Verify getKeys can relogin + Thread.sleep(3100); + KeyProvider kp = createProvider(uri, conf); + kp.getKeys(); + + // Verify addDelegationTokens can relogin + // (different code path inside KMSClientProvider than getKeys) + Thread.sleep(3100); + kp = createProvider(uri, conf); + ((KeyProviderDelegationTokenExtension.DelegationTokenExtension) kp) + .addDelegationTokens("myuser", new Credentials()); + + // Verify getKeys can relogin with proxy user + UserGroupInformation anotherUgi = + UserGroupInformation.createProxyUser("client1", clientUgi); + anotherUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + Thread.sleep(3100); + KeyProvider kp = createProvider(uri, conf); + kp.getKeys(); + return null; + } + }); + return null; + } + }); + return null; + } + }); + } + public void doWebHDFSProxyUserTest(final boolean kerberos) throws Exception { Configuration conf = new Configuration(); conf.set("hadoop.security.authentication", "kerberos"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1674caa/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties b/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties index 5cd037a..b347d27 100644 --- a/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties +++ b/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties @@ -22,7 +22,7 @@ log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n -log4j.rootLogger=WARN, stdout +log4j.rootLogger=INFO, stdout log4j.logger.org.apache.hadoop.conf=ERROR log4j.logger.org.apache.hadoop.crytpo.key.kms.server=ALL log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1674caa/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java b/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java index 9278642..281b3cc 100644 --- a/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java +++ b/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java @@ -147,6 +147,7 @@ public class MiniKdc { public static final String KDC_PORT = "kdc.port"; public static final String INSTANCE = "instance"; public static final String MAX_TICKET_LIFETIME = "max.ticket.lifetime"; + public static final String MIN_TICKET_LIFETIME = "min.ticket.lifetime"; public static final String MAX_RENEWABLE_LIFETIME = "max.renewable.lifetime"; public static final String TRANSPORT = "transport"; public static final String DEBUG = "debug"; @@ -280,7 +281,7 @@ public class MiniKdc { simpleKdc.init(); resetDefaultRealm(); simpleKdc.start(); - LOG.info("MiniKdc stated."); + LOG.info("MiniKdc started."); } private void resetDefaultRealm() throws IOException { @@ -321,6 +322,14 @@ public class MiniKdc { if (conf.getProperty(DEBUG) != null) { krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG, conf.getProperty(DEBUG)); } + if (conf.getProperty(MIN_TICKET_LIFETIME) != null) { + simpleKdc.getKdcConfig().setLong(KdcConfigKey.MINIMUM_TICKET_LIFETIME, + Long.parseLong(conf.getProperty(MIN_TICKET_LIFETIME))); + } + if (conf.getProperty(MAX_TICKET_LIFETIME) != null) { + simpleKdc.getKdcConfig().setLong(KdcConfigKey.MAXIMUM_TICKET_LIFETIME, + Long.parseLong(conf.getProperty(MiniKdc.MAX_TICKET_LIFETIME))); + } } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org