From common-commits-return-82172-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Wed May 2 09:46:06 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 55F6F18065D for ; Wed, 2 May 2018 09:46:05 +0200 (CEST) Received: (qmail 64152 invoked by uid 500); 2 May 2018 07:46:04 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 64143 invoked by uid 99); 2 May 2018 07:46:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 May 2018 07:46:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0ECC4E0AED; Wed, 2 May 2018 07:46:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aajisaka@apache.org To: common-commits@hadoop.apache.org Message-Id: <43c08c9c122c41a3b7a756209b742b19@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal Date: Wed, 2 May 2018 07:46:04 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.9 b5f13f6dc -> e5f47dab7 MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal Signed-off-by: Akira Ajisaka (cherry picked from commit 1a95a4524a8c6c7be601ce8b92640a6a76164a2c) (cherry picked from commit 58bcb90e7039bd4cec116fe66f1b1428dd493372) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5f47dab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5f47dab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5f47dab Branch: refs/heads/branch-2.9 Commit: e5f47dab720e369987cc3a4baff1f8a341a60a35 Parents: b5f13f6 Author: Bibin A Chundatt Authored: Wed May 2 16:14:28 2018 +0900 Committer: Akira Ajisaka Committed: Wed May 2 16:45:58 2018 +0900 ---------------------------------------------------------------------- .../hadoop/mapreduce/security/TokenCache.java | 14 +++++++++----- .../hadoop/mapreduce/security/TestTokenCache.java | 18 +++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5f47dab/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 78f6c16..8cd407c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -96,8 +97,9 @@ public class TokenCache { for(Path p: ps) { fsSet.add(p.getFileSystem(conf)); } + String masterPrincipal = Master.getMasterPrincipal(conf); for (FileSystem fs : fsSet) { - obtainTokensForNamenodesInternal(fs, credentials, conf); + obtainTokensForNamenodesInternal(fs, credentials, conf, masterPrincipal); } } @@ -122,15 +124,17 @@ public class TokenCache { * @param conf * @throws IOException */ - static void obtainTokensForNamenodesInternal(FileSystem fs, - Credentials credentials, Configuration conf) throws IOException { + static void obtainTokensForNamenodesInternal(FileSystem fs, + Credentials credentials, Configuration conf, String renewer) + throws IOException { // RM skips renewing token with empty renewer String delegTokenRenewer = ""; if (!isTokenRenewalExcluded(fs, conf)) { - delegTokenRenewer = Master.getMasterPrincipal(conf); - if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { + if (StringUtils.isEmpty(renewer)) { throw new IOException( "Can't get Master Kerberos principal for use as renewer"); + } else { + delegTokenRenewer = renewer; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5f47dab/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java index 127f8ae..a44e533 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -56,8 +56,8 @@ public class TestTokenCache { @Test public void testObtainTokens() throws Exception { Credentials credentials = new Credentials(); - FileSystem fs = mock(FileSystem.class); - TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf); + FileSystem fs = mock(FileSystem.class); + TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf, renewer); verify(fs).addDelegationTokens(eq(renewer), eq(credentials)); } @@ -105,23 +105,23 @@ public class TestTokenCache { checkToken(creds, newerToken1); // get token for fs1, see that fs2's token was loaded - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); checkToken(creds, newerToken1, token2); // get token for fs2, nothing should change since already present - TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); checkToken(creds, newerToken1, token2); // get token for fs3, should only add token for fs3 - TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); Token token3 = creds.getToken(new Text(fs3.getCanonicalServiceName())); assertTrue(token3 != null); checkToken(creds, newerToken1, token2, token3); // be paranoid, check one last time that nothing changes - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); - TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); - TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); checkToken(creds, newerToken1, token2, token3); } @@ -202,7 +202,7 @@ public class TestTokenCache { // wait to set, else the obtain tokens call above will fail with FNF conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); String fs_addr = fs1.getCanonicalServiceName(); Token nnt = TokenCache.getDelegationToken(creds, fs_addr); assertNotNull("Token for nn is null", nnt); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org