From common-commits-return-82169-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Wed May 2 09:25:09 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id C2D1318065D for ; Wed, 2 May 2018 09:25:08 +0200 (CEST) Received: (qmail 32283 invoked by uid 500); 2 May 2018 07:25:07 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 32270 invoked by uid 99); 2 May 2018 07:25:07 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 May 2018 07:25:07 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 74249DFF7E; Wed, 2 May 2018 07:25:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aajisaka@apache.org To: common-commits@hadoop.apache.org Message-Id: <9aaa97aaee95420e81b3e9d075da6867@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal Date: Wed, 2 May 2018 07:25:07 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-3.1 5fef28d0d -> 254933cf3 MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal Signed-off-by: Akira Ajisaka (cherry picked from commit 1a95a4524a8c6c7be601ce8b92640a6a76164a2c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/254933cf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/254933cf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/254933cf Branch: refs/heads/branch-3.1 Commit: 254933cf310fddc0975217575f946c8edac77170 Parents: 5fef28d Author: Bibin A Chundatt Authored: Wed May 2 16:14:28 2018 +0900 Committer: Akira Ajisaka Committed: Wed May 2 16:15:18 2018 +0900 ---------------------------------------------------------------------- .../hadoop/mapreduce/security/TokenCache.java | 14 +++++++++----- .../hadoop/mapreduce/security/TestTokenCache.java | 18 +++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/254933cf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 12fced9..1156c67 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -96,8 +97,9 @@ public class TokenCache { for(Path p: ps) { fsSet.add(p.getFileSystem(conf)); } + String masterPrincipal = Master.getMasterPrincipal(conf); for (FileSystem fs : fsSet) { - obtainTokensForNamenodesInternal(fs, credentials, conf); + obtainTokensForNamenodesInternal(fs, credentials, conf, masterPrincipal); } } @@ -122,15 +124,17 @@ public class TokenCache { * @param conf * @throws IOException */ - static void obtainTokensForNamenodesInternal(FileSystem fs, - Credentials credentials, Configuration conf) throws IOException { + static void obtainTokensForNamenodesInternal(FileSystem fs, + Credentials credentials, Configuration conf, String renewer) + throws IOException { // RM skips renewing token with empty renewer String delegTokenRenewer = ""; if (!isTokenRenewalExcluded(fs, conf)) { - delegTokenRenewer = Master.getMasterPrincipal(conf); - if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { + if (StringUtils.isEmpty(renewer)) { throw new IOException( "Can't get Master Kerberos principal for use as renewer"); + } else { + delegTokenRenewer = renewer; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/254933cf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java index 127f8ae..a44e533 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -56,8 +56,8 @@ public class TestTokenCache { @Test public void testObtainTokens() throws Exception { Credentials credentials = new Credentials(); - FileSystem fs = mock(FileSystem.class); - TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf); + FileSystem fs = mock(FileSystem.class); + TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf, renewer); verify(fs).addDelegationTokens(eq(renewer), eq(credentials)); } @@ -105,23 +105,23 @@ public class TestTokenCache { checkToken(creds, newerToken1); // get token for fs1, see that fs2's token was loaded - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); checkToken(creds, newerToken1, token2); // get token for fs2, nothing should change since already present - TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); checkToken(creds, newerToken1, token2); // get token for fs3, should only add token for fs3 - TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); Token token3 = creds.getToken(new Text(fs3.getCanonicalServiceName())); assertTrue(token3 != null); checkToken(creds, newerToken1, token2, token3); // be paranoid, check one last time that nothing changes - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); - TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); - TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); checkToken(creds, newerToken1, token2, token3); } @@ -202,7 +202,7 @@ public class TestTokenCache { // wait to set, else the obtain tokens call above will fail with FNF conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); String fs_addr = fs1.getCanonicalServiceName(); Token nnt = TokenCache.getDelegationToken(creds, fs_addr); assertNotNull("Token for nn is null", nnt); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org