hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aajis...@apache.org
Subject hadoop git commit: MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal
Date Wed, 02 May 2018 07:35:22 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 221db515a -> e6e2146ae


MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal

Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
(cherry picked from commit 1a95a4524a8c6c7be601ce8b92640a6a76164a2c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6e2146a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6e2146a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6e2146a

Branch: refs/heads/branch-3.0
Commit: e6e2146ae0acaade1c65110427e3c7aa5eadbc28
Parents: 221db51
Author: Bibin A Chundatt <bibinchundatt@apache.org>
Authored: Wed May 2 16:14:28 2018 +0900
Committer: Akira Ajisaka <aajisaka@apache.org>
Committed: Wed May 2 16:35:18 2018 +0900

----------------------------------------------------------------------
 .../hadoop/mapreduce/security/TokenCache.java     | 14 +++++++++-----
 .../hadoop/mapreduce/security/TestTokenCache.java | 18 +++++++++---------
 2 files changed, 18 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e2146a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
index 12fced9..1156c67 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -96,8 +97,9 @@ public class TokenCache {
     for(Path p: ps) {
       fsSet.add(p.getFileSystem(conf));
     }
+    String masterPrincipal = Master.getMasterPrincipal(conf);
     for (FileSystem fs : fsSet) {
-      obtainTokensForNamenodesInternal(fs, credentials, conf);
+      obtainTokensForNamenodesInternal(fs, credentials, conf, masterPrincipal);
     }
   }
 
@@ -122,15 +124,17 @@ public class TokenCache {
    * @param conf
    * @throws IOException
    */
-  static void obtainTokensForNamenodesInternal(FileSystem fs, 
-      Credentials credentials, Configuration conf) throws IOException {
+  static void obtainTokensForNamenodesInternal(FileSystem fs,
+      Credentials credentials, Configuration conf, String renewer)
+      throws IOException {
     // RM skips renewing token with empty renewer
     String delegTokenRenewer = "";
     if (!isTokenRenewalExcluded(fs, conf)) {
-      delegTokenRenewer = Master.getMasterPrincipal(conf);
-      if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
+      if (StringUtils.isEmpty(renewer)) {
         throw new IOException(
             "Can't get Master Kerberos principal for use as renewer");
+      } else {
+        delegTokenRenewer = renewer;
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e2146a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
index 127f8ae..a44e533 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
@@ -56,8 +56,8 @@ public class TestTokenCache {
   @Test
   public void testObtainTokens() throws Exception {
     Credentials credentials = new Credentials();
-    FileSystem fs = mock(FileSystem.class);  
-    TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf);
+    FileSystem fs = mock(FileSystem.class);
+    TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf, renewer);
     verify(fs).addDelegationTokens(eq(renewer), eq(credentials));
   }
 
@@ -105,23 +105,23 @@ public class TestTokenCache {
     checkToken(creds, newerToken1);
     
     // get token for fs1, see that fs2's token was loaded 
-    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
+    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer);
     checkToken(creds, newerToken1, token2);
     
     // get token for fs2, nothing should change since already present
-    TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf);
+    TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer);
     checkToken(creds, newerToken1, token2);
     
     // get token for fs3, should only add token for fs3
-    TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf);
+    TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer);
     Token<?> token3 = creds.getToken(new Text(fs3.getCanonicalServiceName()));
     assertTrue(token3 != null);
     checkToken(creds, newerToken1, token2, token3);
     
     // be paranoid, check one last time that nothing changes
-    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
-    TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf);
-    TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf);
+    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer);
+    TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer);
+    TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer);
     checkToken(creds, newerToken1, token2, token3);
   }
 
@@ -202,7 +202,7 @@ public class TestTokenCache {
     // wait to set, else the obtain tokens call above will fail with FNF
     conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile);
     creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
-    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
+    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer);
     String fs_addr = fs1.getCanonicalServiceName();
     Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr);
     assertNotNull("Token for nn is null", nnt);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message