hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aajis...@apache.org
Subject hadoop git commit: HDFS-7597. DelegationTokenIdentifier should cache the TokenIdentifier to UGI mapping. Contributed by Daryn Sharp, Bob Hansen, and Xiao Chen.
Date Wed, 22 Jun 2016 02:03:18 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk d8107fcd1 -> d433b16ce


HDFS-7597. DelegationTokenIdentifier should cache the TokenIdentifier to UGI mapping. Contributed
by Daryn Sharp, Bob Hansen, and Xiao Chen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d433b16c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d433b16c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d433b16c

Branch: refs/heads/trunk
Commit: d433b16ce6d74f1a44bc29446c74b1cb5f8a10fa
Parents: d8107fc
Author: Akira Ajisaka <aajisaka@apache.org>
Authored: Wed Jun 22 11:00:16 2016 +0900
Committer: Akira Ajisaka <aajisaka@apache.org>
Committed: Wed Jun 22 11:00:16 2016 +0900

----------------------------------------------------------------------
 .../delegation/DelegationTokenIdentifier.java   | 26 ++++++++++++++++
 .../web/webhdfs/DataNodeUGIProvider.java        |  7 +++++
 .../hdfs/security/TestDelegationToken.java      | 31 +++++++++++++++++++-
 .../hdfs/server/common/TestJspHelper.java       |  8 ++++-
 .../web/webhdfs/TestDataNodeUGIProvider.java    |  1 +
 5 files changed, 71 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
index 7bb0748..d753174 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
@@ -21,13 +21,20 @@ package org.apache.hadoop.hdfs.security.token.delegation;
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
 
+import org.apache.commons.collections.map.LRUMap;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.web.WebHdfsConstants;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * A delegation token identifier that is specific to HDFS.
  */
@@ -37,6 +44,15 @@ public class DelegationTokenIdentifier
   public static final Text HDFS_DELEGATION_KIND =
       new Text("HDFS_DELEGATION_TOKEN");
 
+  @SuppressWarnings("unchecked")
+  private static Map<TokenIdentifier, UserGroupInformation> ugiCache =
+      Collections.synchronizedMap(new LRUMap(64));
+
+  @VisibleForTesting
+  public void clearCache() {
+    ugiCache.clear();
+  }
+
   /**
    * Create an empty delegation token identifier for reading into.
    */
@@ -59,6 +75,16 @@ public class DelegationTokenIdentifier
   }
 
   @Override
+  public UserGroupInformation getUser() {
+    UserGroupInformation ugi = ugiCache.get(this);
+    if (ugi == null) {
+      ugi = super.getUser();
+      ugiCache.put(this, ugi);
+    }
+    return ugi;
+  }
+
+  @Override
   public String toString() {
     StringBuilder sbld = new StringBuilder();
     sbld

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
index 233ba69..fa6f676 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
@@ -61,6 +61,13 @@ public class DataNodeUGIProvider {
     }
   }
 
+  @VisibleForTesting
+  void clearCache() throws IOException {
+    if (UserGroupInformation.isSecurityEnabled()) {
+      params.delegationToken().decodeIdentifier().clearCache();
+    }
+  }
+
   UserGroupInformation ugi() throws IOException {
     UserGroupInformation ugi;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
index 0600536..6112b6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
@@ -239,7 +239,36 @@ public class TestDelegationToken {
       }
     });
   }
-  
+
+  @Test
+  public void testDelegationTokenUgi() throws Exception {
+    final DistributedFileSystem dfs = cluster.getFileSystem();
+    Token<?>[] tokens = dfs.addDelegationTokens("renewer", null);
+    Assert.assertEquals(1, tokens.length);
+    Token<?> token1 = tokens[0];
+    DelegationTokenIdentifier ident =
+        (DelegationTokenIdentifier) token1.decodeIdentifier();
+    UserGroupInformation expectedUgi = ident.getUser();
+
+    // get 2 new instances (clones) of the identifier, query their ugi
+    // twice each, all ugi instances should be equivalent
+    for (int i=0; i<2; i++) {
+      DelegationTokenIdentifier identClone =
+          (DelegationTokenIdentifier)token1.decodeIdentifier();
+      Assert.assertEquals(ident, identClone);
+      Assert.assertNotSame(ident, identClone);
+      Assert.assertSame(expectedUgi, identClone.getUser());
+      Assert.assertSame(expectedUgi, identClone.getUser());
+    }
+
+    // a new token must decode to a different ugi instance than the first token
+    tokens = dfs.addDelegationTokens("renewer", null);
+    Assert.assertEquals(1, tokens.length);
+    Token<?> token2 = tokens[0];
+    Assert.assertNotEquals(token1, token2);
+    Assert.assertNotSame(expectedUgi, token2.decodeIdentifier().getUser());
+  }
+
   /**
    * Test that the delegation token secret manager only runs when the
    * NN is out of safe mode. This is because the secret manager

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
index d0d8d3e..119db8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
@@ -109,6 +109,7 @@ public class TestJspHelper {
     
     //Test attribute name.node.address 
     //Set the nnaddr url parameter to null.
+    token.decodeIdentifier().clearCache();
     when(request.getParameter(JspHelper.NAMENODE_ADDRESS)).thenReturn(null);
     InetSocketAddress addr = new InetSocketAddress("localhost", 2222);
     when(context.getAttribute(NameNodeHttpServer.NAMENODE_ADDRESS_ATTRIBUTE_KEY))
@@ -116,7 +117,12 @@ public class TestJspHelper {
     verifyServiceInToken(context, request, addr.getAddress().getHostAddress()
         + ":2222");
     
-    //Test service already set in the token
+    //Test service already set in the token and DN doesn't change service
+    //when it doesn't know the NN service addr
+    userText = new Text(user+"2");
+    dtId = new DelegationTokenIdentifier(userText, userText, null);
+    token = new Token<DelegationTokenIdentifier>(
+        dtId, new DummySecretManager(0, 0, 0, 0));
     token.setService(new Text("3.3.3.3:3333"));
     tokenString = token.encodeToUrlString();
     //Set the name.node.address attribute in Servlet context to null

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
index bce5422..70618b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
@@ -121,6 +121,7 @@ public class TestDataNodeUGIProvider {
         "With UGI cache, two UGIs for the different token should not be same",
         ugi11, url22);
 
+    ugiProvider2.clearCache();
     awaitCacheEmptyDueToExpiration();
     ugi12 = ugiProvider1.ugi();
     url22 = ugiProvider2.ugi();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message