Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8EB2A73AC for ; Sat, 13 Aug 2011 20:22:12 +0000 (UTC) Received: (qmail 37976 invoked by uid 500); 13 Aug 2011 20:22:12 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 37879 invoked by uid 500); 13 Aug 2011 20:22:11 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 37868 invoked by uid 99); 13 Aug 2011 20:22:11 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 Aug 2011 20:22:11 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 Aug 2011 20:22:09 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id D8804238885D; Sat, 13 Aug 2011 20:21:50 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1157421 - in /hadoop/common/branches/MR-279/mapreduce: ./ mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/ src/test/mapred/org/apache/hadoop/mapreduce/security/ Date: Sat, 13 Aug 2011 20:21:50 -0000 To: mapreduce-commits@hadoop.apache.org From: acmurthy@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110813202150.D8804238885D@eris.apache.org> Author: acmurthy Date: Sat Aug 13 20:21:50 2011 New Revision: 1157421 URL: http://svn.apache.org/viewvc?rev=1157421&view=rev Log: MAPREDUCE-2839. Fixed TokenCache to get delegation tokens using both new and old apis. Contributed by Siddharth Seth. Modified: hadoop/common/branches/MR-279/mapreduce/CHANGES.txt hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java hadoop/common/branches/MR-279/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java Modified: hadoop/common/branches/MR-279/mapreduce/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/CHANGES.txt?rev=1157421&r1=1157420&r2=1157421&view=diff ============================================================================== --- hadoop/common/branches/MR-279/mapreduce/CHANGES.txt (original) +++ hadoop/common/branches/MR-279/mapreduce/CHANGES.txt Sat Aug 13 20:21:50 2011 @@ -4,6 +4,9 @@ Trunk (unreleased changes) MAPREDUCE-279 + MAPREDUCE-2839. Fixed TokenCache to get delegation tokens using both new + and old apis. (Siddharth Seth via acmurthy) + MAPREDUCE-2639. Bug fixes in speculate.DataStatistics. (Josh Wills via acmurthy) Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java?rev=1157421&r1=1157420&r2=1157421&view=diff ============================================================================== --- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java (original) +++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java Sat Aug 13 20:21:50 2011 @@ -19,7 +19,7 @@ package org.apache.hadoop.mapreduce.security; import java.io.IOException; -import java.net.URI; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -28,10 +28,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Master; @@ -131,6 +128,16 @@ public class TokenCache { return; } } + List> tokens = fs.getDelegationTokens(delegTokenRenewer); + if (tokens != null) { + for (Token token : tokens) { + credentials.addToken(token.getService(), token); + LOG.info("Got dt for " + fs.getUri() + ";uri="+ fsName + + ";t.service="+token.getService()); + } + } + //Call getDelegationToken as well for now - for FS implementations + // which may not have implmented getDelegationTokens (hftp) Token token = fs.getDelegationToken(delegTokenRenewer); if (token != null) { Text fsNameText = new Text(fsName); Modified: hadoop/common/branches/MR-279/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1157421&r1=1157420&r2=1157421&view=diff ============================================================================== --- hadoop/common/branches/MR-279/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java (original) +++ hadoop/common/branches/MR-279/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java Sat Aug 13 20:21:50 2011 @@ -30,7 +30,9 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.NoSuchAlgorithmException; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.crypto.KeyGenerator; @@ -38,8 +40,10 @@ import javax.crypto.spec.SecretKeySpec; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.viewfs.ViewFileSystem; import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -151,6 +155,7 @@ public class TestTokenCache { @BeforeClass public static void setUp() throws Exception { + Configuration conf = new Configuration(); conf.set("hadoop.security.auth_to_local", "RULE:[2:$1]"); dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null); @@ -336,6 +341,14 @@ public class TestTokenCache { return t; }}).when(hfs).getDelegationToken(renewer); + //when(hfs.getDelegationTokens()).thenReturn((Token) t); + Mockito.doAnswer(new Answer>>(){ + @Override + public List> answer(InvocationOnMock invocation) + throws Throwable { + return Collections.singletonList(t); + }}).when(hfs).getDelegationTokens(renewer); + //when(hfs.getCanonicalServiceName).thenReturn(fs_addr); Mockito.doAnswer(new Answer(){ @Override @@ -380,4 +393,40 @@ public class TestTokenCache { assertEquals("Failed to substitute HOSTNAME_PATTERN with hostName", serviceName + hostName + domainName, Master.getMasterPrincipal(conf)); } + + @Test + public void testGetTokensForViewFS() throws IOException, URISyntaxException { + Configuration conf = new Configuration(jConf); + FileSystem dfs = dfsCluster.getFileSystem(); + String serviceName = dfs.getCanonicalServiceName(); + + Path p1 = new Path("/mount1"); + Path p2 = new Path("/mount2"); + p1 = dfs.makeQualified(p1); + p2 = dfs.makeQualified(p2); + + conf.set("fs.viewfs.mounttable.default.link./dir1", p1.toString()); + conf.set("fs.viewfs.mounttable.default.link./dir2", p2.toString()); + Credentials credentials = new Credentials(); + Path lp1 = new Path("viewfs:///dir1"); + Path lp2 = new Path("viewfs:///dir2"); + Path[] paths = new Path[2]; + paths[0] = lp1; + paths[1] = lp2; + TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf); + + Collection> tns = + credentials.getAllTokens(); + assertEquals("number of tokens is not 1", 1, tns.size()); + + boolean found = false; + for (Token tt : tns) { + System.out.println("token=" + tt); + if (tt.getKind().equals(DelegationTokenIdentifier.HDFS_DELEGATION_KIND) + && tt.getService().equals(new Text(serviceName))) { + found = true; + } + assertTrue("didn't find token for [" + lp1 + ", " + lp2 + "]", found); + } + } }