Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BD0E49C5E for ; Thu, 30 Aug 2012 19:58:53 +0000 (UTC) Received: (qmail 74807 invoked by uid 500); 30 Aug 2012 19:58:53 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 74743 invoked by uid 500); 30 Aug 2012 19:58:53 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 74735 invoked by uid 99); 30 Aug 2012 19:58:53 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 Aug 2012 19:58:53 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 Aug 2012 19:58:51 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A4A612388980; Thu, 30 Aug 2012 19:58:08 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1379100 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/security/ Date: Thu, 30 Aug 2012 19:58:08 -0000 To: common-commits@hadoop.apache.org From: bobby@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120830195808.A4A612388980@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: bobby Date: Thu Aug 30 19:58:07 2012 New Revision: 1379100 URL: http://svn.apache.org/viewvc?rev=1379100&view=rev Log: HADOOP-8726. The Secrets in Credentials are not available to MR tasks (daryn and Benoy Antony via bobby) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1379100&r1=1379099&r2=1379100&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 30 19:58:07 2012 @@ -993,6 +993,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8725. MR is broken when security is off (daryn via bobby) + HADOOP-8726. The Secrets in Credentials are not available to MR tasks + (daryn and Benoy Antony via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java?rev=1379100&r1=1379099&r2=1379100&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java Thu Aug 30 19:58:07 2012 @@ -274,10 +274,4 @@ public class Credentials implements Writ } } } - - public void addTokensToUGI(UserGroupInformation ugi) { - for (Map.Entry> token: tokenMap.entrySet()) { - ugi.addToken(token.getKey(), token.getValue()); - } - } } Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1379100&r1=1379099&r2=1379100&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java Thu Aug 30 19:58:07 2012 @@ -27,7 +27,6 @@ import java.security.Principal; import java.security.PrivilegedAction; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -646,7 +645,7 @@ public class UserGroupInformation { // user. Credentials cred = Credentials.readTokenStorageFile( new Path("file:///" + fileLocation), conf); - cred.addTokensToUGI(loginUser); + loginUser.addCredentials(cred); } loginUser.spawnAutoRenewalThreadForUserCreds(); } catch (LoginException le) { @@ -1176,41 +1175,6 @@ public class UserGroupInformation { public synchronized Set getTokenIdentifiers() { return subject.getPublicCredentials(TokenIdentifier.class); } - - // wrapper to retain the creds key for the token - private class NamedToken { - Text alias; - Token token; - NamedToken(Text alias, Token token) { - this.alias = alias; - this.token = token; - } - @Override - public boolean equals(Object o) { - boolean equals; - if (o == this) { - equals = true; - } else if (!(o instanceof NamedToken)) { - equals = false; - } else { - Text otherAlias = ((NamedToken)o).alias; - if (alias == otherAlias) { - equals = true; - } else { - equals = (otherAlias != null && otherAlias.equals(alias)); - } - } - return equals; - } - @Override - public int hashCode() { - return (alias != null) ? alias.hashCode() : -1; - } - @Override - public String toString() { - return "NamedToken: alias="+alias+" token="+token; - } - } /** * Add a token to this UGI @@ -1219,7 +1183,7 @@ public class UserGroupInformation { * @return true on successful add of new token */ public synchronized boolean addToken(Token token) { - return addToken(token.getService(), token); + return (token != null) ? addToken(token.getService(), token) : false; } /** @@ -1231,10 +1195,8 @@ public class UserGroupInformation { */ public synchronized boolean addToken(Text alias, Token token) { - NamedToken namedToken = new NamedToken(alias, token); - Collection ugiCreds = subject.getPrivateCredentials(); - ugiCreds.remove(namedToken); // allow token to be replaced - return ugiCreds.add(new NamedToken(alias, token)); + getCredentialsInternal().addToken(alias, token); + return true; } /** @@ -1244,8 +1206,8 @@ public class UserGroupInformation { */ public synchronized Collection> getTokens() { - return Collections.unmodifiableList( - new ArrayList>(getCredentials().getAllTokens())); + return Collections.unmodifiableCollection( + getCredentialsInternal().getAllTokens()); } /** @@ -1254,11 +1216,26 @@ public class UserGroupInformation { * @return Credentials of tokens associated with this user */ public synchronized Credentials getCredentials() { - final Credentials credentials = new Credentials(); - final Set namedTokens = - subject.getPrivateCredentials(NamedToken.class); - for (final NamedToken namedToken : namedTokens) { - credentials.addToken(namedToken.alias, namedToken.token); + return new Credentials(getCredentialsInternal()); + } + + /** + * Add the given Credentials to this user. + * @param credentials of tokens and secrets + */ + public synchronized void addCredentials(Credentials credentials) { + getCredentialsInternal().addAll(credentials); + } + + private synchronized Credentials getCredentialsInternal() { + final Credentials credentials; + final Set credentialsSet = + subject.getPrivateCredentials(Credentials.class); + if (!credentialsSet.isEmpty()){ + credentials = credentialsSet.iterator().next(); + } else { + credentials = new Credentials(); + subject.getPrivateCredentials().add(credentials); } return credentials; } Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java?rev=1379100&r1=1379099&r2=1379100&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java Thu Aug 30 19:58:07 2012 @@ -220,7 +220,7 @@ public class TestCredentials { for (int i=0; i < service.length; i++) { creds.addToken(service[i], token[i]); } - creds.addTokensToUGI(ugi); + ugi.addCredentials(creds); creds = ugi.getCredentials(); for (int i=0; i < service.length; i++) { Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java?rev=1379100&r1=1379099&r2=1379100&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java Thu Aug 30 19:58:07 2012 @@ -250,6 +250,70 @@ public class TestUserGroupInformation { ugi.addToken(t1); checkTokens(ugi, t1, t2, t3); } + + @SuppressWarnings("unchecked") // from Mockito mocks + @Test + public void testGetCreds() throws Exception { + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("someone"); + + Text service = new Text("service"); + Token t1 = mock(Token.class); + when(t1.getService()).thenReturn(service); + Token t2 = mock(Token.class); + when(t2.getService()).thenReturn(new Text("service2")); + Token t3 = mock(Token.class); + when(t3.getService()).thenReturn(service); + + // add token to ugi + ugi.addToken(t1); + ugi.addToken(t2); + checkTokens(ugi, t1, t2); + + Credentials creds = ugi.getCredentials(); + creds.addToken(t3.getService(), t3); + assertSame(t3, creds.getToken(service)); + // check that ugi wasn't modified + checkTokens(ugi, t1, t2); + } + + @SuppressWarnings("unchecked") // from Mockito mocks + @Test + public void testAddCreds() throws Exception { + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("someone"); + + Text service = new Text("service"); + Token t1 = mock(Token.class); + when(t1.getService()).thenReturn(service); + Token t2 = mock(Token.class); + when(t2.getService()).thenReturn(new Text("service2")); + byte[] secret = new byte[]{}; + Text secretKey = new Text("sshhh"); + + // fill credentials + Credentials creds = new Credentials(); + creds.addToken(t1.getService(), t1); + creds.addToken(t2.getService(), t2); + creds.addSecretKey(secretKey, secret); + + // add creds to ugi, and check ugi + ugi.addCredentials(creds); + checkTokens(ugi, t1, t2); + assertSame(secret, ugi.getCredentials().getSecretKey(secretKey)); + } + + @SuppressWarnings("unchecked") // from Mockito mocks + @Test + public void testGetCredsNotSame() + throws Exception { + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("someone"); + Credentials creds = ugi.getCredentials(); + // should always get a new copy + assertNotSame(creds, ugi.getCredentials()); + } + private void checkTokens(UserGroupInformation ugi, Token ... tokens) { // check the ugi's token collection @@ -299,13 +363,22 @@ public class TestUserGroupInformation { Token t2 = mock(Token.class); when(t2.getService()).thenReturn(new Text("t2")); + Credentials creds = new Credentials(); + byte[] secretKey = new byte[]{}; + Text secretName = new Text("shhh"); + creds.addSecretKey(secretName, secretKey); + ugi.addToken(t1); ugi.addToken(t2); + ugi.addCredentials(creds); Collection> z = ugi.getTokens(); assertTrue(z.contains(t1)); assertTrue(z.contains(t2)); assertEquals(2, z.size()); + Credentials ugiCreds = ugi.getCredentials(); + assertSame(secretKey, ugiCreds.getSecretKey(secretName)); + assertEquals(1, ugiCreds.numberOfSecretKeys()); try { z.remove(t1);