hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject svn commit: r1376622 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/security/
Date Thu, 23 Aug 2012 18:09:15 GMT
Author: bobby
Date: Thu Aug 23 18:09:14 2012
New Revision: 1376622

URL: http://svn.apache.org/viewvc?rev=1376622&view=rev
Log:
svn merge -c 1376618 FIXES: HADOOP-8225. DistCp fails when invoked by Oozie (daryn via bobby)

Modified:
    hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
    hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
    hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
    hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java

Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1376622&r1=1376621&r2=1376622&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 23 18:09:14 2012
@@ -674,6 +674,8 @@ Release 0.23.3 - UNRELEASED
     HADOOP-8611. Allow fall-back to the shell-based implementation when 
     JNI-based users-group mapping fails (Robert Parker via bobby) 
 
+    HADOOP-8225. DistCp fails when invoked by Oozie (daryn via bobby)
+
 Release 0.23.2 - UNRELEASED 
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java?rev=1376622&r1=1376621&r2=1376622&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
(original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
Thu Aug 23 18:09:14 2012
@@ -274,4 +274,10 @@ public class Credentials implements Writ
       }
     }
   }
+  
+  public void addTokensToUGI(UserGroupInformation ugi) {
+    for (Map.Entry<Text, Token<?>> token: tokenMap.entrySet()) {
+      ugi.addToken(token.getKey(), token.getValue());
+    }
+  }
 }

Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1376622&r1=1376621&r2=1376622&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Thu Aug 23 18:09:14 2012
@@ -55,6 +55,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -646,9 +647,7 @@ public class UserGroupInformation {
           // user.
           Credentials cred = Credentials.readTokenStorageFile(
               new Path("file:///" + fileLocation), conf);
-          for (Token<?> token: cred.getAllTokens()) {
-            loginUser.addToken(token);
-          }
+          cred.addTokensToUGI(loginUser);
         }
         loginUser.spawnAutoRenewalThreadForUserCreds();
       } catch (LoginException le) {
@@ -1177,6 +1176,41 @@ public class UserGroupInformation {
   public synchronized Set<TokenIdentifier> getTokenIdentifiers() {
     return subject.getPublicCredentials(TokenIdentifier.class);
   }
+
+  // wrapper to retain the creds key for the token
+  private class NamedToken {
+    Text alias;
+    Token<? extends TokenIdentifier> token;
+    NamedToken(Text alias, Token<? extends TokenIdentifier> token) {
+      this.alias = alias;
+      this.token = token;
+    }
+    @Override
+    public boolean equals(Object o) {
+      boolean equals;
+      if (o == this) {
+        equals = true;
+      } else if (!(o instanceof NamedToken)) {
+        equals = false;
+      } else {
+        Text otherAlias = ((NamedToken)o).alias;
+        if (alias == otherAlias) {
+          equals = true;
+        } else {
+          equals = (otherAlias != null && otherAlias.equals(alias));
+        }
+      }
+      return equals;
+    }
+    @Override
+    public int hashCode() {
+      return (alias != null) ? alias.hashCode() : -1; 
+    }
+    @Override
+    public String toString() {
+      return "NamedToken: alias="+alias+" token="+token;
+    }
+  }
   
   /**
    * Add a token to this UGI
@@ -1185,7 +1219,22 @@ public class UserGroupInformation {
    * @return true on successful add of new token
    */
   public synchronized boolean addToken(Token<? extends TokenIdentifier> token) {
-    return subject.getPrivateCredentials().add(token);
+    return addToken(token.getService(), token);
+  }
+
+  /**
+   * Add a named token to this UGI
+   * 
+   * @param alias Name of the token
+   * @param token Token to be added
+   * @return true on successful add of new token
+   */
+  public synchronized boolean addToken(Text alias,
+                                       Token<? extends TokenIdentifier> token) {
+    NamedToken namedToken = new NamedToken(alias, token);
+    Collection<Object> ugiCreds = subject.getPrivateCredentials();
+    ugiCreds.remove(namedToken); // allow token to be replaced
+    return ugiCreds.add(new NamedToken(alias, token));
   }
   
   /**
@@ -1195,14 +1244,23 @@ public class UserGroupInformation {
    */
   public synchronized
   Collection<Token<? extends TokenIdentifier>> getTokens() {
-    Set<Object> creds = subject.getPrivateCredentials();
-    List<Token<?>> result = new ArrayList<Token<?>>(creds.size());
-    for(Object o: creds) {
-      if (o instanceof Token<?>) {
-        result.add((Token<?>) o);
-      }
+    return Collections.unmodifiableList(
+        new ArrayList<Token<?>>(getCredentials().getAllTokens()));
+  }
+
+  /**
+   * Obtain the tokens in credentials form associated with this user.
+   * 
+   * @return Credentials of tokens associated with this user
+   */
+  public synchronized Credentials getCredentials() {
+    final Credentials credentials = new Credentials();
+    final Set<NamedToken> namedTokens =
+        subject.getPrivateCredentials(NamedToken.class);
+    for (final NamedToken namedToken : namedTokens) {
+      credentials.addToken(namedToken.alias, namedToken.token);
     }
-    return Collections.unmodifiableList(result);
+    return credentials;
   }
 
   /**

Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java?rev=1376622&r1=1376621&r2=1376622&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
(original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
Thu Aug 23 18:09:14 2012
@@ -213,5 +213,22 @@ public class TestCredentials {
     // new token & secret should be added
     assertEquals(token[2], creds.getToken(service[2]));
     assertEquals(secret[2], new Text(creds.getSecretKey(secret[2])));
- }
+  }
+  
+  @Test
+  public void testAddTokensToUGI() {
+    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone");
+    Credentials creds = new Credentials();
+    
+    for (int i=0; i < service.length; i++) {
+      creds.addToken(service[i], token[i]);
+    }
+    creds.addTokensToUGI(ugi);
+
+    creds = ugi.getCredentials();
+    for (int i=0; i < service.length; i++) {
+      assertSame(token[i], creds.getToken(service[i]));
+    }
+    assertEquals(service.length, creds.numberOfTokens());
+  }
 }
\ No newline at end of file

Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java?rev=1376622&r1=1376621&r2=1376622&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
(original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
Thu Aug 23 18:09:14 2012
@@ -19,8 +19,7 @@ package org.apache.hadoop.security;
 import static org.junit.Assert.*;
 import org.junit.*;
 
-import org.mockito.Mockito;
-import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.*;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -35,6 +34,7 @@ import javax.security.auth.login.AppConf
 import javax.security.auth.login.LoginContext;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.token.Token;
@@ -194,8 +194,6 @@ public class TestUserGroupInformation {
   public void testEqualsWithRealUser() throws Exception {
     UserGroupInformation realUgi1 = UserGroupInformation.createUserForTesting(
         "RealUser", GROUP_NAMES);
-    UserGroupInformation realUgi2 = UserGroupInformation.createUserForTesting(
-        "RealUser", GROUP_NAMES);
     UserGroupInformation proxyUgi1 = UserGroupInformation.createProxyUser(
         USER_NAME, realUgi1);
     UserGroupInformation proxyUgi2 =
@@ -213,7 +211,82 @@ public class TestUserGroupInformation {
     assertArrayEquals(new String[]{GROUP1_NAME, GROUP2_NAME, GROUP3_NAME},
                       uugi.getGroupNames());
   }
+
+  @SuppressWarnings("unchecked") // from Mockito mocks
+  @Test
+  public <T extends TokenIdentifier> void testAddToken() throws Exception {
+    UserGroupInformation ugi = 
+        UserGroupInformation.createRemoteUser("someone"); 
+    
+    Token<T> t1 = mock(Token.class);
+    Token<T> t2 = mock(Token.class);
+    Token<T> t3 = mock(Token.class);
+    
+    // add token to ugi
+    ugi.addToken(t1);
+    checkTokens(ugi, t1);
+
+    // replace token t1 with t2 - with same key (null)
+    ugi.addToken(t2);
+    checkTokens(ugi, t2);
+    
+    // change t1 service and add token
+    when(t1.getService()).thenReturn(new Text("t1"));
+    ugi.addToken(t1);
+    checkTokens(ugi, t1, t2);
   
+    // overwrite t1 token with t3 - same key (!null)
+    when(t3.getService()).thenReturn(new Text("t1"));
+    ugi.addToken(t3);
+    checkTokens(ugi, t2, t3);
+
+    // just try to re-add with new name
+    when(t1.getService()).thenReturn(new Text("t1.1"));
+    ugi.addToken(t1);
+    checkTokens(ugi, t1, t2, t3);    
+
+    // just try to re-add with new name again
+    ugi.addToken(t1);
+    checkTokens(ugi, t1, t2, t3);    
+  }
+  
+  private void checkTokens(UserGroupInformation ugi, Token<?> ... tokens) {
+    // check the ugi's token collection
+    Collection<Token<?>> ugiTokens = ugi.getTokens();
+    for (Token<?> t : tokens) {
+      assertTrue(ugiTokens.contains(t));
+    }
+    assertEquals(tokens.length, ugiTokens.size());
+
+    // check the ugi's credentials
+    Credentials ugiCreds = ugi.getCredentials();
+    for (Token<?> t : tokens) {
+      assertSame(t, ugiCreds.getToken(t.getService()));
+    }
+    assertEquals(tokens.length, ugiCreds.numberOfTokens());
+  }
+
+  @SuppressWarnings("unchecked") // from Mockito mocks
+  @Test
+  public <T extends TokenIdentifier> void testAddNamedToken() throws Exception {
+    UserGroupInformation ugi = 
+        UserGroupInformation.createRemoteUser("someone"); 
+    
+    Token<T> t1 = mock(Token.class);
+    Text service1 = new Text("t1");
+    Text service2 = new Text("t2");
+    when(t1.getService()).thenReturn(service1);
+    
+    // add token
+    ugi.addToken(service1, t1);
+    assertSame(t1, ugi.getCredentials().getToken(service1));
+
+    // add token with another name
+    ugi.addToken(service2, t1);
+    assertSame(t1, ugi.getCredentials().getToken(service1));
+    assertSame(t1, ugi.getCredentials().getToken(service2));
+  }
+
   @SuppressWarnings("unchecked") // from Mockito mocks
   @Test
   public <T extends TokenIdentifier> void testUGITokens() throws Exception {
@@ -221,7 +294,9 @@ public class TestUserGroupInformation {
       UserGroupInformation.createUserForTesting("TheDoctor", 
                                                 new String [] { "TheTARDIS"});
     Token<T> t1 = mock(Token.class);
+    when(t1.getService()).thenReturn(new Text("t1"));
     Token<T> t2 = mock(Token.class);
+    when(t2.getService()).thenReturn(new Text("t2"));
     
     ugi.addToken(t1);
     ugi.addToken(t2);



Mime
View raw message