hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077139 - in /hadoop/common/branches/branch-0.20-security-patches/src: core/org/apache/hadoop/security/TokenStorage.java core/org/apache/hadoop/security/UserGroupInformation.java test/org/apache/hadoop/security/TestTokenStorage.java
Date Fri, 04 Mar 2011 03:45:07 GMT
Author: omalley
Date: Fri Mar  4 03:45:07 2011
New Revision: 1077139

URL: http://svn.apache.org/viewvc?rev=1077139&view=rev
Log:
commit 0f7a852cc25e9e49753d15d3f3d9af6ab4162ac3
Author: Jitendra Nath Pandey <jitendra@yahoo-inc.com>
Date:   Mon Feb 1 10:28:11 2010 -0800

    HADOOP-6520 from https://issues.apache.org/jira/secure/attachment/12434423/HADOOP-6520-0_20.2.patch
    
    +++ b/YAHOO-CHANGES.txt
    +    HADOOP-6520. UGI should load tokens from the environment. (jitendra)
    +

Added:
    hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java
Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java

Added: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java?rev=1077139&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java
(added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java
Fri Mar  4 03:45:07 2011
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A class that provides the facilities of reading and writing 
+ * secret keys and Tokens.
+ */
+public class TokenStorage implements Writable {
+
+  private  Map<Text, byte[]> secretKeysMap = new HashMap<Text, byte[]>();
+  private  Map<Text, Token<? extends TokenIdentifier>> tokenMap = 
+    new HashMap<Text, Token<? extends TokenIdentifier>>(); 
+
+  /**
+   * Returns the key bytes for the alias
+   * @param alias the alias for the key
+   * @return key for this alias
+   */
+  public byte[] getSecretKey(Text alias) {
+    return secretKeysMap.get(alias);
+  }
+  
+  /**
+   * Returns the Token object for the alias
+   * @param alias the alias for the Token
+   * @return token for this alias
+   */
+  public Token<? extends TokenIdentifier> getToken(Text alias) {
+    return tokenMap.get(alias);
+  }
+  
+  /**
+   * Add a token in the storage (in memory)
+   * @param alias the alias for the key
+   * @param t the token object
+   */
+  public void addToken(Text alias, Token<? extends TokenIdentifier> t) {
+    tokenMap.put(alias, t);
+  }
+  
+  /**
+   * Return all the tokens in the in-memory map
+   */
+  public Collection<Token<? extends TokenIdentifier>> getAllTokens() {
+    return tokenMap.values();
+  }
+  
+  /**
+   * @return number of Tokens in the in-memory map
+   */
+  public int numberOfTokens() {
+    return tokenMap.size();
+  }
+  
+  /**
+   * @return number of keys in the in-memory map
+   */
+  public int numberOfSecretKeys() {
+    return secretKeysMap.size();
+  }
+  
+  /**
+   * Set the key for an alias
+   * @param alias the alias for the key
+   * @param key the key bytes
+   */
+  public void addSecretKey(Text alias, byte[] key) {
+    secretKeysMap.put(alias, key);
+  }
+ 
+  /**
+   * Convenience method for reading a file, and loading the Tokens
+   * therein in the passed UGI
+   * @param filename
+   * @param conf
+   * @param ugi
+   * @throws IOException
+   */
+  public static void readTokensAndLoadInUGI(String filename, Configuration conf, 
+      UserGroupInformation ugi) throws IOException {
+    Path localTokensFile = new Path (filename);
+    FileSystem localFS = FileSystem.getLocal(conf);
+    FSDataInputStream in = localFS.open(localTokensFile);
+    TokenStorage ts = new TokenStorage();
+    ts.readFields(in);
+    for (Token<? extends TokenIdentifier> token : ts.getAllTokens()) {
+      ugi.addToken(token);
+    }
+  }
+  /**
+   * Stores all the keys to DataOutput
+   * @param out
+   * @throws IOException
+   */
+  @Override
+  public void write(DataOutput out) throws IOException {
+    // write out tokens first
+    WritableUtils.writeVInt(out, tokenMap.size());
+    for(Map.Entry<Text, 
+        Token<? extends TokenIdentifier>> e: tokenMap.entrySet()) {
+      e.getKey().write(out);
+      e.getValue().write(out);
+    }
+    
+    // now write out secret keys
+    WritableUtils.writeVInt(out, secretKeysMap.size());
+    for(Map.Entry<Text, byte[]> e : secretKeysMap.entrySet()) {
+      e.getKey().write(out);
+      WritableUtils.writeCompressedByteArray(out, e.getValue());  
+    }
+  }
+  
+  /**
+   * Loads all the keys
+   * @param in
+   * @throws IOException
+   */
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    secretKeysMap.clear();
+    tokenMap.clear();
+    
+    int size = WritableUtils.readVInt(in);
+    for(int i=0; i<size; i++) {
+      Text alias = new Text();
+      alias.readFields(in);
+      Token<? extends TokenIdentifier> t = new Token<TokenIdentifier>();
+      t.readFields(in);
+      tokenMap.put(alias, t);
+    }
+    
+    size = WritableUtils.readVInt(in);
+    for(int i=0; i<size; i++) {
+      Text alias = new Text();
+      alias.readFields(in);
+      byte[] key = WritableUtils.readCompressedByteArray(in);
+      secretKeysMap.put(alias, key);
+    }
+  }
+}

Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1077139&r1=1077138&r2=1077139&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
Fri Mar  4 03:45:07 2011
@@ -30,7 +30,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -127,6 +126,10 @@ public class UserGroupInformation {
   /** Server-side groups fetching service */
   private static Groups groups;
   
+  /**Environment variable pointing to the token cache file*/
+  public static final String HADOOP_TOKEN_FILE_LOCATION = 
+    "HADOOP_TOKEN_FILE_LOCATION";
+  
   /** 
    * A method to initialize the fields that depend on a configuration.
    * Must be called before useKerberos or groups is used.
@@ -316,6 +319,10 @@ public class UserGroupInformation {
         }
         login.login();
         loginUser = new UserGroupInformation(login.getSubject());
+        String tokenFile = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
+        if (tokenFile != null && isSecurityEnabled()) {
+          TokenStorage.readTokensAndLoadInUGI(tokenFile, new Configuration(), loginUser);
+        }
       } catch (LoginException le) {
         throw new IOException("failure to login", le);
       }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java?rev=1077139&r1=1077138&r2=1077139&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java
Fri Mar  4 03:45:07 2011
@@ -29,19 +29,23 @@ import java.io.IOException;
 import java.security.Key;
 import java.security.NoSuchAlgorithmException;
 import java.util.HashMap;
+import java.util.List;
+import java.util.ArrayList;
 import java.util.Map;
+import java.util.Collection;
+
+import static org.mockito.Mockito.mock;
 
 import javax.crypto.KeyGenerator;
 
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
-import org.apache.hadoop.record.Utils;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.security.TokenStorage;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
 import org.junit.Before;
 import org.junit.Test;
 import static org.junit.Assert.*;
-import org.apache.hadoop.mapreduce.security.TokenStorage;
 
 public class TestTokenStorage {
   private static final String DEFAULT_HMAC_ALGORITHM = "HmacSHA1";
@@ -52,19 +56,27 @@ public class TestTokenStorage {
   public void setUp() {
     tmpDir.mkdir();
   }
-
+  
+  @SuppressWarnings("unchecked")
   @Test 
-  public void testReadWriteStorage() throws IOException, NoSuchAlgorithmException{
+  public <T extends TokenIdentifier> void testReadWriteStorage() 
+  throws IOException, NoSuchAlgorithmException{
     // create tokenStorage Object
     TokenStorage ts = new TokenStorage();
     
-    // create a token
-    JobTokenSecretManager jtSecretManager = new JobTokenSecretManager();
-    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text("fakeJobId"));
-    Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>(identifier,
-        jtSecretManager);
-    // store it
-    ts.setJobToken(jt);
+    Token<T> token1 = new Token();
+    Token<T> token2 = new Token();
+    Text service1 = new Text("service1");
+    Text service2 = new Text("service2");
+    Collection<Text> services = new ArrayList<Text>();
+    
+    services.add(service1);
+    services.add(service2);
+    
+    token1.setService(service1);
+    token2.setService(service2);
+    ts.addToken(new Text("sometoken1"), token1);
+    ts.addToken(new Text("sometoken2"), token2);
     
     // create keys and put it in
     final KeyGenerator kg = KeyGenerator.getInstance(DEFAULT_HMAC_ALGORITHM);
@@ -78,33 +90,44 @@ public class TestTokenStorage {
    
     // create file to store
     File tmpFileName = new File(tmpDir, "tokenStorageTest");
-    DataOutputStream dos = new DataOutputStream(new FileOutputStream(tmpFileName));
+    DataOutputStream dos = 
+      new DataOutputStream(new FileOutputStream(tmpFileName));
     ts.write(dos);
     dos.close();
     
     // open and read it back
-    DataInputStream dis = new DataInputStream(new FileInputStream(tmpFileName));    
+    DataInputStream dis = 
+      new DataInputStream(new FileInputStream(tmpFileName));    
     ts = new TokenStorage();
     ts.readFields(dis);
     dis.close();
     
-    // get the token and compare the passwords
-    byte[] tp1 = ts.getJobToken().getPassword();
-    byte[] tp2 = jt.getPassword();
-    int comp = Utils.compareBytes(tp1, 0, tp1.length, tp2, 0, tp2.length);
-    assertTrue("shuffleToken doesn't match", comp==0);
-    
+    // get the tokens and compare the services
+    Collection<Token<? extends TokenIdentifier>> list = ts.getAllTokens();
+    assertEquals("getAllTokens should return collection of size 2", 
+        list.size(), 2);
+    boolean foundFirst = false;
+    boolean foundSecond = false;
+    for (Token<? extends TokenIdentifier> token : list) {
+      if (token.getService().equals(service1)) {
+        foundFirst = true;
+      }
+      if (token.getService().equals(service2)) {
+        foundSecond = true;
+      }
+    }
+    assertTrue("Tokens for services service1 and service2 must be present", 
+        foundFirst && foundSecond);
     // compare secret keys
     int mapLen = m.size();
-    assertEquals("wrong number of keys in the Storage", mapLen, ts.numberOfSecretKeys());
+    assertEquals("wrong number of keys in the Storage", 
+        mapLen, ts.numberOfSecretKeys());
     for(Text a : m.keySet()) {
       byte [] kTS = ts.getSecretKey(a);
       byte [] kLocal = m.get(a);
       assertTrue("keys don't match for " + a, 
-          Utils.compareBytes(kTS, 0, kTS.length, kLocal, 0, kLocal.length)==0);
-    }  
-    
-    assertEquals("All tokens should return collection of size 1", 
-        ts.getAllTokens().size(), 1);
+          WritableComparator.compareBytes(kTS, 0, kTS.length, kLocal,
+              0, kLocal.length)==0);
+    }
   }
  }



Mime
View raw message