hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077296 - in /hadoop/common/branches/branch-0.20-security-patches/src: core/org/apache/hadoop/io/ core/org/apache/hadoop/security/ core/org/apache/hadoop/util/ mapred/org/apache/hadoop/mapred/ mapred/org/apache/hadoop/mapreduce/security/ t...
Date Fri, 04 Mar 2011 04:00:49 GMT
Author: omalley
Date: Fri Mar  4 04:00:48 2011
New Revision: 1077296

URL: http://svn.apache.org/viewvc?rev=1077296&view=rev
Log:
commit fe9b4196a61122b54bf12944cf1f9662dfb17849
Author: Owen O'Malley <omalley@apache.org>
Date:   Fri Mar 5 15:18:03 2010 -0800

    MAPREDUCE-1566. Mechanism to import tokens and secrets from a file in to
    the submitted job. (omalley)
    
    +++ b/YAHOO-CHANGES.txt
    +    MAPREDUCE-1566. Mechanism to import tokens and secrets from a file in to
    +    the submitted job. (omalley)
    +

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/WritableUtils.java
    hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/Credentials.java
    hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
    hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/Child.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapreduce/security/TokenCache.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/WritableUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/WritableUtils.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/WritableUtils.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/WritableUtils.java
Fri Mar  4 04:00:48 2011
@@ -53,7 +53,8 @@ public final class WritableUtils  {
     }
   }
 
-  public static int  writeCompressedByteArray(DataOutput out, byte[] bytes) throws IOException
{
+  public static int  writeCompressedByteArray(DataOutput out, 
+                                              byte[] bytes) throws IOException {
     if (bytes != null) {
       ByteArrayOutputStream bos =  new ByteArrayOutputStream();
       GZIPOutputStream gzout = new GZIPOutputStream(bos);

Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/Credentials.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/Credentials.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/Credentials.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/Credentials.java
Fri Mar  4 04:00:48 2011
@@ -21,11 +21,13 @@ package org.apache.hadoop.security;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
@@ -103,24 +105,41 @@ public class Credentials implements Writ
   }
  
   /**
-   * Convenience method for reading a file, and loading the Tokens
+   * Convenience method for reading a token storage file, and loading the Tokens
    * therein in the passed UGI
    * @param filename
    * @param conf
-   * @param ugi
    * @throws IOException
    */
-  public static void readTokensAndLoadInUGI(String filename, Configuration conf, 
-      UserGroupInformation ugi) throws IOException {
-    Path localTokensFile = new Path (filename);
-    FileSystem localFS = FileSystem.getLocal(conf);
-    FSDataInputStream in = localFS.open(localTokensFile);
-    Credentials ts = new Credentials();
-    ts.readFields(in);
-    for (Token<? extends TokenIdentifier> token : ts.getAllTokens()) {
-      ugi.addToken(token);
+  public void readTokenStorageFile(Path filename, 
+                                   Configuration conf) throws IOException {
+    FSDataInputStream in = filename.getFileSystem(conf).open(filename);
+    byte[] magic = new byte[TOKEN_STORAGE_MAGIC.length];
+    in.readFully(magic);
+    if (!Arrays.equals(magic, TOKEN_STORAGE_MAGIC)) {
+      throw new IOException("Bad header found in token storage " + filename);
     }
+    byte version = in.readByte();
+    if (version != TOKEN_STORAGE_VERSION) {
+      throw new IOException("Unknown version " + version + 
+                            " in token storage " + filename);
+    }
+    readFields(in);
+    in.close();
+  }
+  
+  private static final byte[] TOKEN_STORAGE_MAGIC = "HDTS".getBytes();
+  private static final byte TOKEN_STORAGE_VERSION = 0;
+  
+  public void writeTokenStorageFile(Path filename, 
+                                    Configuration conf) throws IOException {
+    FSDataOutputStream os = filename.getFileSystem(conf).create(filename);
+    os.write(TOKEN_STORAGE_MAGIC);
+    os.write(TOKEN_STORAGE_VERSION);
+    write(os);
+    os.close();
   }
+
   /**
    * Stores all the keys to DataOutput
    * @param out
@@ -140,7 +159,8 @@ public class Credentials implements Writ
     WritableUtils.writeVInt(out, secretKeysMap.size());
     for(Map.Entry<Text, byte[]> e : secretKeysMap.entrySet()) {
       e.getKey().write(out);
-      WritableUtils.writeCompressedByteArray(out, e.getValue());  
+      WritableUtils.writeVInt(out, e.getValue().length);
+      out.write(e.getValue());
     }
   }
   
@@ -167,8 +187,23 @@ public class Credentials implements Writ
     for(int i=0; i<size; i++) {
       Text alias = new Text();
       alias.readFields(in);
-      byte[] key = WritableUtils.readCompressedByteArray(in);
-      secretKeysMap.put(alias, key);
+      int len = WritableUtils.readVInt(in);
+      byte[] value = new byte[len];
+      in.readFully(value);
+      secretKeysMap.put(alias, value);
+    }
+  }
+ 
+  /**
+   * Copy all of the credentials from one credential object into another.
+   * @param other the credentials to copy
+   */
+  public void addAll(Credentials other) {
+    for(Map.Entry<Text, byte[]> secret: other.secretKeysMap.entrySet()) {
+      secretKeysMap.put(secret.getKey(), secret.getValue());
+    }
+    for(Map.Entry<Text, Token<?>> token: other.tokenMap.entrySet()){
+      tokenMap.put(token.getKey(), token.getValue());
     }
   }
 }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
Fri Mar  4 04:00:48 2011
@@ -48,6 +48,7 @@ import javax.security.auth.spi.LoginModu
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -129,6 +130,8 @@ public class UserGroupInformation {
   private static Groups groups;
   /** The last authentication time */
   private static long lastUnsuccessfulAuthenticationAttemptTime;
+  /** The configuration to use */
+  private static Configuration conf;
   
   public static final long MIN_TIME_BEFORE_RELOGIN = 10 * 60 * 1000L;
   
@@ -171,6 +174,7 @@ public class UserGroupInformation {
     javax.security.auth.login.Configuration.setConfiguration
         (new HadoopConfiguration());
     isInitialized = true;
+    UserGroupInformation.conf = conf;
   }
 
   /**
@@ -364,9 +368,15 @@ public class UserGroupInformation {
         }
         login.login();
         loginUser = new UserGroupInformation(login.getSubject());
-        String tokenFile = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
-        if (tokenFile != null && isSecurityEnabled()) {
-          Credentials.readTokensAndLoadInUGI(tokenFile, new Configuration(), loginUser);
+        String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
+        if (fileLocation != null && isSecurityEnabled()) {
+          // load the token storage file and put all of the tokens into the
+          // user.
+          Credentials cred = new Credentials();
+          cred.readTokenStorageFile(new Path("file:///" + fileLocation), conf);
+          for (Token<?> token: cred.getAllTokens()) {
+            loginUser.addToken(token);
+          }
         }
       } catch (LoginException le) {
         throw new IOException("failure to login", le);

Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java
Fri Mar  4 04:00:48 2011
@@ -308,7 +308,8 @@ public class GenericOptionsParser {
         }
 
         LOG.debug("setting conf tokensFile: " + fileName);
-        conf.set("tokenCacheFile", localFs.makeQualified(p).toString());
+        conf.set("mapreduce.job.credentials.json", 
+                 localFs.makeQualified(p).toString());
       } catch (IOException e) {
         throw new RuntimeException(e);
       }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/Child.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/Child.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/Child.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/Child.java
Fri Mar  4 04:00:48 2011
@@ -71,6 +71,7 @@ class Child {
     // file name is passed thru env
     String jobTokenFile = 
       System.getenv().get(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
+    System.err.println("JOB TOKEN FILE " + jobTokenFile);
     Credentials credentials = 
       TokenCache.loadTokens(jobTokenFile, defaultConf);
     LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() + 

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
Fri Mar  4 04:00:48 2011
@@ -759,6 +759,13 @@ public class JobClient extends Configure
         jobCopy.set("mapreduce.job.dir", submitJobDir.toString());
         JobStatus status = null;
         try {
+          // load the binary file, if the user has one
+          String binaryTokenFilename = 
+            jobCopy.get("mapreduce.job.credentials.binary");
+          if (binaryTokenFilename != null) {
+            jobCopy.getCredentials().readTokenStorageFile
+               (new Path("file:///" +  binaryTokenFilename), jobCopy);
+          }
 
           copyAndConfigureFiles(jobCopy, submitJobDir);
 
@@ -815,7 +822,8 @@ public class JobClient extends Configure
         } finally {
           if (status == null) {
             LOG.info("Cleaning up the staging area " + submitJobDir);
-            fs.delete(submitJobDir, true);
+            if (fs != null && submitJobDir != null)
+              fs.delete(submitJobDir, true);
           }
         }
       }
@@ -1883,7 +1891,7 @@ public class JobClient extends Configure
   private void populateTokenCache(Configuration conf, Credentials credentials) 
   throws IOException{
     // create TokenStorage object with user secretKeys
-    String tokensFileName = conf.get("tokenCacheFile");
+    String tokensFileName = conf.get("mapreduce.job.credentials.json");
     if(tokensFileName != null) {
       LOG.info("loading user's secret keys from " + tokensFileName);
       String localFileName = new Path(tokensFileName).toUri().getPath();
@@ -1907,7 +1915,8 @@ public class JobClient extends Configure
       if(json_error)
         LOG.warn("couldn't parse Token Cache JSON file with user secret keys");
     }
-
+    
+ 
     // add the delegation tokens from configuration
     String [] nameNodes = conf.getStrings(JobContext.JOB_NAMENODES);
     LOG.info("adding the following namenodes' delegation tokens:" + Arrays.toString(nameNodes));

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Mar  4 04:00:48 2011
@@ -3207,8 +3207,9 @@ public class JobInProgress {
   private void generateAndStoreTokens() throws IOException {
     Path jobDir = jobtracker.getSystemDirectoryForJob(jobId);
     Path keysFile = new Path(jobDir, TokenCache.JOB_TOKEN_HDFS_FILE);
-    // we need to create this file using the jobtracker's filesystem
-    FSDataOutputStream os = jobtracker.getFileSystem().create(keysFile);
+    if (tokenStorage == null) {
+      tokenStorage = new Credentials();
+    }
     //create JobToken file and write token to it
     JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(jobId
         .toString()));
@@ -3216,15 +3217,10 @@ public class JobInProgress {
         jobtracker.getJobTokenSecretManager());
     token.setService(identifier.getJobId());
     
-    // add this token to the tokenStorage
-    if(tokenStorage == null)
-      tokenStorage = new Credentials();
-
     TokenCache.setJobToken(token, tokenStorage);
         
     // write TokenStorage out
-    tokenStorage.write(os);
-    os.close();
+    tokenStorage.writeTokenStorageFile(keysFile, conf);
     LOG.info("jobToken generated and stored with users keys in "
         + keysFile.toUri().getPath());
   }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Fri Mar  4 04:00:48 2011
@@ -1018,7 +1018,6 @@ public class TaskTracker 
     String localJobTokenFile = localizeJobTokenFile(t.getUser(), jobId);
     rjob.ugi = UserGroupInformation.createRemoteUser(t.getUser());
     
-    
     Credentials ts = TokenCache.loadTokens(localJobTokenFile, fConf);
     Token<JobTokenIdentifier> jt = TokenCache.getJobToken(ts);
     if (jt != null) { //could be null in the case of some unit tests

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapreduce/security/TokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapreduce/security/TokenCache.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapreduce/security/TokenCache.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapreduce/security/TokenCache.java
Fri Mar  4 04:00:48 2011
@@ -160,18 +160,16 @@ public class TokenCache {
   //@InterfaceAudience.Private
   public static Credentials loadTokens(String jobTokenFile, JobConf conf) 
   throws IOException {
-    Path localJobTokenFile = new Path (jobTokenFile);
-    FileSystem localFS = FileSystem.getLocal(conf);
-    FSDataInputStream in = localFS.open(localJobTokenFile);
+    Path localJobTokenFile = new Path ("file:///" + jobTokenFile);
     
     Credentials ts = new Credentials();
-    ts.readFields(in);
+    ts.readTokenStorageFile(localJobTokenFile, conf);
 
     if(LOG.isDebugEnabled()) {
       LOG.debug("Task: Loaded jobTokenFile from: "+localJobTokenFile.toUri().getPath() 
-        +"; num of sec keys  = " + ts.numberOfSecretKeys());
+        +"; num of sec keys  = " + ts.numberOfSecretKeys() + " Number of tokens " + 
+        ts.numberOfTokens());
     }
-    in.close();
     return ts;
   }
 

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1077296&r1=1077295&r2=1077296&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java
Fri Mar  4 04:00:48 2011
@@ -244,7 +244,7 @@ public class TestTokenCache {
   public void testLocalJobTokenCache() throws NoSuchAlgorithmException, IOException {
     // this is local job
     String[] args = {"-m", "1", "-r", "1", "-mt", "1", "-rt", "1"}; 
-    jConf.set("tokenCacheFile", tokenFileName.toString());
+    jConf.set("mapreduce.job.credentials.json", tokenFileName.toString());
 
     int res = -1;
     try {



Mime
View raw message