hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1130079 - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ yarn/yarn-se...
Date Wed, 01 Jun 2011 10:02:45 GMT
Author: vinodkv
Date: Wed Jun  1 10:02:45 2011
New Revision: 1130079

URL: http://svn.apache.org/viewvc?rev=1130079&view=rev
Log:
Allowing hdfs calls from streaming/pipes tasks. Contributed by Vinod Kumar Vavilapalli.

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1130079&r1=1130078&r2=1130079&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Wed Jun  1 10:02:45 2011
@@ -3,6 +3,9 @@ Hadoop MapReduce Change Log
 Trunk (unreleased changes)
 
   MAPREDUCE-279
+
+    Allowing hdfs calls from streaming/pipes tasks. (vinodkv)
+
     Reduce ramp up and zero maps support. (sharad)
 
     Adding some more logging for AM expiry logs (mahadev)

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1130079&r1=1130078&r2=1130079&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
Wed Jun  1 10:02:45 2011
@@ -198,8 +198,11 @@ class YarnChild {
   private static Token<JobTokenIdentifier> loadCredentials(JobConf conf,
       InetSocketAddress address) throws IOException {
     //load token cache storage
+    String tokenFileLocation =
+        System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
     String jobTokenFile =
-      new Path("container_tokens").makeQualified(FileSystem.getLocal(conf)).toUri().getPath();
+        new Path(tokenFileLocation).makeQualified(FileSystem.getLocal(conf))
+            .toUri().getPath();
     Credentials credentials =
       TokenCache.loadTokens(jobTokenFile, conf);
     LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() +

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1130079&r1=1130078&r2=1130079&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
Wed Jun  1 10:02:45 2011
@@ -74,7 +74,7 @@ public class DefaultContainerExecutor ex
     // TODO: Why pick first app dir. The same in LCE why not random?
     Path appStorageDir = getFirstApplicationDir(localDirs, user, appId);
 
-    String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_FMT, locId);
+    String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
     Path tokenDst = new Path(appStorageDir, tokenFn);
     lfs.util().copy(nmPrivateContainerTokensPath, tokenDst);
     lfs.setWorkingDirectory(appStorageDir);
@@ -114,7 +114,7 @@ public class DefaultContainerExecutor ex
 
     // copy container tokens to work dir
     Path tokenDst =
-      new Path(containerWorkDir, ContainerLaunch.CONTAINER_TOKENS);
+      new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
     lfs.util().copy(nmPrivateTokensPath, tokenDst);
 
     // create log dir under app

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1130079&r1=1130078&r2=1130079&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
Wed Jun  1 10:02:45 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.LocalDirAllo
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -63,7 +64,7 @@ public class ContainerLaunch implements 
   private static final Log LOG = LogFactory.getLog(ContainerLaunch.class);
 
   public static final String CONTAINER_SCRIPT = "task.sh";
-  public static final String CONTAINER_TOKENS = "container_tokens";
+  public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
 
   private final Dispatcher dispatcher;
   private final ContainerExecutor exec;
@@ -136,11 +137,19 @@ public class ContainerLaunch implements 
                   + Path.SEPARATOR
                   + containerIdStr
                   + Path.SEPARATOR
-                  + String.format(ContainerLocalizer.TOKEN_FILE_FMT,
+                  + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                       containerIdStr), this.conf);
       DataOutputStream containerScriptOutStream = null;
       DataOutputStream tokensOutStream = null;
 
+      // Select the working directory for the container
+      Path containerWorkDir =
+          lDirAllocator.getLocalPathForWrite(ContainerLocalizer.USERCACHE
+              + Path.SEPARATOR + user + Path.SEPARATOR
+              + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
+              + Path.SEPARATOR + containerIdStr,
+              LocalDirAllocator.SIZE_UNKNOWN, this.conf, false);
+
       try {
         // /////////// Write out the container-script in the nmPrivate space.
         String[] localDirs =
@@ -156,6 +165,11 @@ public class ContainerLaunch implements 
         containerScriptOutStream =
           lfs.create(nmPrivateContainerScriptPath,
               EnumSet.of(CREATE, OVERWRITE));
+
+        // Set the token location too.
+        env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, new Path(
+            containerWorkDir, FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
+
         writeLaunchEnv(containerScriptOutStream, env, localResources,
             command, appDirs);
         // /////////// End of writing out container-script
@@ -170,14 +184,6 @@ public class ContainerLaunch implements 
         IOUtils.cleanup(LOG, containerScriptOutStream, tokensOutStream);
       }
 
-      // Select the working directory for the container
-      Path containerWorkDir =
-          lDirAllocator.getLocalPathForWrite(ContainerLocalizer.USERCACHE
-              + Path.SEPARATOR + user + Path.SEPARATOR
-              + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
-              + Path.SEPARATOR + containerIdStr,
-              LocalDirAllocator.SIZE_UNKNOWN, this.conf, false);
-
       // LaunchContainer is a blocking call. We are here almost means the
       // container is launched, so send out the event.
       dispatcher.getEventHandler().handle(new ContainerEvent(

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1130079&r1=1130078&r2=1130079&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
Wed Jun  1 10:02:45 2011
@@ -74,7 +74,7 @@ public class ContainerLocalizer {
   public static final String APPCACHE = "appcache";
   public static final String USERCACHE = "usercache";
   public static final String OUTPUTDIR = "output";
-  public static final String TOKEN_FILE_FMT = "%s.tokens";
+  public static final String TOKEN_FILE_NAME_FMT = "%s.tokens";
   public static final String WORKDIR = "work";
   private static final String APPCACHE_CTXT_FMT = "%s.app.cache.dirs";
   private static final String USERCACHE_CTXT_FMT = "%s.user.cache.dirs";
@@ -135,7 +135,7 @@ public class ContainerLocalizer {
       // assume credentials in cwd
       // TODO: Fix
       credFile = lfs.open(
-          new Path(String.format(TOKEN_FILE_FMT, localizerId)));
+          new Path(String.format(TOKEN_FILE_NAME_FMT, localizerId)));
       creds.readTokenStorageStream(credFile);
     } finally  {
       if (credFile != null) {

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1130079&r1=1130078&r2=1130079&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
Wed Jun  1 10:02:45 2011
@@ -673,7 +673,7 @@ public class ResourceLocalizationService
             localDirsSelector.getLocalPathForWrite(
                 NM_PRIVATE_DIR
                     + Path.SEPARATOR
-                    + String.format(ContainerLocalizer.TOKEN_FILE_FMT,
+                    + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                         localizerId), getConfig());
         // 0) init queue, etc.
         // 1) write credentials to private dir

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java?rev=1130079&r1=1130078&r2=1130079&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
Wed Jun  1 10:02:45 2011
@@ -107,7 +107,7 @@ public class TestContainerLocalizer {
     DataInputBuffer appTokens = createFakeCredentials(r, 10);
     Path tokenPath =
       lfs.makeQualified(new Path(
-            String.format(ContainerLocalizer.TOKEN_FILE_FMT, cId)));
+            String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, cId)));
     doReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens))
         ).when(spylfs).open(tokenPath);
 



Mime
View raw message