hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r1133266 - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/
Date Wed, 08 Jun 2011 07:22:54 GMT
Author: mahadev
Date: Wed Jun  8 07:22:53 2011
New Revision: 1133266

URL: http://svn.apache.org/viewvc?rev=1133266&view=rev
Log:
Fully resolve paths when launching containers. (Siddharth Seth)

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1133266&r1=1133265&r2=1133266&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Wed Jun  8 07:22:53 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
 
 
     MAPREDUCE-279
+  
+    Fully resolve paths when launching containers. (Siddharth Seth)
    
     MAPREDUCE-2566. YarnConfiguration should reloadConfiguration if 
     instantiated with a non YarnConfiguration object. (siddharth seth)

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1133266&r1=1133265&r2=1133266&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Wed Jun  8 07:22:53 2011
@@ -483,9 +483,9 @@ public abstract class TaskAttemptImpl im
 
       // //////////// Set up JobJar to be localized properly on the remote NM.
       if (conf.get(MRJobConfig.JAR) != null) {
-        Path remoteJobJar =
+        Path remoteJobJar = remoteFS.getDefaultFileSystem().resolvePath(
           remoteFS.makeQualified(new Path(remoteTask.getConf().get(
-              MRJobConfig.JAR)));
+              MRJobConfig.JAR))));
         container.setLocalResource(
             MRConstants.JOB_JAR,
             createLocalResource(remoteFS, recordFactory, remoteJobJar,
@@ -506,8 +506,8 @@ public abstract class TaskAttemptImpl im
               .getCurrentUser().getShortUserName());
       Path remoteJobSubmitDir =
           new Path(path, oldJobId.toString());
-      Path remoteJobConfPath =
-          new Path(remoteJobSubmitDir, MRConstants.JOB_CONF_FILE);
+      Path remoteJobConfPath = remoteFS.getDefaultFileSystem().resolvePath(
+          new Path(remoteJobSubmitDir, MRConstants.JOB_CONF_FILE));
       container.setLocalResource(
           MRConstants.JOB_CONF_FILE,
           createLocalResource(remoteFS, recordFactory, remoteJobConfPath,
@@ -642,8 +642,9 @@ public abstract class TaskAttemptImpl im
       for (int i = 0; i < uris.length; ++i) {
         URI u = uris[i];
         Path p = new Path(u);
-        p = p.makeQualified(remoteFS.getDefaultFileSystem()
-              .getUri(), remoteFS.getWorkingDirectory());
+        p = remoteFS.getDefaultFileSystem().resolvePath(
+            p.makeQualified(remoteFS.getDefaultFileSystem().getUri(),
+                remoteFS.getWorkingDirectory()));
         // Add URI fragment or just the filename
         Path name = new Path((null == u.getFragment())
           ? p.getName()

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1133266&r1=1133265&r2=1133266&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Wed Jun  8 07:22:53 2011
@@ -252,7 +252,8 @@ public class YARNRunner implements Clien
       throws IOException {
     LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
     FileStatus rsrcStat = fs.getFileStatus(p);
-    rsrc.setResource(ConverterUtils.getYarnUrlFromPath(rsrcStat.getPath()));
+    rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
+        .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
     rsrc.setSize(rsrcStat.getLen());
     rsrc.setTimestamp(rsrcStat.getModificationTime());
     rsrc.setType(LocalResourceType.FILE);
@@ -274,9 +275,10 @@ public class YARNRunner implements Clien
 
     Path jobConfPath = new Path(jobSubmitDir, MRConstants.JOB_CONF_FILE);
     
-    URL yarnUrlForJobSubmitDir =
-        ConverterUtils.getYarnUrlFromPath(defaultFileContext.makeQualified(new Path(
-            jobSubmitDir)));
+    URL yarnUrlForJobSubmitDir = ConverterUtils
+        .getYarnUrlFromPath(defaultFileContext.getDefaultFileSystem()
+            .resolvePath(
+                defaultFileContext.makeQualified(new Path(jobSubmitDir))));
     LOG.debug("Creating setup context, jobSubmitDir url is "
         + yarnUrlForJobSubmitDir);
 
@@ -399,7 +401,7 @@ public class YARNRunner implements Clien
   private void parseDistributedCacheArtifacts(
       ApplicationSubmissionContext container, LocalResourceType type,
       URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[], 
-      Path[] pathsToPutOnClasspath) {
+      Path[] pathsToPutOnClasspath) throws IOException {
 
     if (uris != null) {
       // Sanity check
@@ -424,8 +426,9 @@ public class YARNRunner implements Clien
       for (int i = 0; i < uris.length; ++i) {
         URI u = uris[i];
         Path p = new Path(u);
-        p = p.makeQualified(this.defaultFileContext.getDefaultFileSystem()
-              .getUri(), this.defaultFileContext.getWorkingDirectory());
+        p = defaultFileContext.getDefaultFileSystem().resolvePath(
+            p.makeQualified(this.defaultFileContext.getDefaultFileSystem()
+                .getUri(), this.defaultFileContext.getWorkingDirectory()));
         // Add URI fragment or just the filename
         Path name = new Path((null == u.getFragment())
           ? p.getName()
@@ -464,9 +467,9 @@ public class YARNRunner implements Clien
     return result;
   }
   
-  private static LocalResource createLocalResource(URI uri, 
+  private LocalResource createLocalResource(URI uri, 
       LocalResourceType type, LocalResourceVisibility visibility, 
-      long size, long timestamp) {
+      long size, long timestamp) throws IOException {
     LocalResource resource = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(LocalResource.class);
     resource.setResource(ConverterUtils.getYarnUrlFromURI(uri));
     resource.setType(type);



Mime
View raw message