hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r1134497 - in /hadoop/mapreduce/branches/MR-279: CHANGES.txt mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Date Sat, 11 Jun 2011 01:39:35 GMT
Author: ddas
Date: Sat Jun 11 01:39:35 2011
New Revision: 1134497

URL: http://svn.apache.org/viewvc?rev=1134497&view=rev
Log:
Changes a couple of usages of FileContext to FileSystem in YarnRunner to handle distributed
cache path resolutions on non-default filesystems. Contributed by Devaraj Das.

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1134497&r1=1134496&r2=1134497&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Sat Jun 11 01:39:35 2011
@@ -5,6 +5,10 @@ Trunk (unreleased changes)
 
     MAPREDUCE-279
 
+    Changes a couple of usages of FileContext to FileSystem in YarnRunner
+    to handle distributed cache path resolutions on non-default filesystems.
+    (ddas)
+
     Fix to exclude images dir into the tar distribution (luke lu via gkesavan)
 
     Fix more rounding off problems in reduce ramp up. Also fix a bug preventing

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1134497&r1=1134496&r2=1134497&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Sat Jun 11 01:39:35 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -380,7 +381,7 @@ public class YARNRunner implements Clien
       ApplicationSubmissionContext container) throws IOException {
     
     // Cache archives
-    parseDistributedCacheArtifacts(container, LocalResourceType.ARCHIVE, 
+    parseDistributedCacheArtifacts(conf, container, LocalResourceType.ARCHIVE, 
         DistributedCache.getCacheArchives(conf), 
         DistributedCache.getArchiveTimestamps(conf), 
         getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), 
@@ -388,7 +389,7 @@ public class YARNRunner implements Clien
         DistributedCache.getArchiveClassPaths(conf));
     
     // Cache files
-    parseDistributedCacheArtifacts(container, LocalResourceType.FILE, 
+    parseDistributedCacheArtifacts(conf, container, LocalResourceType.FILE, 
         DistributedCache.getCacheFiles(conf),
         DistributedCache.getFileTimestamps(conf),
         getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
@@ -398,7 +399,7 @@ public class YARNRunner implements Clien
 
   // TODO - Move this to MR!
   // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], long[], boolean[],
Path[], FileType)
-  private void parseDistributedCacheArtifacts(
+  private void parseDistributedCacheArtifacts(Configuration conf,
       ApplicationSubmissionContext container, LocalResourceType type,
       URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[], 
       Path[] pathsToPutOnClasspath) throws IOException {
@@ -418,17 +419,17 @@ public class YARNRunner implements Clien
       Map<String, Path> classPaths = new HashMap<String, Path>();
       if (pathsToPutOnClasspath != null) {
         for (Path p : pathsToPutOnClasspath) {
-          p = p.makeQualified(this.defaultFileContext.getDefaultFileSystem()
-                .getUri(), this.defaultFileContext.getWorkingDirectory());
+          FileSystem fs = p.getFileSystem(conf);
+          p = p.makeQualified(fs.getUri(), fs.getWorkingDirectory());
           classPaths.put(p.toUri().getPath().toString(), p);
         }
       }
       for (int i = 0; i < uris.length; ++i) {
         URI u = uris[i];
         Path p = new Path(u);
-        p = defaultFileContext.getDefaultFileSystem().resolvePath(
-            p.makeQualified(this.defaultFileContext.getDefaultFileSystem()
-                .getUri(), this.defaultFileContext.getWorkingDirectory()));
+        FileSystem fs = p.getFileSystem(conf);
+        p = fs.resolvePath(
+            p.makeQualified(fs.getUri(), fs.getWorkingDirectory()));
         // Add URI fragment or just the filename
         Path name = new Path((null == u.getFragment())
           ? p.getName()



Mime
View raw message