hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1384878 - in /hadoop/common/branches/branch-1-win: CHANGES.branch-1-win.txt src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java
Date Fri, 14 Sep 2012 18:26:24 GMT
Author: vinodkv
Date: Fri Sep 14 18:26:23 2012
New Revision: 1384878

URL: http://svn.apache.org/viewvc?rev=1384878&view=rev
Log:
HADOOP-8734. Modify LocalJobRunner to support private distributed cache. Contributed by Ivan
Mitic.

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java

Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1384878&r1=1384877&r2=1384878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Fri Sep 14 18:26:23 2012
@@ -1,4 +1,6 @@
-branch-hadoop-1-win - unreleased
+Branch-hadoop-1-win - unreleased
+
+  NEW FEATURES
 
   IMPROVEMENTS
 
@@ -84,6 +86,11 @@ branch-hadoop-1-win - unreleased
     MAPREDUCE-4374. Completed support for configurable environment for child
     map/reduce tasks. (Chuan Liu via acmurthy)
 
+    HADOOP-8734. Modify LocalJobRunner to support private distributed cache.
+    (Ivan Mitic via vinodkv)
+
+OPTIMAZATIONS
+
 BUG FIXES
 
     HDFS-6527. Backport HADOOP-7389: Use of TestingGroups by tests causes

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java?rev=1384878&r1=1384877&r2=1384878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
Fri Sep 14 18:26:23 2012
@@ -122,6 +122,8 @@ class LocalJobRunner implements JobSubmi
         trackerDistributedCacheManager.newTaskDistributedCacheManager(
             jobid, conf);
       taskDistributedCacheManager.setupCache(conf, "archive", "archive");
+      // Localize private distributed cache
+      JobLocalizer.downloadPrivateCache(conf);
       
       if (DistributedCache.getSymlink(conf)) {
         // This is not supported largely because,

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java?rev=1384878&r1=1384877&r2=1384878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java
Fri Sep 14 18:26:23 2012
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
@@ -133,6 +134,10 @@ public class TestMRWithDistributedCache 
         makeJar(new Path(TEST_ROOT_DIR, "distributed.third.jar"), 3);
     Path fourth =
         makeJar(new Path(TEST_ROOT_DIR, "distributed.fourth.jar"), 4);
+    // Change permissions on one file to be private (others cannot read
+    // the file) to make sure private distributed cache works fine with
+    // the LocalJobRunner.
+    FileUtil.chmod(fourth.toUri().getPath(), "700");
 
     // Creates the Job Configuration
     DistributedCache.addCacheFile(



Mime
View raw message