hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r1087209 - in /hadoop/common/branches/branch-0.20-security: CHANGES.txt src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
Date Thu, 31 Mar 2011 08:12:48 GMT
Author: cdouglas
Date: Thu Mar 31 08:12:48 2011
New Revision: 1087209

URL: http://svn.apache.org/viewvc?rev=1087209&view=rev
Log:
MAPREDUCE-2409. Distinguish distributed cache artifacts localized as
files, archives. Contributed by Siddharth Seth

Modified:
    hadoop/common/branches/branch-0.20-security/CHANGES.txt
    hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
    hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java

Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1087209&r1=1087208&r2=1087209&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Thu Mar 31 08:12:48 2011
@@ -18,7 +18,10 @@ Release 0.20.204.0 - unreleased
     HDFS-1773. Do not show decommissioned datanodes, which are not in both
     include and exclude lists, on web and JMX interfaces.
     (Tanping Wang via szetszwo)
-  
+
+    MAPREDUCE-2409. Distinguish distributed cache artifacts localized as
+    files, archives. (Siddharth Seth via cdouglas)
+
   IMPROVEMENTS
 
     HDFS-1541. Not marking datanodes dead when namenode in safemode.

Modified: hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java?rev=1087209&r1=1087208&r2=1087209&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Thu Mar 31 08:12:48 2011
@@ -147,7 +147,7 @@ public class TrackerDistributedCacheMana
                      boolean isPublic, CacheFile file) throws IOException {
     String key;
     String user = getLocalizedCacheOwner(isPublic);
-    key = getKey(cache, conf, confFileStamp, user);
+    key = getKey(cache, conf, confFileStamp, user, isArchive);
     CacheStatus lcacheStatus;
     Path localizedPath = null;
     Path localPath = null;
@@ -573,11 +573,12 @@ public class TrackerDistributedCacheMana
     return true;
   }
 
-  String getKey(URI cache, Configuration conf, long timeStamp, String user) 
-      throws IOException {
-    return makeRelative(cache, conf) + String.valueOf(timeStamp) + user;
+  String getKey(URI cache, Configuration conf, long timeStamp, String user,
+      boolean isArchive) throws IOException {
+    return (isArchive ? "a" : "f") + "^" + makeRelative(cache, conf)
+        + String.valueOf(timeStamp) + user;
   }
-  
+
   /**
    * This method create symlinks for all files in a given dir in another 
    * directory.

Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java?rev=1087209&r1=1087208&r2=1087209&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
Thu Mar 31 08:12:48 2011
@@ -319,7 +319,51 @@ public class TestTrackerDistributedCache
     checkLocalizedPath(true);
     checkLocalizedPath(false);
   }
-  
+
+  public void testSameNameFileArchiveCache() throws IOException,
+      InterruptedException {
+    if (!canRun()) {
+      return;
+    }
+    TrackerDistributedCacheManager manager = new TrackerDistributedCacheManager(
+        conf, taskController);
+    String userName = getJobOwnerName();
+    File workDir = new File(TEST_ROOT_DIR, "workdir");
+    Path cacheFile = new Path(TEST_ROOT_DIR, "fileArchiveCacheFile");
+
+    createPublicTempFile(cacheFile);
+    Configuration conf1 = new Configuration(conf);
+    conf1.set("user.name", userName);
+
+    DistributedCache.addCacheFile(cacheFile.toUri(), conf1);
+    DistributedCache.addCacheArchive(cacheFile.toUri(), conf1);
+    TrackerDistributedCacheManager.determineTimestamps(conf1);
+    TrackerDistributedCacheManager.determineCacheVisibilities(conf1);
+    dumpState(conf1);
+
+    TaskDistributedCacheManager handle = manager
+        .newTaskDistributedCacheManager(new JobID("jt", 1), conf1);
+    handle.setupCache(conf1, TaskTracker.getPublicDistributedCacheDir(),
+        TaskTracker.getPrivateDistributedCacheDir(userName));
+
+    TaskDistributedCacheManager.CacheFile cFile = handle.getCacheFiles().get(0);
+    TaskDistributedCacheManager.CacheFile cArchive = handle.getCacheFiles()
+        .get(1);
+
+    String distCacheDir = TaskTracker.getPublicDistributedCacheDir();
+
+    Path localizedPathForFile = manager.getLocalCache(cacheFile.toUri(), conf1,
+        distCacheDir, fs.getFileStatus(cacheFile), false, cFile.timestamp,
+        true, cFile);
+
+    Path localizedPathForArchive = manager.getLocalCache(cacheFile.toUri(),
+        conf1, distCacheDir, fs.getFileStatus(cacheFile), true,
+        cArchive.timestamp, true, cArchive);
+    assertNotSame("File and Archive resolve to the same path: "
+        + localizedPathForFile + ". Should differ.", localizedPathForFile,
+        localizedPathForArchive);
+  }
+
   private void appendStringArray(StringBuilder buffer, String[] data) {
     if (data != null && data.length != 0) {
       buffer.append(data[0]);



Mime
View raw message