hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077102 - in /hadoop/common/branches/branch-0.20-security-patches/src: core/org/apache/hadoop/filecache/DistributedCache.java mapred/org/apache/hadoop/mapred/TaskRunner.java
Date Fri, 04 Mar 2011 03:40:45 GMT
Author: omalley
Date: Fri Mar  4 03:40:45 2011
New Revision: 1077102

URL: http://svn.apache.org/viewvc?rev=1077102&view=rev
Log:
commit d1705ffb5c5cc115bf9d95281b4cee1cfac15a30
Author: Hemanth Yamijala <yhemanth@yahoo-inc.com>
Date:   Mon Jan 11 20:28:02 2010 +0530

    Reverting patch https://issues.apache.org/jira/secure/attachment/12426383/patch-1140-2-ydist.txt
for MAPREDUCE:1140

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java?rev=1077102&r1=1077101&r2=1077102&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java
Fri Mar  4 03:40:45 2011
@@ -216,49 +216,38 @@ public class DistributedCache {
       lcacheStatus.refcount++;
     }
     
-    boolean initSuccessful = false;
-    try {
-      synchronized (lcacheStatus) {
-        if (!lcacheStatus.isInited()) {
-          localizedPath = localizeCache(conf, cache, confFileStamp,
-              lcacheStatus, fileStatus, isArchive);
-          lcacheStatus.initComplete();
-        } else {
-          localizedPath = checkCacheStatusValidity(conf, cache, confFileStamp,
-              lcacheStatus, fileStatus, isArchive);
-        }
-        createSymlink(conf, cache, lcacheStatus, isArchive, currentWorkDir,
-            honorSymLinkConf);
-      }
-
-      // try deleting stuff if you can
-      long size = 0;
-      synchronized (lcacheStatus) {
-        synchronized (baseDirSize) {
-          Long get = baseDirSize.get(lcacheStatus.getBaseDir());
-          if (get != null) {
-            size = get.longValue();
-          } else {
-            LOG.warn("Cannot find size of baseDir: "
-                + lcacheStatus.getBaseDir());
-          }
-        }
+    synchronized (lcacheStatus) {
+      if (!lcacheStatus.isInited()) {
+        localizedPath = localizeCache(conf, cache, confFileStamp, lcacheStatus,
+          fileStatus, isArchive);
+        lcacheStatus.initComplete();
+      } else {
+        localizedPath = checkCacheStatusValidity(conf, cache, confFileStamp,
+          lcacheStatus, fileStatus, isArchive);
       }
-      // setting the cache size to a default of 10GB
-      long allowedSize = conf.getLong("local.cache.size", DEFAULT_CACHE_SIZE);
-      if (allowedSize < size) {
-        // try some cache deletions
-        deleteCache(conf);
-      }
-      initSuccessful = true;
-      return localizedPath;
-    } finally {
-      if (!initSuccessful) {
-        synchronized (cachedArchives) {
-          lcacheStatus.refcount--;
+      createSymlink(conf, cache, lcacheStatus, isArchive,
+        currentWorkDir, honorSymLinkConf);
+    }
+ 
+    // try deleting stuff if you can
+    long size = 0;
+    synchronized (lcacheStatus) {
+      synchronized (baseDirSize) {
+        Long get = baseDirSize.get(lcacheStatus.getBaseDir());
+        if ( get != null ) {
+    	    size = get.longValue();
+        } else {
+          LOG.warn("Cannot find size of baseDir: " + lcacheStatus.getBaseDir());
         }
       }
     }
+    // setting the cache size to a default of 10GB
+    long allowedSize = conf.getLong("local.cache.size", DEFAULT_CACHE_SIZE);
+    if (allowedSize < size) {
+      // try some cache deletions
+      deleteCache(conf);
+    }
+    return localizedPath;
   }
 
   

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1077102&r1=1077101&r2=1077102&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
Fri Mar  4 03:40:45 2011
@@ -141,19 +141,9 @@ abstract class TaskRunner extends Thread
     return jobConf.get(JobConf.MAPRED_TASK_ENV);
   }
   
-  private static class CacheFile {
-    URI uri;
-    long timeStamp;
-    CacheFile (URI uri, long timeStamp) {
-      this.uri = uri;
-      this.timeStamp = timeStamp;
-    }
-  }
-  
   @Override
   public final void run() {
     String errorInfo = "Child Error";
-    List<CacheFile> localizedCacheFiles = new ArrayList<CacheFile>();
     try {
       
       //before preparing the job localize 
@@ -198,8 +188,6 @@ abstract class TaskRunner extends Thread
                                                         getAbsolutePath()), 
                                                   false,
                                                   lDirAlloc);
-            localizedCacheFiles.add(new CacheFile(archives[i], Long
-                .parseLong(archivesTimestamps[i])));
             
           }
           DistributedCache.setLocalArchives(conf, stringifyPathArray(p));
@@ -220,8 +208,6 @@ abstract class TaskRunner extends Thread
                                                         getAbsolutePath()), 
                                                   false,
                                                   lDirAlloc);
-            localizedCacheFiles.add(new CacheFile(files[i], Long
-                .parseLong(fileTimestamps[i])));
           }
           DistributedCache.setLocalFiles(conf, stringifyPathArray(p));
         }
@@ -542,8 +528,22 @@ abstract class TaskRunner extends Thread
       }
     } finally {
       try{
-        for (CacheFile cf : localizedCacheFiles){
-          DistributedCache.releaseCache(cf.uri, conf, cf.timeStamp);
+        URI[] archives = DistributedCache.getCacheArchives(conf);
+        URI[] files = DistributedCache.getCacheFiles(conf);
+        String[] archivesTimestamps = 
+          DistributedCache.getArchiveTimestamps(conf);
+        String[] fileTimestamps = DistributedCache.getFileTimestamps(conf);
+        if (archives != null){
+          for (int i = 0; i < archives.length; i++){
+            DistributedCache.releaseCache(archives[i], conf,
+              Long.parseLong(archivesTimestamps[i]));
+          }
+        }
+        if (files != null){
+          for(int i = 0; i < files.length; i++){
+            DistributedCache.releaseCache(files[i], conf,
+              Long.parseLong(fileTimestamps[i]));
+          }
         }
       }catch(IOException ie){
         LOG.warn("Error releasing caches : Cache files might not have been cleaned up");



Mime
View raw message