hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject svn commit: r1525774 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/CleanupQueue.java src/mapred/org/apache/hadoop/mapred/JobInProgress.java src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
Date Tue, 24 Sep 2013 05:05:01 GMT
Author: cnauroth
Date: Tue Sep 24 05:05:00 2013
New Revision: 1525774

URL: http://svn.apache.org/r1525774
Log:
MAPREDUCE-5508. JobTracker memory leak caused by unreleased FileSystem objects in JobInProgress#cleanupJob.
Contributed by Xi Fang.

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1525774&r1=1525773&r2=1525774&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Sep 24 05:05:00 2013
@@ -139,6 +139,9 @@ Release 1.3.0 - unreleased
     FileSystem#close can cause return of a closed DistributedFileSystem instance
     from the FileSystem cache. (cnauroth)
 
+    MAPREDUCE-5508. JobTracker memory leak caused by unreleased FileSystem
+    objects in JobInProgress#cleanupJob. (Xi Fang via cnauroth)
+
 Release 1.2.2 - unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java?rev=1525774&r1=1525773&r2=1525774&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
(original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
Tue Sep 24 05:05:00 2013
@@ -60,14 +60,10 @@ public class CleanupQueue {
     final Configuration conf;
     final UserGroupInformation ugi;
     final JobID jobIdTokenRenewalToCancel;
+    FileSystem fs;
 
     public PathDeletionContext(Path fullPath, Configuration conf) {
-      this(fullPath, conf, null, null);
-    }
-
-    public PathDeletionContext(Path fullPath, Configuration conf,
-        UserGroupInformation ugi) {
-      this(fullPath, conf, ugi, null);
+      this(fullPath, conf, null, null, null);
     }
     
     /**
@@ -86,11 +82,12 @@ public class CleanupQueue {
      *                                  <code>null</code>
      */
     public PathDeletionContext(Path fullPath, Configuration conf,
-        UserGroupInformation ugi, JobID jobIdTokenRenewalToCancel) {
+        UserGroupInformation ugi, JobID jobIdTokenRenewalToCancel, FileSystem fs) {
       this.fullPath = fullPath;
       this.conf = conf;
       this.ugi = ugi;
       this.jobIdTokenRenewalToCancel = jobIdTokenRenewalToCancel;
+      this.fs = fs;
     }
     
     protected Path getPathForCleanup() {
@@ -106,7 +103,7 @@ public class CleanupQueue {
       (ugi == null ? UserGroupInformation.getLoginUser() : ugi).doAs(
           new PrivilegedExceptionAction<Object>() {
             public Object run() throws IOException {
-              FileSystem fs = p.getFileSystem(conf);
+              fs = (fs == null ?  p.getFileSystem(conf) : fs);
               try {
                 fs.delete(p, true);
                 return null;

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1525774&r1=1525773&r2=1525774&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Tue Sep 24 05:05:00 2013
@@ -3330,10 +3330,17 @@ public class JobInProgress {
         String jobTempDir = conf.get("mapreduce.job.dir");
         if (jobTempDir != null && conf.getKeepTaskFilesPattern() == null &&
             !conf.getKeepFailedTaskFiles()) {
-          Path jobTempDirPath = new Path(jobTempDir);
-          tempDirFs = jobTempDirPath.getFileSystem(conf);
+          final Path jobTempDirPath = new Path(jobTempDir);
+          try {
+            tempDirFs = userUGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
+              public FileSystem run() throws IOException {
+                return jobTempDirPath.getFileSystem(conf);
+              }});
+          } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+          }
           CleanupQueue.getInstance().addToQueue(
-              new PathDeletionContext(jobTempDirPath, conf, userUGI, jobId));
+              new PathDeletionContext(jobTempDirPath, conf, userUGI, jobId, tempDirFs));
         }
 
       } catch (IOException e) {

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java?rev=1525774&r1=1525773&r2=1525774&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
Tue Sep 24 05:05:00 2013
@@ -44,7 +44,7 @@ public class TestCleanupQueue {
     // With UGI, should close FileSystem
     CleanupQueue cleanupQueue = new CleanupQueue();
     PathDeletionContext context = new PathDeletionContext(path, conf,
-        UserGroupInformation.getLoginUser());
+        UserGroupInformation.getLoginUser(), null, null);
     cleanupQueue.addToQueue(context);
     
     while (FileSystem.getCacheSize() > 0) {



Mime
View raw message