hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r723722 - in /hadoop/core/branches/branch-0.18: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobInProgress.java src/mapred/org/apache/hadoop/mapred/JobTracker.java src/mapred/org/apache/hadoop/mapred/Task.java
Date Fri, 05 Dec 2008 12:28:09 GMT
Author: ddas
Date: Fri Dec  5 04:28:07 2008
New Revision: 723722

URL: http://svn.apache.org/viewvc?rev=723722&view=rev
Log:
HADOOP-4654. Removes temporary output directory for failed and killed tasks in the JobTracker's
task commit thread. Contributed by Amareshwari Sriramadasu.

Modified:
    hadoop/core/branches/branch-0.18/CHANGES.txt
    hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java

Modified: hadoop/core/branches/branch-0.18/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/CHANGES.txt?rev=723722&r1=723721&r2=723722&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.18/CHANGES.txt Fri Dec  5 04:28:07 2008
@@ -71,6 +71,10 @@
 
     HADOOP-4746. Job output directory should be normalized. (hairong)
 
+    HADOOP-4654. Removes temporary output directory for failed and killed
+    tasks in the JobTracker's task commit thread. 
+    (Amareshwari Sriramadasu via ddas)
+
 Release 0.18.2 - 2008-11-03
 
   BUG FIXES

Modified: hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=723722&r1=723721&r2=723722&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Dec  5 04:28:07 2008
@@ -524,6 +524,10 @@
         return;
       } else if (state == TaskStatus.State.FAILED ||
                  state == TaskStatus.State.KILLED) {
+        // To remove the temporary output of failed/killed tasks
+        JobWithTaskContext j = new JobWithTaskContext(this, tip, 
+                                     status.getTaskID(), metrics);
+        jobtracker.addToCommitQueue(j);
         // Get the event number for the (possibly) previously successful
         // task. If there exists one, then set that status to OBSOLETE 
         int eventNumber;

Modified: hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=723722&r1=723721&r2=723722&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/JobTracker.java Fri
Dec  5 04:28:07 2008
@@ -2318,6 +2318,15 @@
                     isTipComplete[index] = true;
                   }
                 }
+              } else if (states[index] == TaskStatus.State.FAILED || 
+                         states[index] == TaskStatus.State.KILLED) {
+                try {
+                  tasks[index].removeTaskOutput();
+                } catch (IOException e) {
+                  LOG.info("Failed to remove temporary directory of "
+                           + status[index].getTaskID() + " with " 
+                           + StringUtils.stringifyException(e));
+                }
               }
             } catch (IOException ioe) {
               // Oops! Failed to copy the task's output to its final place;

Modified: hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java?rev=723722&r1=723721&r2=723722&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/Task.java Fri Dec
 5 04:28:07 2008
@@ -571,6 +571,14 @@
     }
   }
   
+  void removeTaskOutput() throws IOException {
+    if (taskOutputPath != null) {
+      FileSystem fs = taskOutputPath.getFileSystem(conf);
+      // Delete the temporary task-specific output directory
+      fs.delete(taskOutputPath, true);
+    }
+  }
+  
   private Path getFinalPath(Path jobOutputDir, Path taskOutput) {
     URI relativePath = taskOutputPath.toUri().relativize(taskOutput.toUri());
     if (relativePath.getPath().length() > 0) {



Mime
View raw message