hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1476464 - in /hadoop/common/branches/branch-1.2: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobInProgress.java src/mapred/org/apache/hadoop/mapred/JobTracker.java src/test/org/apache/hadoop/mapred/TestRecoveryManager.java
Date Sat, 27 Apr 2013 01:43:49 GMT
Author: acmurthy
Date: Sat Apr 27 01:43:49 2013
New Revision: 1476464

URL: http://svn.apache.org/r1476464
Log:
Merge -c 1476463 from branch-1 to branch-1.2 to fix MAPREDUCE-5158. Cleanup job-submission
related files on HDFS during JobTracker restart for jobs which set mapreduce.job.restart.recover
to false to indicate they don't want to be recovered. Contributed by Mayank Bansal.

Modified:
    hadoop/common/branches/branch-1.2/CHANGES.txt
    hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java

Modified: hadoop/common/branches/branch-1.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/CHANGES.txt?rev=1476464&r1=1476463&r2=1476464&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.2/CHANGES.txt Sat Apr 27 01:43:49 2013
@@ -594,6 +594,11 @@ Release 1.2.0 - 2013.04.16
     HADOOP-9502. chmod/chown do not return error exit codes for some exceptions.
     (szetszwo)
 
+    MAPREDUCE-5158. Cleanup job-submission related files on HDFS during
+    JobTracker restart for jobs which set mapreduce.job.restart.recover to
+    false to indicate they don't want to be recovered. (Mayank Bansal via
+    acmurthy) 
+
 Release 1.1.2 - 2013.01.30
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1476464&r1=1476463&r2=1476464&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Sat Apr 27 01:43:49 2013
@@ -3298,6 +3298,16 @@ public class JobInProgress {
       jobtracker.storeCompletedJob(this);
       jobtracker.finalizeJob(this);
 
+    }
+    cleanupJob();
+  }
+  
+  /**
+   * The job is dead. We're now cleaning it, getting rid of job directories and
+   * removing all delegation token etc.
+   */
+  void cleanupJob() {
+    synchronized (this) {
       try {
         // Definitely remove the local-disk copy of the job file
         if (localJobFile != null) {

Modified: hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1476464&r1=1476463&r2=1476464&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Sat Apr 27 01:43:49 2013
@@ -3609,6 +3609,8 @@ public class JobTracker implements MRCon
         !job.getJobConf().getBoolean(
             JobConf.MAPREDUCE_RECOVER_JOB, 
             JobConf.DEFAULT_MAPREDUCE_RECOVER_JOB)) {
+      LOG.info("Job "+ jobId.toString() + " is not enable for recovery, cleaning up job files");
+      job.cleanupJob();
       return null;
     }
     

Modified: hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=1476464&r1=1476463&r2=1476464&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java
(original)
+++ hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java
Sat Apr 27 01:43:49 2013
@@ -257,6 +257,8 @@ public class TestRecoveryManager {
       UtilsForTests.waitFor(100);
     }
     Assert.assertTrue("Task should be successful", rJob1.isSuccessful());
+    Assert.assertTrue("Job should be cleaned up", !fs.exists(new Path(job1.get("mapreduce.job.dir"))));
+    Assert.assertTrue("Job should be cleaned up", !fs.exists(new Path(job2.get("mapreduce.job.dir"))));
   }
 
   public static class TestJobTrackerInstrumentation extends JobTrackerInstrumentation {



Mime
View raw message