Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: mapreduce-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r788637 - in /hadoop/mapreduce/trunk: CHANGES.txt
 src/java/org/apache/hadoop/mapred/JobHistory.java
 src/java/org/apache/hadoop/mapred/JobTracker.java
 src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
Date: Fri, 26 Jun 2009 09:00:36 -0000
To: mapreduce-commits@hadoop.apache.org
From: sharad@apache.org
Message-Id: <20090626090036.4A1BD2388865@eris.apache.org>

Author: sharad
Date: Fri Jun 26 09:00:35 2009
New Revision: 788637

URL: http://svn.apache.org/viewvc?rev=788637&view=rev
Log:
MAPREDUCE-130. Delete the jobconf copy from the log directory of the JobTracker when the job is retired. Contributed by Amar Kamat.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=788637&r1=788636&r2=788637&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Jun 26 09:00:35 2009
@@ -42,3 +42,6 @@
 
     MAPREDUCE-2. Fixes a bug in KeyFieldBasedPartitioner in handling empty
     keys. (Amar Kamat via sharad)
+
+    MAPREDUCE-130. Delete the jobconf copy from the log directory of the 
+    JobTracker when the job is retired. (Amar Kamat via sharad)

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java?rev=788637&r1=788636&r2=788637&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobHistory.java Fri Jun 26 09:00:35 2009
@@ -846,6 +846,19 @@
     }
 
     /**
+     * Deletes job data from the local disk.
+     * For now just deletes the localized copy of job conf
+     */
+    static void cleanupJob(JobID id) {
+      String localJobFilePath =  JobInfo.getLocalJobFilePath(id);
+      File f = new File (localJobFilePath);
+      LOG.info("Deleting localized job conf at " + f);
+      if (!f.delete()) {
+        LOG.debug("Failed to delete file " + f);
+      }
+    }
+
+    /**
      * Log job submitted event to history. Creates a new file in history 
      * for the job. if history file creation fails, it disables history 
      * for all other events. 

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=788637&r1=788636&r2=788637&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Jun 26 09:00:35 2009
@@ -454,6 +454,9 @@
                     LOG.info("Retired job with id: '" + 
                              job.getProfile().getJobID() + "' of user '" +
                              jobUser + "'");
+
+                    // clean up job files from the local disk
+                    JobHistory.JobInfo.cleanupJob(job.getProfile().getJobID());
                   }
                 }
               }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java?rev=788637&r1=788636&r2=788637&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java Fri Jun 26 09:00:35 2009
@@ -788,10 +788,14 @@
   public void testJobHistoryFile() throws IOException {
     MiniMRCluster mr = null;
     try {
-      mr = new MiniMRCluster(2, "file:///", 3);
+      JobConf conf = new JobConf();
+      // keep for less time
+      conf.setLong("mapred.jobtracker.retirejob.check", 1000);
+      conf.setLong("mapred.jobtracker.retirejob.interval", 1000);
+      mr = new MiniMRCluster(2, "file:///", 3, null, null, conf);
 
       // run the TCs
-      JobConf conf = mr.createJobConf();
+      conf = mr.createJobConf();
 
       FileSystem fs = FileSystem.get(conf);
       // clean up
@@ -812,6 +816,15 @@
       validateJobHistoryFileFormat(job.getID(), conf, "SUCCESS", false);
       validateJobHistoryFileContent(mr, job, conf);
 
+      // get the job conf filename
+      String name = JobHistory.JobInfo.getLocalJobFilePath(job.getID());
+      File file = new File(name);
+
+      // check if the file get deleted
+      while (file.exists()) {
+        LOG.info("Waiting for " + file + " to be deleted");
+        UtilsForTests.waitFor(100);
+      }
     } finally {
       if (mr != null) {
         cleanupLocalFiles(mr);