Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: common-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1076957 - in
 /hadoop/common/branches/branch-0.20-security-patches/src:
 mapred/org/apache/hadoop/mapred/JobHistory.java
 mapred/org/apache/hadoop/mapred/JobTracker.java
 test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
Date: Fri, 04 Mar 2011 03:25:52 -0000
To: common-commits@hadoop.apache.org
From: omalley@apache.org
Message-Id: <20110304032553.103662388901@eris.apache.org>

Author: omalley
Date: Fri Mar  4 03:25:52 2011
New Revision: 1076957

URL: http://svn.apache.org/viewvc?rev=1076957&view=rev
Log:
commit 95b712ea53d706cee2450155f670bef9848ec9f9
Author: Lee Tucker <ltucker@yahoo-inc.com>
Date:   Thu Jul 30 17:40:44 2009 -0700

    Applying patch 2871577.mr693.patch

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1076957&r1=1076956&r2=1076957&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java Fri Mar  4 03:25:52 2011
@@ -104,6 +104,15 @@ public class JobHistory {
   private static JobConf jtConf;
   private static Path DONE = null; // folder for completed jobs
   /**
+   * A filter for conf files
+   */  
+  private static final PathFilter CONF_FILTER = new PathFilter() {
+    public boolean accept(Path path) {
+      return path.getName().endsWith("_conf.xml");
+    }
+  };
+
+  /**
    * A class that manages all the files related to a job. For now 
    *   - writers : list of open files
    *   - job history filename
@@ -938,6 +947,19 @@ public class JobHistory {
     }
 
     /**
+     * Delete job conf from the history folder.
+     */
+    static void deleteConfFiles() throws IOException {
+      LOG.info("Cleaning up config files from the job history folder");
+      FileSystem fs = new Path(LOG_DIR).getFileSystem(jtConf);
+      FileStatus[] status = fs.listStatus(new Path(LOG_DIR), CONF_FILTER);
+      for (FileStatus s : status) {
+        LOG.info("Deleting conf file " + s.getPath());
+        fs.delete(s.getPath(), false);
+      }
+    }
+
+    /**
      * Move the completed job into the completed folder.
      * This assumes that the jobhistory file is closed and all operations on the
      * jobhistory file is complete.

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1076957&r1=1076956&r2=1076957&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java Fri Mar  4 03:25:52 2011
@@ -1430,6 +1430,14 @@ public class JobTracker implements MRCon
       Iterator<JobID> idIter = jobsToRecover.iterator();
       JobInProgress job = null;
       File jobIdFile = null;
+
+      // 0. Cleanup
+      try {
+        JobHistory.JobInfo.deleteConfFiles();
+      } catch (IOException ioe) {
+        LOG.info("Error in cleaning up job history folder", ioe);
+      }
+
       while (idIter.hasNext()) {
         JobID id = idIter.next();
         LOG.info("Trying to recover details of job " + id);

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java?rev=1076957&r1=1076956&r2=1076957&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java Fri Mar  4 03:25:52 2011
@@ -487,6 +487,11 @@ public class TestJobTrackerRestart exten
     String history = 
       JobHistory.JobInfo.getJobHistoryFileName(jip.getJobConf(), id);
     Path historyPath = JobHistory.JobInfo.getJobHistoryLogLocation(history);
+    // get the conf file name
+    String parts[] = history.split("_");
+    // jobtracker-hostname_jobtracker-identifier_conf.xml
+    String jobUniqueString = parts[0] + "_" + parts[1] + "_" +  id;
+    Path confPath = new Path(historyPath.getParent(), jobUniqueString + "_conf.xml");
     
     //  make sure that setup is launched
     while (jip.runningMaps() == 0) {
@@ -521,6 +526,10 @@ public class TestJobTrackerRestart exten
     
     job1.waitForCompletion();
     job2.waitForCompletion();
+
+    // check if the old files are deleted
+    assertFalse("Old jobhistory file is not deleted", historyFS.exists(historyPath));
+    assertFalse("Old jobconf file is not deleted", historyFS.exists(confPath));
   }
   
   public void testJobTrackerRestart() throws IOException {