hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r777025 - in /hadoop/core/branches/branch-0.20: ./ CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobHistory.java src/mapred/org/apache/hadoop/mapred/JobInProgress.java src/mapred/org/apache/hadoop/mapred/JobTracker.java
Date Thu, 21 May 2009 09:00:07 GMT
Author: ddas
Date: Thu May 21 09:00:06 2009
New Revision: 777025

URL: http://svn.apache.org/viewvc?rev=777025&view=rev
Log:
Merge -r 772883:772884 from trunk onto 0.20 branch. Fixes HADOOP-4372.

Modified:
    hadoop/core/branches/branch-0.20/   (props changed)
    hadoop/core/branches/branch-0.20/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java

Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 21 09:00:06 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772920,773889,776638
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638

Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=777025&r1=777024&r2=777025&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Thu May 21 09:00:06 2009
@@ -20,6 +20,9 @@
     HADOOP-4674. Fix fs help messages for -test, -text, -tail, -stat 
     and -touchz options.  (Ravi Phulari via szetszwo)
 
+    HADOOP-4372. Improves the way history filenames are obtained and manipulated.
+    (Amar Kamat via ddas)
+
   OPTIMIZATIONS
 
   BUG FIXES

Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 21 09:00:06 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,772844,772876,772920,773889,776638
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,772844,772876,772884,772920,773889,776638

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=777025&r1=777024&r2=777025&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobHistory.java Thu
May 21 09:00:06 2009
@@ -666,12 +666,9 @@
       };
       
       FileStatus[] statuses = fs.listStatus(new Path(LOG_DIR), filter);
-      String filename;
+      String filename = null;
       if (statuses.length == 0) {
-        filename = 
-          encodeJobHistoryFileName(getNewJobHistoryFileName(jobConf, id));
-        LOG.info("Nothing to recover! Generating a new filename " + filename 
-                 + " for job " + id);
+        LOG.info("Nothing to recover for job " + id);
       } else {
         // return filename considering that fact the name can be a 
         // secondary filename like filename.recover
@@ -792,6 +789,9 @@
     throws IOException {
       String masterLogFileName = 
         JobHistory.JobInfo.getJobHistoryFileName(conf, id);
+      if (masterLogFileName == null) {
+        return;
+      }
       Path masterLogPath = 
         JobHistory.JobInfo.getJobHistoryLogLocation(masterLogFileName);
       String tmpLogFileName = getSecondaryJobHistoryFile(masterLogFileName);
@@ -834,10 +834,19 @@
      * @param jobConfPath path to job conf xml file in HDFS.
      * @param submitTime time when job tracker received the job
      * @throws IOException
+     * @deprecated Use 
+     *     {@link #logSubmitted(JobID, JobConf, String, long, boolean)} instead.
      */
     public static void logSubmitted(JobID jobId, JobConf jobConf, 
                                     String jobConfPath, long submitTime) 
     throws IOException {
+      logSubmitted(jobId, jobConf, jobConfPath, submitTime, true);
+    }
+    
+    public static void logSubmitted(JobID jobId, JobConf jobConf, 
+                                    String jobConfPath, long submitTime, 
+                                    boolean restarted) 
+    throws IOException {
       FileSystem fs = null;
       String userLogDir = null;
       String jobUniqueString = JOBTRACKER_UNIQUE_STRING + jobId;
@@ -850,8 +859,13 @@
         String user = getUserName(jobConf);
         
         // get the history filename
-        String logFileName = 
-          getJobHistoryFileName(jobConf, jobId);
+        String logFileName = null;
+        if (restarted) {
+          logFileName = getJobHistoryFileName(jobConf, jobId);
+        } else {
+          logFileName = 
+            encodeJobHistoryFileName(getNewJobHistoryFileName(jobConf, jobId));
+        }
 
         // setup the history log file for this job
         Path logFile = getJobHistoryLogLocation(logFileName);
@@ -869,8 +883,10 @@
             // create output stream for logging in hadoop.job.history.location
             fs = new Path(LOG_DIR).getFileSystem(jobConf);
             
-            logFile = recoverJobHistoryFile(jobConf, logFile);
-            logFileName = logFile.getName();
+            if (restarted) {
+              logFile = recoverJobHistoryFile(jobConf, logFile);
+              logFileName = logFile.getName();
+            }
             
             int defaultBufferSize = 
               fs.getConf().getInt("io.file.buffer.size", 4096);

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=777025&r1=777024&r2=777025&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Thu May 21 09:00:06 2009
@@ -373,6 +373,10 @@
     return tasksInited.get();
   }
   
+  boolean hasRestarted() {
+    return restartCount > 0;
+  }
+
   /**
    * Construct the splits, etc.  This is invoked from an async
    * thread so that split-computation doesn't block anyone.
@@ -392,7 +396,7 @@
 
     // log job info
     JobHistory.JobInfo.logSubmitted(getJobID(), conf, jobFile.toString(), 
-                                    this.startTime);
+                                    this.startTime, hasRestarted());
     // log the job priority
     setPriority(this.priority);
     

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=777025&r1=777024&r2=777025&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java Thu
May 21 09:00:06 2009
@@ -1234,18 +1234,23 @@
           // 3. Get the log file and the file path
           String logFileName = 
             JobHistory.JobInfo.getJobHistoryFileName(job.getJobConf(), id);
-          Path jobHistoryFilePath = 
-            JobHistory.JobInfo.getJobHistoryLogLocation(logFileName);
-
-          // 4. Recover the history file. This involved
-          //     - deleting file.recover if file exists
-          //     - renaming file.recover to file if file doesnt exist
-          // This makes sure that the (master) file exists
-          JobHistory.JobInfo.recoverJobHistoryFile(job.getJobConf(), 
-                                                   jobHistoryFilePath);
+          if (logFileName != null) {
+            Path jobHistoryFilePath = 
+              JobHistory.JobInfo.getJobHistoryLogLocation(logFileName);
+
+            // 4. Recover the history file. This involved
+            //     - deleting file.recover if file exists
+            //     - renaming file.recover to file if file doesnt exist
+            // This makes sure that the (master) file exists
+            JobHistory.JobInfo.recoverJobHistoryFile(job.getJobConf(), 
+                                                     jobHistoryFilePath);
           
-          // 5. Cache the history file name as it costs one dfs access
-          jobHistoryFilenameMap.put(job.getJobID(), jobHistoryFilePath);
+            // 5. Cache the history file name as it costs one dfs access
+            jobHistoryFilenameMap.put(job.getJobID(), jobHistoryFilePath);
+          } else {
+            LOG.info("No history file found for job " + id);
+            idIter.remove(); // remove from recovery list
+          }
 
           // 6. Sumbit the job to the jobtracker
           addJob(id, job);
@@ -2020,10 +2025,12 @@
 
     // start the merge of log files
     JobID id = job.getStatus().getJobID();
-    try {
-      JobHistory.JobInfo.finalizeRecovery(id, job.getJobConf());
-    } catch (IOException ioe) {
-      LOG.info("Failed to finalize the log file recovery for job " + id, ioe);
+    if (job.hasRestarted()) {
+      try {
+        JobHistory.JobInfo.finalizeRecovery(id, job.getJobConf());
+      } catch (IOException ioe) {
+        LOG.info("Failed to finalize the log file recovery for job " + id, ioe);
+      }
     }
 
     final JobTrackerInstrumentation metrics = getInstrumentation();



Mime
View raw message