hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r700021 - in /hadoop/core/trunk: CHANGES.txt conf/hadoop-default.xml src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java src/mapred/org/apache/hadoop/mapred/JobHistory.java src/webapps/job/loadhistory.jsp
Date Mon, 29 Sep 2008 08:08:50 GMT
Author: ddas
Date: Mon Sep 29 01:08:50 2008
New Revision: 700021

URL: http://svn.apache.org/viewvc?rev=700021&view=rev
Log:
HADOOP-4189. Fixes the history blocksize & intertracker protocol version issues introduced
as part of HADOOP-3245. Contributed by Amar Kamat.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/conf/hadoop-default.xml
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
    hadoop/core/trunk/src/webapps/job/loadhistory.jsp

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Sep 29 01:08:50 2008
@@ -776,6 +776,9 @@
     memory management in TaskTracker are disabled on Windows.
     (Vinod K V via rangadi)
 
+    HADOOP-4189. Fixes the history blocksize & intertracker protocol version
+    issues introduced as part of HADOOP-3245. (Amar Kamat via ddas)
+
 Release 0.18.2 - Unreleased
 
   BUG FIXES

Modified: hadoop/core/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/hadoop-default.xml?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/conf/hadoop-default.xml (original)
+++ hadoop/core/trunk/conf/hadoop-default.xml Mon Sep 29 01:08:50 2008
@@ -762,19 +762,11 @@
 
 <property>
   <name>mapred.jobtracker.job.history.block.size</name>
-  <value>0</value>
+  <value>3145728></value>
   <description>The block size of the job history file. Since the job recovery
                uses job history, its important to dump job history to disk as 
-               soon as possible.
-  </description>
-</property>
-
-<property>
-  <name>mapred.jobtracker.job.history.buffer.size</name>
-  <value>4096</value>
-  <description>The buffer size for the job history file. Since the job 
-               recovery uses job history, its important to frequently flush the 
-               job history to disk. This will minimize the loss in recovery.
+               soon as possible. Note that this is an expert level parameter.
+               The default value is set to 3 MB.
   </description>
 </property>
 

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java Mon Sep
29 01:08:50 2008
@@ -48,6 +48,8 @@
    * Version 16: adds ResourceStatus to TaskTrackerStatus for HADOOP-3759
    * Version 17: Changed format of Task and TaskStatus for HADOOP-3150
    * Version 18: Changed status message due to changes in TaskStatus
+   *             Changed heartbeat to piggyback JobTracker restart information
+                 so that the TaskTracker can synchronize itself.
    */
   public static final long versionID = 18L;
   

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java Mon Sep 29 01:08:50
2008
@@ -85,7 +85,6 @@
   private static boolean disableHistory = false; 
   private static final String SECONDARY_FILE_SUFFIX = ".recover";
   private static long jobHistoryBlockSize = 0;
-  private static int jobHistoryBufferSize = 0;
   private static String jobtrackerHostname;
   /**
    * Record types are identifiers for each line of log in history files. 
@@ -148,14 +147,10 @@
       }
       conf.set("hadoop.job.history.location", LOG_DIR);
       disableHistory = false;
-      // set the job history block size
+      // set the job history block size (default is 3MB)
       jobHistoryBlockSize = 
         conf.getLong("mapred.jobtracker.job.history.block.size", 
-                     fs.getDefaultBlockSize());
-      // set the job history buffer size
-      jobHistoryBufferSize = 
-        Integer.parseInt(conf.get("mapred.jobtracker.job.history.buffer.size", 
-                                  "4096"));
+                     3 * 1024 * 1024);
     } catch(IOException e) {
         LOG.error("Failed to initialize JobHistory log file", e); 
         disableHistory = true;
@@ -729,8 +724,10 @@
             
             logFile = recoverJobHistoryFile(jobConf, logFile);
             
+            int defaultBufferSize = 
+              fs.getConf().getInt("io.file.buffer.size", 4096);
             out = fs.create(logFile, FsPermission.getDefault(), true, 
-                            jobHistoryBufferSize, 
+                            defaultBufferSize, 
                             fs.getDefaultReplication(), 
                             jobHistoryBlockSize, null);
             writer = new PrintWriter(out);

Modified: hadoop/core/trunk/src/webapps/job/loadhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/loadhistory.jsp?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/src/webapps/job/loadhistory.jsp (original)
+++ hadoop/core/trunk/src/webapps/job/loadhistory.jsp Mon Sep 29 01:08:50 2008
@@ -24,9 +24,16 @@
     // then remove the attribute
     // if the job has not yet finished, remove the attribute sothat it 
     // gets refreshed.
+    boolean isJobComplete = false;
+    if (null != job) {
+      String jobStatus = job.get(Keys.JOB_STATUS);
+      isJobComplete = Values.SUCCESS.name() == jobStatus
+                      || Values.FAILED.name() == jobStatus
+                      || Values.KILLED.name() == jobStatus;
+    }
     if (null != job && 
        (!jobId.equals(job.get(Keys.JOBID)) 
-         || job.get(Keys.JOB_STATUS) == "")) {
+         || !isJobComplete)) {
       // remove jobInfo from session, keep only one job in session at a time
       request.getSession().removeAttribute("job"); 
       job = null ; 



Mime
View raw message