hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r608373 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/mapred/
Date Thu, 03 Jan 2008 08:15:33 GMT
Author: ddas
Date: Thu Jan  3 00:15:30 2008
New Revision: 608373

URL: http://svn.apache.org/viewvc?rev=608373&view=rev
Log:
HADOOP-2208. Counter update frequency (from TaskTracker to JobTracker) is capped at 1 minute.
Contributed by Amareshwari Sri Ramadasu.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Jan  3 00:15:30 2008
@@ -152,6 +152,9 @@
     HADOOP-1900. Makes the heartbeat and task event queries interval 
     dependent on the cluster size.  (Amareshwari Sri Ramadasu via ddas)
 
+    HADOOP-2208. Counter update frequency (from TaskTracker to JobTracker) is 
+    capped at 1 minute.  (Amareshwari Sri Ramadasu via ddas)
+
   BUG FIXES
 
     HADOOP-2100.  Remove faulty check for existence of $HADOOP_PID_DIR and let

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java Thu Jan
 3 00:15:30 2008
@@ -39,8 +39,9 @@
    * TaskTrackerStatus for HADOOP-1274
    * Version 8: HeartbeatResponse is added with the next heartbeat interval.
    * version 9 changes the counter representation for HADOOP-2248
+   * version 10 changes the TaskStatus representation for HADOOP-2208
    */
-  public static final long versionID = 9L;
+  public static final long versionID = 10L;
   
   public final static int TRACKERS_OK = 0;
   public final static int UNKNOWN_TASKTRACKER = 1;

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java Thu Jan  3 00:15:30
2008
@@ -29,6 +29,8 @@
   
   public static final int CLUSTER_INCREMENT = 50;
 
+  public static final long COUNTER_UPDATE_INTERVAL = 60 * 1000;
+
   //for the inmemory filesystem (to do in-memory merge)
   /**
    * Constant denoting when a merge of in memory files will be triggered 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java Thu Jan  3 00:15:30
2008
@@ -55,6 +55,7 @@
     
   private Phase phase = Phase.STARTING; 
   private Counters counters;
+  private boolean includeCounters;
 
   public TaskStatus() {}
 
@@ -70,6 +71,7 @@
     this.taskTracker = taskTracker;
     this.phase = phase;
     this.counters = counters;
+    this.includeCounters = true;
   }
   
   public String getTaskId() { return taskid; }
@@ -181,6 +183,15 @@
     }
     this.phase = phase; 
   }
+  
+  public boolean getIncludeCounters() {
+    return includeCounters; 
+  }
+  
+  public void setIncludeCounters(boolean send) {
+    includeCounters = send;
+  }
+  
   /**
    * Get task's counters.
    */
@@ -281,7 +292,10 @@
     WritableUtils.writeEnum(out, phase);
     out.writeLong(startTime);
     out.writeLong(finishTime);
-    counters.write(out);
+    out.writeBoolean(includeCounters);
+    if (includeCounters) {
+      counters.write(out);
+    }
   }
 
   public void readFields(DataInput in) throws IOException {
@@ -294,7 +308,10 @@
     this.startTime = in.readLong(); 
     this.finishTime = in.readLong(); 
     counters = new Counters();
-    counters.readFields(in);
+    this.includeCounters = in.readBoolean();
+    if (includeCounters) {
+      counters.readFields(in);
+    }
   }
   
   //////////////////////////////////////////////////////////////////////////////

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Thu Jan  3 00:15:30
2008
@@ -958,12 +958,25 @@
     return State.NORMAL;
   }
 
+  private long previousUpdate = 0;
+
   /**
    * Build and transmit the heart beat to the JobTracker
    * @return false if the tracker was unknown
    * @throws IOException
    */
   private HeartbeatResponse transmitHeartBeat() throws IOException {
+    // Send Counters in the status once every COUNTER_UPDATE_INTERVAL
+    long now = System.currentTimeMillis();
+    boolean sendCounters;
+    if (now > (previousUpdate + COUNTER_UPDATE_INTERVAL)) {
+      sendCounters = true;
+      previousUpdate = now;
+    }
+    else {
+      sendCounters = false;
+    }
+
     // 
     // Check if the last heartbeat got through... 
     // if so then build the heartbeat information for the JobTracker;
@@ -972,7 +985,9 @@
     if (status == null) {
       synchronized (this) {
         status = new TaskTrackerStatus(taskTrackerName, localHostname, 
-                                       httpPort, cloneAndResetRunningTaskStatuses(), 
+                                       httpPort, 
+                                       cloneAndResetRunningTaskStatuses(
+                                         sendCounters), 
                                        failures, 
                                        maxCurrentMapTasks,
                                        maxCurrentReduceTasks); 
@@ -2060,10 +2075,16 @@
     return taskTrackerName;
   }
     
-  private synchronized List<TaskStatus> cloneAndResetRunningTaskStatuses() {
+  private synchronized List<TaskStatus> cloneAndResetRunningTaskStatuses(
+                                          boolean sendCounters) {
     List<TaskStatus> result = new ArrayList<TaskStatus>(runningTasks.size());
     for(TaskInProgress tip: runningTasks.values()) {
       TaskStatus status = tip.getStatus();
+      status.setIncludeCounters(sendCounters);
+      // send counters for finished or failed tasks.
+      if (status.getRunState() != TaskStatus.State.RUNNING) {
+        status.setIncludeCounters(true);
+      }
       result.add((TaskStatus)status.clone());
       status.clearStatus();
     }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java Thu Jan
 3 00:15:30 2008
@@ -37,8 +37,9 @@
    *         org.apache.hadoop.mapred.TaskStatus.Phase, Counters) 
    *         with {@link #statusUpdate(String, TaskStatus)}
    * Version 5 changed counters representation for HADOOP-2248
+   * Version 6 changes the TaskStatus representation for HADOOP-2208
    * */
-  public static final long versionID = 5L;
+  public static final long versionID = 6L;
   
   /** Called when a child task process starts, to get its task.*/
   Task getTask(String taskid) throws IOException;



Mime
View raw message