hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1194856 - in /hadoop/common/branches/branch-0.20-security-204: ./ src/core/org/apache/hadoop/fs/ src/mapred/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/fs/ src/tools/org/apache/hadoop/tools/
Date Sat, 29 Oct 2011 09:44:19 GMT
Author: acmurthy
Date: Sat Oct 29 09:44:19 2011
New Revision: 1194856

URL: http://svn.apache.org/viewvc?rev=1194856&view=rev
Log:
Merge -c 1194854 from branch-0.20-security to branch-0.20-security-204 to fix MAPREDUCE-2355.

Modified:
    hadoop/common/branches/branch-0.20-security-204/   (props changed)
    hadoop/common/branches/branch-0.20-security-204/CHANGES.txt   (props changed)
    hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/HarFileSystem.java
  (props changed)
    hadoop/common/branches/branch-0.20-security-204/src/mapred/   (props changed)
    hadoop/common/branches/branch-0.20-security-204/src/mapred/mapred-default.xml
    hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/fs/TestHarFileSystem.java
  (props changed)
    hadoop/common/branches/branch-0.20-security-204/src/tools/org/apache/hadoop/tools/HadoopArchives.java
  (props changed)

Propchange: hadoop/common/branches/branch-0.20-security-204/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 29 09:44:19 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/branch-0.20:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
-/hadoop/common/branches/branch-0.20-security:1097202,1098837,1100336,1134140,1150528,1150860
+/hadoop/common/branches/branch-0.20-security:1097202,1098837,1100336,1134140,1150528,1150860,1194854
 /hadoop/common/branches/branch-0.20-security-203:1096071,1097012-1099333,1102071,1128115
 /hadoop/core/branches/branch-0.19:713112
 /hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569

Propchange: hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 29 09:44:19 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/branch-0.20/CHANGES.txt:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
-/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336,1101315,1101629,1101729,1102378,1102869,1103940,1125139,1125170,1125587,1125589,1127362,1130409,1131277,1131280,1131286,1131290,1131299,1131461,1131737,1134140,1150528,1150860,1151287
+/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336,1101315,1101629,1101729,1102378,1102869,1103940,1125139,1125170,1125587,1125589,1127362,1130409,1131277,1131280,1131286,1131290,1131299,1131461,1131737,1134140,1150528,1150860,1151287,1194854
 /hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097012-1099333,1102071,1128115
 /hadoop/common/branches/branch-0.20-security-205/CHANGES.txt:1132788,1133133,1133274,1133282,1133475
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226

Propchange: hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/HarFileSystem.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 29 09:44:19 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/branch-0.20/src/core/org/apache/hadoop/fs/HarFileSystem.java:826138,826568,833001,880632,898713
-/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/HarFileSystem.java:1100336,1150528,1150860
+/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/HarFileSystem.java:1100336,1150528,1150860,1194854
 /hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/fs/HarFileSystem.java:1128115
 /hadoop/common/trunk/src/core/org/apache/hadoop/fs/HarFileSystem.java:910709
 /hadoop/common/trunk/src/java/org/apache/hadoop/fs/HarFileSystem.java:979485

Propchange: hadoop/common/branches/branch-0.20-security-204/src/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 29 09:44:19 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/branch-0.20/src/mapred:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
-/hadoop/common/branches/branch-0.20-security/src/mapred:1097202,1098837,1100336,1134140,1150528,1150860
+/hadoop/common/branches/branch-0.20-security/src/mapred:1097202,1098837,1100336,1134140,1150528,1150860,1194854
 /hadoop/common/branches/branch-0.20-security-203/src/mapred:1096071,1097012-1099333,1128115
 /hadoop/core/branches/branch-0.19/src/mapred:713112
 /hadoop/core/trunk/src/mapred:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569

Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/mapred-default.xml?rev=1194856&r1=1194855&r2=1194856&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/mapred-default.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/mapred-default.xml Sat Oct
29 09:44:19 2011
@@ -242,6 +242,24 @@
 </property>
 
 <property>
+  <name>mapreduce.tasktracker.outofband.heartbeat.damper</name>
+  <value>1000000</value>
+  <description>When out-of-band heartbeats are enabled, provides
+  damping to avoid overwhelming the JobTracker if too many out-of-band
+  heartbeats would occur. The damping is calculated such that the
+  heartbeat interval is divided by (T*D + 1) where T is the number
+  of completed tasks and D is the damper value.
+  
+  Setting this to a high value like the default provides no damping --
+  as soon as any task finishes, a heartbeat will be sent. Setting this
+  parameter to 0 is equivalent to disabling the out-of-band heartbeat feature.
+  A value of 1 would indicate that, after one task has completed, the
+  time to wait before the next heartbeat would be 1/2 the usual time.
+  After two tasks have finished, it would be 1/3 the usual time, etc.
+  </description>
+</property>
+
+<property>
   <name>mapred.jobtracker.restart.recover</name>
   <value>false</value>
   <description>"true" to enable (job) recovery upon restart,

Modified: hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1194856&r1=1194855&r2=1194856&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/common/branches/branch-0.20-security-204/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Sat Oct 29 09:44:19 2011
@@ -45,6 +45,7 @@ import java.util.Vector;
 import java.util.Map.Entry;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 
 import javax.crypto.SecretKey;
@@ -360,9 +361,13 @@ public class TaskTracker implements MRCo
   static final String TT_OUTOFBAND_HEARBEAT =
     "mapreduce.tasktracker.outofband.heartbeat";
   private volatile boolean oobHeartbeatOnTaskCompletion;
+  static final String TT_OUTOFBAND_HEARTBEAT_DAMPER = 
+    "mapreduce.tasktracker.outofband.heartbeat.damper";
+  static private final int DEFAULT_OOB_HEARTBEAT_DAMPER = 1000000;
+  private volatile int oobHeartbeatDamper;
   
   // Track number of completed tasks to send an out-of-band heartbeat
-  private IntWritable finishedCount = new IntWritable(0);
+  private AtomicInteger finishedCount = new AtomicInteger(0);
   
   private MapEventsFetcherThread mapEventsFetcher;
   final int workerThreads;
@@ -854,6 +859,9 @@ public class TaskTracker implements MRCo
     
     oobHeartbeatOnTaskCompletion = 
       fConf.getBoolean(TT_OUTOFBAND_HEARBEAT, false);
+    oobHeartbeatDamper = 
+      fConf.getInt(TT_OUTOFBAND_HEARTBEAT_DAMPER, 
+          DEFAULT_OOB_HEARTBEAT_DAMPER);
   }
 
   /**
@@ -1555,25 +1563,39 @@ public class TaskTracker implements MRCo
     return recentMapEvents;
   }
 
+  private long getHeartbeatInterval(int numFinishedTasks) {
+    return (heartbeatInterval / (numFinishedTasks * oobHeartbeatDamper + 1));
+  }
+  
   /**
    * Main service loop.  Will stay in this loop forever.
    */
   State offerService() throws Exception {
-    long lastHeartbeat = 0;
+    long lastHeartbeat = System.currentTimeMillis();
 
     while (running && !shuttingDown) {
       try {
         long now = System.currentTimeMillis();
-
-        long waitTime = heartbeatInterval - (now - lastHeartbeat);
-        if (waitTime > 0) {
+        
+        // accelerate to account for multiple finished tasks up-front
+        long remaining = 
+          (lastHeartbeat + getHeartbeatInterval(finishedCount.get())) - now;
+        while (remaining > 0) {
           // sleeps for the wait time or 
-          // until there are empty slots to schedule tasks
+          // until there are *enough* empty slots to schedule tasks
           synchronized (finishedCount) {
-            if (finishedCount.get() == 0) {
-              finishedCount.wait(waitTime);
+            finishedCount.wait(remaining);
+            
+            // Recompute
+            now = System.currentTimeMillis();
+            remaining = 
+              (lastHeartbeat + getHeartbeatInterval(finishedCount.get())) - now;
+            
+            if (remaining <= 0) {
+              // Reset count 
+              finishedCount.set(0);
+              break;
             }
-            finishedCount.set(0);
           }
         }
 
@@ -2349,8 +2371,7 @@ public class TaskTracker implements MRCo
   private void notifyTTAboutTaskCompletion() {
     if (oobHeartbeatOnTaskCompletion) {
       synchronized (finishedCount) {
-        int value = finishedCount.get();
-        finishedCount.set(value+1);
+        finishedCount.incrementAndGet();
         finishedCount.notify();
       }
     }

Propchange: hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/fs/TestHarFileSystem.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 29 09:44:19 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:826138,826568,833001,880632,898713
-/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:1100336,1150528,1150860
+/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:1100336,1150528,1150860,1194854
 /hadoop/common/branches/branch-0.20-security-203/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:1128115
 /hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestHarFileSystem.java:979485
 /hadoop/common/trunk/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:910709

Propchange: hadoop/common/branches/branch-0.20-security-204/src/tools/org/apache/hadoop/tools/HadoopArchives.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 29 09:44:19 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/branch-0.20/src/tools/org/apache/hadoop/tools/HadoopArchives.java:826138,826568,833001,880632,898713
-/hadoop/common/branches/branch-0.20-security/src/tools/org/apache/hadoop/tools/HadoopArchives.java:1100336,1150528,1150860
+/hadoop/common/branches/branch-0.20-security/src/tools/org/apache/hadoop/tools/HadoopArchives.java:1100336,1150528,1150860,1194854
 /hadoop/common/branches/branch-0.20-security-203/src/tools/org/apache/hadoop/tools/HadoopArchives.java:1128115
 /hadoop/common/trunk/src/tools/org/apache/hadoop/tools/HadoopArchives.java:910709
 /hadoop/core/branches/branch-0.19/src/tools/org/apache/hadoop/tools/HadoopArchives.java:713112



Mime
View raw message