hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r889498 - in /hadoop/mapreduce/branches/branch-0.21: ./ conf/ src/c++/ src/contrib/ src/contrib/block_forensics/ src/contrib/capacity-scheduler/ src/contrib/data_join/ src/contrib/dynamic-scheduler/ src/contrib/eclipse-plugin/ src/contrib/f...
Date Fri, 11 Dec 2009 04:11:54 GMT
Author: acmurthy
Date: Fri Dec 11 04:11:53 2009
New Revision: 889498

URL: http://svn.apache.org/viewvc?rev=889498&view=rev
Log:
Merge -r 889495:889496 from trunk to branch-0.21 to fix MAPREDUCE-1171.

Modified:
    hadoop/mapreduce/branches/branch-0.21/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/.gitignore   (props changed)
    hadoop/mapreduce/branches/branch-0.21/CHANGES.txt   (contents, props changed)
    hadoop/mapreduce/branches/branch-0.21/conf/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/conf/capacity-scheduler.xml.template   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/c++/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/block_forensics/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/build-contrib.xml   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/build.xml   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/data_join/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/dynamic-scheduler/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/eclipse-plugin/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/index/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/mrunit/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/sqoop/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/contrib/vaidya/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/examples/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/java/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/JobContext.java
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/fs/   (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/hdfs/   (props
changed)
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/io/FileBench.java
  (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
  (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/ipc/   (props
changed)
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
  (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
  (props changed)
    hadoop/mapreduce/branches/branch-0.21/src/webapps/job/   (props changed)

Propchange: hadoop/mapreduce/branches/branch-0.21/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/mapred:713112
-/hadoop/mapreduce/trunk:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
 /hadoop/core/trunk/.gitignore:784664-785643
-/hadoop/mapreduce/trunk/.gitignore:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/.gitignore:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=889498&r1=889497&r2=889498&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Fri Dec 11 04:11:53 2009
@@ -869,3 +869,7 @@
 
     MAPREDUCE-1230. Fix handling of null records in VerticaInputFormat. (Omer
     Trajman via cdouglas)
+
+    MAPREDUCE-1171. Allow shuffle retries and read-error reporting to be
+    configurable. (Amareshwari Sriramadasu via acmurthy)
+

Propchange: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
 /hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
-/hadoop/mapreduce/trunk/CHANGES.txt:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/CHANGES.txt:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/conf:713112
 /hadoop/core/trunk/conf:784664-785643
-/hadoop/mapreduce/trunk/conf:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/conf:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/conf/capacity-scheduler.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/conf/capacity-scheduler.xml.template:713112
 /hadoop/core/trunk/conf/capacity-scheduler.xml.template:776175-785643
-/hadoop/mapreduce/trunk/conf/capacity-scheduler.xml.template:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/conf/capacity-scheduler.xml.template:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++:713112
 /hadoop/core/trunk/src/c++:776175-784663
-/hadoop/mapreduce/trunk/src/c++:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/c++:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
 /hadoop/core/trunk/src/contrib:784664-785643
-/hadoop/mapreduce/trunk/src/contrib:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/block_forensics:713112
 /hadoop/core/trunk/src/contrib/block_forensics:784664-785643
-/hadoop/mapreduce/trunk/src/contrib/block_forensics:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/block_forensics:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
 /hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/build-contrib.xml:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/build-contrib.xml:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
 /hadoop/core/trunk/src/contrib/build.xml:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/build.xml:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/build.xml:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/capacity-scheduler:713112
 /hadoop/core/trunk/src/contrib/capacity-scheduler:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
 /hadoop/core/trunk/src/contrib/data_join:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/data_join:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/data_join:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/dynamic-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/src/contrib/dynamic-scheduler:713112
 /hadoop/core/trunk/src/contrib/dynamic-scheduler:784975-786373
-/hadoop/mapreduce/trunk/src/contrib/dynamic-scheduler:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/dynamic-scheduler:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
 /hadoop/core/trunk/src/contrib/eclipse-plugin:776175-784663
-/hadoop/mapreduce/trunk/src/contrib/eclipse-plugin:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/eclipse-plugin:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/fairscheduler:713112
 /hadoop/core/trunk/src/contrib/fairscheduler:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/fairscheduler:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/fairscheduler:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
 /hadoop/core/trunk/src/contrib/index:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/index:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/index:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/mrunit/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/mrunit:713112
 /hadoop/core/trunk/src/contrib/mrunit:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/mrunit:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/mrunit:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/sqoop/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/src/contrib/sqoop:713112
 /hadoop/core/trunk/src/contrib/sqoop:784975-786373
-/hadoop/mapreduce/trunk/src/contrib/sqoop:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/sqoop:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/streaming:713112
 /hadoop/core/trunk/src/contrib/streaming:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/streaming:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/streaming:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
 /hadoop/core/trunk/src/contrib/vaidya:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/vaidya:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/contrib/vaidya:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/examples:713112
 /hadoop/core/trunk/src/examples:776175-784663
-/hadoop/mapreduce/trunk/src/examples:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/examples:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/java:713112
 /hadoop/core/trunk/src/mapred:776175-785643
-/hadoop/mapreduce/trunk/src/java:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/java:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/JobContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/JobContext.java?rev=889498&r1=889497&r2=889498&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/JobContext.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/JobContext.java
Fri Dec 11 04:11:53 2009
@@ -212,6 +212,10 @@
     "mapreduce.reduce.shuffle.connect.timeout";
   public static final String SHUFFLE_READ_TIMEOUT = 
     "mapreduce.reduce.shuffle.read.timeout";
+  public static final String SHUFFLE_FETCH_FAILURES = 
+    "mapreduce.reduce.shuffle.maxfetchfailures";
+  public static final String SHUFFLE_NOTIFY_READERROR = 
+    "mapreduce.reduce.shuffle.notify.readerror";
   public static final String REDUCE_SKIP_INCR_PROC_COUNT = 
     "mapreduce.reduce.skip.proc-count.auto-incr";
   public static final String REDUCE_SKIP_MAXGROUPS = 

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java?rev=889498&r1=889497&r2=889498&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java
Fri Dec 11 04:11:53 2009
@@ -37,6 +37,7 @@
 import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.TaskStatus;
+import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskID;
 import org.apache.hadoop.mapreduce.task.reduce.MapHost.State;
@@ -82,10 +83,12 @@
   
   private int maxMapRuntime = 0;
   private int maxFailedUniqueFetches = 5;
+  private int maxFetchFailuresBeforeReporting;
   
   private long totalBytesShuffledTillNow = 0;
   private DecimalFormat  mbpsFormat = new DecimalFormat("0.00");
 
+  private boolean reportReadErrorImmediately = true;
   
   public ShuffleScheduler(JobConf job, TaskStatus status,
                           ExceptionReporter reporter,
@@ -108,6 +111,10 @@
     referee.start();
     this.maxFailedUniqueFetches = Math.min(totalMaps,
         this.maxFailedUniqueFetches);
+    this.maxFetchFailuresBeforeReporting = job.getInt(
+        JobContext.SHUFFLE_FETCH_FAILURES, REPORT_FAILURE_LIMIT);
+    this.reportReadErrorImmediately = job.getBoolean(
+        JobContext.SHUFFLE_NOTIFY_READERROR, true);
   }
 
   public synchronized void copySucceeded(TaskAttemptID mapId, 
@@ -175,7 +182,6 @@
       }
     }
     
-    // Notify the JobTracker after every 'reportFailureLimit' failures
     checkAndInformJobTracker(failures, mapId, readError);
 
     checkReducerHealth();
@@ -188,9 +194,14 @@
     failedShuffleCounter.increment(1);
   }
   
+  // Notify the JobTracker  
+  // after every read error, if 'reportReadErrorImmediately' is true or
+  // after every 'maxFetchFailuresBeforeReporting' failures
   private void checkAndInformJobTracker(
       int failures, TaskAttemptID mapId, boolean readError) {
-    if (readError || ((failures % REPORT_FAILURE_LIMIT) == 0)) {
+    if ((reportReadErrorImmediately && readError)
+        || ((failures % maxFetchFailuresBeforeReporting) == 0)) {
+      LOG.info("Reporting fetch failure for " + mapId + " to jobtracker.");
       status.addFetchFailedMap((org.apache.hadoop.mapred.TaskAttemptID) mapId);
     }
   }

Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
 /hadoop/core/trunk/src/test/mapred:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/test/mapred:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:817878-835934,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:817878-835934,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/io/FileBench.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/FileBench.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -3,4 +3,4 @@
 /hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/ipc:776175-785643
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs-with-mr/org/apache/hadoop/ipc:796829-820463
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/ipc:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/ipc:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:817878-835934,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:817878-835934,835968,836063,883594,884468-884469,884531,889496

Propchange: hadoop/mapreduce/branches/branch-0.21/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 11 04:11:53 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
 /hadoop/core/trunk/src/webapps/job:776175-785643
-/hadoop/mapreduce/trunk/src/webapps/job:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531
+/hadoop/mapreduce/trunk/src/webapps/job:818355,818918,818946,824750,828081,835968,836063,883594,884468-884469,884531,889496



Mime
View raw message