hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r700059 - in /hadoop/core/branches/branch-0.19: CHANGES.txt src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
Date Mon, 29 Sep 2008 11:02:25 GMT
Author: ddas
Date: Mon Sep 29 04:02:24 2008
New Revision: 700059

URL: http://svn.apache.org/viewvc?rev=700059&view=rev
Log:
Merge -r 700055:700056 from trunk onto branch 0.19. Fixes HADOOP-4237.

Modified:
    hadoop/core/branches/branch-0.19/CHANGES.txt
    hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java

Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=700059&r1=700058&r2=700059&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Mon Sep 29 04:02:24 2008
@@ -757,6 +757,9 @@
     HADOOP-4190. Fixes the backward compatibility issue with Job History.
     introduced by HADOOP-3245 and HADOOP-2403. (Amar Kamat via ddas)
 
+    HADOOP-4237. Fixes the TestStreamingBadRecords.testNarrowDown testcase.
+    (Sharad Agarwal via ddas)
+
 Release 0.18.2 - Unreleased
 
   BUG FIXES

Modified: hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java?rev=700059&r1=700058&r2=700059&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
(original)
+++ hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
Mon Sep 29 04:02:24 2008
@@ -170,7 +170,6 @@
       "-jobconf", "mapred.skip.reduce.max.skip.groups="+Long.MAX_VALUE,
       "-jobconf", "mapred.map.tasks=1",
       "-jobconf", "mapred.reduce.tasks=1",
-      "-jobconf", "mapred.task.timeout=30000",
       "-jobconf", "fs.default.name="+clusterConf.get("fs.default.name"),
       "-jobconf", "mapred.job.tracker="+clusterConf.get("mapred.job.tracker"),
       "-jobconf", "mapred.job.tracker.http.address="
@@ -197,13 +196,15 @@
       "-verbose",
       "-inputformat", "org.apache.hadoop.mapred.KeyValueTextInputFormat",
       "-jobconf", "mapred.skip.attempts.to.start.skipping=1",
-      "-jobconf", "mapred.map.max.attempts=12",
-      "-jobconf", "mapred.reduce.max.attempts=8",
+      //actually fewer attempts are required than specified
+      //but to cater to the case of slow processed counter update, need to 
+      //have more attempts
+      "-jobconf", "mapred.map.max.attempts=20",
+      "-jobconf", "mapred.reduce.max.attempts=15",
       "-jobconf", "mapred.skip.map.max.skip.records=1",
       "-jobconf", "mapred.skip.reduce.max.skip.groups=1",
       "-jobconf", "mapred.map.tasks=1",
       "-jobconf", "mapred.reduce.tasks=1",
-      "-jobconf", "mapred.task.timeout=30000",
       "-jobconf", "fs.default.name="+clusterConf.get("fs.default.name"),
       "-jobconf", "mapred.job.tracker="+clusterConf.get("mapred.job.tracker"),
       "-jobconf", "mapred.job.tracker.http.address="
@@ -275,7 +276,7 @@
       }
       else if(badRecords.size()>2 && line.contains(badRecords.get(2))) {
         LOG.warn("Encountered BAD record");
-        Thread.sleep(15*60*1000);
+        System.exit(-1);
       }
       super.processLine(line);
     }



Mime
View raw message