hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1078932 - in /hadoop/mapreduce/branches/branch-0.22: CHANGES.txt src/test/mapred/org/apache/hadoop/mapreduce/TestLocalRunner.java
Date Mon, 07 Mar 2011 20:39:40 GMT
Author: todd
Date: Mon Mar  7 20:39:39 2011
New Revision: 1078932

URL: http://svn.apache.org/viewvc?rev=1078932&view=rev
Log:
MAPREDUCE-2284. TestLocalRunner.testMultiMaps times out. Contributed by Todd Lipcon

Modified:
    hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
    hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapreduce/TestLocalRunner.java

Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1078932&r1=1078931&r2=1078932&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Mon Mar  7 20:39:39 2011
@@ -505,6 +505,8 @@ Release 0.22.0 - Unreleased
     MAPREDUCE-2251. Remove unused mapreduce.job.userhistorylocation config.
     (Harsh J Chouraria via todd)
 
+    MAPREDUCE-2284. TestLocalRunner.testMultiMaps times out (todd)
+
 Release 0.21.1 - Unreleased
 
   NEW FEATURES

Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapreduce/TestLocalRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapreduce/TestLocalRunner.java?rev=1078932&r1=1078931&r2=1078932&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapreduce/TestLocalRunner.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapreduce/TestLocalRunner.java
Mon Mar  7 20:39:39 2011
@@ -49,6 +49,13 @@ public class TestLocalRunner extends Tes
 
   private static final Log LOG = LogFactory.getLog(TestLocalRunner.class);
 
+  private static int INPUT_SIZES[] =
+    new int[] { 50000, 500, 500, 20,  5000, 500};
+  private static int OUTPUT_SIZES[] =
+    new int[] { 1,     500, 500, 500, 500, 500};
+  private static int SLEEP_INTERVALS[] =
+    new int[] { 10000, 15,  15, 20, 250, 60 };
+
   private static class StressMapper
       extends Mapper<LongWritable, Text, LongWritable, Text> {
 
@@ -75,49 +82,12 @@ public class TestLocalRunner extends Tes
     public void map(LongWritable key, Text val, Context c)
         throws IOException, InterruptedException {
 
-      switch(threadId) {
-      case 0:
-        // Write a single value and be done.
+      // Write many values quickly.
+      for (int i = 0; i < OUTPUT_SIZES[threadId]; i++) {
         c.write(new LongWritable(0), val);
-        break;
-      case 1:
-      case 2:
-        // Write many values quickly.
-        for (int i = 0; i < 500; i++) {
-          c.write(new LongWritable(0), val);
-        }
-        break;
-      case 3:
-        // Write many values, using thread sleeps to delay this.
-        for (int i = 0; i < 50; i++) {
-          for (int j = 0; j < 10; j++) {
-            c.write(new LongWritable(0), val);
-          }
+        if (i % SLEEP_INTERVALS[threadId] == 1) {
           Thread.sleep(1);
         }
-        break;
-      case 4:
-        // Write many values, using busy-loops to delay this.
-        for (int i = 0; i < 500; i++) {
-          for (int j = 0; j < 10000; j++) {
-            this.exposedState++;
-          }
-          c.write(new LongWritable(0), val);
-        }
-        break;
-      case 5:
-        // Write many values, using very slow busy-loops to delay this.
-        for (int i = 0; i < 500; i++) {
-          for (int j = 0; j < 100000; j++) {
-            this.exposedState++;
-          }
-          c.write(new LongWritable(0), val);
-        }
-        break;
-      default:
-        // Write a single value and be done.
-        c.write(new LongWritable(0), val);
-        break;
       }
     }
 
@@ -194,12 +164,12 @@ public class TestLocalRunner extends Tes
   // This is the total number of map output records we expect to generate,
   // based on input file sizes (see createMultiMapsInput()) and the behavior
   // of the different StressMapper threads.
-  private final static int TOTAL_RECORDS = 50000
-      + (500 * 500)
-      + (500 * 500)
-      + (20 * 500)
-      + (5000 * 500)
-      + (500 * 500);
+  private static int TOTAL_RECORDS = 0;
+  static {
+    for (int i = 0; i < 6; i++) {
+      TOTAL_RECORDS += INPUT_SIZES[i] * OUTPUT_SIZES[i];
+    }
+  }
 
   private final String INPUT_DIR = "multiMapInput";
   private final String OUTPUT_DIR = "multiMapOutput";
@@ -238,13 +208,9 @@ public class TestLocalRunner extends Tes
 
     // Create input files, with sizes calibrated based on
     // the amount of work done in each mapper.
-    createInputFile(inputPath, 0, 50000);
-    createInputFile(inputPath, 1, 500);
-    createInputFile(inputPath, 2, 500);
-    createInputFile(inputPath, 3, 20);
-    createInputFile(inputPath, 4, 5000);
-    createInputFile(inputPath, 5, 500);
-
+    for (int i = 0; i < 6; i++) {
+      createInputFile(inputPath, i, INPUT_SIZES[i]);
+    }
     return inputPath;
   }
 



Mime
View raw message