hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r700918 - in /hadoop/core/trunk: CHANGES.txt src/test/org/apache/hadoop/mapred/TestReduceFetch.java
Date Wed, 01 Oct 2008 20:57:37 GMT
Author: cdouglas
Date: Wed Oct  1 13:57:36 2008
New Revision: 700918

URL: http://svn.apache.org/viewvc?rev=700918&view=rev
Log:
HADOOP-4302. Fix a race condition in TestReduceFetch that can yield false
negatvies.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestReduceFetch.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=700918&r1=700917&r2=700918&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Oct  1 13:57:36 2008
@@ -819,6 +819,9 @@
     HADOOP-4232. Fix race condition in JVM reuse when multiple slots become
     free. (ddas via acmurthy) 
 
+    HADOOP-4302. Fix a race condition in TestReduceFetch that can yield false
+    negatvies. (cdouglas)
+
 Release 0.18.2 - Unreleased
 
   BUG FIXES

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestReduceFetch.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestReduceFetch.java?rev=700918&r1=700917&r2=700918&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestReduceFetch.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestReduceFetch.java Wed Oct  1 13:57:36
2008
@@ -67,11 +67,11 @@
       Text key = new Text();
       Text val = new Text();
       key.set("KEYKEYKEYKEYKEYKEYKEYKEY");
-      byte[] b = new byte[1024];
+      byte[] b = new byte[1000];
       Arrays.fill(b, (byte)'V');
       val.set(b);
       b = null;
-      for (int i = 0; i < 1024; ++i) {
+      for (int i = 0; i < 4 * 1024; ++i) {
         output.collect(key, val);
       }
     }
@@ -84,7 +84,6 @@
     conf.setReducerClass(IdentityReducer.class);
     conf.setOutputKeyClass(Text.class);
     conf.setOutputValueClass(Text.class);
-    conf.setNumMapTasks(3);
     conf.setNumReduceTasks(1);
     conf.setInputFormat(FakeIF.class);
     FileInputFormat.setInputPaths(conf, new Path("/in"));
@@ -106,25 +105,41 @@
   public void testReduceFromDisk() throws Exception {
     JobConf job = mrCluster.createJobConf();
     job.set("mapred.job.reduce.input.buffer.percent", "0.0");
+    job.setNumMapTasks(3);
     Counters c = runJob(job);
-    assertTrue(c.findCounter(HDFS_WRITE).getCounter() <=
-               c.findCounter(LOCAL_READ).getCounter());
+    final long hdfsWritten = c.findCounter(HDFS_WRITE).getCounter();
+    final long localRead = c.findCounter(LOCAL_READ).getCounter();
+    assertTrue("Expected more bytes read from local (" +
+        localRead + ") than written to HDFS (" + hdfsWritten + ")",
+        hdfsWritten <= localRead);
   }
 
   public void testReduceFromPartialMem() throws Exception {
     JobConf job = mrCluster.createJobConf();
-    job.setInt("mapred.inmem.merge.threshold", 2);
+    job.setNumMapTasks(5);
+    job.setInt("mapred.inmem.merge.threshold", 0);
     job.set("mapred.job.reduce.input.buffer.percent", "1.0");
+    job.setInt("mapred.reduce.parallel.copies", 1);
+    job.setInt("io.sort.mb", 10);
+    job.set("mapred.child.java.opts", "-Xmx128m");
+    job.set("mapred.job.shuffle.input.buffer.percent", "0.14");
+    job.setNumTasksToExecutePerJvm(1);
+    job.set("mapred.job.shuffle.merge.percent", "1.0");
     Counters c = runJob(job);
-    assertTrue(c.findCounter(HDFS_WRITE).getCounter() >=
-               c.findCounter(LOCAL_READ).getCounter() + 1024 * 1024);
+    final long hdfsWritten = c.findCounter(HDFS_WRITE).getCounter();
+    final long localRead = c.findCounter(LOCAL_READ).getCounter();
+    assertTrue("Expected at least 1MB fewer bytes read from local (" +
+        localRead + ") than written to HDFS (" + hdfsWritten + ")",
+        hdfsWritten >= localRead + 1024 * 1024);
   }
 
   public void testReduceFromMem() throws Exception {
     JobConf job = mrCluster.createJobConf();
     job.set("mapred.job.reduce.input.buffer.percent", "1.0");
+    job.setNumMapTasks(3);
     Counters c = runJob(job);
-    assertTrue(c.findCounter(LOCAL_READ).getCounter() == 0);
+    final long localRead = c.findCounter(LOCAL_READ).getCounter();
+    assertTrue("Non-zero read from local: " + localRead, localRead == 0);
   }
 
 }



Mime
View raw message