hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r762218 - in /hadoop/core/branches/branch-0.20: ./ CHANGES.txt src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
Date Mon, 06 Apr 2009 06:31:11 GMT
Author: omalley
Date: Mon Apr  6 06:31:11 2009
New Revision: 762218

URL: http://svn.apache.org/viewvc?rev=762218&view=rev
Log:
HADOOP-5437. Fix TestMiniMRDFSSort to properly test jvm-reuse. (omalley)

Modified:
    hadoop/core/branches/branch-0.20/   (props changed)
    hadoop/core/branches/branch-0.20/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java

Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Apr  6 06:31:11 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632,762216

Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=762218&r1=762217&r2=762218&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Mon Apr  6 06:31:11 2009
@@ -356,6 +356,8 @@
     HADOOP-5468. Add sub-menus to forrest documentation and make some minor
     edits.  (Corinne Chandel via szetszwo)
 
+    HADOOP-5437. Fix TestMiniMRDFSSort to properly test jvm-reuse. (omalley)
+
   OPTIMIZATIONS
 
     HADOOP-3293. Fixes FileInputFormat to do provide locations for splits

Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Apr  6 06:31:11 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632,762216

Modified: hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java?rev=762218&r1=762217&r2=762218&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
(original)
+++ hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
Mon Apr  6 06:31:11 2009
@@ -18,10 +18,18 @@
 
 package org.apache.hadoop.mapred;
 
+import java.io.IOException;
+
+import junit.extensions.TestSetup;
+import junit.framework.Test;
 import junit.framework.TestCase;
+import junit.framework.TestSuite;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ToolRunner;
@@ -39,9 +47,30 @@
 
   // Knobs to control randomwriter; and hence sort
   private static final int NUM_HADOOP_SLAVES = 3;
-  private static final int RW_BYTES_PER_MAP = 2 * 1024 * 1024;
+  // make it big enough to cause a spill in the map
+  private static final int RW_BYTES_PER_MAP = 3 * 1024 * 1024;
   private static final int RW_MAPS_PER_HOST = 2;
 
+  private static MiniMRCluster mrCluster = null;
+  private static MiniDFSCluster dfsCluster = null;
+  private static FileSystem dfs = null;
+  public static Test suite() {
+    TestSetup setup = new TestSetup(new TestSuite(TestMiniMRDFSSort.class)) {
+      protected void setUp() throws Exception {
+        Configuration conf = new Configuration();
+        dfsCluster = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null);
+        dfs = dfsCluster.getFileSystem();
+        mrCluster = new MiniMRCluster(NUM_HADOOP_SLAVES, 
+                                      dfs.getUri().toString(), 1);
+      }
+      protected void tearDown() throws Exception {
+        if (dfsCluster != null) { dfsCluster.shutdown(); }
+        if (mrCluster != null) { mrCluster.shutdown(); }
+      }
+    };
+    return setup;
+  }
+
   private static void runRandomWriter(JobConf job, Path sortInput) 
   throws Exception {
     // Scale down the default settings for RandomWriter for the test-case
@@ -57,8 +86,10 @@
   private static void runSort(JobConf job, Path sortInput, Path sortOutput) 
   throws Exception {
 
+    job.setInt("mapred.job.reuse.jvm.num.tasks", -1);
     job.setInt("io.sort.mb", 1);
-    job.setLong("mapred.min.split.size", Long.MAX_VALUE);
+    job.setNumMapTasks(12);
+
     // Setup command-line arguments to 'sort'
     String[] sortArgs = {sortInput.toString(), sortOutput.toString()};
     
@@ -75,34 +106,66 @@
     // Run Sort-Validator
     assertEquals(ToolRunner.run(job, new SortValidator(), svArgs), 0);
   }
-  Configuration conf = new Configuration();
-  public void testMapReduceSort() throws Exception {
-    MiniDFSCluster dfs = null;
-    MiniMRCluster mr = null;
-    FileSystem fileSys = null;
-    try {
-
-      // Start the mini-MR and mini-DFS clusters
-      dfs = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null);
-      fileSys = dfs.getFileSystem();
-      mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(), 1);
-
-      // Run randomwriter to generate input for 'sort'
-      runRandomWriter(mr.createJobConf(), SORT_INPUT_PATH);
-      
-      // Run sort
-      runSort(mr.createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
-      
-      // Run sort-validator to check if sort worked correctly
-      runSortValidator(mr.createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
-    } finally {
-      if (dfs != null) { dfs.shutdown(); }
-      if (mr != null) { mr.shutdown();
-      }
+  
+  private static class ReuseDetector extends MapReduceBase
+      implements Mapper<BytesWritable,BytesWritable, Text, Text> {
+    static int instances = 0;
+    Reporter reporter = null;
+
+    @Override
+    public void map(BytesWritable key, BytesWritable value,
+                    OutputCollector<Text, Text> output, 
+                    Reporter reporter) throws IOException {
+      this.reporter = reporter;
+    }
+    
+    public void close() throws IOException {
+      reporter.incrCounter("jvm", "use", ++instances);
     }
   }
-  public void testMapReduceSortWithJvmReuse() throws Exception {
-    conf.setInt("mapred.job.reuse.jvm.num.tasks", -1);
-    testMapReduceSort();
+
+  private static void runJvmReuseTest(JobConf job,
+                                      boolean reuse) throws IOException {
+    // setup a map-only job that reads the input and only sets the counters
+    // based on how many times the jvm was reused.
+    job.setInt("mapred.job.reuse.jvm.num.tasks", reuse ? -1 : 1);
+    FileInputFormat.setInputPaths(job, SORT_INPUT_PATH);
+    job.setInputFormat(SequenceFileInputFormat.class);
+    job.setOutputFormat(NullOutputFormat.class);
+    job.setMapperClass(ReuseDetector.class);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(Text.class);
+    job.setNumMapTasks(24);
+    job.setNumReduceTasks(0);
+    RunningJob result = JobClient.runJob(job);
+    long uses = result.getCounters().findCounter("jvm", "use").getValue();
+    System.out.println("maps = " + job.getNumMapTasks());
+    System.out.println(result.getCounters());
+    int maps = job.getNumMapTasks();
+    if (reuse) {
+      assertTrue("maps = " + maps + ", uses = " + uses, maps < uses);
+    } else {
+      assertEquals("uses should be number of maps", job.getNumMapTasks(), uses);
+    }
+  }
+
+  public void testMapReduceSort() throws Exception {
+    // Run randomwriter to generate input for 'sort'
+    runRandomWriter(mrCluster.createJobConf(), SORT_INPUT_PATH);
+
+    // Run sort
+    runSort(mrCluster.createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
+
+    // Run sort-validator to check if sort worked correctly
+    runSortValidator(mrCluster.createJobConf(), SORT_INPUT_PATH, 
+                     SORT_OUTPUT_PATH);
+  }
+  
+  public void testJvmReuse() throws Exception {
+    runJvmReuseTest(mrCluster.createJobConf(), true);
+  }
+
+  public void testNoJvmReuse() throws Exception {
+    runJvmReuseTest(mrCluster.createJobConf(), false);
   }
 }



Mime
View raw message