hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yhema...@apache.org
Subject svn commit: r812546 - in /hadoop/mapreduce/trunk: CHANGES.txt src/test/mapred/org/apache/hadoop/mapred/MRBench.java src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java
Date Tue, 08 Sep 2009 15:13:39 GMT
Author: yhemanth
Date: Tue Sep  8 15:13:39 2009
New Revision: 812546

URL: http://svn.apache.org/viewvc?rev=812546&view=rev
Log:
MAPREDUCE-945. Modifies MRBench and TestMapRed to use ToolRunner so that options such as queue
name can be passed via command line. Contributed by Sreekanth Ramakrishnan.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MRBench.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=812546&r1=812545&r2=812546&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Sep  8 15:13:39 2009
@@ -307,6 +307,10 @@
     a task is killed due to exceeding memory limits.
     (Vinod Kumar Vavilapalli via yhemanth)
 
+    MAPREDUCE-945. Modifies MRBench and TestMapRed to use ToolRunner so that
+    options such as queue name can be passed via command line.
+    (Sreekanth Ramakrishnan via yhemanth)
+
   BUG FIXES
 
     MAPREDUCE-878. Rename fair scheduler design doc to 

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MRBench.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MRBench.java?rev=812546&r1=812545&r2=812546&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MRBench.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MRBench.java Tue Sep 
8 15:13:39 2009
@@ -26,16 +26,19 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.UTF8;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * Runs a job multiple times and takes average of all runs.
  */
-public class MRBench {
+public class MRBench extends Configured implements Tool{
   
   private static final Log LOG = LogFactory.getLog(MRBench.class);
   private static Path BASE_DIR =
@@ -87,7 +90,7 @@
    * an appropriate number of leading '0' (zero) characters.  The order of
    * generated data is one of ascending, descending, or random.
    */
-  public static void generateTextFile(FileSystem fs, Path inputFile, 
+  public void generateTextFile(FileSystem fs, Path inputFile, 
                                       long numLines, Order sortOrder) throws IOException

   {
     LOG.info("creating control file: "+numLines+" numLines, "+sortOrder+" sortOrder");
@@ -137,8 +140,9 @@
   /**
    * Create the job configuration.
    */
-  private static JobConf setupJob(int numMaps, int numReduces, String jarFile) {
-    JobConf jobConf = new JobConf(MRBench.class);
+  private JobConf setupJob(int numMaps, int numReduces, String jarFile) {
+    JobConf jobConf = new JobConf(getConf());
+    jobConf.setJarByClass(MRBench.class);
     FileInputFormat.addInputPath(jobConf, INPUT_DIR);
     
     jobConf.setInputFormat(TextInputFormat.class);
@@ -165,8 +169,7 @@
    * Runs a MapReduce task, given number of times. The input to each run
    * is the same file.
    */
-  private static ArrayList<Long> runJobInSequence(JobConf masterJobConf, int numRuns)
throws IOException {
-    Path intrimData = null; 
+  private ArrayList<Long> runJobInSequence(JobConf masterJobConf, int numRuns) throws
IOException {
     Random rand = new Random();
     ArrayList<Long> execTimes = new ArrayList<Long>(); 
     
@@ -204,7 +207,13 @@
    *    [-verbose]
    * </pre>
    */
-  public static void main (String[] args) throws IOException {
+  public static void main (String[] args) throws Exception {
+    int res = ToolRunner.run(new MRBench(), args);
+    System.exit(res);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
     String version = "MRBenchmark.0.0.2";
     System.out.println(version);
 
@@ -265,7 +274,7 @@
         inputSortOrder == null)
       {
         System.err.println(usage);
-        System.exit(-1);
+        return -1;
       }
 
     JobConf jobConf = setupJob(numMaps, numReduces, jarFile);
@@ -303,6 +312,7 @@
     System.out.println("DataLines\tMaps\tReduces\tAvgTime (milliseconds)");
     System.out.println(inputLines + "\t\t" + numMaps + "\t" + 
                        numReduces + "\t" + avgTime);
+    return 0;
   }
   
 }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java?rev=812546&r1=812545&r2=812546&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapRed.java Tue Sep
 8 15:13:39 2009
@@ -28,9 +28,8 @@
 import java.util.Iterator;
 import java.util.Random;
 
-import junit.framework.TestCase;
-
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -43,6 +42,13 @@
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.mapred.lib.IdentityMapper;
 import org.apache.hadoop.mapred.lib.IdentityReducer;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 /**********************************************************
  * MapredLoadTest generates a bunch of work that exercises
@@ -77,7 +83,7 @@
  * 7) A mapred job integrates all the count files into a single one.
  *
  **********************************************************/
-public class TestMapRed extends TestCase {
+public class TestMapRed extends Configured implements Tool {
   /**
    * Modified to make it a junit test.
    * The RandomGen Job does the actual work of creating
@@ -248,6 +254,7 @@
      }
   **/
 
+  @Test
   public void testMapred() throws Exception {
     launch();
   }
@@ -312,6 +319,7 @@
     }
   }
 
+  @Test
   public void testPartitioner() throws Exception {
     JobConf conf = new JobConf(TestMapRed.class);
     conf.setPartitionerClass(BadPartitioner.class);
@@ -361,6 +369,7 @@
     public void close() { }
   }
 
+  @Test
   public void testNullKeys() throws Exception {
     JobConf conf = new JobConf(TestMapRed.class);
     FileSystem fs = FileSystem.getLocal(conf);
@@ -452,7 +461,8 @@
       fs.delete(testdir, true);
     }
   }
-    
+  
+  @Test 
   public void testCompression() throws Exception {
     EnumSet<SequenceFile.CompressionType> seq =
       EnumSet.allOf(SequenceFile.CompressionType.class);
@@ -468,11 +478,19 @@
   /**
    * 
    */
-  public static void launch() throws Exception {
+  public void launch() throws Exception {
     //
     // Generate distribution of ints.  This is the answer key.
     //
-    JobConf conf = new JobConf(TestMapRed.class);
+    JobConf conf = null;
+    //Check to get configuration and check if it is configured thro' Configured
+    //interface. This would happen when running testcase thro' command line.
+    if(getConf() == null) {
+      conf = new JobConf();
+    } else {
+      conf = new JobConf(getConf());
+    }
+    conf.setJarByClass(TestMapRed.class);
     int countsToGo = counts;
     int dist[] = new int[range];
     for (int i = 0; i < range; i++) {
@@ -734,23 +752,15 @@
    * Launches all the tasks in order.
    */
   public static void main(String[] argv) throws Exception {
-    if (argv.length < 2) {
-      System.err.println("Usage: TestMapRed <range> <counts>");
-      System.err.println();
-      System.err.println("Note: a good test will have a <counts> value that is substantially
larger than the <range>");
-      return;
-    }
-
-    int i = 0;
-    range = Integer.parseInt(argv[i++]);
-    counts = Integer.parseInt(argv[i++]);
-    launch();
+    int res = ToolRunner.run(new TestMapRed(), argv);
+    System.exit(res);
   }
-    
+  @Test  
   public void testSmallInput(){
     runJob(100);
   }
-
+  
+  @Test
   public void testBiggerInput(){
     runJob(1000);
   }
@@ -797,7 +807,24 @@
 
       JobClient.runJob(conf);
     } catch (Exception e) {
-      fail("Threw exception:" + e);
+      assertTrue("Threw exception:" + e,false);
     }
   }
+
+  @Override
+  public int run(String[] argv) throws Exception {
+    if (argv.length < 2) {
+      System.err.println("Usage: TestMapRed <range> <counts>");
+      System.err.println();
+      System.err.println("Note: a good test will have a " +
+      		"<counts> value that is substantially larger than the <range>");
+      return -1;
+    }
+
+    int i = 0;
+    range = Integer.parseInt(argv[i++]);
+    counts = Integer.parseInt(argv[i++]);
+    launch();
+    return 0;
+  }
 }



Mime
View raw message