Return-Path: Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: (qmail 91842 invoked from network); 4 Mar 2011 03:29:54 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 4 Mar 2011 03:29:54 -0000 Received: (qmail 16791 invoked by uid 500); 4 Mar 2011 03:29:53 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 16763 invoked by uid 500); 4 Mar 2011 03:29:53 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 16752 invoked by uid 99); 4 Mar 2011 03:29:53 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Mar 2011 03:29:53 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Mar 2011 03:29:49 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 856CD2388C1F; Fri, 4 Mar 2011 03:29:28 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1077002 - in /hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred: MRBench.java TestMapRed.java Date: Fri, 04 Mar 2011 03:29:28 -0000 To: common-commits@hadoop.apache.org From: omalley@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110304032928.856CD2388C1F@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: omalley Date: Fri Mar 4 03:29:28 2011 New Revision: 1077002 URL: http://svn.apache.org/viewvc?rev=1077002&view=rev Log: commit 3060afa4458ef9554f23247e77a99da7e02cfe72 Author: Yahoo\! Date: Thu Sep 24 16:06:26 2009 -0700 MAPREDUCE-945. Modifies MRBench and TestMapRed to use ToolRunner so that options such as queue name can be passed via command line. Contributed by Sreekanth Ramakrishnan. from http://issues.apache.org/jira/secure/attachment/12418910/mapreduce-945-internal-3.8.patch.txt +++ b/YAHOO-CHANGES.txt +54. MAPREDUCE-945. Modifies MRBench and TestMapRed to use + ToolRunner so that options such as queue name can be + passed via command line. Contributed by Sreekanth Ramakrishnan. + Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/MRBench.java hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMapRed.java Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/MRBench.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/MRBench.java?rev=1077002&r1=1077001&r2=1077002&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/MRBench.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/MRBench.java Fri Mar 4 03:29:28 2011 @@ -26,16 +26,19 @@ import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.UTF8; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; /** * Runs a job multiple times and takes average of all runs. */ -public class MRBench { +public class MRBench extends Configured implements Tool{ private static final Log LOG = LogFactory.getLog(MRBench.class); private static Path BASE_DIR = @@ -87,7 +90,7 @@ public class MRBench { * an appropriate number of leading '0' (zero) characters. The order of * generated data is one of ascending, descending, or random. */ - public static void generateTextFile(FileSystem fs, Path inputFile, + public void generateTextFile(FileSystem fs, Path inputFile, long numLines, Order sortOrder) throws IOException { LOG.info("creating control file: "+numLines+" numLines, "+sortOrder+" sortOrder"); @@ -137,8 +140,9 @@ public class MRBench { /** * Create the job configuration. */ - private static JobConf setupJob(int numMaps, int numReduces, String jarFile) { - JobConf jobConf = new JobConf(MRBench.class); + private JobConf setupJob(int numMaps, int numReduces, String jarFile) { + JobConf jobConf = new JobConf(getConf()); + jobConf.setJarByClass(MRBench.class); FileInputFormat.addInputPath(jobConf, INPUT_DIR); jobConf.setInputFormat(TextInputFormat.class); @@ -165,8 +169,7 @@ public class MRBench { * Runs a MapReduce task, given number of times. The input to each run * is the same file. */ - private static ArrayList runJobInSequence(JobConf masterJobConf, int numRuns) throws IOException { - Path intrimData = null; + private ArrayList runJobInSequence(JobConf masterJobConf, int numRuns) throws IOException { Random rand = new Random(); ArrayList execTimes = new ArrayList(); @@ -204,7 +207,13 @@ public class MRBench { * [-verbose] * */ - public static void main (String[] args) throws IOException { + public static void main (String[] args) throws Exception { + int res = ToolRunner.run(new MRBench(), args); + System.exit(res); + } + + @Override + public int run(String[] args) throws Exception { String version = "MRBenchmark.0.0.2"; System.out.println(version); @@ -265,7 +274,7 @@ public class MRBench { inputSortOrder == null) { System.err.println(usage); - System.exit(-1); + return -1; } JobConf jobConf = setupJob(numMaps, numReduces, jarFile); @@ -303,6 +312,7 @@ public class MRBench { System.out.println("DataLines\tMaps\tReduces\tAvgTime (milliseconds)"); System.out.println(inputLines + "\t\t" + numMaps + "\t" + numReduces + "\t" + avgTime); + return 0; } } Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMapRed.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMapRed.java?rev=1077002&r1=1077001&r2=1077002&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMapRed.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestMapRed.java Fri Mar 4 03:29:28 2011 @@ -31,6 +31,7 @@ import java.util.Random; import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -43,6 +44,9 @@ import org.apache.hadoop.io.WritableComp import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.hadoop.mapred.lib.IdentityReducer; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + /********************************************************** * MapredLoadTest generates a bunch of work that exercises @@ -77,7 +81,7 @@ import org.apache.hadoop.mapred.lib.Iden * 7) A mapred job integrates all the count files into a single one. * **********************************************************/ -public class TestMapRed extends TestCase { +public class TestMapRed extends TestCase implements Tool { /** * Modified to make it a junit test. * The RandomGen Job does the actual work of creating @@ -248,6 +252,7 @@ public class TestMapRed extends TestCase } **/ + public void testMapred() throws Exception { launch(); } @@ -314,6 +319,7 @@ public class TestMapRed extends TestCase } } + public void testPartitioner() throws Exception { JobConf conf = new JobConf(TestMapRed.class); conf.setPartitionerClass(BadPartitioner.class); @@ -363,6 +369,7 @@ public class TestMapRed extends TestCase public void close() { } } + public void testNullKeys() throws Exception { JobConf conf = new JobConf(TestMapRed.class); FileSystem fs = FileSystem.getLocal(conf); @@ -454,7 +461,8 @@ public class TestMapRed extends TestCase fs.delete(testdir, true); } } - + + public void testCompression() throws Exception { EnumSet seq = EnumSet.allOf(SequenceFile.CompressionType.class); @@ -470,11 +478,19 @@ public class TestMapRed extends TestCase /** * */ - public static void launch() throws Exception { + public void launch() throws Exception { // // Generate distribution of ints. This is the answer key. // - JobConf conf = new JobConf(TestMapRed.class); + JobConf conf = null; + //Check to get configuration and check if it is configured thro' Configured + //interface. This would happen when running testcase thro' command line. + if(getConf() == null) { + conf = new JobConf(); + } else { + conf = new JobConf(getConf()); + } + conf.setJarByClass(TestMapRed.class); int countsToGo = counts; int dist[] = new int[range]; for (int i = 0; i < range; i++) { @@ -736,23 +752,15 @@ public class TestMapRed extends TestCase * Launches all the tasks in order. */ public static void main(String[] argv) throws Exception { - if (argv.length < 2) { - System.err.println("Usage: TestMapRed "); - System.err.println(); - System.err.println("Note: a good test will have a value that is substantially larger than the "); - return; - } - - int i = 0; - range = Integer.parseInt(argv[i++]); - counts = Integer.parseInt(argv[i++]); - launch(); + int res = ToolRunner.run(new TestMapRed(), argv); + System.exit(res); } public void testSmallInput(){ runJob(100); } - + + public void testBiggerInput(){ runJob(1000); } @@ -799,7 +807,36 @@ public class TestMapRed extends TestCase JobClient.runJob(conf); } catch (Exception e) { - fail("Threw exception:" + e); + assertTrue("Threw exception:" + e,false); } } + + @Override + public int run(String[] argv) throws Exception { + if (argv.length < 2) { + System.err.println("Usage: TestMapRed "); + System.err.println(); + System.err.println("Note: a good test will have a " + + " value that is substantially larger than the "); + return -1; + } + + int i = 0; + range = Integer.parseInt(argv[i++]); + counts = Integer.parseInt(argv[i++]); + launch(); + return 0; + } + + Configuration myConf = null; + + @Override + public Configuration getConf() { + return myConf; + } + + @Override + public void setConf(Configuration conf) { + myConf = conf; + } }