Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 15293 invoked from network); 21 Aug 2009 14:51:42 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 21 Aug 2009 14:51:42 -0000 Received: (qmail 15338 invoked by uid 500); 21 Aug 2009 14:52:04 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 15295 invoked by uid 500); 21 Aug 2009 14:52:04 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 15285 invoked by uid 99); 21 Aug 2009 14:52:04 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Aug 2009 14:52:04 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Aug 2009 14:52:01 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 9793123888EC; Fri, 21 Aug 2009 14:51:41 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r806577 [2/2] - in /hadoop/mapreduce/trunk: ./ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/ src/co... Date: Fri, 21 Aug 2009 14:51:40 -0000 To: mapreduce-commits@hadoop.apache.org From: tomwhite@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090821145141.9793123888EC@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce; + +import static org.apache.hadoop.mrunit.testutil.ExtendedAssert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mrunit.types.Pair; +import org.junit.Before; +import org.junit.Test; + +public class TestMapDriver extends TestCase { + + private Mapper mapper; + private MapDriver driver; + + @Before + public void setUp() { + mapper = new Mapper(); // default action is identity mapper. + driver = new MapDriver(mapper); + } + + @Test + public void testRun() { + List> out = null; + + try { + out = driver.withInput(new Text("foo"), new Text("bar")).run(); + } catch (IOException ioe) { + fail(); + } + + List> expected = new ArrayList>(); + expected.add(new Pair(new Text("foo"), new Text("bar"))); + + assertListEquals(out, expected); + } + + @Test + public void testTestRun1() { + driver.withInput(new Text("foo"), new Text("bar")) + .withOutput(new Text("foo"), new Text("bar")) + .runTest(); + } + + @Test + public void testTestRun2() { + try { + driver.withInput(new Text("foo"), new Text("bar")) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun3() { + try { + driver.withInput(new Text("foo"), new Text("bar")) + .withOutput(new Text("foo"), new Text("bar")) + .withOutput(new Text("foo"), new Text("bar")) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun4() { + try { + driver.withInput(new Text("foo"), new Text("bar")) + .withOutput(new Text("foo"), new Text("bar")) + .withOutput(new Text("bonusfoo"), new Text("bar")) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + + } + @Test + public void testTestRun5() { + try { + driver.withInput(new Text("foo"), new Text("bar")) + .withOutput(new Text("foo"), new Text("somethingelse")) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun6() { + try { + driver.withInput(new Text("foo"), new Text("bar")) + .withOutput(new Text("someotherkey"), new Text("bar")) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun7() { + try { + driver.withInput(new Text("foo"), new Text("bar")) + .withOutput(new Text("someotherkey"), new Text("bar")) + .withOutput(new Text("foo"), new Text("bar")) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testSetInput() { + try { + driver.setInput(new Pair(new Text("foo"), new Text("bar"))); + } catch (Exception e) { + fail(); + } + + assertEquals(driver.getInputKey(), new Text("foo")); + assertEquals(driver.getInputValue(), new Text("bar")); + } + + @Test + public void testSetInputNull() { + try { + driver.setInput((Pair) null); + fail(); + } catch (Exception e) { + // expect this. + } + } + + @Test + public void testEmptyInput() { + // MapDriver will forcibly map (null, null) as undefined input; + // identity mapper expects (null, null) back. + driver.withOutput(null, null).runTest(); + } + + @Test + public void testEmptyInput2() { + // it is an error to expect no output because we expect + // the mapper to be fed (null, null) as an input if the + // user doesn't set any input. + try { + driver.runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + +} + Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mrunit.mapreduce; + +import static org.apache.hadoop.mrunit.testutil.ExtendedAssert.assertListEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer; +import org.apache.hadoop.mrunit.types.Pair; +import org.junit.Before; +import org.junit.Test; + +public class TestMapReduceDriver extends TestCase { + + private static final int FOO_IN_A = 42; + private static final int FOO_IN_B = 10; + private static final int BAR_IN = 12; + private static final int FOO_OUT = 52; + + private Mapper mapper; + private Reducer reducer; + private MapReduceDriver driver; + + private MapReduceDriver driver2; + + @Before + public void setUp() throws Exception { + mapper = new Mapper(); // This is the IdentityMapper + reducer = new LongSumReducer(); + driver = new MapReduceDriver( + mapper, reducer); + // for shuffle tests + driver2 = new MapReduceDriver(); + } + + @Test + public void testRun() { + List> out = null; + try { + out = driver + .withInput(new Text("foo"), new LongWritable(FOO_IN_A)) + .withInput(new Text("foo"), new LongWritable(FOO_IN_B)) + .withInput(new Text("bar"), new LongWritable(BAR_IN)) + .run(); + } catch (IOException ioe) { + fail(); + } + + List> expected = + new ArrayList>(); + expected.add(new Pair(new Text("bar"), + new LongWritable(BAR_IN))); + expected.add(new Pair(new Text("foo"), + new LongWritable(FOO_OUT))); + + assertListEquals(out, expected); + } + + @Test + public void testTestRun1() { + driver + .withInput(new Text("foo"), new LongWritable(FOO_IN_A)) + .withInput(new Text("foo"), new LongWritable(FOO_IN_B)) + .withInput(new Text("bar"), new LongWritable(BAR_IN)) + .withOutput(new Text("bar"), new LongWritable(BAR_IN)) + .withOutput(new Text("foo"), new LongWritable(FOO_OUT)) + .runTest(); + } + + @Test + public void testTestRun2() { + driver + .withInput(new Text("foo"), new LongWritable(FOO_IN_A)) + .withInput(new Text("bar"), new LongWritable(BAR_IN)) + .withInput(new Text("foo"), new LongWritable(FOO_IN_B)) + .withOutput(new Text("bar"), new LongWritable(BAR_IN)) + .withOutput(new Text("foo"), new LongWritable(FOO_OUT)) + .runTest(); + } + + @Test + public void testTestRun3() { + try { + driver + .withInput(new Text("foo"), new LongWritable(FOO_IN_A)) + .withInput(new Text("bar"), new LongWritable(BAR_IN)) + .withInput(new Text("foo"), new LongWritable(FOO_IN_B)) + .withOutput(new Text("foo"), new LongWritable(FOO_OUT)) + .withOutput(new Text("bar"), new LongWritable(BAR_IN)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected + } + } + + @Test + public void testEmptyInput() { + driver.runTest(); + } + + @Test + public void testEmptyInputWithOutputFails() { + try { + driver + .withOutput(new Text("foo"), new LongWritable(FOO_OUT)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testEmptyShuffle() { + List> inputs = new ArrayList>(); + List>> outputs = driver2.shuffle(inputs); + assertEquals(0, outputs.size()); + } + + // just shuffle a single (k, v) pair + @Test + public void testSingleShuffle() { + List> inputs = new ArrayList>(); + inputs.add(new Pair(new Text("a"), new Text("b"))); + + List>> outputs = driver2.shuffle(inputs); + + List>> expected = new ArrayList>>(); + List sublist = new ArrayList(); + sublist.add(new Text("b")); + expected.add(new Pair>(new Text("a"), sublist)); + + assertListEquals(expected, outputs); + } + + // shuffle multiple values from the same key. + @Test + public void testShuffleOneKey() { + List> inputs = new ArrayList>(); + inputs.add(new Pair(new Text("a"), new Text("b"))); + inputs.add(new Pair(new Text("a"), new Text("c"))); + + List>> outputs = driver2.shuffle(inputs); + + List>> expected = new ArrayList>>(); + List sublist = new ArrayList(); + sublist.add(new Text("b")); + sublist.add(new Text("c")); + expected.add(new Pair>(new Text("a"), sublist)); + + assertListEquals(expected, outputs); + } + + // shuffle multiple keys + @Test + public void testMultiShuffle1() { + List> inputs = new ArrayList>(); + inputs.add(new Pair(new Text("a"), new Text("x"))); + inputs.add(new Pair(new Text("b"), new Text("z"))); + inputs.add(new Pair(new Text("b"), new Text("w"))); + inputs.add(new Pair(new Text("a"), new Text("y"))); + + List>> outputs = driver2.shuffle(inputs); + + List>> expected = new ArrayList>>(); + List sublist1 = new ArrayList(); + sublist1.add(new Text("x")); + sublist1.add(new Text("y")); + expected.add(new Pair>(new Text("a"), sublist1)); + + List sublist2 = new ArrayList(); + sublist2.add(new Text("z")); + sublist2.add(new Text("w")); + expected.add(new Pair>(new Text("b"), sublist2)); + + assertListEquals(expected, outputs); + } + + + // shuffle multiple keys that are out-of-order to start. + @Test + public void testMultiShuffle2() { + List> inputs = new ArrayList>(); + inputs.add(new Pair(new Text("b"), new Text("z"))); + inputs.add(new Pair(new Text("a"), new Text("x"))); + inputs.add(new Pair(new Text("b"), new Text("w"))); + inputs.add(new Pair(new Text("a"), new Text("y"))); + + List>> outputs = driver2.shuffle(inputs); + + List>> expected = new ArrayList>>(); + List sublist1 = new ArrayList(); + sublist1.add(new Text("x")); + sublist1.add(new Text("y")); + expected.add(new Pair>(new Text("a"), sublist1)); + + List sublist2 = new ArrayList(); + sublist2.add(new Text("z")); + sublist2.add(new Text("w")); + expected.add(new Pair>(new Text("b"), sublist2)); + + assertListEquals(expected, outputs); + } + +} + Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,227 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce; + +import static org.apache.hadoop.mrunit.testutil.ExtendedAssert.assertListEquals; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer; +import org.apache.hadoop.mrunit.types.Pair; +import org.junit.Before; +import org.junit.Test; + +public class TestReduceDriver extends TestCase { + + private static final int IN_A = 4; + private static final int IN_B = 6; + private static final int OUT_VAL = 10; + private static final int INCORRECT_OUT = 12; + private static final int OUT_EMPTY = 0; + + private Reducer reducer; + private ReduceDriver driver; + + @Before + public void setUp() throws Exception { + reducer = new LongSumReducer(); + driver = new ReduceDriver( + reducer); + } + + @Test + public void testRun() { + List> out = null; + + try { + out = driver.withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .run(); + } catch (IOException ioe) { + fail(); + } + + List> expected = + new ArrayList>(); + expected.add(new Pair(new Text("foo"), + new LongWritable(OUT_VAL))); + + assertListEquals(out, expected); + + } + + @Test + public void testTestRun1() { + driver + .withInputKey(new Text("foo")) + .withOutput(new Text("foo"), new LongWritable(0)) + .runTest(); + } + + @Test + public void testTestRun2() { + driver + .withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .withOutput(new Text("foo"), new LongWritable(OUT_VAL)) + .runTest(); + } + + @Test + public void testTestRun3() { + try { + driver + .withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .withOutput(new Text("bar"), new LongWritable(OUT_VAL)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + + } + + @Test + public void testTestRun4() { + try { + driver + .withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .withOutput(new Text("foo"), new LongWritable(INCORRECT_OUT)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun5() { + try { + driver + .withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .withOutput(new Text("foo"), new LongWritable(IN_A)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun6() { + try { + driver + .withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .withOutput(new Text("foo"), new LongWritable(IN_A)) + .withOutput(new Text("foo"), new LongWritable(IN_B)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun7() { + try { + driver + .withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .withOutput(new Text("foo"), new LongWritable(OUT_VAL)) + .withOutput(new Text("foo"), new LongWritable(OUT_VAL)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun8() { + try { + driver + .withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .withOutput(new Text("bar"), new LongWritable(OUT_VAL)) + .withOutput(new Text("foo"), new LongWritable(OUT_VAL)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testTestRun9() { + try { + driver + .withInputKey(new Text("foo")) + .withInputValue(new LongWritable(IN_A)) + .withInputValue(new LongWritable(IN_B)) + .withOutput(new Text("foo"), new LongWritable(OUT_VAL)) + .withOutput(new Text("bar"), new LongWritable(OUT_VAL)) + .runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } + + @Test + public void testEmptyInput() { + // (null, ) will be forcibly fed as input + // since we use LongSumReducer, expect (null, 0) out. + driver + .withOutput(null, new LongWritable(OUT_EMPTY)) + .runTest(); + } + + @Test + public void testEmptyInput2() { + // because a null key with zero inputs will be fed as input + // to this reducer, do not accept no outputs. + try { + driver.runTest(); + fail(); + } catch (RuntimeException re) { + // expected. + } + } +} +