Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 15323 invoked from network); 21 Aug 2009 14:51:43 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 21 Aug 2009 14:51:43 -0000 Received: (qmail 15404 invoked by uid 500); 21 Aug 2009 14:52:05 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 15360 invoked by uid 500); 21 Aug 2009 14:52:05 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 15350 invoked by uid 99); 21 Aug 2009 14:52:05 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Aug 2009 14:52:05 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Aug 2009 14:52:01 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 8E87D23888A0; Fri, 21 Aug 2009 14:51:41 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r806577 [1/2] - in /hadoop/mapreduce/trunk: ./ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/ src/co... Date: Fri, 21 Aug 2009 14:51:40 -0000 To: mapreduce-commits@hadoop.apache.org From: tomwhite@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090821145141.8E87D23888A0@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: tomwhite Date: Fri Aug 21 14:51:39 2009 New Revision: 806577 URL: http://svn.apache.org/viewvc?rev=806577&view=rev Log: MAPREDUCE-800. MRUnit should support the new API. Contributed by Aaron Kimball. Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriverBase.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriverBase.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockInputSplit.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockOutputCommitter.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockRawKeyValueIterator.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java Modified: hadoop/mapreduce/trunk/CHANGES.txt hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/TestDriver.java hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java Modified: hadoop/mapreduce/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=806577&r1=806576&r2=806577&view=diff ============================================================================== --- hadoop/mapreduce/trunk/CHANGES.txt (original) +++ hadoop/mapreduce/trunk/CHANGES.txt Fri Aug 21 14:51:39 2009 @@ -62,6 +62,9 @@ MAPREDUCE-814. Provide a way to configure completed job history files to be on HDFS. (sharad) + MAPREDUCE-800. MRUnit should support the new API. (Aaron Kimball via + tomwhite) + IMPROVEMENTS MAPREDUCE-816. Rename "local" mysql import to "direct" in Sqoop. Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java?rev=806577&r1=806576&r2=806577&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java (original) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java Fri Aug 21 14:51:39 2009 @@ -38,15 +38,12 @@ * single (k, v) -> (k, v)* case from the Mapper, representing a single unit * test. Multiple input (k, v) pairs should go in separate unit tests. */ -public class MapDriver extends TestDriver { +public class MapDriver extends MapDriverBase { public static final Log LOG = LogFactory.getLog(MapDriver.class); private Mapper myMapper; - private K1 inputKey; - private V1 inputVal; - public MapDriver(final Mapper m) { myMapper = m; } @@ -54,7 +51,6 @@ public MapDriver() { } - /** * Set the Mapper instance to use with this test driver * @@ -78,18 +74,6 @@ } /** - * Sets the input key to send to the mapper - * - */ - public void setInputKey(K1 key) { - inputKey = key; - } - - public K1 getInputKey() { - return inputKey; - } - - /** * Identical to setInputKey() but with fluent programming style * * @return this @@ -100,19 +84,6 @@ } /** - * Sets the input value to send to the mapper - * - * @param val - */ - public void setInputValue(V1 val) { - inputVal = val; - } - - public V1 getInputValue() { - return inputVal; - } - - /** * Identical to setInputValue() but with fluent programming style * * @param val @@ -124,15 +95,6 @@ } /** - * Sets the input to send to the mapper - * - */ - public void setInput(K1 key, V1 val) { - setInputKey(key); - setInputValue(val); - } - - /** * Identical to setInput() but returns self for fluent programming style * * @return this @@ -143,21 +105,6 @@ } /** - * Sets the input to send to the mapper - * - * @param inputRecord - * a (key, val) pair - */ - public void setInput(Pair inputRecord) { - if (null != inputRecord) { - setInputKey(inputRecord.getFirst()); - setInputValue(inputRecord.getSecond()); - } else { - throw new IllegalArgumentException("null inputRecord in setInput()"); - } - } - - /** * Identical to setInput() but returns self for fluent programming style * * @param inputRecord @@ -169,20 +116,6 @@ } /** - * Adds an output (k, v) pair we expect from the Mapper - * - * @param outputRecord - * The (k, v) pair to add - */ - public void addOutput(Pair outputRecord) { - if (null != outputRecord) { - expectedOutputs.add(outputRecord); - } else { - throw new IllegalArgumentException("Tried to add null outputRecord"); - } - } - - /** * Works like addOutput(), but returns self for fluent style * * @param outputRecord @@ -194,14 +127,6 @@ } /** - * Adds a (k, v) pair we expect as output from the mapper - * - */ - public void addOutput(K2 key, V2 val) { - addOutput(new Pair(key, val)); - } - - /** * Functions like addOutput() but returns self for fluent programming * style * @@ -213,30 +138,6 @@ } /** - * Expects an input of the form "key \t val" Forces the Mapper input types - * to Text. - * - * @param input - * A string of the form "key \t val". - */ - public void setInputFromString(String input) { - if (null == input) { - throw new IllegalArgumentException("null input given to setInputFromString"); - } else { - Pair inputPair = parseTabbedPair(input); - if (null != inputPair) { - // I know this is not type-safe, but I don't know a better way to do - // this. - setInputKey((K1) inputPair.getFirst()); - setInputValue((V1) inputPair.getSecond()); - } else { - throw new IllegalArgumentException( - "Could not parse input pair in setInputFromString"); - } - } - } - - /** * Identical to setInputFromString, but with a fluent programming style * * @param input @@ -249,28 +150,6 @@ } /** - * Expects an input of the form "key \t val" Forces the Mapper output types - * to Text. - * - * @param output - * A string of the form "key \t val". Trims any whitespace. - */ - public void addOutputFromString(String output) { - if (null == output) { - throw new IllegalArgumentException("null input given to setOutput"); - } else { - Pair outputPair = parseTabbedPair(output); - if (null != outputPair) { - // I know this is not type-safe, but I don't know a better way to do - // this. - addOutput((Pair) outputPair); - } else { - throw new IllegalArgumentException("Could not parse output pair in setOutput"); - } - } - } - - /** * Identical to addOutputFromString, but with a fluent programming style * * @param output @@ -294,33 +173,6 @@ } @Override - public void runTest() throws RuntimeException { - String inputKeyStr = "(null)"; - String inputValStr = "(null)"; - - if (null != inputKey) { - inputKeyStr = inputKey.toString(); - } - - if (null != inputVal) { - inputValStr = inputVal.toString(); - } - - LOG.debug("Mapping input (" + inputKeyStr + ", " + inputValStr + ")"); - - List> outputs = null; - - try { - outputs = run(); - validate(outputs); - } catch (IOException ioe) { - LOG.error("IOException in mapper: " + ioe.toString()); - LOG.debug("Setting success to false based on IOException"); - throw new RuntimeException(); - } - } - - @Override public String toString() { return "MapDriver (" + myMapper + ")"; } Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriverBase.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriverBase.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriverBase.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriverBase.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mrunit; + + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mrunit.mock.MockOutputCollector; +import org.apache.hadoop.mrunit.mock.MockReporter; +import org.apache.hadoop.mrunit.types.Pair; + +/** + * Harness that allows you to test a Mapper instance. You provide the input + * key and value that should be sent to the Mapper, and outputs you expect to + * be sent by the Mapper to the collector for those inputs. By calling + * runTest(), the harness will deliver the input to the Mapper and will check + * its outputs against the expected results. This is designed to handle a + * single (k, v) -> (k, v)* case from the Mapper, representing a single unit + * test. Multiple input (k, v) pairs should go in separate unit tests. + */ +public abstract class MapDriverBase extends TestDriver { + + public static final Log LOG = LogFactory.getLog(MapDriverBase.class); + + protected K1 inputKey; + protected V1 inputVal; + + + /** + * Sets the input key to send to the mapper + * + */ + public void setInputKey(K1 key) { + inputKey = key; + } + + public K1 getInputKey() { + return inputKey; + } + + /** + * Sets the input value to send to the mapper + * + * @param val + */ + public void setInputValue(V1 val) { + inputVal = val; + } + + public V1 getInputValue() { + return inputVal; + } + + /** + * Sets the input to send to the mapper + * + */ + public void setInput(K1 key, V1 val) { + setInputKey(key); + setInputValue(val); + } + + /** + * Sets the input to send to the mapper + * + * @param inputRecord + * a (key, val) pair + */ + public void setInput(Pair inputRecord) { + if (null != inputRecord) { + setInputKey(inputRecord.getFirst()); + setInputValue(inputRecord.getSecond()); + } else { + throw new IllegalArgumentException("null inputRecord in setInput()"); + } + } + + /** + * Adds an output (k, v) pair we expect from the Mapper + * + * @param outputRecord + * The (k, v) pair to add + */ + public void addOutput(Pair outputRecord) { + if (null != outputRecord) { + expectedOutputs.add(outputRecord); + } else { + throw new IllegalArgumentException("Tried to add null outputRecord"); + } + } + + /** + * Adds a (k, v) pair we expect as output from the mapper + * + */ + public void addOutput(K2 key, V2 val) { + addOutput(new Pair(key, val)); + } + + /** + * Expects an input of the form "key \t val" Forces the Mapper input types + * to Text. + * + * @param input + * A string of the form "key \t val". + */ + public void setInputFromString(String input) { + if (null == input) { + throw new IllegalArgumentException("null input given to setInputFromString"); + } else { + Pair inputPair = parseTabbedPair(input); + if (null != inputPair) { + // I know this is not type-safe, but I don't know a better way to do + // this. + setInputKey((K1) inputPair.getFirst()); + setInputValue((V1) inputPair.getSecond()); + } else { + throw new IllegalArgumentException( + "Could not parse input pair in setInputFromString"); + } + } + } + + /** + * Expects an input of the form "key \t val" Forces the Mapper output types + * to Text. + * + * @param output + * A string of the form "key \t val". Trims any whitespace. + */ + public void addOutputFromString(String output) { + if (null == output) { + throw new IllegalArgumentException("null input given to setOutput"); + } else { + Pair outputPair = parseTabbedPair(output); + if (null != outputPair) { + // I know this is not type-safe, but I don't know a better way to do + // this. + addOutput((Pair) outputPair); + } else { + throw new IllegalArgumentException("Could not parse output pair in setOutput"); + } + } + } + + public abstract List> run() throws IOException; + + @Override + public void runTest() throws RuntimeException { + String inputKeyStr = "(null)"; + String inputValStr = "(null)"; + + if (null != inputKey) { + inputKeyStr = inputKey.toString(); + } + + if (null != inputVal) { + inputValStr = inputVal.toString(); + } + + LOG.debug("Mapping input (" + inputKeyStr + ", " + inputValStr + ")"); + + List> outputs = null; + + try { + outputs = run(); + validate(outputs); + } catch (IOException ioe) { + LOG.error("IOException in mapper: " + ioe.toString()); + LOG.debug("Setting success to false based on IOException"); + throw new RuntimeException(); + } + } +} + Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java?rev=806577&r1=806576&r2=806577&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java (original) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java Fri Aug 21 14:51:39 2009 @@ -48,7 +48,7 @@ * the Mapper and before the Reducer. */ public class MapReduceDriver - extends TestDriver { + extends MapReduceDriverBase { public static final Log LOG = LogFactory.getLog(MapReduceDriver.class); @@ -56,13 +56,10 @@ private Reducer myReducer; private Reducer myCombiner; - private List> inputList; - public MapReduceDriver(final Mapper m, final Reducer r) { myMapper = m; myReducer = r; - inputList = new ArrayList>(); } public MapReduceDriver(final Mapper m, @@ -71,11 +68,9 @@ myMapper = m; myReducer = r; myCombiner = c; - inputList = new ArrayList>(); } public MapReduceDriver() { - inputList = new ArrayList>(); } /** Set the Mapper instance to use with this test driver @@ -151,15 +146,6 @@ } /** - * Adds an input to send to the mapper - * @param key - * @param val - */ - public void addInput(K1 key, V1 val) { - inputList.add(new Pair(key, val)); - } - - /** * Identical to addInput() but returns self for fluent programming style * @param key * @param val @@ -171,18 +157,6 @@ } /** - * Adds an input to send to the Mapper - * @param input The (k, v) pair to add to the input list. - */ - public void addInput(Pair input) { - if (null == input) { - throw new IllegalArgumentException("Null input in addInput()"); - } - - inputList.add(input); - } - - /** * Identical to addInput() but returns self for fluent programming style * @param input The (k, v) pair to add * @return this @@ -194,18 +168,6 @@ } /** - * Adds an output (k, v) pair we expect from the Reducer - * @param outputRecord The (k, v) pair to add - */ - public void addOutput(Pair outputRecord) { - if (null != outputRecord) { - expectedOutputs.add(outputRecord); - } else { - throw new IllegalArgumentException("Tried to add null outputRecord"); - } - } - - /** * Works like addOutput(), but returns self for fluent style * @param outputRecord * @return this @@ -217,15 +179,6 @@ } /** - * Adds a (k, v) pair we expect as output from the Reducer - * @param key - * @param val - */ - public void addOutput(K3 key, V3 val) { - addOutput(new Pair(key, val)); - } - - /** * Functions like addOutput() but returns self for fluent programming style * @param key * @param val @@ -237,26 +190,6 @@ } /** - * Expects an input of the form "key \t val" - * Forces the Mapper input types to Text. - * @param input A string of the form "key \t val". Trims any whitespace. - */ - public void addInputFromString(String input) { - if (null == input) { - throw new IllegalArgumentException("null input given to setInput"); - } else { - Pair inputPair = parseTabbedPair(input); - if (null != inputPair) { - // I know this is not type-safe, but I don't - // know a better way to do this. - addInput((Pair) inputPair); - } else { - throw new IllegalArgumentException("Could not parse input pair in addInput"); - } - } - } - - /** * Identical to addInputFromString, but with a fluent programming style * @param input A string of the form "key \t val". Trims any whitespace. * @return this @@ -267,27 +200,6 @@ } /** - * Expects an input of the form "key \t val" - * Forces the Reducer output types to Text. - * @param output A string of the form "key \t val". Trims any whitespace. - */ - public void addOutputFromString(String output) { - if (null == output) { - throw new IllegalArgumentException("null input given to setOutput"); - } else { - Pair outputPair = parseTabbedPair(output); - if (null != outputPair) { - // I know this is not type-safe, - // but I don't know a better way to do this. - addOutput((Pair) outputPair); - } else { - throw new IllegalArgumentException( - "Could not parse output pair in setOutput"); - } - } - } - - /** * Identical to addOutputFromString, but with a fluent programming style * @param output A string of the form "key \t val". Trims any whitespace. * @return this @@ -347,58 +259,7 @@ } @Override - public void runTest() throws RuntimeException { - List> reduceOutputs = null; - boolean succeeded; - - try { - reduceOutputs = run(); - validate(reduceOutputs); - } catch (IOException ioe) { - LOG.error("IOException: " + ioe.toString()); - LOG.debug("Setting success to false based on IOException"); - throw new RuntimeException(); - } - } - - /** Take the outputs from the Mapper, combine all values for the - * same key, and sort them by key. - * @param mapOutputs An unordered list of (key, val) pairs from the mapper - * @return the sorted list of (key, list(val))'s to present to the reducer - */ - List>> shuffle(List> mapOutputs) { - HashMap> reducerInputs = new HashMap>(); - - // step 1: condense all values with the same key into a list. - for (Pair mapOutput : mapOutputs) { - List valuesForKey = reducerInputs.get(mapOutput.getFirst()); - - if (null == valuesForKey) { - // this is the first (k, v) pair for this key. Add it to the list. - valuesForKey = new ArrayList(); - valuesForKey.add(mapOutput.getSecond()); - reducerInputs.put(mapOutput.getFirst(), valuesForKey); - } else { - // add this value to the existing list for this key - valuesForKey.add(mapOutput.getSecond()); - } - } - - // build a list out of these (k, list(v)) pairs - List>> finalInputs = new ArrayList>>(); - Set>> entries = reducerInputs.entrySet(); - for (Map.Entry> entry : entries) { - K2 key = entry.getKey(); - List vals = entry.getValue(); - finalInputs.add(new Pair>(key, vals)); - } - - // and then sort the output list by key - if (finalInputs.size() > 0) { - Collections.sort(finalInputs, - finalInputs.get(0).new FirstElemComparator()); - } - - return finalInputs; + public String toString() { + return "MapReduceDriver (" + myMapper + ", " + myReducer + ")"; } } Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,195 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mrunit; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mrunit.types.Pair; + +/** + * Harness that allows you to test a Mapper and a Reducer instance together + * You provide the input key and value that should be sent to the Mapper, and + * outputs you expect to be sent by the Reducer to the collector for those + * inputs. By calling runTest(), the harness will deliver the input to the + * Mapper, feed the intermediate results to the Reducer (without checking + * them), and will check the Reducer's outputs against the expected results. + * This is designed to handle a single (k, v)* -> (k, v)* case from the + * Mapper/Reducer pair, representing a single unit test. + */ +public abstract class MapReduceDriverBase + extends TestDriver { + + public static final Log LOG = LogFactory.getLog(MapReduceDriverBase.class); + + protected List> inputList; + + public MapReduceDriverBase() { + inputList = new ArrayList>(); + } + + /** + * Adds an input to send to the mapper + * @param key + * @param val + */ + public void addInput(K1 key, V1 val) { + inputList.add(new Pair(key, val)); + } + + /** + * Adds an input to send to the Mapper + * @param input The (k, v) pair to add to the input list. + */ + public void addInput(Pair input) { + if (null == input) { + throw new IllegalArgumentException("Null input in addInput()"); + } + + inputList.add(input); + } + + /** + * Adds an output (k, v) pair we expect from the Reducer + * @param outputRecord The (k, v) pair to add + */ + public void addOutput(Pair outputRecord) { + if (null != outputRecord) { + expectedOutputs.add(outputRecord); + } else { + throw new IllegalArgumentException("Tried to add null outputRecord"); + } + } + + /** + * Adds a (k, v) pair we expect as output from the Reducer + * @param key + * @param val + */ + public void addOutput(K3 key, V3 val) { + addOutput(new Pair(key, val)); + } + + /** + * Expects an input of the form "key \t val" + * Forces the Mapper input types to Text. + * @param input A string of the form "key \t val". Trims any whitespace. + */ + public void addInputFromString(String input) { + if (null == input) { + throw new IllegalArgumentException("null input given to setInput"); + } else { + Pair inputPair = parseTabbedPair(input); + if (null != inputPair) { + // I know this is not type-safe, but I don't + // know a better way to do this. + addInput((Pair) inputPair); + } else { + throw new IllegalArgumentException("Could not parse input pair in addInput"); + } + } + } + + /** + * Expects an input of the form "key \t val" + * Forces the Reducer output types to Text. + * @param output A string of the form "key \t val". Trims any whitespace. + */ + public void addOutputFromString(String output) { + if (null == output) { + throw new IllegalArgumentException("null input given to setOutput"); + } else { + Pair outputPair = parseTabbedPair(output); + if (null != outputPair) { + // I know this is not type-safe, + // but I don't know a better way to do this. + addOutput((Pair) outputPair); + } else { + throw new IllegalArgumentException( + "Could not parse output pair in setOutput"); + } + } + } + + public abstract List> run() throws IOException; + + @Override + public void runTest() throws RuntimeException { + List> reduceOutputs = null; + boolean succeeded; + + try { + reduceOutputs = run(); + validate(reduceOutputs); + } catch (IOException ioe) { + LOG.error("IOException: " + ioe.toString()); + LOG.debug("Setting success to false based on IOException"); + throw new RuntimeException(); + } + } + + /** Take the outputs from the Mapper, combine all values for the + * same key, and sort them by key. + * @param mapOutputs An unordered list of (key, val) pairs from the mapper + * @return the sorted list of (key, list(val))'s to present to the reducer + */ + public List>> shuffle(List> mapOutputs) { + HashMap> reducerInputs = new HashMap>(); + + // step 1: condense all values with the same key into a list. + for (Pair mapOutput : mapOutputs) { + List valuesForKey = reducerInputs.get(mapOutput.getFirst()); + + if (null == valuesForKey) { + // this is the first (k, v) pair for this key. Add it to the list. + valuesForKey = new ArrayList(); + valuesForKey.add(mapOutput.getSecond()); + reducerInputs.put(mapOutput.getFirst(), valuesForKey); + } else { + // add this value to the existing list for this key + valuesForKey.add(mapOutput.getSecond()); + } + } + + // build a list out of these (k, list(v)) pairs + List>> finalInputs = new ArrayList>>(); + Set>> entries = reducerInputs.entrySet(); + for (Map.Entry> entry : entries) { + K2 key = entry.getKey(); + List vals = entry.getValue(); + finalInputs.add(new Pair>(key, vals)); + } + + // and then sort the output list by key + if (finalInputs.size() > 0) { + Collections.sort(finalInputs, + finalInputs.get(0).new FirstElemComparator()); + } + + return finalInputs; + } +} Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java?rev=806577&r1=806576&r2=806577&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java (original) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java Fri Aug 21 14:51:39 2009 @@ -40,22 +40,17 @@ * (k, v*) -> (k, v)* case from the Reducer, representing a single unit test. * Multiple input (k, v*) sets should go in separate unit tests. */ -public class ReduceDriver extends TestDriver { +public class ReduceDriver extends ReduceDriverBase { public static final Log LOG = LogFactory.getLog(ReduceDriver.class); private Reducer myReducer; - private K1 inputKey; - private List inputValues; - public ReduceDriver(final Reducer r) { myReducer = r; - inputValues = new ArrayList(); } public ReduceDriver() { - inputValues = new ArrayList(); } /** @@ -85,14 +80,6 @@ } /** - * Sets the input key to send to the Reducer - * - */ - public void setInputKey(K1 key) { - inputKey = key; - } - - /** * Identical to setInputKey() but with fluent programming style * * @return this @@ -103,15 +90,6 @@ } /** - * adds an input value to send to the reducer - * - * @param val - */ - public void addInputValue(V1 val) { - inputValues.add(val); - } - - /** * Identical to addInputValue() but with fluent programming style * * @param val @@ -123,25 +101,6 @@ } /** - * Sets the input values to send to the reducer; overwrites existing ones - * - * @param values - */ - public void setInputValues(List values) { - inputValues.clear(); - inputValues.addAll(values); - } - - /** - * Adds a set of input values to send to the reducer - * - * @param values - */ - public void addInputValues(List values) { - inputValues.addAll(values); - } - - /** * Identical to addInputValues() but with fluent programming style * * @param values @@ -153,16 +112,6 @@ } /** - * Sets the input to send to the reducer - * - * @param values - */ - public void setInput(K1 key, List values) { - setInputKey(key); - setInputValues(values); - } - - /** * Identical to setInput() but returns self for fluent programming style * * @return this @@ -173,20 +122,6 @@ } /** - * Adds an output (k, v) pair we expect from the Reducer - * - * @param outputRecord - * The (k, v) pair to add - */ - public void addOutput(Pair outputRecord) { - if (null != outputRecord) { - expectedOutputs.add(outputRecord); - } else { - throw new IllegalArgumentException("Tried to add null outputRecord"); - } - } - - /** * Works like addOutput(), but returns self for fluent style * * @param outputRecord @@ -198,16 +133,6 @@ } /** - * Adds an output (k, v) pair we expect from the Reducer - * - * @param key The key part of a (k, v) pair to add - * @param val The val part of a (k, v) pair to add - */ - public void addOutput(K2 key, V2 val) { - addOutput(new Pair(key, val)); - } - - /** * Works like addOutput(), but returns self for fluent style * * @param key The key part of a (k, v) pair to add @@ -220,31 +145,6 @@ } /** - * Expects an input of the form "key \t val, val, val..." Forces the Reducer - * input types to Text. - * - * @param input - * A string of the form "key \t val,val,val". Trims any whitespace. - */ - public void setInputFromString(String input) { - if (null == input) { - throw new IllegalArgumentException("null input given to setInputFromString"); - } else { - Pair inputPair = parseTabbedPair(input); - if (null != inputPair) { - // I know this is not type-safe, but I don't know a better way to do - // this. - setInputKey((K1) inputPair.getFirst()); - setInputValues((List) parseCommaDelimitedList(inputPair.getSecond() - .toString())); - } else { - throw new IllegalArgumentException( - "Could not parse input pair in setInputFromString"); - } - } - } - - /** * Identical to setInput, but with a fluent programming style * * @param input @@ -257,28 +157,6 @@ } /** - * Expects an input of the form "key \t val" Forces the Reducer output types - * to Text. - * - * @param output - * A string of the form "key \t val". Trims any whitespace. - */ - public void addOutputFromString(String output) { - if (null == output) { - throw new IllegalArgumentException("null input given to setOutput"); - } else { - Pair outputPair = parseTabbedPair(output); - if (null != outputPair) { - // I know this is not type-safe, but I don't know a better way to do - // this. - addOutput((Pair) outputPair); - } else { - throw new IllegalArgumentException("Could not parse output pair in setOutput"); - } - } - } - - /** * Identical to addOutput, but with a fluent programming style * * @param output @@ -292,7 +170,6 @@ @Override public List> run() throws IOException { - MockOutputCollector outputCollector = new MockOutputCollector(); MockReporter reporter = new MockReporter(MockReporter.ReporterType.Reducer); @@ -305,39 +182,8 @@ } @Override - public void runTest() throws RuntimeException { - - String inputKeyStr = "(null)"; - - if (null != inputKey) { - inputKeyStr = inputKey.toString(); - } - - StringBuilder sb = new StringBuilder(); - formatValueList(inputValues, sb); - - LOG.debug("Reducing input (" + inputKeyStr + ", " + sb.toString() + ")"); - - List> outputs = null; - try { - outputs = run(); - validate(outputs); - } catch (IOException ioe) { - LOG.error("IOException in reducer: " + ioe.toString()); - LOG.debug("Setting success to false based on IOException"); - throw new RuntimeException(); - } - } - - @Override public String toString() { - String reducerStr = "null"; - - if (null != myReducer) { - reducerStr = myReducer.toString(); - } - - return "ReduceDriver (" + reducerStr + ")"; + return "ReduceDriver (" + myReducer + ")"; } } Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriverBase.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriverBase.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriverBase.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriverBase.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,195 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mrunit; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mrunit.mock.MockOutputCollector; +import org.apache.hadoop.mrunit.mock.MockReporter; +import org.apache.hadoop.mrunit.types.Pair; + +/** + * Harness that allows you to test a Reducer instance. You provide a key and a + * set of intermediate values for that key that represent inputs that should + * be sent to the Reducer (as if they came from a Mapper), and outputs you + * expect to be sent by the Reducer to the collector. By calling runTest(), + * the harness will deliver the input to the Reducer and will check its + * outputs against the expected results. This is designed to handle a single + * (k, v*) -> (k, v)* case from the Reducer, representing a single unit test. + * Multiple input (k, v*) sets should go in separate unit tests. + */ +public abstract class ReduceDriverBase extends TestDriver { + + protected K1 inputKey; + protected List inputValues; + + public ReduceDriverBase() { + inputValues = new ArrayList(); + } + + /** + * Sets the input key to send to the Reducer + * + */ + public void setInputKey(K1 key) { + inputKey = key; + } + + /** + * adds an input value to send to the reducer + * + * @param val + */ + public void addInputValue(V1 val) { + inputValues.add(val); + } + + /** + * Sets the input values to send to the reducer; overwrites existing ones + * + * @param values + */ + public void setInputValues(List values) { + inputValues.clear(); + inputValues.addAll(values); + } + + /** + * Adds a set of input values to send to the reducer + * + * @param values + */ + public void addInputValues(List values) { + inputValues.addAll(values); + } + + /** + * Sets the input to send to the reducer + * + * @param values + */ + public void setInput(K1 key, List values) { + setInputKey(key); + setInputValues(values); + } + + /** + * Adds an output (k, v) pair we expect from the Reducer + * + * @param outputRecord + * The (k, v) pair to add + */ + public void addOutput(Pair outputRecord) { + if (null != outputRecord) { + expectedOutputs.add(outputRecord); + } else { + throw new IllegalArgumentException("Tried to add null outputRecord"); + } + } + + /** + * Adds an output (k, v) pair we expect from the Reducer + * + * @param key The key part of a (k, v) pair to add + * @param val The val part of a (k, v) pair to add + */ + public void addOutput(K2 key, V2 val) { + addOutput(new Pair(key, val)); + } + + /** + * Expects an input of the form "key \t val, val, val..." Forces the Reducer + * input types to Text. + * + * @param input + * A string of the form "key \t val,val,val". Trims any whitespace. + */ + public void setInputFromString(String input) { + if (null == input) { + throw new IllegalArgumentException("null input given to setInputFromString"); + } else { + Pair inputPair = parseTabbedPair(input); + if (null != inputPair) { + // I know this is not type-safe, but I don't know a better way to do + // this. + setInputKey((K1) inputPair.getFirst()); + setInputValues((List) parseCommaDelimitedList(inputPair.getSecond() + .toString())); + } else { + throw new IllegalArgumentException( + "Could not parse input pair in setInputFromString"); + } + } + } + + /** + * Expects an input of the form "key \t val" Forces the Reducer output types + * to Text. + * + * @param output + * A string of the form "key \t val". Trims any whitespace. + */ + public void addOutputFromString(String output) { + if (null == output) { + throw new IllegalArgumentException("null input given to setOutput"); + } else { + Pair outputPair = parseTabbedPair(output); + if (null != outputPair) { + // I know this is not type-safe, but I don't know a better way to do + // this. + addOutput((Pair) outputPair); + } else { + throw new IllegalArgumentException("Could not parse output pair in setOutput"); + } + } + } + + public abstract List> run() throws IOException; + + @Override + public void runTest() throws RuntimeException { + + String inputKeyStr = "(null)"; + + if (null != inputKey) { + inputKeyStr = inputKey.toString(); + } + + StringBuilder sb = new StringBuilder(); + formatValueList(inputValues, sb); + + LOG.debug("Reducing input (" + inputKeyStr + ", " + sb.toString() + ")"); + + List> outputs = null; + try { + outputs = run(); + validate(outputs); + } catch (IOException ioe) { + LOG.error("IOException in reducer: " + ioe.toString()); + LOG.debug("Setting success to false based on IOException"); + throw new RuntimeException(); + } + } +} + Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/TestDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/TestDriver.java?rev=806577&r1=806576&r2=806577&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/TestDriver.java (original) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/TestDriver.java Fri Aug 21 14:51:39 2009 @@ -70,7 +70,7 @@ * Split "key \t val" into Pair(Text(key), Text(val)) * @param tabSeparatedPair */ - static Pair parseTabbedPair(String tabSeparatedPair) { + public static Pair parseTabbedPair(String tabSeparatedPair) { String key, val; Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mrunit.MapDriverBase; +import org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper; +import org.apache.hadoop.mrunit.types.Pair; + +/** + * Harness that allows you to test a Mapper instance. You provide the input + * key and value that should be sent to the Mapper, and outputs you expect to + * be sent by the Mapper to the collector for those inputs. By calling + * runTest(), the harness will deliver the input to the Mapper and will check + * its outputs against the expected results. This is designed to handle a + * single (k, v) -> (k, v)* case from the Mapper, representing a single unit + * test. Multiple input (k, v) pairs should go in separate unit tests. + */ +public class MapDriver extends MapDriverBase { + + public static final Log LOG = LogFactory.getLog(MapDriver.class); + + private Mapper myMapper; + + public MapDriver(final Mapper m) { + myMapper = m; + } + + public MapDriver() { + } + + + /** + * Set the Mapper instance to use with this test driver + * + * @param m the Mapper instance to use + */ + public void setMapper(Mapper m) { + myMapper = m; + } + + /** Sets the Mapper instance to use and returns self for fluent style */ + public MapDriver withMapper(Mapper m) { + setMapper(m); + return this; + } + + /** + * @return the Mapper object being used by this test + */ + public Mapper getMapper() { + return myMapper; + } + + /** + * Identical to setInputKey() but with fluent programming style + * + * @return this + */ + public MapDriver withInputKey(K1 key) { + setInputKey(key); + return this; + } + + /** + * Identical to setInputValue() but with fluent programming style + * + * @param val + * @return this + */ + public MapDriver withInputValue(V1 val) { + setInputValue(val); + return this; + } + + /** + * Identical to setInput() but returns self for fluent programming style + * + * @return this + */ + public MapDriver withInput(K1 key, V1 val) { + setInput(key, val); + return this; + } + + /** + * Identical to setInput() but returns self for fluent programming style + * + * @param inputRecord + * @return this + */ + public MapDriver withInput(Pair inputRecord) { + setInput(inputRecord); + return this; + } + + /** + * Works like addOutput(), but returns self for fluent style + * + * @param outputRecord + * @return this + */ + public MapDriver withOutput(Pair outputRecord) { + addOutput(outputRecord); + return this; + } + + /** + * Functions like addOutput() but returns self for fluent programming + * style + * + * @return this + */ + public MapDriver withOutput(K2 key, V2 val) { + addOutput(key, val); + return this; + } + + /** + * Identical to setInputFromString, but with a fluent programming style + * + * @param input + * A string of the form "key \t val". Trims any whitespace. + * @return this + */ + public MapDriver withInputFromString(String input) { + setInputFromString(input); + return this; + } + + /** + * Identical to addOutputFromString, but with a fluent programming style + * + * @param output + * A string of the form "key \t val". Trims any whitespace. + * @return this + */ + public MapDriver withOutputFromString(String output) { + addOutputFromString(output); + return this; + } + + @Override + public List> run() throws IOException { + List> inputs = new ArrayList>(); + inputs.add(new Pair(inputKey, inputVal)); + + try { + MockMapContextWrapper wrapper = new MockMapContextWrapper(); + MockMapContextWrapper.MockMapContext context = + wrapper.getMockContext(inputs); + + myMapper.run(context); + return context.getOutputs(); + } catch (InterruptedException ie) { + throw new IOException(ie); + } + } + + @Override + public String toString() { + return "MapDriver (0.20+) (" + myMapper + ")"; + } +} + Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mrunit.mapreduce; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mrunit.MapReduceDriverBase; +import org.apache.hadoop.mrunit.types.Pair; + +/** + * Harness that allows you to test a Mapper and a Reducer instance together + * You provide the input key and value that should be sent to the Mapper, and + * outputs you expect to be sent by the Reducer to the collector for those + * inputs. By calling runTest(), the harness will deliver the input to the + * Mapper, feed the intermediate results to the Reducer (without checking + * them), and will check the Reducer's outputs against the expected results. + * This is designed to handle a single (k, v)* -> (k, v)* case from the + * Mapper/Reducer pair, representing a single unit test. + */ +public class MapReduceDriver + extends MapReduceDriverBase { + + public static final Log LOG = LogFactory.getLog(MapReduceDriver.class); + + private Mapper myMapper; + private Reducer myReducer; + + public MapReduceDriver(final Mapper m, + final Reducer r) { + myMapper = m; + myReducer = r; + } + + public MapReduceDriver() { + } + + /** Set the Mapper instance to use with this test driver + * @param m the Mapper instance to use */ + public void setMapper(Mapper m) { + myMapper = m; + } + + /** Sets the Mapper instance to use and returns self for fluent style */ + public MapReduceDriver withMapper( + Mapper m) { + setMapper(m); + return this; + } + + /** + * @return the Mapper object being used by this test + */ + public Mapper getMapper() { + return myMapper; + } + + /** + * Sets the reducer object to use for this test + * @param r The reducer object to use + */ + public void setReducer(Reducer r) { + myReducer = r; + } + + /** + * Identical to setReducer(), but with fluent programming style + * @param r The Reducer to use + * @return this + */ + public MapReduceDriver withReducer( + Reducer r) { + setReducer(r); + return this; + } + + /** + * @return the Reducer object being used for this test + */ + public Reducer getReducer() { + return myReducer; + } + + /** + * Identical to addInput() but returns self for fluent programming style + * @param key + * @param val + * @return this + */ + public MapReduceDriver withInput(K1 key, V1 val) { + addInput(key, val); + return this; + } + + /** + * Identical to addInput() but returns self for fluent programming style + * @param input The (k, v) pair to add + * @return this + */ + public MapReduceDriver withInput( + Pair input) { + addInput(input); + return this; + } + + /** + * Works like addOutput(), but returns self for fluent style + * @param outputRecord + * @return this + */ + public MapReduceDriver withOutput( + Pair outputRecord) { + addOutput(outputRecord); + return this; + } + + /** + * Functions like addOutput() but returns self for fluent programming style + * @param key + * @param val + * @return this + */ + public MapReduceDriver withOutput(K3 key, V3 val) { + addOutput(key, val); + return this; + } + + /** + * Identical to addInputFromString, but with a fluent programming style + * @param input A string of the form "key \t val". Trims any whitespace. + * @return this + */ + public MapReduceDriver withInputFromString(String input) { + addInputFromString(input); + return this; + } + + /** + * Identical to addOutputFromString, but with a fluent programming style + * @param output A string of the form "key \t val". Trims any whitespace. + * @return this + */ + public MapReduceDriver withOutputFromString(String output) { + addOutputFromString(output); + return this; + } + + public List> run() throws IOException { + + List> mapOutputs = new ArrayList>(); + + // run map component + for (Pair input : inputList) { + LOG.debug("Mapping input " + input.toString() + ")"); + + mapOutputs.addAll(new MapDriver(myMapper).withInput( + input).run()); + } + + List>> reduceInputs = shuffle(mapOutputs); + List> reduceOutputs = new ArrayList>(); + + for (Pair> input : reduceInputs) { + K2 inputKey = input.getFirst(); + List inputValues = input.getSecond(); + StringBuilder sb = new StringBuilder(); + formatValueList(inputValues, sb); + LOG.debug("Reducing input (" + inputKey.toString() + ", " + + sb.toString() + ")"); + + reduceOutputs.addAll(new ReduceDriver(myReducer) + .withInputKey(inputKey).withInputValues(inputValues).run()); + } + + return reduceOutputs; + } + + @Override + public String toString() { + return "MapReduceDriver (0.20+) (" + myMapper + ", " + myReducer + ")"; + } +} Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mrunit.ReduceDriverBase; +import org.apache.hadoop.mrunit.mapreduce.mock.MockReduceContextWrapper; +import org.apache.hadoop.mrunit.types.Pair; + +/** + * Harness that allows you to test a Reducer instance. You provide a key and a + * set of intermediate values for that key that represent inputs that should + * be sent to the Reducer (as if they came from a Mapper), and outputs you + * expect to be sent by the Reducer to the collector. By calling runTest(), + * the harness will deliver the input to the Reducer and will check its + * outputs against the expected results. This is designed to handle a single + * (k, v*) -> (k, v)* case from the Reducer, representing a single unit test. + * Multiple input (k, v*) sets should go in separate unit tests. + */ +public class ReduceDriver extends ReduceDriverBase { + + public static final Log LOG = LogFactory.getLog(ReduceDriver.class); + + private Reducer myReducer; + + public ReduceDriver(final Reducer r) { + myReducer = r; + } + + public ReduceDriver() { + } + + /** + * Sets the reducer object to use for this test + * + * @param r + * The reducer object to use + */ + public void setReducer(Reducer r) { + myReducer = r; + } + + /** + * Identical to setReducer(), but with fluent programming style + * + * @param r + * The Reducer to use + * @return this + */ + public ReduceDriver withReducer(Reducer r) { + setReducer(r); + return this; + } + + public Reducer getReducer() { + return myReducer; + } + + /** + * Identical to setInputKey() but with fluent programming style + * + * @return this + */ + public ReduceDriver withInputKey(K1 key) { + setInputKey(key); + return this; + } + + /** + * Identical to addInputValue() but with fluent programming style + * + * @param val + * @return this + */ + public ReduceDriver withInputValue(V1 val) { + addInputValue(val); + return this; + } + + /** + * Identical to addInputValues() but with fluent programming style + * + * @param values + * @return this + */ + public ReduceDriver withInputValues(List values) { + addInputValues(values); + return this; + } + + /** + * Identical to setInput() but returns self for fluent programming style + * + * @return this + */ + public ReduceDriver withInput(K1 key, List values) { + setInput(key, values); + return this; + } + + /** + * Works like addOutput(), but returns self for fluent style + * + * @param outputRecord + * @return this + */ + public ReduceDriver withOutput(Pair outputRecord) { + addOutput(outputRecord); + return this; + } + + /** + * Works like addOutput(), but returns self for fluent style + * + * @param key The key part of a (k, v) pair to add + * @param val The val part of a (k, v) pair to add + * @return this + */ + public ReduceDriver withOutput(K2 key, V2 val) { + addOutput(key, val); + return this; + } + + /** + * Identical to setInput, but with a fluent programming style + * + * @param input + * A string of the form "key \t val". Trims any whitespace. + * @return this + */ + public ReduceDriver withInputFromString(String input) { + setInputFromString(input); + return this; + } + + /** + * Identical to addOutput, but with a fluent programming style + * + * @param output + * A string of the form "key \t val". Trims any whitespace. + * @return this + */ + public ReduceDriver withOutputFromString(String output) { + addOutputFromString(output); + return this; + } + + @Override + public List> run() throws IOException { + List>> inputs = new ArrayList>>(); + inputs.add(new Pair>(inputKey, inputValues)); + + try { + MockReduceContextWrapper wrapper = new MockReduceContextWrapper(); + MockReduceContextWrapper.MockReduceContext context = + wrapper.getMockContext(inputs); + + myReducer.run(context); + return context.getOutputs(); + } catch (InterruptedException ie) { + throw new IOException(ie); + } + } + + @Override + public String toString() { + return "ReduceDriver (0.20+) (" + myReducer + ")"; + } +} + Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockInputSplit.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockInputSplit.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockInputSplit.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockInputSplit.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce.mock; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.lib.input.FileSplit; + +import java.io.IOException; + +/** + * Mock implementation of InputSplit that does nothing. + */ +public class MockInputSplit extends FileSplit { + + private static final Path MOCK_PATH = new Path("somefile"); + + public MockInputSplit() { + super(MOCK_PATH, 0, 0, (String []) null); + } + + public String toString() { + return "MockInputSplit"; + } + + /** + * Return the path object represented by this as a FileSplit. + */ + public static Path getMockPath() { + return MOCK_PATH; + } +} + Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce.mock; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mrunit.types.Pair; +import org.apache.hadoop.mrunit.mock.MockOutputCollector; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +/** + * o.a.h.mapreduce.Mapper.map() expects to use a Mapper.Context + * object as a parameter. We want to override the functionality + * of a lot of Context to have it send the results back to us, etc. + * But since Mapper.Context is an inner class of Mapper, we need to + * put any subclasses of Mapper.Context in a subclass of Mapper. + * + * This wrapper class exists for that purpose. + */ +public class MockMapContextWrapper + extends Mapper { + + public static final Log LOG = LogFactory.getLog(MockMapContextWrapper.class); + + /** + * Mock context instance that provides input to and receives output from + * the Mapper instance under test. + */ + public class MockMapContext extends Mapper.Context { + + private Iterator> inputIter; + private Pair curInput; + private MockOutputCollector output; + + public MockMapContext(final List> in) + throws IOException, InterruptedException { + + super(new Configuration(), + new TaskAttemptID("mrunit-jt", 0, TaskType.MAP, 0, 0), + null, null, new MockOutputCommitter(), null, null); + this.inputIter = in.iterator(); + this.output = new MockOutputCollector(); + } + + @Override + public InputSplit getInputSplit() { + return new MockInputSplit(); + } + + @Override + public KEYIN getCurrentKey() { + return curInput.getFirst(); + } + + @Override + public VALUEIN getCurrentValue() { + return curInput.getSecond(); + } + + @Override + public boolean nextKeyValue() throws IOException { + if (this.inputIter.hasNext()) { + this.curInput = this.inputIter.next(); + return true; + } else { + return false; + } + } + + public void write(KEYOUT key, VALUEOUT value) throws IOException { + output.collect(key, value); + } + + /** This method does nothing in the mock version. */ + public Counter getCounter(Enum counterName) { + return null; + } + + @Override + /** This method does nothing in the mock version. */ + public Counter getCounter(String groupName, String counterName) { + return null; + } + + @Override + /** This method does nothing in the mock version. */ + public void progress() { + } + + @Override + /** This method does nothing in the mock version. */ + public void setStatus(String status) { + } + + /** + * @return the outputs from the MockOutputCollector back to + * the test harness. + */ + public List> getOutputs() { + return output.getOutputs(); + } + } + + public MockMapContext getMockContext(List> inputs) + throws IOException, InterruptedException { + return new MockMapContext(inputs); + } +} + Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockOutputCommitter.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockOutputCommitter.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockOutputCommitter.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockOutputCommitter.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce.mock; + +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; + +/** + * Mock implementation of OutputCommitter that does nothing. + */ +public class MockOutputCommitter extends OutputCommitter { + + public void setupJob(JobContext jobContext) { + } + + public void cleanupJob(JobContext jobContext) { + } + + public void setupTask(TaskAttemptContext taskContext) { + } + + public boolean needsTaskCommit(TaskAttemptContext taskContext) { + return false; + } + + public void commitTask(TaskAttemptContext taskContext) { + } + + public void abortTask(TaskAttemptContext taskContext) { + } +} + Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockRawKeyValueIterator.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockRawKeyValueIterator.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockRawKeyValueIterator.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockRawKeyValueIterator.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce.mock; + +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.util.Progress; +import org.apache.hadoop.mapred.RawKeyValueIterator; + +import java.io.IOException; + +/** + * Mock implementation of RawKeyValueIterator that does nothing. + */ +public class MockRawKeyValueIterator implements RawKeyValueIterator { + public DataInputBuffer getKey() { + return null; + } + + public DataInputBuffer getValue() { + return null; + } + + public boolean next() { + return false; + } + + public void close() { + } + + public Progress getProgress() { + return null; + } +} + Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce.mock; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.ReduceContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mrunit.types.Pair; +import org.apache.hadoop.mrunit.mock.MockOutputCollector; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +/** + * o.a.h.mapreduce.Reducer.reduce() expects to use a Reducer.Context + * object as a parameter. We want to override the functionality + * of a lot of Context to have it send the results back to us, etc. + * But since Reducer.Context is an inner class of Reducer, we need to + * put any subclasses of Reducer.Context in a subclass of Reducer. + * + * This wrapper class exists for that purpose. + */ +public class MockReduceContextWrapper + extends Reducer { + + public static final Log LOG = LogFactory.getLog(MockReduceContextWrapper.class); + + /** + * Mock context instance that provides input to and receives output from + * the Mapper instance under test. + */ + public class MockReduceContext extends Reducer.Context { + + // The iterator over the input key, list(val). + private Iterator>> inputIter; + + // The current key and list of values. + private KEYIN curKey; + private InspectableIterable curValueIterable; + + private MockOutputCollector output; + + public MockReduceContext(final List>> in) + throws IOException, InterruptedException { + + super(new Configuration(), + new TaskAttemptID("mrunit-jt", 0, TaskType.REDUCE, 0, 0), + new MockRawKeyValueIterator(), null, null, + new MockOutputCommitter(), null, null, + (Class) Text.class, (Class) Text.class); + this.inputIter = in.iterator(); + this.output = new MockOutputCollector(); + } + + + /** + * A private iterable/iterator implementation that wraps around the + * underlying iterable/iterator used by the input value list. This + * memorizes the last value we saw so that we can return it in getCurrentValue(). + */ + private class InspectableIterable implements Iterable { + private Iterable base; + private VALUEIN lastVal; + + public InspectableIterable(final Iterable baseCollection) { + this.base = baseCollection; + } + + public Iterator iterator() { + return new InspectableIterator(this.base.iterator()); + } + + public VALUEIN getLastVal() { + return lastVal; + } + + private class InspectableIterator + extends ReduceContext.ValueIterator + implements Iterator { + private Iterator iter; + public InspectableIterator(final Iterator baseIter) { + iter = baseIter; + } + + public VALUEIN next() { + InspectableIterable.this.lastVal = iter.next(); + return InspectableIterable.this.lastVal; + } + + public boolean hasNext() { + return iter.hasNext(); + } + + public void remove() { + iter.remove(); + } + } + } + + @Override + public boolean nextKey() { + if (inputIter.hasNext()) { + // Advance to the next key and list of values + Pair> p = inputIter.next(); + curKey = p.getFirst(); + + // Reset the value iterator + curValueIterable = new InspectableIterable(p.getSecond()); + return true; + } else { + return false; + } + } + + @Override + public boolean nextKeyValue() { + return nextKey(); + } + + @Override + public KEYIN getCurrentKey() { + return curKey; + } + + @Override + public VALUEIN getCurrentValue() { + return curValueIterable.getLastVal(); + } + + @Override + public Iterable getValues() { + return curValueIterable; + } + + public void write(KEYOUT key, VALUEOUT value) throws IOException { + output.collect(key, value); + } + + /** This method does nothing in the mock version. */ + public Counter getCounter(Enum counterName) { + return null; + } + + @Override + /** This method does nothing in the mock version. */ + public Counter getCounter(String groupName, String counterName) { + return null; + } + + @Override + /** This method does nothing in the mock version. */ + public void progress() { + } + + @Override + /** This method does nothing in the mock version. */ + public void setStatus(String status) { + } + + /** + * @return the outputs from the MockOutputCollector back to + * the test harness. + */ + public List> getOutputs() { + return output.getOutputs(); + } + } + + public MockReduceContext getMockContext(List>> inputs) + throws IOException, InterruptedException { + return new MockReduceContext(inputs); + } +} + Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java?rev=806577&r1=806576&r2=806577&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java (original) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java Fri Aug 21 14:51:39 2009 @@ -43,6 +43,7 @@ suite.addTestSuite(TestExample.class); suite.addTest(org.apache.hadoop.mrunit.types.AllTests.suite()); + suite.addTest(org.apache.hadoop.mrunit.mapreduce.AllTests.suite()); return suite; } Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java?rev=806577&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java (added) +++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java Fri Aug 21 14:51:39 2009 @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mrunit.mapreduce; + +import junit.framework.Test; +import junit.framework.TestSuite; + +/** + * All tests for the new 0.20+ mapreduce API versions of the test harness. + */ +public final class AllTests { + + private AllTests() { } + + public static Test suite() { + TestSuite suite = new TestSuite("Test for org.apache.hadoop.mrunit.mapreduce"); + + suite.addTestSuite(TestMapDriver.class); + suite.addTestSuite(TestReduceDriver.class); + suite.addTestSuite(TestMapReduceDriver.class); + + return suite; + } + +} +