hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r807942 - in /hadoop/mapreduce/trunk: ./ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/ src/contrib/...
Date Wed, 26 Aug 2009 09:42:01 GMT
Author: cdouglas
Date: Wed Aug 26 09:42:00 2009
New Revision: 807942

URL: http://svn.apache.org/viewvc?rev=807942&view=rev
Log:
MAPREDUCE-910. Support counters in MRUnit. Contributed by Aaron Kimball

Added:
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Aug 26 09:42:00 2009
@@ -269,6 +269,8 @@
     MAPREDUCE-825. JobClient completion poll interval of 5s causes slow tests
     in local mode. (Aaron Kimball via tomwhite)
 
+    MAPREDUCE-910. Support counters in MRUnit. (Aaron Kimball via cdouglas)
+
   BUG FIXES
 
     MAPREDUCE-878. Rename fair scheduler design doc to 

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java
Wed Aug 26 09:42:00 2009
@@ -24,6 +24,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.Mapper;
 import org.apache.hadoop.mrunit.mock.MockOutputCollector;
 import org.apache.hadoop.mrunit.mock.MockReporter;
@@ -43,12 +44,33 @@
   public static final Log LOG = LogFactory.getLog(MapDriver.class);
 
   private Mapper<K1, V1, K2, V2> myMapper;
+  private Counters counters;
 
   public MapDriver(final Mapper<K1, V1, K2, V2> m) {
     myMapper = m;
+    counters = new Counters();
   }
 
   public MapDriver() {
+    counters = new Counters();
+  }
+
+  /** @return the counters used in this test */
+  public Counters getCounters() {
+    return counters;
+  }
+
+  /** Sets the counters object to use for this test.
+   * @param ctrs The counters object to use.
+   */
+  public void setCounters(final Counters ctrs) {
+    this.counters = ctrs;
+  }
+
+  /** Sets the counters to use and returns self for fluent style */
+  public MapDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+    setCounters(ctrs);
+    return this;
   }
 
   /**
@@ -165,7 +187,7 @@
   public List<Pair<K2, V2>> run() throws IOException {
     MockOutputCollector<K2, V2> outputCollector =
       new MockOutputCollector<K2, V2>();
-    MockReporter reporter = new MockReporter(MockReporter.ReporterType.Mapper);
+    MockReporter reporter = new MockReporter(MockReporter.ReporterType.Mapper, getCounters());
 
     myMapper.map(inputKey, inputVal, outputCollector, reporter);
 

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java
Wed Aug 26 09:42:00 2009
@@ -29,6 +29,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.Mapper;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mrunit.types.Pair;
@@ -55,11 +56,13 @@
   private Mapper<K1, V1, K2, V2> myMapper;
   private Reducer<K2, V2, K3, V3> myReducer;
   private Reducer<K2, V2, K2, V2> myCombiner;
+  private Counters counters;
 
   public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
                          final Reducer<K2, V2, K3, V3> r) {
     myMapper = m;
     myReducer = r;
+    counters = new Counters();
   }
 
   public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
@@ -68,9 +71,29 @@
     myMapper = m;
     myReducer = r;
     myCombiner = c;
+    counters = new Counters();
   }
 
   public MapReduceDriver() {
+    counters = new Counters();
+  }
+
+  /** @return the counters used in this test */
+  public Counters getCounters() {
+    return counters;
+  }
+
+  /** Sets the counters object to use for this test.
+   * @param ctrs The counters object to use.
+   */
+  public void setCounters(final Counters ctrs) {
+    this.counters = ctrs;
+  }
+
+  /** Sets the counters to use and returns self for fluent style */
+  public MapReduceDriver<K1, V1, K2, V2, K3, V3> withCounters(final Counters ctrs)
{
+    setCounters(ctrs);
+    return this;
   }
 
   /** Set the Mapper instance to use with this test driver
@@ -227,7 +250,10 @@
             + sb.toString() + ")");
 
         reduceOutputs.addAll(new ReduceDriver<K2, V2, OUTKEY, OUTVAL>(reducer)
-                .withInputKey(inputKey).withInputValues(inputValues).run());
+                .withCounters(getCounters())
+                .withInputKey(inputKey)
+                .withInputValues(inputValues)
+                .run());
       }
 
       return reduceOutputs;
@@ -243,7 +269,7 @@
       LOG.debug("Mapping input " + input.toString() + ")");
 
       mapOutputs.addAll(new MapDriver<K1, V1, K2, V2>(myMapper).withInput(
-              input).run());
+              input).withCounters(getCounters()).run());
     }
 
     if (myCombiner != null) {

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java
Wed Aug 26 09:42:00 2009
@@ -25,6 +25,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.Mapper;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mrunit.types.Pair;
@@ -56,15 +57,18 @@
 
   private List<Pair<Mapper, Reducer>> mapReducePipeline;
   private List<Pair<K1, V1>> inputList;
+  private Counters counters;
 
   public PipelineMapReduceDriver(final List<Pair<Mapper, Reducer>> pipeline)
{
     this.mapReducePipeline = copyMapReduceList(pipeline);
     this.inputList = new ArrayList<Pair<K1, V1>>();
+    this.counters = new Counters();
   }
 
   public PipelineMapReduceDriver() {
     this.mapReducePipeline = new ArrayList<Pair<Mapper, Reducer>>();
     this.inputList = new ArrayList<Pair<K1, V1>>();
+    this.counters = new Counters();
   }
 
   private List<Pair<Mapper, Reducer>> copyMapReduceList(List<Pair<Mapper,
Reducer>> lst) {
@@ -77,6 +81,25 @@
     return outList;
   }
 
+  /** @return the counters used in this test */
+  public Counters getCounters() {
+    return counters;
+  }
+
+  /** Sets the counters object to use for this test.
+   * @param ctrs The counters object to use.
+   */
+  public void setCounters(final Counters ctrs) {
+    this.counters = ctrs;
+  }
+
+  /** Sets the counters to use and returns self for fluent style */
+  public PipelineMapReduceDriver<K1, V1, K2, V2> withCounters(final Counters ctrs)
{
+    setCounters(ctrs);
+    return this;
+  }
+
+
   /** Add a Mapper and Reducer instance to the pipeline to use with this test driver
    * @param m The Mapper instance to add to the pipeline
    * @param r The Reducer instance to add to the pipeline
@@ -282,6 +305,8 @@
       // Create a MapReduceDriver to run this phase of the pipeline.
       MapReduceDriver mrDriver = new MapReduceDriver(job.getFirst(), job.getSecond());
 
+      mrDriver.setCounters(getCounters());
+
       // Add the inputs from the user, or from the previous stage of the pipeline.
       for (Object input : inputs) {
         mrDriver.addInput((Pair) input);

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java
Wed Aug 26 09:42:00 2009
@@ -25,6 +25,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mrunit.mock.MockOutputCollector;
 import org.apache.hadoop.mrunit.mock.MockReporter;
@@ -45,14 +46,36 @@
   public static final Log LOG = LogFactory.getLog(ReduceDriver.class);
 
   private Reducer<K1, V1, K2, V2> myReducer;
+  private Counters counters;
 
   public ReduceDriver(final Reducer<K1, V1, K2, V2> r) {
     myReducer = r;
+    counters = new Counters();
   }
 
   public ReduceDriver() {
+    counters = new Counters();
   }
 
+  /** @return the counters used in this test */
+  public Counters getCounters() {
+    return counters;
+  }
+
+  /** Sets the counters object to use for this test.
+   * @param ctrs The counters object to use.
+   */
+  public void setCounters(final Counters ctrs) {
+    this.counters = ctrs;
+  }
+
+  /** Sets the counters to use and returns self for fluent style */
+  public ReduceDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+    setCounters(ctrs);
+    return this;
+  }
+
+
   /**
    * Sets the reducer object to use for this test
    *
@@ -172,7 +195,7 @@
   public List<Pair<K2, V2>> run() throws IOException {
     MockOutputCollector<K2, V2> outputCollector =
       new MockOutputCollector<K2, V2>();
-    MockReporter reporter = new MockReporter(MockReporter.ReporterType.Reducer);
+    MockReporter reporter = new MockReporter(MockReporter.ReporterType.Reducer, getCounters());
 
     myReducer.reduce(inputKey, inputValues.iterator(), outputCollector,
             reporter);

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
Wed Aug 26 09:42:00 2009
@@ -26,6 +26,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mrunit.MapDriverBase;
 import org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper;
@@ -45,12 +46,15 @@
   public static final Log LOG = LogFactory.getLog(MapDriver.class);
 
   private Mapper<K1, V1, K2, V2> myMapper;
+  private Counters counters;
 
   public MapDriver(final Mapper<K1, V1, K2, V2> m) {
     myMapper = m;
+    counters = new Counters();
   }
 
   public MapDriver() {
+    counters = new Counters();
   }
 
 
@@ -76,6 +80,24 @@
     return myMapper;
   }
 
+  /** @return the counters used in this test */
+  public Counters getCounters() {
+    return counters;
+  }
+
+  /** Sets the counters object to use for this test.
+   * @param ctrs The counters object to use.
+   */
+  public void setCounters(final Counters ctrs) {
+    this.counters = ctrs;
+  }
+
+  /** Sets the counters to use and returns self for fluent style */
+  public MapDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+    setCounters(ctrs);
+    return this;
+  }
+
   /**
    * Identical to setInputKey() but with fluent programming style
    *
@@ -172,7 +194,7 @@
     try {
       MockMapContextWrapper<K1, V1, K2, V2> wrapper = new MockMapContextWrapper();
       MockMapContextWrapper<K1, V1, K2, V2>.MockMapContext context =
-          wrapper.getMockContext(inputs);
+          wrapper.getMockContext(inputs, getCounters());
 
       myMapper.run(context);
       return context.getOutputs();

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
Wed Aug 26 09:42:00 2009
@@ -29,6 +29,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mrunit.MapReduceDriverBase;
@@ -51,14 +52,17 @@
 
   private Mapper<K1, V1, K2, V2> myMapper;
   private Reducer<K2, V2, K3, V3> myReducer;
+  private Counters counters;
 
   public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
                          final Reducer<K2, V2, K3, V3> r) {
     myMapper = m;
     myReducer = r;
+    counters = new Counters();
   }
 
   public MapReduceDriver() {
+    counters = new Counters();
   }
 
   /** Set the Mapper instance to use with this test driver
@@ -107,6 +111,24 @@
     return myReducer;
   }
 
+  /** @return the counters used in this test */
+  public Counters getCounters() {
+    return counters;
+  }
+
+  /** Sets the counters object to use for this test.
+   * @param ctrs The counters object to use.
+   */
+  public void setCounters(final Counters ctrs) {
+    this.counters = ctrs;
+  }
+
+  /** Sets the counters to use and returns self for fluent style */
+  public MapReduceDriver<K1, V1, K2, V2, K3, V3> withCounters(final Counters ctrs)
{
+    setCounters(ctrs);
+    return this;
+  }
+
   /**
    * Identical to addInput() but returns self for fluent programming style
    * @param key
@@ -180,7 +202,7 @@
       LOG.debug("Mapping input " + input.toString() + ")");
 
       mapOutputs.addAll(new MapDriver<K1, V1, K2, V2>(myMapper).withInput(
-              input).run());
+              input).withCounters(getCounters()).run());
     }
 
     List<Pair<K2, List<V2>>> reduceInputs = shuffle(mapOutputs);
@@ -195,6 +217,7 @@
           + sb.toString() + ")");
 
       reduceOutputs.addAll(new ReduceDriver<K2, V2, K3, V3>(myReducer)
+              .withCounters(getCounters())
               .withInputKey(inputKey).withInputValues(inputValues).run());
     }
 

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java
Wed Aug 26 09:42:00 2009
@@ -26,6 +26,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mrunit.ReduceDriverBase;
 import org.apache.hadoop.mrunit.mapreduce.mock.MockReduceContextWrapper;
@@ -46,12 +47,15 @@
   public static final Log LOG = LogFactory.getLog(ReduceDriver.class);
 
   private Reducer<K1, V1, K2, V2> myReducer;
+  private Counters counters;
 
   public ReduceDriver(final Reducer<K1, V1, K2, V2> r) {
     myReducer = r;
+    counters = new Counters();
   }
 
   public ReduceDriver() {
+    counters = new Counters();
   }
 
   /**
@@ -80,6 +84,24 @@
     return myReducer;
   }
 
+  /** @return the counters used in this test */
+  public Counters getCounters() {
+    return counters;
+  }
+
+  /** Sets the counters object to use for this test.
+   * @param ctrs The counters object to use.
+   */
+  public void setCounters(final Counters ctrs) {
+    this.counters = ctrs;
+  }
+
+  /** Sets the counters to use and returns self for fluent style */
+  public ReduceDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+    setCounters(ctrs);
+    return this;
+  }
+
   /**
    * Identical to setInputKey() but with fluent programming style
    *
@@ -177,7 +199,7 @@
     try {
       MockReduceContextWrapper<K1, V1, K2, V2> wrapper = new MockReduceContextWrapper();
       MockReduceContextWrapper<K1, V1, K2, V2>.MockReduceContext context =
-          wrapper.getMockContext(inputs);
+          wrapper.getMockContext(inputs, getCounters());
 
       myReducer.run(context);
       return context.getOutputs();

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java
Wed Aug 26 09:42:00 2009
@@ -21,7 +21,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -57,12 +57,12 @@
     private Pair<KEYIN, VALUEIN> curInput;
     private MockOutputCollector<KEYOUT, VALUEOUT> output;
 
-    public MockMapContext(final List<Pair<KEYIN, VALUEIN>> in)
+    public MockMapContext(final List<Pair<KEYIN, VALUEIN>> in, final Counters
counters)
         throws IOException, InterruptedException {
 
       super(new Configuration(),
             new TaskAttemptID("mrunit-jt", 0, TaskType.MAP, 0, 0),
-            null, null, new MockOutputCommitter(), null, null);
+            null, null, new MockOutputCommitter(), new MockReporter(counters), null);
       this.inputIter = in.iterator();
       this.output = new MockOutputCollector<KEYOUT, VALUEOUT>();
     }
@@ -96,17 +96,6 @@
       output.collect(key, value);
     }
 
-    /** This method does nothing in the mock version. */
-    public Counter getCounter(Enum<?> counterName) {
-      return null;
-    }
-
-    @Override
-    /** This method does nothing in the mock version. */
-    public Counter getCounter(String groupName, String counterName) {
-      return null;
-    }
-
     @Override
     /** This method does nothing in the mock version. */
     public void progress() {
@@ -126,9 +115,9 @@
     }
   }
 
-  public MockMapContext getMockContext(List<Pair<KEYIN, VALUEIN>> inputs)
+  public MockMapContext getMockContext(List<Pair<KEYIN, VALUEIN>> inputs, Counters
counters)
       throws IOException, InterruptedException {
-    return new MockMapContext(inputs);
+    return new MockMapContext(inputs, counters);
   }
 }
 

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java
Wed Aug 26 09:42:00 2009
@@ -22,7 +22,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.ReduceContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -63,13 +63,13 @@
 
     private MockOutputCollector<KEYOUT, VALUEOUT> output;
 
-    public MockReduceContext(final List<Pair<KEYIN, List<VALUEIN>>> in)
+    public MockReduceContext(final List<Pair<KEYIN, List<VALUEIN>>> in,
final Counters counters)
         throws IOException, InterruptedException {
 
       super(new Configuration(),
             new TaskAttemptID("mrunit-jt", 0, TaskType.REDUCE, 0, 0),
             new MockRawKeyValueIterator(), null, null,
-            new MockOutputCommitter(), null, null,
+            new MockOutputCommitter(), new MockReporter(counters), null,
             (Class) Text.class, (Class) Text.class);
       this.inputIter = in.iterator();
       this.output = new MockOutputCollector<KEYOUT, VALUEOUT>();
@@ -159,17 +159,6 @@
       output.collect(key, value);
     }
 
-    /** This method does nothing in the mock version. */
-    public Counter getCounter(Enum<?> counterName) {
-      return null;
-    }
-
-    @Override
-    /** This method does nothing in the mock version. */
-    public Counter getCounter(String groupName, String counterName) {
-      return null;
-    }
-
     @Override
     /** This method does nothing in the mock version. */
     public void progress() {
@@ -189,9 +178,10 @@
     }
   }
 
-  public MockReduceContext getMockContext(List<Pair<KEYIN, List<VALUEIN>>>
inputs)
+  public MockReduceContext getMockContext(List<Pair<KEYIN, List<VALUEIN>>>
inputs,
+      Counters counters)
       throws IOException, InterruptedException {
-    return new MockReduceContext(inputs);
+    return new MockReduceContext(inputs, counters);
   }
 }
 

Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java?rev=807942&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java
(added)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java
Wed Aug 26 09:42:00 2009
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit.mapreduce.mock;
+
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.StatusReporter;
+
+public class MockReporter extends StatusReporter {
+
+  private Counters counters;
+
+  public MockReporter(final Counters ctrs) {
+    this.counters = ctrs;
+  }
+
+  @Override
+  public void setStatus(String status) {
+    // do nothing.
+  }
+
+  @Override
+  public void progress() {
+    // do nothing.
+  }
+
+  @Override
+  public Counter getCounter(String group, String name) {
+    Counter counter = null;
+    if (counters != null) {
+      counter = counters.findCounter(group, name);
+    }
+
+    return counter;
+  }
+
+  @Override
+  public Counter getCounter(Enum key) {
+    Counter counter = null;
+    if (counters != null) {
+      counter = counters.findCounter(key);
+    }
+
+    return counter;
+  }
+}
+

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java
Wed Aug 26 09:42:00 2009
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.mrunit.mock;
 
+import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.Counters.Counter;
@@ -24,6 +25,7 @@
 public class MockReporter implements Reporter {
 
   private MockInputSplit inputSplit = new MockInputSplit();
+  private Counters counters;
 
   public enum ReporterType {
     Mapper,
@@ -32,8 +34,9 @@
 
   private ReporterType typ;
 
-  public MockReporter(final ReporterType kind) {
+  public MockReporter(final ReporterType kind, final Counters ctrs) {
     this.typ = kind;
+    this.counters = ctrs;
   }
 
   @Override
@@ -48,12 +51,16 @@
 
   @Override
   public void incrCounter(Enum key, long amount) {
-    // do nothing.
+    if (null != counters) {
+      counters.incrCounter(key, amount);
+    }
   }
 
   @Override
   public void incrCounter(String group, String counter, long amount) {
-    // do nothing.
+    if (null != counters) {
+      counters.incrCounter(group, counter, amount);
+    }
   }
 
   @Override
@@ -67,15 +74,23 @@
   }
 
   @Override
-  public Counter getCounter(String s1, String s2) {
-    // do nothing
-    return null;
+  public Counter getCounter(String group, String name) {
+    Counters.Counter counter = null;
+    if (counters != null) {
+      counter = counters.findCounter(group, name);
+    }
+
+    return counter;
   }
 
   @Override
   public Counter getCounter(Enum key) {
-    // do nothing
-    return null;
+    Counters.Counter counter = null;
+    if (counters != null) {
+      counter = counters.findCounter(key);
+    }
+
+    return counter;
   }
 }
 

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
Wed Aug 26 09:42:00 2009
@@ -42,6 +42,7 @@
     suite.addTestSuite(TestReduceDriver.class);
     suite.addTestSuite(TestTestDriver.class);
     suite.addTestSuite(TestExample.class);
+    suite.addTestSuite(TestCounters.class);
 
     suite.addTest(org.apache.hadoop.mrunit.types.AllTests.suite());
     suite.addTest(org.apache.hadoop.mrunit.mapreduce.AllTests.suite());

Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java?rev=807942&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java
(added)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java
Wed Aug 26 09:42:00 2009
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test counters usage in various drivers.
+ */
+public class TestCounters extends TestCase {
+
+  private static final String GROUP = "GROUP";
+  private static final String ELEM = "ELEM";
+
+  private class CounterMapper extends MapReduceBase implements Mapper<Text, Text, Text,
Text> {
+    public void map(Text k, Text v, OutputCollector<Text, Text> out, Reporter r)
+        throws IOException {
+
+      r.incrCounter(GROUP, ELEM, 1);
+
+      // Emit the same (k, v) pair twice.
+      out.collect(k, v);
+      out.collect(k, v);
+    }
+  }
+
+  private class CounterReducer extends MapReduceBase implements Reducer<Text, Text, Text,
Text> {
+    public void reduce(Text k, Iterator<Text> vals, OutputCollector<Text, Text>
out, Reporter r)
+        throws IOException {
+
+      while (vals.hasNext()) {
+        r.incrCounter(GROUP, ELEM, 1);
+        out.collect(k, vals.next());
+      }
+    }
+  }
+
+  @Test
+  public void testMapper() throws IOException {
+    Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+    MapDriver<Text, Text, Text, Text> driver = new MapDriver<Text, Text, Text, Text>(mapper);
+    driver.withInput(new Text("foo"), new Text("bar")).run();
+    assertEquals("Expected 1 counter increment", 1,
+        driver.getCounters().findCounter(GROUP, ELEM).getValue());
+  }
+
+  @Test
+  public void testReducer() throws IOException {
+    Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+    ReduceDriver<Text, Text, Text, Text> driver = new ReduceDriver<Text, Text, Text,
Text>(reducer);
+    driver.withInputKey(new Text("foo"))
+          .withInputValue(new Text("bar"))
+          .run();
+    assertEquals("Expected 1 counter increment", 1,
+        driver.getCounters().findCounter(GROUP, ELEM).getValue());
+  }
+
+  @Test
+  public void testMapReduce() throws IOException {
+    Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+    Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+    MapReduceDriver<Text, Text, Text, Text, Text, Text> driver =
+        new MapReduceDriver<Text, Text, Text, Text, Text, Text>(mapper, reducer);
+
+    driver.withInput(new Text("foo"), new Text("bar"))
+          .run();
+
+    assertEquals("Expected counter=3", 3,
+        driver.getCounters().findCounter(GROUP, ELEM).getValue());
+  }
+
+  @Test
+  public void testPipeline() throws IOException {
+    Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+    Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+    PipelineMapReduceDriver<Text, Text, Text, Text> driver =
+        new PipelineMapReduceDriver<Text, Text, Text, Text>();
+
+    driver.withMapReduce(mapper, reducer)
+          .withMapReduce(mapper, reducer)
+          .withInput(new Text("foo"), new Text("bar"))
+          .run();
+
+    assertEquals("Expected counter=9", 9,
+        driver.getCounters().findCounter(GROUP, ELEM).getValue());
+  }
+}
+

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java
Wed Aug 26 09:42:00 2009
@@ -34,6 +34,7 @@
     suite.addTestSuite(TestMapDriver.class);
     suite.addTestSuite(TestReduceDriver.class);
     suite.addTestSuite(TestMapReduceDriver.class);
+    suite.addTestSuite(TestCounters.class);
 
     return suite;
   }

Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java?rev=807942&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java
(added)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java
Wed Aug 26 09:42:00 2009
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mrunit.mapreduce;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test counters usage in various drivers.
+ */
+public class TestCounters extends TestCase {
+
+  private static final String GROUP = "GROUP";
+  private static final String ELEM = "ELEM";
+
+  private class CounterMapper extends Mapper<Text, Text, Text, Text> {
+    public void map(Text k, Text v, Context context)
+        throws IOException, InterruptedException {
+
+      context.getCounter(GROUP, ELEM).increment(1);
+
+      // Emit the same (k, v) pair twice.
+      context.write(k, v);
+      context.write(k, v);
+    }
+  }
+
+  private class CounterReducer extends Reducer<Text, Text, Text, Text> {
+    public void reduce(Text k, Iterable<Text> vals, Context context)
+        throws IOException, InterruptedException {
+
+      for(Text val : vals) {
+        context.getCounter(GROUP, ELEM).increment(1);
+        context.write(k, val);
+      }
+    }
+  }
+
+  @Test
+  public void testMapper() throws IOException {
+    Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+    MapDriver<Text, Text, Text, Text> driver = new MapDriver<Text, Text, Text, Text>(mapper);
+    driver.withInput(new Text("foo"), new Text("bar")).run();
+    assertEquals("Expected 1 counter increment", 1,
+        driver.getCounters().findCounter(GROUP, ELEM).getValue());
+  }
+
+  @Test
+  public void testReducer() throws IOException {
+    Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+    ReduceDriver<Text, Text, Text, Text> driver = new ReduceDriver<Text, Text, Text,
Text>(reducer);
+    driver.withInputKey(new Text("foo"))
+          .withInputValue(new Text("bar"))
+          .run();
+    assertEquals("Expected 1 counter increment", 1,
+        driver.getCounters().findCounter(GROUP, ELEM).getValue());
+  }
+
+  @Test
+  public void testMapReduce() throws IOException {
+    Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+    Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+    MapReduceDriver<Text, Text, Text, Text, Text, Text> driver =
+        new MapReduceDriver<Text, Text, Text, Text, Text, Text>(mapper, reducer);
+
+    driver.withInput(new Text("foo"), new Text("bar"))
+          .run();
+
+    assertEquals("Expected counter=3", 3,
+        driver.getCounters().findCounter(GROUP, ELEM).getValue());
+  }
+}
+

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java
Wed Aug 26 09:42:00 2009
@@ -27,7 +27,7 @@
 
   @Test
   public void testGetInputSplitForMapper() {
-    InputSplit split = new MockReporter(MockReporter.ReporterType.Mapper).getInputSplit();
+    InputSplit split = new MockReporter(MockReporter.ReporterType.Mapper, null).getInputSplit();
     assertTrue(null != split);
   }
 
@@ -36,7 +36,7 @@
   @Test
   public void testGetInputSplitForReducer() {
     try {
-      new MockReporter(MockReporter.ReporterType.Reducer).getInputSplit();
+      new MockReporter(MockReporter.ReporterType.Reducer, null).getInputSplit();
       fail(); // shouldn't get here
     } catch (UnsupportedOperationException uoe) {
       // expected this.



Mime
View raw message