hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jo...@apache.org
Subject svn commit: r794324 - in /hadoop/mapreduce/trunk: ./ src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/ src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/ src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/
Date Wed, 15 Jul 2009 16:42:07 GMT
Author: johan
Date: Wed Jul 15 16:42:06 2009
New Revision: 794324

URL: http://svn.apache.org/viewvc?rev=794324&view=rev
Log:
MAPREDUCE-680. Fix so MRUnit can handle reuse of Writable objects. (Aaron Kimball via johan)

Added:
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockOutputCollector.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockOutputCollector.java
    hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=794324&r1=794323&r2=794324&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Jul 15 16:42:06 2009
@@ -199,3 +199,6 @@
 
     MAPREDUCE-565. Fix partitioner to work with new API. (Owen O'Malley via
     cdouglas)
+    
+    MAPREDUCE-680. Fix so MRUnit can handle reuse of Writable objects.
+    (Aaron Kimball via johan)

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockOutputCollector.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockOutputCollector.java?rev=794324&r1=794323&r2=794324&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockOutputCollector.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockOutputCollector.java
Wed Jul 15 16:42:06 2009
@@ -22,38 +22,92 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.serializer.Deserializer;
+import org.apache.hadoop.io.serializer.SerializationFactory;
+import org.apache.hadoop.io.serializer.Serializer;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mrunit.types.Pair;
+import org.apache.hadoop.util.ReflectionUtils;
+
 
 /**
  * OutputCollector to use in the test framework for Mapper and Reducer
  * classes. Accepts a set of output (k, v) pairs and returns them to the
  * framework for validation.
- *
- * BUG: Currently, this does not make deep copies of values passed to collect().
- * So emitting the same Text object (for instance) repeatedly, with different
- * internal string data each time, is not tested in the same way that Hadoop's
- * OutputCollector works.
- *
  */
 public class MockOutputCollector<K, V> implements OutputCollector<K, V> {
 
   private ArrayList<Pair<K, V>> collectedOutputs;
+  private SerializationFactory serializationFactory;
+  private DataOutputBuffer outBuffer;
+  private DataInputBuffer inBuffer;
+  private Configuration conf;
+
 
   public MockOutputCollector() {
     collectedOutputs = new ArrayList<Pair<K, V>>();
+
+    outBuffer = new DataOutputBuffer();
+    inBuffer = new DataInputBuffer();
+
+    conf = new Configuration();
+    serializationFactory = new SerializationFactory(conf);
+  }
+
+
+  private Object getInstance(Class klazz) {
+    return ReflectionUtils.newInstance(klazz, conf);
+  }
+
+
+  private Object deepCopy(Object obj) throws IOException {
+
+    if (null == obj) {
+      return null;
+    }
+
+    Class klazz = obj.getClass();
+    Object out = getInstance(klazz); // the output object to return.
+    Serializer s = serializationFactory.getSerializer(klazz);
+    Deserializer ds = serializationFactory.getDeserializer(klazz);
+
+    try {
+      s.open(outBuffer);
+      ds.open(inBuffer);
+
+      outBuffer.reset();
+      s.serialize(obj);
+
+      byte [] data = outBuffer.getData();
+      int len = outBuffer.getLength();
+      inBuffer.reset(data, len);
+
+      out = ds.deserialize(out);
+
+      return out;
+    } finally {
+      try {
+        s.close();
+      } catch (IOException ioe) {
+        // ignore this; we're closing.
+      }
+
+      try {
+        ds.close();
+      } catch (IOException ioe) {
+        // ignore this; we're closing.
+      }
+    }
   }
 
   /**
    * Accepts another (key, value) pair as an output of this mapper/reducer.
-   *
-   * BUG: Currently, this does not make deep copies of values passed to collect().
-   * So emitting the same Text object (for instance) repeatedly, with different
-   * internal string data each time, is not tested in the same way that Hadoop's
-   * OutputCollector works.
    */
   public void collect(K key, V value) throws IOException {
-    collectedOutputs.add(new Pair<K, V>(key, value));
+    collectedOutputs.add(new Pair<K, V>((K) deepCopy(key), (V) deepCopy(value)));
   }
 
   /**

Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java?rev=794324&r1=794323&r2=794324&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
Wed Jul 15 16:42:06 2009
@@ -18,6 +18,7 @@
 package org.apache.hadoop.mrunit;
 
 import org.apache.hadoop.mrunit.mock.TestMockReporter;
+import org.apache.hadoop.mrunit.mock.TestMockOutputCollector;
 
 import junit.framework.Test;
 import junit.framework.TestSuite;
@@ -36,6 +37,7 @@
     suite.addTestSuite(TestMapDriver.class);
     suite.addTestSuite(TestMapReduceDriver.class);
     suite.addTestSuite(TestMockReporter.class);
+    suite.addTestSuite(TestMockOutputCollector.class);
     suite.addTestSuite(TestReduceDriver.class);
     suite.addTestSuite(TestTestDriver.class);
     suite.addTestSuite(TestExample.class);

Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockOutputCollector.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockOutputCollector.java?rev=794324&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockOutputCollector.java
(added)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockOutputCollector.java
Wed Jul 15 16:42:06 2009
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit.mock;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mrunit.MapDriver;
+
+import org.junit.Test;
+
+
+public class TestMockOutputCollector extends TestCase {
+
+  /**
+   * A mapper that reuses the same key and val objects to emit multiple values
+   */
+  class RepeatMapper extends MapReduceBase implements Mapper<Text, Text, Text, Text>
{
+    public void map(Text k, Text v, OutputCollector<Text, Text> out, Reporter r)
+        throws IOException {
+      Text outKey = new Text();
+      Text outVal = new Text();
+
+      outKey.set("1");
+      outVal.set("a");
+      out.collect(outKey, outVal);
+
+      outKey.set("2");
+      outVal.set("b");
+      out.collect(outKey, outVal);
+
+      outKey.set("3");
+      outVal.set("c");
+      out.collect(outKey, outVal);
+    }
+  }
+
+  @Test
+  public void testRepeatedObjectUse() {
+    Mapper<Text, Text, Text, Text> mapper = new RepeatMapper();
+    MapDriver<Text, Text, Text, Text> driver = new MapDriver(mapper);
+
+    driver.withInput(new Text("inK"), new Text("inV"))
+          .withOutput(new Text("1"), new Text("a"))
+          .withOutput(new Text("2"), new Text("b"))
+          .withOutput(new Text("3"), new Text("c"))
+          .runTest();
+  }
+}
+



Mime
View raw message