avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1435355 - in /avro/trunk: ./ lang/java/mapred/src/main/java/org/apache/avro/mapred/ lang/java/mapred/src/test/java/org/apache/avro/mapred/ lang/java/mapred/src/test/java/org/apache/avro/mapreduce/
Date Fri, 18 Jan 2013 21:32:55 GMT
Author: cutting
Date: Fri Jan 18 21:32:55 2013
New Revision: 1435355

URL: http://svn.apache.org/viewvc?rev=1435355&view=rev
Log:
AVRO-1232. Java: Add a toString() method to AvroWrapper so that it works with TextOutputFormat.
 Contributed by Garrett Wu.

Added:
    avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
  (with props)
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java
    avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1435355&r1=1435354&r2=1435355&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Jan 18 21:32:55 2013
@@ -22,6 +22,9 @@ Trunk (not yet released)
     AVRO-1223. Java: Add a static method to generated classes that
     returns its schema, getClassSchema().  (cutting)
 
+    AVRO-1232. Java: Add a toString() method to AvroWrapper so that it
+    works with TextOutputFormat.  (Garrett Wu via cutting)
+
   BUG FIXES
 
     AVRO-1231. Java: Fix Trevni shredder to work on non-recursive

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java?rev=1435355&r1=1435354&r2=1435355&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroWrapper.java Fri
Jan 18 21:32:55 2013
@@ -54,4 +54,9 @@ public class AvroWrapper<T> {
     return true;
   }
     
+  /** Get the wrapped datum as JSON. */
+  @Override
+  public String toString() {
+    return datum.toString();
+  }
 }

Added: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java?rev=1435355&view=auto
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
(added)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
Fri Jan 18 21:32:55 2013
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.avro.mapred;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class TestAvroWrapper {
+  @Test
+  public void testToString() {
+    String datum = "my string";
+    AvroWrapper<CharSequence> wrapper = new AvroWrapper<CharSequence>(datum);
+    assertEquals(datum, wrapper.toString());
+  }
+}

Propchange: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroWrapper.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java?rev=1435355&r1=1435354&r2=1435355&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java
(original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapreduce/TestWordCount.java
Fri Jan 18 21:32:55 2013
@@ -18,7 +18,10 @@
 
 package org.apache.avro.mapreduce;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -27,6 +30,7 @@ import org.apache.avro.file.DataFileRead
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.mapred.AvroKey;
+import org.apache.avro.mapred.AvroValue;
 import org.apache.avro.mapred.FsInput;
 import org.apache.avro.specific.SpecificDatumReader;
 import org.apache.avro.reflect.ReflectData;
@@ -45,6 +49,7 @@ import org.apache.hadoop.mapreduce.Reduc
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 
 import org.junit.Assert;
 import org.junit.Rule;
@@ -126,6 +131,19 @@ public class TestWordCount {
     }
   }
 
+  private static class AvroSumReducer
+      extends Reducer<Text, IntWritable, AvroKey<CharSequence>, AvroValue<Integer>>
{
+    @Override
+    protected void reduce(Text key, Iterable<IntWritable> counts, Context context)
+        throws IOException, InterruptedException {
+      int sum = 0;
+      for (IntWritable count : counts) {
+        sum += count.get();
+      }
+      context.write(new AvroKey<CharSequence>(key.toString()), new AvroValue<Integer>(sum));
+    }
+  }
+
   private static class GenericStatsReducer
       extends Reducer<Text, IntWritable, AvroKey<GenericData.Record>, NullWritable>
{
     private AvroKey<GenericData.Record> mStats;
@@ -455,4 +473,49 @@ public class TestWordCount {
     Assert.assertEquals(2, counts.get("banana").intValue());
     Assert.assertEquals(1, counts.get("carrot").intValue());
   }
+
+  /**
+   * Tests the MR output to text files when using AvroKey and AvroValue records.
+   */
+  @Test
+  public void testAvroUsingTextFileOutput() throws Exception {
+    Job job = new Job();
+
+    FileInputFormat.setInputPaths(job, new Path(getClass()
+            .getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt")
+            .toURI().toString()));
+    job.setInputFormatClass(TextInputFormat.class);
+
+    job.setMapperClass(LineCountMapper.class);
+    job.setMapOutputKeyClass(Text.class);
+    job.setMapOutputValueClass(IntWritable.class);
+
+    job.setReducerClass(AvroSumReducer.class);
+    AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.STRING));
+    AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT));
+
+    job.setOutputFormatClass(TextOutputFormat.class);
+    Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-text");
+    FileOutputFormat.setOutputPath(job, outputPath);
+
+    Assert.assertTrue(job.waitForCompletion(true));
+
+    // Check that the results from the MapReduce were as expected.
+    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
+    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*"));
+    Assert.assertEquals(1, outputFiles.length);
+    Path filePath = outputFiles[0].getPath();
+    InputStream inputStream = filePath.getFileSystem(job.getConfiguration()).open(filePath);
+    Assert.assertNotNull(inputStream);
+    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+    try {
+      Assert.assertTrue(reader.ready());
+      Assert.assertEquals("apple\t3", reader.readLine());
+      Assert.assertEquals("banana\t2", reader.readLine());
+      Assert.assertEquals("carrot\t1", reader.readLine());
+      Assert.assertFalse(reader.ready());
+    } finally {
+      reader.close();
+    }
+  }
 }



Mime
View raw message