hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r707262 - in /hadoop/core/trunk: CHANGES.txt src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java src/test/org/apache/hadoop/mapred/TestJavaSerialization.java
Date Thu, 23 Oct 2008 03:59:21 GMT
Author: acmurthy
Date: Wed Oct 22 20:59:20 2008
New Revision: 707262

URL: http://svn.apache.org/viewvc?rev=707262&view=rev
Log:
HADOOP-4466. Ensure that SequenceFileOutputFormat isn't tied to Writables and can be used
with other Serialization frameworks. Contributed by Chris Wensel.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=707262&r1=707261&r2=707262&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Oct 22 20:59:20 2008
@@ -1018,6 +1018,10 @@
     HADOOP-4387. TestHDFSFileSystemContract fails on windows nightly builds.
     (Raghu Angadi)
 
+    HADOOP-4466. Ensure that SequenceFileOutputFormat isn't tied to Writables
+    and can be used with other Serialization frameworks. (Chris Wensel via
+    acmurthy)
+
 Release 0.18.2 - Unreleased
 
   BUG FIXES

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java?rev=707262&r1=707261&r2=707262&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java Wed
Oct 22 20:59:20 2008
@@ -26,8 +26,6 @@
 import org.apache.hadoop.fs.FileUtil;
 
 import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.DefaultCodec;
@@ -58,8 +56,8 @@
     }
     final SequenceFile.Writer out = 
       SequenceFile.createWriter(fs, job, file,
-                                job.getOutputKeyClass().asSubclass(WritableComparable.class),
-                                job.getOutputValueClass().asSubclass(Writable.class),
+                                job.getOutputKeyClass(),
+                                job.getOutputValueClass(),
                                 compressionType,
                                 codec,
                                 progress);

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java?rev=707262&r1=707261&r2=707262&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java Wed Oct
22 20:59:20 2008
@@ -106,4 +106,46 @@
     reader.close();
   }
 
+  /**
+   * HADOOP-4466:
+   * This test verifies the JavSerialization impl can write to SequenceFiles. by virtue other
+   * SequenceFileOutputFormat is not coupled to Writable types, if so, the job will fail.
+   *
+   */
+  public void testWriteToSequencefile() throws Exception {
+    OutputStream os = getFileSystem().create(new Path(getInputDir(),
+        "text.txt"));
+    Writer wr = new OutputStreamWriter(os);
+    wr.write("b a\n");
+    wr.close();
+
+    JobConf conf = createJobConf();
+    conf.setJobName("JavaSerialization");
+
+    conf.set("io.serializations",
+    "org.apache.hadoop.io.serializer.JavaSerialization," +
+    "org.apache.hadoop.io.serializer.WritableSerialization");
+
+    conf.setInputFormat(TextInputFormat.class);
+    conf.setOutputFormat(SequenceFileOutputFormat.class); // test we can write to sequence
files
+
+    conf.setOutputKeyClass(String.class);
+    conf.setOutputValueClass(Long.class);
+    conf.setOutputKeyComparatorClass(JavaSerializationComparator.class);
+
+    conf.setMapperClass(WordCountMapper.class);
+    conf.setReducerClass(SumReducer.class);
+
+    FileInputFormat.setInputPaths(conf, getInputDir());
+
+    FileOutputFormat.setOutputPath(conf, getOutputDir());
+
+    JobClient.runJob(conf);
+
+    Path[] outputFiles = FileUtil.stat2Paths(
+                           getFileSystem().listStatus(getOutputDir(),
+                           new OutputLogFilter()));
+    assertEquals(1, outputFiles.length);
+}
+
 }



Mime
View raw message