hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r812109 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/streaming/src/java/org/apache/hadoop/typedbytes/TypedBytesInput.java src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java
Date Mon, 07 Sep 2009 12:29:11 GMT
Author: tomwhite
Date: Mon Sep  7 12:29:11 2009
New Revision: 812109

URL: http://svn.apache.org/viewvc?rev=812109&view=rev
Log:
MAPREDUCE-764. TypedBytesInput's readRaw() does not preserve custom type codes. Contributed
by Klaas Bosteels.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/streaming/src/java/org/apache/hadoop/typedbytes/TypedBytesInput.java
    hadoop/mapreduce/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=812109&r1=812108&r2=812109&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Sep  7 12:29:11 2009
@@ -533,3 +533,6 @@
 
     MAPREDUCE-943. Removes a testcase in TestNodeRefresh that doesn't make 
     sense in the new Job recovery model. (Amar Kamat via ddas)
+
+    MAPREDUCE-764. TypedBytesInput's readRaw() does not preserve custom type
+    codes. (Klaas Bosteels via tomwhite)

Modified: hadoop/mapreduce/trunk/src/contrib/streaming/src/java/org/apache/hadoop/typedbytes/TypedBytesInput.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/streaming/src/java/org/apache/hadoop/typedbytes/TypedBytesInput.java?rev=812109&r1=812108&r2=812109&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/streaming/src/java/org/apache/hadoop/typedbytes/TypedBytesInput.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/streaming/src/java/org/apache/hadoop/typedbytes/TypedBytesInput.java
Mon Sep  7 12:29:11 2009
@@ -149,7 +149,7 @@
     } else if (code == Type.MARKER.code) {
       return null;
     } else if (50 <= code && code <= 200) { // application-specific typecodes
-      return readRawBytes();
+      return readRawBytes(code);
     } else {
       throw new RuntimeException("unknown type");
     }
@@ -202,14 +202,15 @@
   }
 
   /**
-   * Reads the raw bytes following a <code>Type.BYTES</code> code.
+   * Reads the raw bytes following a custom code.
+   * @param code the custom type code
    * @return the obtained bytes sequence
    * @throws IOException
    */
-  public byte[] readRawBytes() throws IOException {
+  public byte[] readRawBytes(int code) throws IOException {
     int length = in.readInt();
     byte[] bytes = new byte[5 + length];
-    bytes[0] = (byte) Type.BYTES.code;
+    bytes[0] = (byte) code;
     bytes[1] = (byte) (0xff & (length >> 24));
     bytes[2] = (byte) (0xff & (length >> 16));
     bytes[3] = (byte) (0xff & (length >> 8));
@@ -217,6 +218,15 @@
     in.readFully(bytes, 5, length);
     return bytes;
   }
+  
+  /**
+   * Reads the raw bytes following a <code>Type.BYTES</code> code.
+   * @return the obtained bytes sequence
+   * @throws IOException
+   */
+  public byte[] readRawBytes() throws IOException {
+    return readRawBytes(Type.BYTES.code);
+  }
 
   /**
    * Reads the byte following a <code>Type.BYTE</code> code.

Modified: hadoop/mapreduce/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java?rev=812109&r1=812108&r2=812109&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java
Mon Sep  7 12:29:11 2009
@@ -27,6 +27,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -120,6 +121,24 @@
     istream.close();
   }
 
+  public void testCustomTypesIO() throws IOException {
+    byte[] rawBytes = new byte[] { 100, 0, 0, 0, 3, 1, 2, 3 };
+    
+    FileOutputStream ostream = new FileOutputStream(tmpfile);
+    DataOutputStream dostream = new DataOutputStream(ostream);
+    TypedBytesOutput out = new TypedBytesOutput(dostream);
+    out.writeRaw(rawBytes);
+    dostream.close();
+    ostream.close();
+
+    FileInputStream istream = new FileInputStream(tmpfile);
+    DataInputStream distream = new DataInputStream(istream);
+    TypedBytesInput in = new TypedBytesInput(distream);
+    assertTrue(Arrays.equals(rawBytes, in.readRaw()));
+    distream.close();
+    istream.close();
+  }
+  
   public void testRecordIO() throws IOException {
     RecRecord1 r1 = new RecRecord1();
     r1.setBoolVal(true);



Mime
View raw message