hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1418212 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src: main/java/org/apache/hadoop/io/SequenceFile.java main/java/org/apache/hadoop/io/UTF8.java test/java/org/apache/hadoop/io/TestUTF8.java
Date Fri, 07 Dec 2012 08:17:40 GMT
Author: todd
Date: Fri Dec  7 08:17:38 2012
New Revision: 1418212

URL: http://svn.apache.org/viewvc?rev=1418212&view=rev
Log:
HDFS-4282. TestEditLog.testFuzzSequences FAILED in all pre-commit test. Contributed by Todd
Lipcon.

Modified:
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestUTF8.java

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java?rev=1418212&r1=1418211&r2=1418212&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
Fri Dec  7 08:17:38 2012
@@ -1858,10 +1858,10 @@ public class SequenceFile {
         UTF8 className = new UTF8();
 
         className.readFields(in);
-        keyClassName = className.toString(); // key class name
+        keyClassName = className.toStringChecked(); // key class name
 
         className.readFields(in);
-        valClassName = className.toString(); // val class name
+        valClassName = className.toStringChecked(); // val class name
       } else {
         keyClassName = Text.readString(in);
         valClassName = Text.readString(in);

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java?rev=1418212&r1=1418211&r2=1418212&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java
Fri Dec  7 08:17:38 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.io;
 import java.io.IOException;
 import java.io.DataInput;
 import java.io.DataOutput;
+import java.io.UTFDataFormatException;
 
 import org.apache.hadoop.util.StringUtils;
 
@@ -155,6 +156,21 @@ public class UTF8 implements WritableCom
     }
     return buffer.toString();
   }
+  
+  /**
+   * Convert to a string, checking for valid UTF8.
+   * @return the converted string
+   * @throws UTFDataFormatException if the underlying bytes contain invalid
+   * UTF8 data.
+   */
+  public String toStringChecked() throws IOException {
+    StringBuilder buffer = new StringBuilder(length);
+    synchronized (IBUF) {
+      IBUF.reset(bytes, length);
+      readChars(IBUF, buffer, length);
+    }
+    return buffer.toString();
+  }
 
   /** Returns true iff <code>o</code> is a UTF8 with the same contents.  */
   @Override
@@ -238,7 +254,7 @@ public class UTF8 implements WritableCom
   }
 
   private static void readChars(DataInput in, StringBuilder buffer, int nBytes)
-    throws IOException {
+    throws UTFDataFormatException, IOException {
     DataOutputBuffer obuf = OBUF_FACTORY.get();
     obuf.reset();
     obuf.write(in, nBytes);
@@ -250,15 +266,27 @@ public class UTF8 implements WritableCom
         // 0b0xxxxxxx: 1-byte sequence
         buffer.append((char)(b & 0x7F));
       } else if ((b & 0xE0) == 0xC0) {
+        if (i >= nBytes) {
+          throw new UTFDataFormatException("Truncated UTF8 at " +
+              StringUtils.byteToHexString(bytes, i - 1, 1));
+        }
         // 0b110xxxxx: 2-byte sequence
         buffer.append((char)(((b & 0x1F) << 6)
             | (bytes[i++] & 0x3F)));
       } else if ((b & 0xF0) == 0xE0) {
         // 0b1110xxxx: 3-byte sequence
+        if (i + 1 >= nBytes) {
+          throw new UTFDataFormatException("Truncated UTF8 at " +
+              StringUtils.byteToHexString(bytes, i - 1, 2));
+        }
         buffer.append((char)(((b & 0x0F) << 12)
             | ((bytes[i++] & 0x3F) << 6)
             |  (bytes[i++] & 0x3F)));
       } else if ((b & 0xF8) == 0xF0) {
+        if (i + 2 >= nBytes) {
+          throw new UTFDataFormatException("Truncated UTF8 at " +
+              StringUtils.byteToHexString(bytes, i - 1, 3));
+        }
         // 0b11110xxx: 4-byte sequence
         int codepoint =
             ((b & 0x07) << 18)
@@ -274,8 +302,8 @@ public class UTF8 implements WritableCom
         // Only show the next 6 bytes max in the error code - in case the
         // buffer is large, this will prevent an exceedingly large message.
         int endForError = Math.min(i + 5, nBytes);
-        throw new IOException("Invalid UTF8 at " +
-          StringUtils.byteToHexString(bytes, i - 1, endForError));
+        throw new UTFDataFormatException("Invalid UTF8 at " +
+            StringUtils.byteToHexString(bytes, i - 1, endForError));
       }
     }
   }

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestUTF8.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestUTF8.java?rev=1418212&r1=1418211&r2=1418212&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestUTF8.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestUTF8.java
Fri Dec  7 08:17:38 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.io;
 
 import junit.framework.TestCase;
 import java.io.IOException;
+import java.io.UTFDataFormatException;
 import java.util.Random;
 
 import org.apache.hadoop.test.GenericTestUtils;
@@ -126,9 +127,9 @@ public class TestUTF8 extends TestCase {
     try {
       UTF8.fromBytes(invalid);
       fail("did not throw an exception");
-    } catch (IOException ioe) {
+    } catch (UTFDataFormatException utfde) {
       GenericTestUtils.assertExceptionContains(
-          "Invalid UTF8 at ffff01020304", ioe);
+          "Invalid UTF8 at ffff01020304", utfde);
     }
   }
 
@@ -142,9 +143,27 @@ public class TestUTF8 extends TestCase {
     try {
       UTF8.fromBytes(invalid);
       fail("did not throw an exception");
-    } catch (IOException ioe) {
+    } catch (UTFDataFormatException utfde) {
       GenericTestUtils.assertExceptionContains(
-          "Invalid UTF8 at f88880808004", ioe);
+          "Invalid UTF8 at f88880808004", utfde);
+    }
+  }
+  
+  /**
+   * Test that decoding invalid UTF8 due to truncation yields the correct
+   * exception type.
+   */
+  public void testInvalidUTF8Truncated() throws Exception {
+    // Truncated CAT FACE character -- this is a 4-byte sequence, but we
+    // only have the first three bytes.
+    byte[] truncated = new byte[] {
+        (byte)0xF0, (byte)0x9F, (byte)0x90 };
+    try {
+      UTF8.fromBytes(truncated);
+      fail("did not throw an exception");
+    } catch (UTFDataFormatException utfde) {
+      GenericTestUtils.assertExceptionContains(
+          "Truncated UTF8 at f09f90", utfde);
     }
   }
 }



Mime
View raw message