From common-commits-return-18174-apmail-hadoop-common-commits-archive=hadoop.apache.org@hadoop.apache.org Tue Jun 12 22:13:16 2012 Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DB20895FA for ; Tue, 12 Jun 2012 22:13:16 +0000 (UTC) Received: (qmail 77326 invoked by uid 500); 12 Jun 2012 22:13:16 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 77270 invoked by uid 500); 12 Jun 2012 22:13:16 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 77261 invoked by uid 99); 12 Jun 2012 22:13:16 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Jun 2012 22:13:16 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Jun 2012 22:13:14 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7D52F2388860; Tue, 12 Jun 2012 22:12:54 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1349564 - in /hadoop/common/branches/branch-2/hadoop-common-project: ./ hadoop-auth/ hadoop-common/ hadoop-common/src/main/docs/ hadoop-common/src/main/java/ hadoop-common/src/main/java/org/apache/hadoop/fs/ hadoop-common/src/main/java/org... Date: Tue, 12 Jun 2012 22:12:53 -0000 To: common-commits@hadoop.apache.org From: eli@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120612221254.7D52F2388860@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: eli Date: Tue Jun 12 22:12:53 2012 New Revision: 1349564 URL: http://svn.apache.org/viewvc?rev=1349564&view=rev Log: HADOOP-8507. Avoid OOM while deserializing DelegationTokenIdentifer. Contributed by Colin Patrick McCabe Modified: hadoop/common/branches/branch-2/hadoop-common-project/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-auth/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/core/ (props changed) hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java Propchange: hadoop/common/branches/branch-2/hadoop-common-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project:r1349561 Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-auth/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-auth:r1349561 Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common:r1349561 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1349564&r1=1349563&r2=1349564&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jun 12 22:12:53 2012 @@ -83,6 +83,9 @@ Release 2.0.1-alpha - UNRELEASED HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli) + HADOOP-8507. Avoid OOM while deserializing DelegationTokenIdentifer. + (Colin Patrick McCabe via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1349561 Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1349561 Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1349561 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1349564&r1=1349563&r2=1349564&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java Tue Jun 12 22:12:53 2012 @@ -254,7 +254,7 @@ public class FileStatus implements Writa // Writable ////////////////////////////////////////////////// public void write(DataOutput out) throws IOException { - Text.writeString(out, getPath().toString(), Text.ONE_MEGABYTE); + Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN); out.writeLong(getLen()); out.writeBoolean(isDirectory()); out.writeShort(getReplication()); @@ -262,16 +262,16 @@ public class FileStatus implements Writa out.writeLong(getModificationTime()); out.writeLong(getAccessTime()); getPermission().write(out); - Text.writeString(out, getOwner(), Text.ONE_MEGABYTE); - Text.writeString(out, getGroup(), Text.ONE_MEGABYTE); + Text.writeString(out, getOwner(), Text.DEFAULT_MAX_LEN); + Text.writeString(out, getGroup(), Text.DEFAULT_MAX_LEN); out.writeBoolean(isSymlink()); if (isSymlink()) { - Text.writeString(out, getSymlink().toString(), Text.ONE_MEGABYTE); + Text.writeString(out, getSymlink().toString(), Text.DEFAULT_MAX_LEN); } } public void readFields(DataInput in) throws IOException { - String strPath = Text.readString(in, Text.ONE_MEGABYTE); + String strPath = Text.readString(in, Text.DEFAULT_MAX_LEN); this.path = new Path(strPath); this.length = in.readLong(); this.isdir = in.readBoolean(); @@ -280,10 +280,10 @@ public class FileStatus implements Writa modification_time = in.readLong(); access_time = in.readLong(); permission.readFields(in); - owner = Text.readString(in, Text.ONE_MEGABYTE); - group = Text.readString(in, Text.ONE_MEGABYTE); + owner = Text.readString(in, Text.DEFAULT_MAX_LEN); + group = Text.readString(in, Text.DEFAULT_MAX_LEN); if (in.readBoolean()) { - this.symlink = new Path(Text.readString(in, Text.ONE_MEGABYTE)); + this.symlink = new Path(Text.readString(in, Text.DEFAULT_MAX_LEN)); } else { this.symlink = null; } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java?rev=1349564&r1=1349563&r2=1349564&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java Tue Jun 12 22:12:53 2012 @@ -84,8 +84,8 @@ public class PermissionStatus implements /** {@inheritDoc} */ public void readFields(DataInput in) throws IOException { - username = Text.readString(in, Text.ONE_MEGABYTE); - groupname = Text.readString(in, Text.ONE_MEGABYTE); + username = Text.readString(in, Text.DEFAULT_MAX_LEN); + groupname = Text.readString(in, Text.DEFAULT_MAX_LEN); permission = FsPermission.read(in); } @@ -110,8 +110,8 @@ public class PermissionStatus implements String username, String groupname, FsPermission permission) throws IOException { - Text.writeString(out, username, Text.ONE_MEGABYTE); - Text.writeString(out, groupname, Text.ONE_MEGABYTE); + Text.writeString(out, username, Text.DEFAULT_MAX_LEN); + Text.writeString(out, groupname, Text.DEFAULT_MAX_LEN); permission.write(out); } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java?rev=1349564&r1=1349563&r2=1349564&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java Tue Jun 12 22:12:53 2012 @@ -287,6 +287,20 @@ public class Text extends BinaryComparab in.readFully(bytes, 0, newLength); length = newLength; } + + public void readFields(DataInput in, int maxLength) throws IOException { + int newLength = WritableUtils.readVInt(in); + if (newLength < 0) { + throw new IOException("tried to deserialize " + newLength + + " bytes of data! newLength must be non-negative."); + } else if (newLength >= maxLength) { + throw new IOException("tried to deserialize " + newLength + + " bytes of data, but maxLength = " + maxLength); + } + setCapacity(newLength, false); + in.readFully(bytes, 0, newLength); + length = newLength; + } /** Skips over one Text in the input. */ public static void skip(DataInput in) throws IOException { @@ -304,6 +318,16 @@ public class Text extends BinaryComparab out.write(bytes, 0, length); } + public void write(DataOutput out, int maxLength) throws IOException { + if (length > maxLength) { + throw new IOException("data was too long to write! Expected " + + "less than or equal to " + maxLength + " bytes, but got " + + length + " bytes."); + } + WritableUtils.writeVInt(out, length); + out.write(bytes, 0, length); + } + /** Returns true iff o is a Text with the same contents. */ public boolean equals(Object o) { if (o instanceof Text) @@ -417,7 +441,7 @@ public class Text extends BinaryComparab return bytes; } - static final public int ONE_MEGABYTE = 1024 * 1024; + static final public int DEFAULT_MAX_LEN = 1024 * 1024; /** Read a UTF8 encoded string from in */ @@ -432,7 +456,7 @@ public class Text extends BinaryComparab */ public static String readString(DataInput in, int maxLength) throws IOException { - int length = WritableUtils.readVIntInRange(in, 0, maxLength - 1); + int length = WritableUtils.readVIntInRange(in, 0, maxLength); byte [] bytes = new byte[length]; in.readFully(bytes, 0, length); return decode(bytes); @@ -454,9 +478,9 @@ public class Text extends BinaryComparab throws IOException { ByteBuffer bytes = encode(s); int length = bytes.limit(); - if (length >= maxLength) { + if (length > maxLength) { throw new IOException("string was too long to write! Expected " + - "less than " + maxLength + " bytes, but got " + + "less than or equal to " + maxLength + " bytes, but got " + length + " bytes."); } WritableUtils.writeVInt(out, length); Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java?rev=1349564&r1=1349563&r2=1349564&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java Tue Jun 12 22:12:53 2012 @@ -31,6 +31,8 @@ import org.apache.hadoop.security.Hadoop import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.TokenIdentifier; +import com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public abstract class AbstractDelegationTokenIdentifier @@ -173,16 +175,17 @@ extends TokenIdentifier { throw new IOException("Unknown version of delegation token " + version); } - owner.readFields(in); - renewer.readFields(in); - realUser.readFields(in); + owner.readFields(in, Text.DEFAULT_MAX_LEN); + renewer.readFields(in, Text.DEFAULT_MAX_LEN); + realUser.readFields(in, Text.DEFAULT_MAX_LEN); issueDate = WritableUtils.readVLong(in); maxDate = WritableUtils.readVLong(in); sequenceNumber = WritableUtils.readVInt(in); masterKeyId = WritableUtils.readVInt(in); } - public void write(DataOutput out) throws IOException { + @VisibleForTesting + void writeImpl(DataOutput out) throws IOException { out.writeByte(VERSION); owner.write(out); renewer.write(out); @@ -193,6 +196,19 @@ extends TokenIdentifier { WritableUtils.writeVInt(out, masterKeyId); } + public void write(DataOutput out) throws IOException { + if (owner.getLength() > Text.DEFAULT_MAX_LEN) { + throw new IOException("owner is too long to be serialized!"); + } + if (renewer.getLength() > Text.DEFAULT_MAX_LEN) { + throw new IOException("renewer is too long to be serialized!"); + } + if (realUser.getLength() > Text.DEFAULT_MAX_LEN) { + throw new IOException("realuser is too long to be serialized!"); + } + writeImpl(out); + } + public String toString() { StringBuilder buffer = new StringBuilder(); buffer Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1349561 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java?rev=1349564&r1=1349563&r2=1349564&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java Tue Jun 12 22:12:53 2012 @@ -137,38 +137,38 @@ public class TestText extends TestCase { } } - public void doTestLimitedIO(String str, int strLen) throws IOException { + public void doTestLimitedIO(String str, int len) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); DataInputBuffer in = new DataInputBuffer(); out.reset(); try { - Text.writeString(out, str, strLen); + Text.writeString(out, str, len); fail("expected writeString to fail when told to write a string " + "that was too long! The string was '" + str + "'"); } catch (IOException e) { } - Text.writeString(out, str, strLen + 1); + Text.writeString(out, str, len + 1); // test that it reads correctly in.reset(out.getData(), out.getLength()); - in.mark(strLen); + in.mark(len); String after; try { - after = Text.readString(in, strLen); + after = Text.readString(in, len); fail("expected readString to fail when told to read a string " + "that was too long! The string was '" + str + "'"); } catch (IOException e) { } in.reset(); - after = Text.readString(in, strLen + 1); + after = Text.readString(in, len + 1); assertTrue(str.equals(after)); } public void testLimitedIO() throws Exception { - doTestLimitedIO("abcd", 4); - doTestLimitedIO("", 0); - doTestLimitedIO("1", 1); + doTestLimitedIO("abcd", 3); + doTestLimitedIO("foo bar baz", 10); + doTestLimitedIO("1", 0); } public void testCompare() throws Exception { Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java?rev=1349564&r1=1349563&r2=1349564&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java Tue Jun 12 22:12:53 2012 @@ -19,6 +19,7 @@ package org.apache.hadoop.security.token.delegation; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; @@ -387,4 +388,46 @@ public class TestDelegationToken { } } + private boolean testDelegationTokenIdentiferSerializationRoundTrip(Text owner, + Text renewer, Text realUser) throws IOException { + TestDelegationTokenIdentifier dtid = new TestDelegationTokenIdentifier( + owner, renewer, realUser); + DataOutputBuffer out = new DataOutputBuffer(); + dtid.writeImpl(out); + DataInputBuffer in = new DataInputBuffer(); + in.reset(out.getData(), out.getLength()); + try { + TestDelegationTokenIdentifier dtid2 = + new TestDelegationTokenIdentifier(); + dtid2.readFields(in); + assertTrue(dtid.equals(dtid2)); + return true; + } catch(IOException e){ + return false; + } + } + + @Test + public void testSimpleDtidSerialization() throws IOException { + assertTrue(testDelegationTokenIdentiferSerializationRoundTrip( + new Text("owner"), new Text("renewer"), new Text("realUser"))); + assertTrue(testDelegationTokenIdentiferSerializationRoundTrip( + new Text(""), new Text(""), new Text(""))); + assertTrue(testDelegationTokenIdentiferSerializationRoundTrip( + new Text(""), new Text("b"), new Text(""))); + } + + @Test + public void testOverlongDtidSerialization() throws IOException { + byte[] bigBuf = new byte[Text.DEFAULT_MAX_LEN + 1]; + for (int i = 0; i < bigBuf.length; i++) { + bigBuf[i] = 0; + } + assertFalse(testDelegationTokenIdentiferSerializationRoundTrip( + new Text(bigBuf), new Text("renewer"), new Text("realUser"))); + assertFalse(testDelegationTokenIdentiferSerializationRoundTrip( + new Text("owner"), new Text(bigBuf), new Text("realUser"))); + assertFalse(testDelegationTokenIdentiferSerializationRoundTrip( + new Text("owner"), new Text("renewer"), new Text(bigBuf))); + } }