From hdfs-commits-return-3633-apmail-hadoop-hdfs-commits-archive=hadoop.apache.org@hadoop.apache.org Thu May 10 23:13:29 2012 Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 03009C815 for ; Thu, 10 May 2012 23:13:29 +0000 (UTC) Received: (qmail 40906 invoked by uid 500); 10 May 2012 23:13:28 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 40839 invoked by uid 500); 10 May 2012 23:13:28 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 40828 invoked by uid 99); 10 May 2012 23:13:28 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 May 2012 23:13:28 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 May 2012 23:13:25 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id AA2812388860; Thu, 10 May 2012 23:13:03 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1336943 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/security/token/block/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/ Date: Thu, 10 May 2012 23:13:03 -0000 To: hdfs-commits@hadoop.apache.org From: eli@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120510231303.AA2812388860@eris.apache.org> Author: eli Date: Thu May 10 23:13:02 2012 New Revision: 1336943 URL: http://svn.apache.org/viewvc?rev=1336943&view=rev Log: HDFS-3134. harden edit log loader against malformed or malicious input. Contributed by Colin Patrick McCabe Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1336943&r1=1336942&r2=1336943&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu May 10 23:13:02 2012 @@ -444,6 +444,9 @@ Release 2.0.0 - UNRELEASED HDFS-3369. Rename {get|set|add}INode(..) methods in BlockManager and BlocksMap to {get|set|add}BlockCollection(..). (John George via szetszwo) + HDFS-3134. harden edit log loader against malformed or malicious input. + (Colin Patrick McCabe via eli) + OPTIMIZATIONS HDFS-3024. Improve performance of stringification in addStoredBlock (todd) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java?rev=1336943&r1=1336942&r2=1336943&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java Thu May 10 23:13:02 2012 @@ -148,7 +148,8 @@ public class BlockTokenIdentifier extend userId = WritableUtils.readString(in); blockPoolId = WritableUtils.readString(in); blockId = WritableUtils.readVLong(in); - int length = WritableUtils.readVInt(in); + int length = WritableUtils.readVIntInRange(in, 0, + AccessMode.class.getEnumConstants().length); for (int i = 0; i < length; i++) { modes.add(WritableUtils.readEnum(in, AccessMode.class)); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java?rev=1336943&r1=1336942&r2=1336943&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java Thu May 10 23:13:02 2012 @@ -203,6 +203,10 @@ public abstract class FSEditLogOp { } T setBlocks(Block[] blocks) { + if (blocks.length > MAX_BLOCKS) { + throw new RuntimeException("Can't have more than " + MAX_BLOCKS + + " in an AddCloseOp."); + } this.blocks = blocks; return (T)this; } @@ -296,10 +300,18 @@ public abstract class FSEditLogOp { } } + static final public int MAX_BLOCKS = 1024 * 1024 * 64; + private static Block[] readBlocks( DataInputStream in, int logVersion) throws IOException { int numBlocks = in.readInt(); + if (numBlocks < 0) { + throw new IOException("invalid negative number of blocks"); + } else if (numBlocks > MAX_BLOCKS) { + throw new IOException("invalid number of blocks: " + numBlocks + + ". The maximum number of blocks per file is " + MAX_BLOCKS); + } Block[] blocks = new Block[numBlocks]; for (int i = 0; i < numBlocks; i++) { Block blk = new Block(); @@ -579,6 +591,7 @@ public abstract class FSEditLogOp { String trg; String[] srcs; long timestamp; + final static public int MAX_CONCAT_SRC = 1024 * 1024; private ConcatDeleteOp() { super(OP_CONCAT_DELETE); @@ -594,7 +607,12 @@ public abstract class FSEditLogOp { } ConcatDeleteOp setSources(String[] srcs) { + if (srcs.length > MAX_CONCAT_SRC) { + throw new RuntimeException("ConcatDeleteOp can only have " + + MAX_CONCAT_SRC + " sources at most."); + } this.srcs = srcs; + return this; } @@ -624,8 +642,8 @@ public abstract class FSEditLogOp { if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) { this.length = in.readInt(); if (length < 3) { // trg, srcs.., timestamp - throw new IOException("Incorrect data format. " - + "Concat delete operation."); + throw new IOException("Incorrect data format " + + "for ConcatDeleteOp."); } } this.trg = FSImageSerialization.readString(in); @@ -635,6 +653,15 @@ public abstract class FSEditLogOp { } else { srcSize = this.length - 1 - 1; // trg and timestamp } + if (srcSize < 0) { + throw new IOException("Incorrect data format. " + + "ConcatDeleteOp cannot have a negative number of data " + + " sources."); + } else if (srcSize > MAX_CONCAT_SRC) { + throw new IOException("Incorrect data format. " + + "ConcatDeleteOp can have at most " + MAX_CONCAT_SRC + + " sources, but we tried to have " + (length - 3) + " sources."); + } this.srcs = new String [srcSize]; for(int i=0; i