Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7AEF5C9FA for ; Tue, 15 May 2012 19:04:09 +0000 (UTC) Received: (qmail 96417 invoked by uid 500); 15 May 2012 19:04:09 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 96351 invoked by uid 500); 15 May 2012 19:04:09 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 96343 invoked by uid 99); 15 May 2012 19:04:09 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 May 2012 19:04:09 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 May 2012 19:04:04 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 5FA602388A36; Tue, 15 May 2012 19:03:43 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1338832 - in /hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project: ./ hadoop-hdfs/ hadoop-hdfs/src/main/java/ hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/ba... Date: Tue, 15 May 2012 19:03:42 -0000 To: hdfs-commits@hadoop.apache.org From: eli@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120515190343.5FA602388A36@eris.apache.org> Author: eli Date: Tue May 15 19:03:41 2012 New Revision: 1338832 URL: http://svn.apache.org/viewvc?rev=1338832&view=rev Log: HDFS-3418. svn merge -c 1338830 from trunk Modified: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/ (props changed) hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed) hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed) hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed) hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed) hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed) hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project:r1338830 Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1338830 Modified: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1338832&r1=1338831&r2=1338832&view=diff ============================================================================== --- hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue May 15 19:03:41 2012 @@ -300,6 +300,9 @@ Release 2.0.0 - UNRELEASED HADOOP-8285 HDFS changes for Use ProtoBuf for RpcPayLoadHeader (sanjay radia) + HDFS-3418. Rename BlockWithLocationsProto datanodeIDs field to storageIDs. + (eli) + OPTIMIZATIONS HDFS-2477. Optimize computing the diff between a block report and the Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1338830 Modified: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1338832&r1=1338831&r2=1338832&view=diff ============================================================================== --- hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original) +++ hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Tue May 15 19:03:41 2012 @@ -254,11 +254,11 @@ public class PBHelper { public static BlockWithLocationsProto convert(BlockWithLocations blk) { return BlockWithLocationsProto.newBuilder() .setBlock(convert(blk.getBlock())) - .addAllDatanodeIDs(Arrays.asList(blk.getDatanodes())).build(); + .addAllStorageIDs(Arrays.asList(blk.getStorageIDs())).build(); } public static BlockWithLocations convert(BlockWithLocationsProto b) { - return new BlockWithLocations(convert(b.getBlock()), b.getDatanodeIDsList() + return new BlockWithLocations(convert(b.getBlock()), b.getStorageIDsList() .toArray(new String[0])); } Modified: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java?rev=1338832&r1=1338831&r2=1338832&view=diff ============================================================================== --- hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java (original) +++ hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java Tue May 15 19:03:41 2012 @@ -205,6 +205,7 @@ public class Balancer { private Map globalBlockList = new HashMap(); private MovedBlocks movedBlocks = new MovedBlocks(); + // Map storage IDs to BalancerDatanodes private Map datanodes = new HashMap(); @@ -620,8 +621,8 @@ public class Balancer { synchronized (block) { // update locations - for ( String location : blk.getDatanodes() ) { - BalancerDatanode datanode = datanodes.get(location); + for ( String storageID : blk.getStorageIDs() ) { + BalancerDatanode datanode = datanodes.get(storageID); if (datanode != null) { // not an unknown datanode block.addLocation(datanode); } Modified: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java?rev=1338832&r1=1338831&r2=1338832&view=diff ============================================================================== --- hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java (original) +++ hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java Tue May 15 19:03:41 2012 @@ -21,9 +21,8 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.protocol.Block; -/** A class to implement an array of BlockLocations - * It provide efficient customized serialization/deserialization methods - * in stead of using the default array (de)serialization provided by RPC +/** + * Maintains an array of blocks and their corresponding storage IDs. */ @InterfaceAudience.Private @InterfaceStability.Evolving @@ -36,12 +35,12 @@ public class BlocksWithLocations { @InterfaceStability.Evolving public static class BlockWithLocations { Block block; - String datanodeIDs[]; + String storageIDs[]; /** constructor */ - public BlockWithLocations(Block b, String[] datanodes) { - block = b; - datanodeIDs = datanodes; + public BlockWithLocations(Block block, String[] storageIDs) { + this.block = block; + this.storageIDs = storageIDs; } /** get the block */ @@ -50,15 +49,15 @@ public class BlocksWithLocations { } /** get the block's locations */ - public String[] getDatanodes() { - return datanodeIDs; + public String[] getStorageIDs() { + return storageIDs; } } private BlockWithLocations[] blocks; /** Constructor with one parameter */ - public BlocksWithLocations( BlockWithLocations[] blocks ) { + public BlocksWithLocations(BlockWithLocations[] blocks) { this.blocks = blocks; } Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1338830 Modified: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1338832&r1=1338831&r2=1338832&view=diff ============================================================================== --- hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto (original) +++ hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto Tue May 15 19:03:41 2012 @@ -274,7 +274,7 @@ message BlockProto { */ message BlockWithLocationsProto { required BlockProto block = 1; // Block - repeated string datanodeIDs = 2; // Datanodes with replicas of the block + repeated string storageIDs = 2; // Datanodes with replicas of the block } /** Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1338830 Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1338830 Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1338830 Propchange: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1338830 Modified: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java?rev=1338832&r1=1338831&r2=1338832&view=diff ============================================================================== --- hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java (original) +++ hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java Tue May 15 19:03:41 2012 @@ -101,18 +101,18 @@ public class TestGetBlocks extends TestC BlockWithLocations[] locs; locs = namenode.getBlocks(dataNodes[0], fileLen).getBlocks(); assertEquals(locs.length, 2); - assertEquals(locs[0].getDatanodes().length, 2); - assertEquals(locs[1].getDatanodes().length, 2); + assertEquals(locs[0].getStorageIDs().length, 2); + assertEquals(locs[1].getStorageIDs().length, 2); // get blocks of size BlockSize from dataNodes[0] locs = namenode.getBlocks(dataNodes[0], DEFAULT_BLOCK_SIZE).getBlocks(); assertEquals(locs.length, 1); - assertEquals(locs[0].getDatanodes().length, 2); + assertEquals(locs[0].getStorageIDs().length, 2); // get blocks of size 1 from dataNodes[0] locs = namenode.getBlocks(dataNodes[0], 1).getBlocks(); assertEquals(locs.length, 1); - assertEquals(locs[0].getDatanodes().length, 2); + assertEquals(locs[0].getStorageIDs().length, 2); // get blocks of size 0 from dataNodes[0] getBlocksWithException(namenode, dataNodes[0], 0); Modified: hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java?rev=1338832&r1=1338831&r2=1338832&view=diff ============================================================================== --- hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java (original) +++ hadoop/common/branches/branch-2.0.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java Tue May 15 19:03:41 2012 @@ -161,7 +161,7 @@ public class TestPBHelper { private void compare(BlockWithLocations locs1, BlockWithLocations locs2) { assertEquals(locs1.getBlock(), locs2.getBlock()); - assertTrue(Arrays.equals(locs1.getDatanodes(), locs2.getDatanodes())); + assertTrue(Arrays.equals(locs1.getStorageIDs(), locs2.getStorageIDs())); } @Test