Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F0A921092E for ; Thu, 12 Dec 2013 07:20:51 +0000 (UTC) Received: (qmail 63947 invoked by uid 500); 12 Dec 2013 07:20:45 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 63914 invoked by uid 500); 12 Dec 2013 07:20:45 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 63906 invoked by uid 99); 12 Dec 2013 07:20:45 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Dec 2013 07:20:45 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Dec 2013 07:20:40 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6F5B92388C38; Thu, 12 Dec 2013 07:19:44 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1550363 [7/8] - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/protocol/ src/main/java/org/apache/hadoop/hdfs/protocolPB/ src/main/java/... Date: Thu, 12 Dec 2013 07:17:58 -0000 To: hdfs-commits@hadoop.apache.org From: arp@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131212071944.6F5B92388C38@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Thu Dec 12 07:17:51 2013 @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetworkTopology; @@ -57,41 +58,57 @@ public class TestReplicationPolicyWithNo private BlockPlacementPolicy replicator; private static final String filename = "/dummyfile.txt"; - private final static DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] { - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/n2"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2/n3"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/n3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/n4"), - DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3/n5"), - DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r3/n6") - }; - - private final static DatanodeDescriptor dataNodesInBoundaryCase[] = - new DatanodeDescriptor[] { - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r1/n2"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/n3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/n3") - }; - - private final static DatanodeDescriptor dataNodesInMoreTargetsCase[] = - new DatanodeDescriptor[] { - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/r1/n2"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/r1/n2"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/r1/n3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/r1/n3"), - DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/r2/n4"), - DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/r2/n4"), - DFSTestUtil.getDatanodeDescriptor("9.9.9.9", "/r2/n5"), - DFSTestUtil.getDatanodeDescriptor("10.10.10.10", "/r2/n5"), - DFSTestUtil.getDatanodeDescriptor("11.11.11.11", "/r2/n6"), - DFSTestUtil.getDatanodeDescriptor("12.12.12.12", "/r2/n6"), + private static final DatanodeStorageInfo[] storages; + private static final DatanodeDescriptor[] dataNodes; + static { + final String[] racks = { + "/d1/r1/n1", + "/d1/r1/n1", + "/d1/r1/n2", + "/d1/r2/n3", + "/d1/r2/n3", + "/d1/r2/n4", + "/d2/r3/n5", + "/d2/r3/n6" + }; + storages = DFSTestUtil.createDatanodeStorageInfos(racks); + dataNodes = DFSTestUtil.toDatanodeDescriptor(storages); + } + + private static final DatanodeStorageInfo[] storagesInBoundaryCase; + private static final DatanodeDescriptor[] dataNodesInBoundaryCase; + static { + final String[] racksInBoundaryCase = { + "/d1/r1/n1", + "/d1/r1/n1", + "/d1/r1/n1", + "/d1/r1/n2", + "/d1/r2/n3", + "/d1/r2/n3" + }; + storagesInBoundaryCase = DFSTestUtil.createDatanodeStorageInfos(racksInBoundaryCase); + dataNodesInBoundaryCase = DFSTestUtil.toDatanodeDescriptor(storagesInBoundaryCase); + } + + private static final DatanodeStorageInfo[] storagesInMoreTargetsCase; + private final static DatanodeDescriptor[] dataNodesInMoreTargetsCase; + static { + final String[] racksInMoreTargetsCase = { + "/r1/n1", + "/r1/n1", + "/r1/n2", + "/r1/n2", + "/r1/n3", + "/r1/n3", + "/r2/n4", + "/r2/n4", + "/r2/n5", + "/r2/n5", + "/r2/n6", + "/r2/n6" + }; + storagesInMoreTargetsCase = DFSTestUtil.createDatanodeStorageInfos(racksInMoreTargetsCase); + dataNodesInMoreTargetsCase = DFSTestUtil.toDatanodeDescriptor(storagesInMoreTargetsCase); }; private final static DatanodeDescriptor NODE = @@ -129,9 +146,20 @@ public class TestReplicationPolicyWithNo namenode.stop(); } + private static void updateHeartbeatWithUsage(DatanodeDescriptor dn, + long capacity, long dfsUsed, long remaining, long blockPoolUsed, + long dnCacheCapacity, long dnCacheUsed, int xceiverCount, + int volFailures) { + dn.getStorageInfos()[0].setUtilizationForTesting( + capacity, dfsUsed, remaining, blockPoolUsed); + dn.updateHeartbeat( + BlockManagerTestUtil.getStorageReportsForDatanode(dn), + dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures); + } + private static void setupDataNodeCapacity() { for(int i=0; i targetSet = new HashSet(); - for(DatanodeDescriptor node:targets) { + for(DatanodeStorageInfo storage:targets) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); String nodeGroup = NetworkTopology.getLastHalf(node.getNetworkLocation()); if(targetSet.contains(nodeGroup)) { return false; @@ -156,34 +185,50 @@ public class TestReplicationPolicyWithNo } return true; } - - private DatanodeDescriptor[] chooseTarget(int numOfReplicas) { + + private boolean isOnSameRack(DatanodeStorageInfo left, DatanodeStorageInfo right) { + return isOnSameRack(left.getDatanodeDescriptor(), right); + } + + private boolean isOnSameRack(DatanodeDescriptor left, DatanodeStorageInfo right) { + return cluster.isOnSameRack(left, right.getDatanodeDescriptor()); + } + + private boolean isOnSameNodeGroup(DatanodeStorageInfo left, DatanodeStorageInfo right) { + return isOnSameNodeGroup(left.getDatanodeDescriptor(), right); + } + + private boolean isOnSameNodeGroup(DatanodeDescriptor left, DatanodeStorageInfo right) { + return cluster.isOnSameNodeGroup(left, right.getDatanodeDescriptor()); + } + + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas) { return chooseTarget(numOfReplicas, dataNodes[0]); } - private DatanodeDescriptor[] chooseTarget(int numOfReplicas, + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, DatanodeDescriptor writer) { return chooseTarget(numOfReplicas, writer, - new ArrayList()); + new ArrayList()); } - private DatanodeDescriptor[] chooseTarget(int numOfReplicas, - List chosenNodes) { + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, + List chosenNodes) { return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes); } - private DatanodeDescriptor[] chooseTarget(int numOfReplicas, - DatanodeDescriptor writer, List chosenNodes) { + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, + DatanodeDescriptor writer, List chosenNodes) { return chooseTarget(numOfReplicas, writer, chosenNodes, null); } - private DatanodeDescriptor[] chooseTarget( + private DatanodeStorageInfo[] chooseTarget( int numOfReplicas, DatanodeDescriptor writer, - List chosenNodes, + List chosenNodes, Set excludedNodes) { return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes, - false, excludedNodes, BLOCK_SIZE); + false, excludedNodes, BLOCK_SIZE, StorageType.DEFAULT); } /** @@ -197,49 +242,53 @@ public class TestReplicationPolicyWithNo */ @Test public void testChooseTarget1() throws Exception { - dataNodes[0].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[0], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 4, 0); // overloaded - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertEquals(targets[0], dataNodes[0]); + assertEquals(storages[0], targets[0]); + targets = chooseTarget(2); assertEquals(targets.length, 2); - assertEquals(targets[0], dataNodes[0]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[0], targets[0]); + + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); - assertEquals(targets[0], dataNodes[0]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameNodeGroup(targets[1], targets[2])); + assertEquals(storages[0], targets[0]); + + assertFalse(isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameNodeGroup(targets[1], targets[2])); targets = chooseTarget(4); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[0]); - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertEquals(storages[0], targets[0]); + + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); + assertFalse(isOnSameRack(targets[0], targets[2])); // Make sure no more than one replicas are on the same nodegroup verifyNoTwoTargetsOnSameNodeGroup(targets); - dataNodes[0].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[0], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0); } - private void verifyNoTwoTargetsOnSameNodeGroup(DatanodeDescriptor[] targets) { + private void verifyNoTwoTargetsOnSameNodeGroup(DatanodeStorageInfo[] targets) { Set nodeGroupSet = new HashSet(); - for (DatanodeDescriptor target: targets) { - nodeGroupSet.add(target.getNetworkLocation()); + for (DatanodeStorageInfo target: targets) { + nodeGroupSet.add(target.getDatanodeDescriptor().getNetworkLocation()); } assertEquals(nodeGroupSet.size(), targets.length); } @@ -254,36 +303,37 @@ public class TestReplicationPolicyWithNo */ @Test public void testChooseTarget2() throws Exception { - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator; - List chosenNodes = new ArrayList(); + List chosenNodes = new ArrayList(); Set excludedNodes = new HashSet(); excludedNodes.add(dataNodes[1]); targets = repl.chooseTarget(filename, 4, dataNodes[0], chosenNodes, false, - excludedNodes, BLOCK_SIZE); + excludedNodes, BLOCK_SIZE, StorageType.DEFAULT); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[0]); + assertEquals(storages[0], targets[0]); + assertTrue(cluster.isNodeGroupAware()); // Make sure no replicas are on the same nodegroup for (int i=1;i<4;i++) { - assertFalse(cluster.isOnSameNodeGroup(targets[0], targets[i])); + assertFalse(isOnSameNodeGroup(targets[0], targets[i])); } - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); - assertFalse(cluster.isOnSameRack(targets[1], targets[3])); + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); + assertFalse(isOnSameRack(targets[1], targets[3])); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); - chosenNodes.add(dataNodes[2]); + chosenNodes.add(storages[2]); targets = repl.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true, - excludedNodes, BLOCK_SIZE); + excludedNodes, BLOCK_SIZE, StorageType.DEFAULT); System.out.println("targets=" + Arrays.asList(targets)); assertEquals(2, targets.length); //make sure that the chosen node is in the target. int i = 0; - for(; i < targets.length && !dataNodes[2].equals(targets[i]); i++); + for(; i < targets.length && !storages[2].equals(targets[i]); i++); assertTrue(i < targets.length); } @@ -298,39 +348,39 @@ public class TestReplicationPolicyWithNo @Test public void testChooseTarget3() throws Exception { // make data node 0 to be not qualified to choose - dataNodes[0].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[0], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0L, 0L, 0, 0); // no space - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertEquals(targets[0], dataNodes[1]); + assertEquals(storages[1], targets[0]); targets = chooseTarget(2); assertEquals(targets.length, 2); - assertEquals(targets[0], dataNodes[1]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[1], targets[0]); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); - assertEquals(targets[0], dataNodes[1]); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[1], targets[0]); + assertTrue(isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(4); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[1]); + assertEquals(storages[1], targets[0]); assertTrue(cluster.isNodeGroupAware()); verifyNoTwoTargetsOnSameNodeGroup(targets); - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); - dataNodes[0].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[0], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0); } @@ -348,33 +398,33 @@ public class TestReplicationPolicyWithNo public void testChooseTarget4() throws Exception { // make data node 0-2 to be not qualified to choose: not enough disk space for(int i=0; i<3; i++) { - dataNodes[i].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[i], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0L, 0L, 0, 0); } - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); targets = chooseTarget(2); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); for(int i=0; i<3; i++) { - assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[i])); } verifyNoTwoTargetsOnSameNodeGroup(targets); - assertTrue(cluster.isOnSameRack(targets[0], targets[1]) || - cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertTrue(isOnSameRack(targets[0], targets[1]) || + isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[2])); } /** @@ -387,7 +437,7 @@ public class TestReplicationPolicyWithNo @Test public void testChooseTarget5() throws Exception { setupDataNodeCapacity(); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, NODE); assertEquals(targets.length, 0); @@ -396,12 +446,12 @@ public class TestReplicationPolicyWithNo targets = chooseTarget(2, NODE); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3, NODE); assertEquals(targets.length, 3); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[1])); verifyNoTwoTargetsOnSameNodeGroup(targets); } @@ -415,27 +465,27 @@ public class TestReplicationPolicyWithNo @Test public void testRereplicate1() throws Exception { setupDataNodeCapacity(); - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - DatanodeDescriptor[] targets; + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3, chosenNodes); assertEquals(targets.length, 3); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertTrue(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameNodeGroup(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(targets[0], targets[2])); } /** @@ -448,22 +498,22 @@ public class TestReplicationPolicyWithNo @Test public void testRereplicate2() throws Exception { setupDataNodeCapacity(); - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - chosenNodes.add(dataNodes[1]); + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + chosenNodes.add(storages[1]); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]) && - cluster.isOnSameRack(dataNodes[0], targets[1])); + assertFalse(isOnSameRack(dataNodes[0], targets[0]) && + isOnSameRack(dataNodes[0], targets[1])); } /** @@ -476,33 +526,33 @@ public class TestReplicationPolicyWithNo @Test public void testRereplicate3() throws Exception { setupDataNodeCapacity(); - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - chosenNodes.add(dataNodes[3]); + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + chosenNodes.add(storages[3]); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(dataNodes[3], targets[0])); + assertTrue(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(dataNodes[3], targets[0])); targets = chooseTarget(1, dataNodes[3], chosenNodes); assertEquals(targets.length, 1); - assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0])); - assertFalse(cluster.isOnSameNodeGroup(dataNodes[3], targets[0])); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertTrue(isOnSameRack(dataNodes[3], targets[0])); + assertFalse(isOnSameNodeGroup(dataNodes[3], targets[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0])); + assertTrue(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameNodeGroup(dataNodes[0], targets[0])); targets = chooseTarget(2, dataNodes[3], chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0])); + assertTrue(isOnSameRack(dataNodes[3], targets[0])); } /** @@ -576,16 +626,17 @@ public class TestReplicationPolicyWithNo cluster.add(dataNodesInBoundaryCase[i]); } for(int i=0; i chosenNodes = new ArrayList(); - chosenNodes.add(dataNodesInBoundaryCase[0]); - chosenNodes.add(dataNodesInBoundaryCase[5]); - DatanodeDescriptor[] targets; + List chosenNodes = new ArrayList(); + chosenNodes.add(storagesInBoundaryCase[0]); + chosenNodes.add(storagesInBoundaryCase[5]); + DatanodeStorageInfo[] targets; targets = chooseTarget(1, dataNodesInBoundaryCase[0], chosenNodes); - assertFalse(cluster.isOnSameNodeGroup(targets[0], - dataNodesInBoundaryCase[0])); - assertFalse(cluster.isOnSameNodeGroup(targets[0], - dataNodesInBoundaryCase[5])); + assertFalse(isOnSameNodeGroup(dataNodesInBoundaryCase[0], targets[0])); + assertFalse(isOnSameNodeGroup(dataNodesInBoundaryCase[5], targets[0])); assertTrue(checkTargetsOnDifferentNodeGroup(targets)); } @@ -651,12 +700,12 @@ public class TestReplicationPolicyWithNo } for(int i=0; i live = new ArrayList(); live.add(dnDesc1); live.add(dnDesc2); @@ -615,3 +639,4 @@ public class TestJspHelper { MessageFormat.format(EXPECTED__NOTF_PATTERN, version))); } } + Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Thu Dec 12 07:17:51 2013 @@ -22,11 +22,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Random; import javax.management.NotCompliantMBeanException; import javax.management.ObjectName; @@ -48,7 +48,9 @@ import org.apache.hadoop.hdfs.server.dat import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.DataChecksum; @@ -74,7 +76,7 @@ public class SimulatedFSDataset implemen @Override public SimulatedFSDataset newInstance(DataNode datanode, DataStorage storage, Configuration conf) throws IOException { - return new SimulatedFSDataset(datanode, storage, conf); + return new SimulatedFSDataset(storage, conf); } @Override @@ -136,6 +138,11 @@ public class SimulatedFSDataset implemen } @Override + public String getStorageUuid() { + return storage.getStorageUuid(); + } + + @Override synchronized public long getGenerationStamp() { return theBlock.getGenerationStamp(); } @@ -318,13 +325,15 @@ public class SimulatedFSDataset implemen private static class SimulatedStorage { private Map map = new HashMap(); - private long capacity; // in bytes + private final String storageUuid = "SimulatedStroage-" + DatanodeStorage.generateUuid(); + + private final long capacity; // in bytes synchronized long getFree() { return capacity - getUsed(); } - synchronized long getCapacity() { + long getCapacity() { return capacity; } @@ -379,22 +388,33 @@ public class SimulatedFSDataset implemen } return bpStorage; } + + String getStorageUuid() { + return storageUuid; + } + + synchronized StorageReport getStorageReport(String bpid) { + return new StorageReport(getStorageUuid(), false, getCapacity(), + getUsed(), getFree(), map.get(bpid).getUsed()); + } } private final Map> blockMap = new HashMap>(); private final SimulatedStorage storage; - private final String storageId; + private final String datanodeUuid; - public SimulatedFSDataset(DataNode datanode, DataStorage storage, - Configuration conf) { + public SimulatedFSDataset(DataStorage storage, Configuration conf) { if (storage != null) { - storage.createStorageID(datanode.getXferPort()); - this.storageId = storage.getStorageID(); + for (int i = 0; i < storage.getNumStorageDirs(); ++i) { + storage.createStorageID(storage.getStorageDir(i)); + } + this.datanodeUuid = storage.getDatanodeUuid(); } else { - this.storageId = "unknownStorageId" + new Random().nextInt(); + this.datanodeUuid = "SimulatedDatanode-" + DataNode.generateUuid(); } - registerMBean(storageId); + + registerMBean(datanodeUuid); this.storage = new SimulatedStorage( conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY)); } @@ -451,8 +471,7 @@ public class SimulatedFSDataset implemen } } - @Override - public synchronized BlockListAsLongs getBlockReport(String bpid) { + synchronized BlockListAsLongs getBlockReport(String bpid) { final List blocks = new ArrayList(); final Map map = blockMap.get(bpid); if (map != null) { @@ -465,6 +484,12 @@ public class SimulatedFSDataset implemen return new BlockListAsLongs(blocks, null); } + @Override + public synchronized Map getBlockReports( + String bpid) { + return Collections.singletonMap(new DatanodeStorage(storage.storageUuid), getBlockReport(bpid)); + } + @Override // FsDatasetSpi public List getCacheReport(String bpid) { return new LinkedList(); @@ -661,7 +686,7 @@ public class SimulatedFSDataset implemen } @Override // FsDatasetSpi - public void recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) + public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { final Map map = getMap(b.getBlockPoolId()); BInfo binfo = map.get(b.getLocalBlock()); @@ -675,6 +700,7 @@ public class SimulatedFSDataset implemen map.remove(b.getLocalBlock()); binfo.theBlock.setGenerationStamp(newGS); map.put(binfo.theBlock, binfo); + return binfo.getStorageUuid(); } @Override // FsDatasetSpi @@ -931,7 +957,7 @@ public class SimulatedFSDataset implemen @Override public String getStorageInfo() { - return "Simulated FSDataset-" + storageId; + return "Simulated FSDataset-" + datanodeUuid; } @Override @@ -958,7 +984,8 @@ public class SimulatedFSDataset implemen public String updateReplicaUnderRecovery(ExtendedBlock oldBlock, long recoveryId, long newlength) { - return storageId; + // Caller does not care about the exact Storage UUID returned. + return datanodeUuid; } @Override // FsDatasetSpi @@ -1013,11 +1040,6 @@ public class SimulatedFSDataset implemen } @Override - public String[] getBlockPoolList() { - throw new UnsupportedOperationException(); - } - - @Override public void checkAndUpdate(String bpid, long blockId, File diskFile, File diskMetaFile, FsVolumeSpi vol) { throw new UnsupportedOperationException(); @@ -1029,7 +1051,12 @@ public class SimulatedFSDataset implemen } @Override - public List getFinalizedBlocks(String bpid) { + public StorageReport[] getStorageReports(String bpid) { + return new StorageReport[] {storage.getStorageReport(bpid)}; + } + + @Override + public List getFinalizedBlocks(String bpid) { throw new UnsupportedOperationException(); } @@ -1048,3 +1075,4 @@ public class SimulatedFSDataset implemen throw new UnsupportedOperationException(); } } + Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Thu Dec 12 07:17:51 2013 @@ -104,7 +104,7 @@ public class TestBPOfferService { .when(mockDn).getMetrics(); // Set up a simulated dataset with our fake BP - mockFSDataset = Mockito.spy(new SimulatedFSDataset(null, null, conf)); + mockFSDataset = Mockito.spy(new SimulatedFSDataset(null, conf)); mockFSDataset.addBlockPool(FAKE_BPID, conf); // Wire the dataset to the DN. @@ -180,7 +180,7 @@ public class TestBPOfferService { waitForBlockReport(mockNN2); // When we receive a block, it should report it to both NNs - bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, ""); + bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, "", ""); ReceivedDeletedBlockInfo[] ret = waitForBlockReceived(FAKE_BLOCK, mockNN1); assertEquals(1, ret.length); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Thu Dec 12 07:17:51 2013 @@ -35,6 +35,7 @@ import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -121,7 +122,7 @@ public class TestBlockRecovery { * @throws IOException */ @Before - public void startUp() throws IOException { + public void startUp() throws IOException, URISyntaxException { tearDownDone = false; conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, DATA_DIR); @@ -131,11 +132,12 @@ public class TestBlockRecovery { conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); FileSystem.setDefaultUri(conf, "hdfs://" + NN_ADDR.getHostName() + ":" + NN_ADDR.getPort()); - ArrayList dirs = new ArrayList(); + ArrayList locations = new ArrayList(); File dataDir = new File(DATA_DIR); FileUtil.fullyDelete(dataDir); dataDir.mkdirs(); - dirs.add(dataDir); + StorageLocation location = StorageLocation.parse(dataDir.getPath()); + locations.add(location); final DatanodeProtocolClientSideTranslatorPB namenode = mock(DatanodeProtocolClientSideTranslatorPB.class); @@ -163,7 +165,7 @@ public class TestBlockRecovery { new DatanodeCommand[0], new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1))); - dn = new DataNode(conf, dirs, null) { + dn = new DataNode(conf, locations, null) { @Override DatanodeProtocolClientSideTranslatorPB connectToNN( InetSocketAddress nnAddr) throws IOException { Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java Thu Dec 12 07:17:51 2013 @@ -265,7 +265,7 @@ public class TestBlockReplacement { // sendRequest DataOutputStream out = new DataOutputStream(sock.getOutputStream()); new Sender(out).replaceBlock(block, BlockTokenSecretManager.DUMMY_TOKEN, - source.getStorageID(), sourceProxy); + source.getDatanodeUuid(), sourceProxy); out.flush(); // receiveResponse DataInputStream reply = new DataInputStream(sock.getInputStream()); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java Thu Dec 12 07:17:51 2013 @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.io.File; @@ -25,6 +27,7 @@ import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeoutException; @@ -88,7 +91,7 @@ public class TestBlockReport { private MiniDFSCluster cluster; private DistributedFileSystem fs; - Random rand = new Random(RAND_LIMIT); + private static Random rand = new Random(RAND_LIMIT); private static Configuration conf; @@ -112,6 +115,48 @@ public class TestBlockReport { cluster.shutdown(); } + // Generate a block report, optionally corrupting the generation + // stamp and/or length of one block. + private static StorageBlockReport[] getBlockReports( + DataNode dn, String bpid, boolean corruptOneBlockGs, + boolean corruptOneBlockLen) { + Map perVolumeBlockLists = + dn.getFSDataset().getBlockReports(bpid); + + // Send block report + StorageBlockReport[] reports = + new StorageBlockReport[perVolumeBlockLists.size()]; + boolean corruptedGs = false; + boolean corruptedLen = false; + + int reportIndex = 0; + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + DatanodeStorage dnStorage = kvPair.getKey(); + BlockListAsLongs blockList = kvPair.getValue(); + + // Walk the list of blocks until we find one each to corrupt the + // generation stamp and length, if so requested. + for (int i = 0; i < blockList.getNumberOfBlocks(); ++i) { + if (corruptOneBlockGs && !corruptedGs) { + blockList.corruptBlockGSForTesting(i, rand); + LOG.info("Corrupted the GS for block ID " + i); + corruptedGs = true; + } else if (corruptOneBlockLen && !corruptedLen) { + blockList.corruptBlockLengthForTesting(i, rand); + LOG.info("Corrupted the length for block ID " + i); + corruptedLen = true; + } else { + break; + } + } + + reports[reportIndex++] = + new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs()); + } + + return reports; + } + /** * Test write a file, verifies and closes it. Then the length of the blocks * are messed up and BlockReport is forced. @@ -152,10 +197,8 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N0); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); List blocksAfterReport = DFSTestUtil.getAllBlocks(fs.open(filePath)); @@ -210,7 +253,6 @@ public class TestBlockReport { for (Integer aRemovedIndex : removedIndex) { blocks2Remove.add(lBlocks.get(aRemovedIndex).getBlock()); } - ArrayList blocks = locatedToBlocks(lBlocks, removedIndex); if(LOG.isDebugEnabled()) { LOG.debug("Number of blocks allocated " + lBlocks.size()); @@ -224,8 +266,11 @@ public class TestBlockReport { for (File f : findAllFiles(dataDir, new MyFileFilter(b.getBlockName(), true))) { DataNodeTestUtils.getFSDataset(dn0).unfinalizeBlock(b); - if (!f.delete()) + if (!f.delete()) { LOG.warn("Couldn't delete " + b.getBlockName()); + } else { + LOG.debug("Deleted file " + f.toString()); + } } } @@ -234,10 +279,8 @@ public class TestBlockReport { // all blocks belong to the same file, hence same BP String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn0.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn0, poolId, false, false); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); BlockManagerTestUtil.getComputedDatanodeWork(cluster.getNamesystem() .getBlockManager()); @@ -252,9 +295,8 @@ public class TestBlockReport { /** - * Test writes a file and closes it. Then test finds a block - * and changes its GS to be < of original one. - * New empty block is added to the list of blocks. + * Test writes a file and closes it. + * Block reported is generated with a bad GS for a single block. * Block report is forced and the check for # of corrupted blocks is performed. * * @throws IOException in case of an error @@ -263,50 +305,64 @@ public class TestBlockReport { public void blockReport_03() throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); - - ArrayList blocks = - prepareForRide(filePath, METHOD_NAME, FILE_SIZE); - - // The block with modified GS won't be found. Has to be deleted - blocks.get(0).setGenerationStamp(rand.nextLong()); - // This new block is unknown to NN and will be mark for deletion. - blocks.add(new Block()); + ArrayList blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N0); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; + StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false); DatanodeCommand dnCmd = - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); if(LOG.isDebugEnabled()) { LOG.debug("Got the command: " + dnCmd); } printStats(); - assertEquals("Wrong number of CorruptedReplica+PendingDeletion " + - "blocks is found", 2, - cluster.getNamesystem().getCorruptReplicaBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); } /** - * This test isn't a representative case for BlockReport - * The empty method is going to be left here to keep the naming - * of the test plan in synch with the actual implementation + * Test writes a file and closes it. + * Block reported is generated with an extra block. + * Block report is forced and the check for # of pendingdeletion + * blocks is performed. + * + * @throws IOException in case of an error */ - public void blockReport_04() { - } + @Test + public void blockReport_04() throws IOException { + final String METHOD_NAME = GenericTestUtils.getMethodName(); + Path filePath = new Path("/" + METHOD_NAME + ".dat"); + DFSTestUtil.createFile(fs, filePath, + FILE_SIZE, REPL_FACTOR, rand.nextLong()); + - // Client requests new block from NN. The test corrupts this very block - // and forces new block report. - // The test case isn't specific for BlockReport because it relies on - // BlockScanner which is out of scope of this test - // Keeping the name to be in synch with the test plan - // - public void blockReport_05() { + DataNode dn = cluster.getDataNodes().get(DN_N0); + // all blocks belong to the same file, hence same BP + String poolId = cluster.getNamesystem().getBlockPoolId(); + + // Create a bogus new block which will not be present on the namenode. + ExtendedBlock b = new ExtendedBlock( + poolId, rand.nextLong(), 1024L, rand.nextLong()); + dn.getFSDataset().createRbw(b); + + DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); + DatanodeCommand dnCmd = + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); + if(LOG.isDebugEnabled()) { + LOG.debug("Got the command: " + dnCmd); + } + printStats(); + + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(0L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(1L)); } /** @@ -323,17 +379,15 @@ public class TestBlockReport { Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; - ArrayList blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); + writeFile(METHOD_NAME, FILE_SIZE, filePath); startDNandWait(filePath, true); - // all blocks belong to the same file, hence same BP + // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); printStats(); assertEquals("Wrong number of PendingReplication Blocks", 0, cluster.getNamesystem().getUnderReplicatedBlocks()); @@ -353,68 +407,40 @@ public class TestBlockReport { * @throws IOException in case of an error */ @Test - // Currently this test is failing as expected 'cause the correct behavior is - // not yet implemented (9/15/09) public void blockReport_07() throws Exception { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; // write file and start second node to be "older" than the original - ArrayList blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); + writeFile(METHOD_NAME, FILE_SIZE, filePath); startDNandWait(filePath, true); - int randIndex = rand.nextInt(blocks.size()); - // Get a block and screw its GS - Block corruptedBlock = blocks.get(randIndex); - String secondNode = cluster.getDataNodes().get(DN_N1).getStorageId(); - if(LOG.isDebugEnabled()) { - LOG.debug("Working with " + secondNode); - LOG.debug("BlockGS before " + blocks.get(randIndex).getGenerationStamp()); - } - corruptBlockGS(corruptedBlock); - if(LOG.isDebugEnabled()) { - LOG.debug("BlockGS after " + blocks.get(randIndex).getGenerationStamp()); - LOG.debug("Done corrupting GS of " + corruptedBlock.getBlockName()); - } // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); printStats(); - assertEquals("Wrong number of Corrupted blocks", - 1, cluster.getNamesystem().getCorruptReplicaBlocks() + -// the following might have to be added into the equation if -// the same block could be in two different states at the same time -// and then the expected number of has to be changed to '2' -// cluster.getNamesystem().getPendingReplicationBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); - - // Get another block and screw its length to be less than original - if (randIndex == 0) - randIndex++; - else - randIndex--; - corruptedBlock = blocks.get(randIndex); - corruptBlockLen(corruptedBlock); - if(LOG.isDebugEnabled()) { - LOG.debug("Done corrupting length of " + corruptedBlock.getBlockName()); - } - - report[0] = new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()); - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + assertThat("Wrong number of PendingReplication blocks", + cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); + + reports = getBlockReports(dn, poolId, true, true); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); printStats(); - assertEquals("Wrong number of Corrupted blocks", - 2, cluster.getNamesystem().getCorruptReplicaBlocks() + - cluster.getNamesystem().getPendingReplicationBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(2L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + assertThat("Wrong number of PendingReplication blocks", + cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); printStats(); @@ -457,9 +483,7 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; + StorageBlockReport[] report = getBlockReports(dn, poolId, false, false); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); assertEquals("Wrong number of PendingReplication blocks", @@ -490,14 +514,11 @@ public class TestBlockReport { // write file and start second node to be "older" than the original try { - ArrayList blocks = - writeFile(METHOD_NAME, 12 * bytesChkSum, filePath); + writeFile(METHOD_NAME, 12 * bytesChkSum, filePath); Block bl = findBlock(filePath, 12 * bytesChkSum); BlockChecker bc = new BlockChecker(filePath); bc.start(); - corruptBlockGS(bl); - corruptBlockLen(bl); waitForTempReplica(bl, DN_N1); @@ -505,9 +526,7 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; + StorageBlockReport[] report = getBlockReports(dn, poolId, true, true); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); assertEquals("Wrong number of PendingReplication blocks", @@ -783,38 +802,6 @@ public class TestBlockReport { ((Log4JLogger) TestBlockReport.LOG).getLogger().setLevel(Level.ALL); } - private void corruptBlockLen(final Block block) - throws IOException { - if (block == null) { - throw new IOException("Block isn't suppose to be null"); - } - long oldLen = block.getNumBytes(); - long newLen = oldLen - rand.nextLong(); - assertTrue("Old and new length shouldn't be the same", - block.getNumBytes() != newLen); - block.setNumBytes(newLen); - if(LOG.isDebugEnabled()) { - LOG.debug("Length of " + block.getBlockName() + - " is changed to " + newLen + " from " + oldLen); - } - } - - private void corruptBlockGS(final Block block) - throws IOException { - if (block == null) { - throw new IOException("Block isn't suppose to be null"); - } - long oldGS = block.getGenerationStamp(); - long newGS = oldGS - rand.nextLong(); - assertTrue("Old and new GS shouldn't be the same", - block.getGenerationStamp() != newGS); - block.setGenerationStamp(newGS); - if(LOG.isDebugEnabled()) { - LOG.debug("Generation stamp of " + block.getBlockName() + - " is changed to " + block.getGenerationStamp() + " from " + oldGS); - } - } - private Block findBlock(Path path, long size) throws IOException { Block ret; List lbs = Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java Thu Dec 12 07:17:51 2013 @@ -19,12 +19,14 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.*; -import java.net.URI; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; +import java.util.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.StorageType; import org.junit.Test; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; import static org.mockito.Mockito.*; @@ -34,19 +36,69 @@ import org.apache.hadoop.hdfs.server.dat public class TestDataDirs { - @Test (timeout = 10000) - public void testGetDataDirsFromURIs() throws Throwable { + @Test (timeout = 30000) + public void testDataDirParsing() throws Throwable { + Configuration conf = new Configuration(); + List locations; + File dir0 = new File("/dir0"); + File dir1 = new File("/dir1"); + File dir2 = new File("/dir2"); + File dir3 = new File("/dir3"); + + // Verify that a valid string is correctly parsed, and that storage + // type is not case-sensitive + String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1); + locations = DataNode.getStorageLocations(conf); + assertThat(locations.size(), is(4)); + assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(0).getUri(), is(dir0.toURI())); + assertThat(locations.get(1).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(1).getUri(), is(dir1.toURI())); + assertThat(locations.get(2).getStorageType(), is(StorageType.SSD)); + assertThat(locations.get(2).getUri(), is(dir2.toURI())); + assertThat(locations.get(3).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(3).getUri(), is(dir3.toURI())); + + // Verify that an unrecognized storage type result in an exception. + String locations2 = "[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations2); + try { + locations = DataNode.getStorageLocations(conf); + fail(); + } catch(IllegalArgumentException iae) { + DataNode.LOG.info("The exception is expected.", iae); + } + + // Assert that a string with no storage type specified is + // correctly parsed and the default storage type is picked up. + String locations3 = "/dir0,/dir1"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations3); + locations = DataNode.getStorageLocations(conf); + assertThat(locations.size(), is(2)); + assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(0).getUri(), is(dir0.toURI())); + assertThat(locations.get(1).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(1).getUri(), is(dir1.toURI())); + } + + @Test (timeout = 30000) + public void testDataDirValidation() throws Throwable { DataNodeDiskChecker diskChecker = mock(DataNodeDiskChecker.class); doThrow(new IOException()).doThrow(new IOException()).doNothing() .when(diskChecker).checkDir(any(LocalFileSystem.class), any(Path.class)); LocalFileSystem fs = mock(LocalFileSystem.class); - Collection uris = Arrays.asList(new URI("file:/p1/"), - new URI("file:/p2/"), new URI("file:/p3/")); + AbstractList locations = new ArrayList(); - List dirs = DataNode.getDataDirsFromURIs(uris, fs, diskChecker); - assertEquals("number of valid data dirs", 1, dirs.size()); - String validDir = dirs.iterator().next().getPath(); - assertEquals("p3 should be valid", new File("/p3").getPath(), validDir); + locations.add(StorageLocation.parse("file:/p1/")); + locations.add(StorageLocation.parse("file:/p2/")); + locations.add(StorageLocation.parse("file:/p3/")); + + List checkedLocations = + DataNode.checkStorageLocations(locations, fs, diskChecker); + assertEquals("number of valid data dirs", 1, checkedLocations.size()); + String validDir = checkedLocations.iterator().next().getFile().getPath(); + assertThat("p3 should be valid", new File("/p3/").getPath(), is(validDir)); } } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java Thu Dec 12 07:17:51 2013 @@ -163,7 +163,7 @@ public class TestDataNodeMultipleRegistr for (BPOfferService bpos : dn.getAllBpOs()) { LOG.info("reg: bpid=" + "; name=" + bpos.bpRegistration + "; sid=" - + bpos.bpRegistration.getStorageID() + "; nna=" + + + bpos.bpRegistration.getDatanodeUuid() + "; nna=" + getNNSocketAddress(bpos)); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Thu Dec 12 07:17:51 2013 @@ -42,11 +42,13 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; @@ -151,13 +153,23 @@ public class TestDataNodeVolumeFailure { DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3 String bpid = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid); - final StorageBlockReport[] report = { - new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - DataNodeTestUtils.getFSDataset(dn).getBlockReport(bpid - ).getBlockListAsLongs()) - }; - cluster.getNameNodeRpc().blockReport(dnR, bpid, report); + + Map perVolumeBlockLists = + dn.getFSDataset().getBlockReports(bpid); + + // Send block report + StorageBlockReport[] reports = + new StorageBlockReport[perVolumeBlockLists.size()]; + + int reportIndex = 0; + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + DatanodeStorage dnStorage = kvPair.getKey(); + BlockListAsLongs blockList = kvPair.getValue(); + reports[reportIndex++] = + new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs()); + } + + cluster.getNameNodeRpc().blockReport(dnR, bpid, reports); // verify number of blocks and files... verify(filename, filesize); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java Thu Dec 12 07:17:51 2013 @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; @@ -406,6 +407,16 @@ public class TestDirectoryScanner { public File getFinalizedDir(String bpid) throws IOException { return new File("/base/current/" + bpid + "/finalized"); } + + @Override + public StorageType getStorageType() { + return StorageType.DEFAULT; + } + + @Override + public String getStorageID() { + return ""; + } } private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi(); @@ -436,7 +447,7 @@ public class TestDirectoryScanner { void testScanInfoObject(long blockId) throws Exception { DirectoryScanner.ScanInfo scanInfo = - new DirectoryScanner.ScanInfo(blockId); + new DirectoryScanner.ScanInfo(blockId, null, null, null); assertEquals(blockId, scanInfo.getBlockId()); assertNull(scanInfo.getBlockFile()); assertNull(scanInfo.getMetaFile()); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java Thu Dec 12 07:17:51 2013 @@ -311,7 +311,7 @@ public class TestSimulatedFSDataset { } private SimulatedFSDataset getSimulatedFSDataset() { - SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, null, conf); + SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf); fsdataset.addBlockPool(bpid, conf); return fsdataset; } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Thu Dec 12 07:17:51 2013 @@ -17,6 +17,14 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; @@ -25,28 +33,40 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.protocol.*; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataStorage; -import org.apache.hadoop.hdfs.server.protocol.*; +import org.apache.hadoop.hdfs.server.protocol.BlockCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; +import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.net.DNS; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.Groups; -import org.apache.hadoop.util.*; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.util.VersionInfo; import org.apache.log4j.Level; import org.apache.log4j.LogManager; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.List; - /** * Main class for a series of name-node benchmarks. * @@ -817,17 +837,16 @@ public class NNThroughputBenchmark imple dnRegistration = new DatanodeRegistration( new DatanodeID(DNS.getDefaultIP("default"), DNS.getDefaultHost("default", "default"), - "", getNodePort(dnIdx), + DataNode.generateUuid(), getNodePort(dnIdx), DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT), - new DataStorage(nsInfo, ""), + new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); - DataNode.setNewStorageID(dnRegistration); // register datanode dnRegistration = nameNodeProto.registerDatanode(dnRegistration); //first block reports - storage = new DatanodeStorage(dnRegistration.getStorageID()); + storage = new DatanodeStorage(dnRegistration.getDatanodeUuid()); final StorageBlockReport[] reports = { new StorageBlockReport(storage, new BlockListAsLongs(null, null).getBlockListAsLongs()) @@ -843,7 +862,7 @@ public class NNThroughputBenchmark imple void sendHeartbeat() throws IOException { // register datanode // TODO:FEDERATION currently a single block pool is supported - StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(), + StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(), false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0).getCommands(); @@ -890,7 +909,7 @@ public class NNThroughputBenchmark imple @SuppressWarnings("unused") // keep it for future blockReceived benchmark int replicateBlocks() throws IOException { // register datanode - StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(), + StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(), false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0).getCommands(); @@ -920,14 +939,14 @@ public class NNThroughputBenchmark imple DatanodeInfo dnInfo = blockTargets[t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo, - new DataStorage(nsInfo, dnInfo.getStorageID()), + new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo( blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - receivedDNReg.getStorageID(), rdBlocks) }; + receivedDNReg.getDatanodeUuid(), rdBlocks) }; nameNodeProto.blockReceivedAndDeleted(receivedDNReg, nameNode .getNamesystem().getBlockPoolId(), report); } @@ -1050,7 +1069,7 @@ public class NNThroughputBenchmark imple loc.getBlock().getLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - datanodes[dnIdx].dnRegistration.getStorageID(), rdBlocks) }; + datanodes[dnIdx].dnRegistration.getDatanodeUuid(), rdBlocks) }; nameNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, loc .getBlock().getBlockPoolId(), report); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Thu Dec 12 07:17:51 2013 @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp; @@ -111,8 +112,8 @@ public class NameNodeAdapter { public static HeartbeatResponse sendHeartBeat(DatanodeRegistration nodeReg, DatanodeDescriptor dd, FSNamesystem namesystem) throws IOException { - return namesystem.handleHeartbeat(nodeReg, dd.getCapacity(), - dd.getDfsUsed(), dd.getRemaining(), dd.getBlockPoolUsed(), + return namesystem.handleHeartbeat(nodeReg, + BlockManagerTestUtil.getStorageReportsForDatanode(dd), dd.getCacheCapacity(), dd.getCacheRemaining(), 0, 0, 0); } @@ -243,3 +244,4 @@ public class NameNodeAdapter { return NNStorage.getInProgressEditsFile(sd, startTxId); } } + Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java Thu Dec 12 07:17:51 2013 @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.L import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.net.Node; @@ -99,13 +100,13 @@ public class TestAddBlockRetry { bmField.setAccessible(true); bmField.set(ns, spyBM); - doAnswer(new Answer() { + doAnswer(new Answer() { @Override - public DatanodeDescriptor[] answer(InvocationOnMock invocation) + public DatanodeStorageInfo[] answer(InvocationOnMock invocation) throws Throwable { LOG.info("chooseTarget for " + src); - DatanodeDescriptor[] ret = - (DatanodeDescriptor[]) invocation.callRealMethod(); + DatanodeStorageInfo[] ret = + (DatanodeStorageInfo[]) invocation.callRealMethod(); count++; if(count == 1) { // run second addBlock() LOG.info("Starting second addBlock for " + src); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1550363&r1=1550362&r2=1550363&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java Thu Dec 12 07:17:51 2013 @@ -24,7 +24,7 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.junit.Test; @@ -47,7 +47,7 @@ public class TestCommitBlockSynchronizat throws IOException { Configuration conf = new Configuration(); FSImage image = new FSImage(conf); - DatanodeDescriptor[] targets = new DatanodeDescriptor[0]; + final DatanodeStorageInfo[] targets = {}; FSNamesystem namesystem = new FSNamesystem(conf, image); FSNamesystem namesystemSpy = spy(namesystem);