Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 72199200CB7 for ; Fri, 30 Jun 2017 16:59:55 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 6FEC6160BF6; Fri, 30 Jun 2017 14:59:55 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 955D9160BEB for ; Fri, 30 Jun 2017 16:59:54 +0200 (CEST) Received: (qmail 7308 invoked by uid 500); 30 Jun 2017 14:59:49 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 5202 invoked by uid 99); 30 Jun 2017 14:59:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 30 Jun 2017 14:59:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C8350F323B; Fri, 30 Jun 2017 14:59:42 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: stevel@apache.org To: common-commits@hadoop.apache.org Date: Fri, 30 Jun 2017 14:59:45 -0000 Message-Id: <7fae5f347e42488498b2a4cf9a44619f@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [04/35] hadoop git commit: HDFS-12033. DatanodeManager picking EC recovery tasks should also consider the number of regular replication tasks. Contributed by Lei (Eddy) Xu. archived-at: Fri, 30 Jun 2017 14:59:55 -0000 HDFS-12033. DatanodeManager picking EC recovery tasks should also consider the number of regular replication tasks. Contributed by Lei (Eddy) Xu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/144753e8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/144753e8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/144753e8 Branch: refs/heads/HADOOP-13345 Commit: 144753e87f4a9daa51200be05ff2bb760bf38169 Parents: a9d3412 Author: Lei Xu Authored: Mon Jun 26 15:43:50 2017 -0700 Committer: Lei Xu Committed: Mon Jun 26 15:43:50 2017 -0700 ---------------------------------------------------------------------- .../server/blockmanagement/DatanodeManager.java | 1 + .../blockmanagement/TestDatanodeManager.java | 51 ++++++++++++++++++++ 2 files changed, 52 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/144753e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index a786c6a..1d09751 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1661,6 +1661,7 @@ public class DatanodeManager { if (pendingList != null) { cmds.add(new BlockCommand(DatanodeProtocol.DNA_TRANSFER, blockPoolId, pendingList)); + maxTransfers -= pendingList.size(); } // check pending erasure coding tasks List pendingECList = nodeinfo http://git-wip-us.apache.org/repos/asf/hadoop/blob/144753e8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java index 30e2aaf..de002f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java @@ -44,13 +44,21 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.protocol.BlockCommand; +import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand; +import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; +import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; +import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.util.Shell; import org.junit.Assert; @@ -491,4 +499,47 @@ public class TestDatanodeManager { Assert.assertEquals("Unexpected host or host in unexpected position", "127.0.0.1:23456", bothAgain.get(1).getInfoAddr()); } + + @Test + public void testPendingRecoveryTasks() throws IOException { + FSNamesystem fsn = Mockito.mock(FSNamesystem.class); + Mockito.when(fsn.hasWriteLock()).thenReturn(true); + Configuration conf = new Configuration(); + DatanodeManager dm = Mockito.spy(mockDatanodeManager(fsn, conf)); + + int maxTransfers = 20; + int numPendingTasks = 7; + int numECTasks = maxTransfers - numPendingTasks; + + DatanodeDescriptor nodeInfo = Mockito.mock(DatanodeDescriptor.class); + Mockito.when(nodeInfo.isRegistered()).thenReturn(true); + Mockito.when(nodeInfo.getStorageInfos()) + .thenReturn(new DatanodeStorageInfo[0]); + + List pendingList = + Collections.nCopies(numPendingTasks, new BlockTargetPair(null, null)); + Mockito.when(nodeInfo.getReplicationCommand(maxTransfers)) + .thenReturn(pendingList); + List ecPendingList = + Collections.nCopies(numECTasks, null); + + Mockito.when(nodeInfo.getErasureCodeCommand(numECTasks)) + .thenReturn(ecPendingList); + DatanodeRegistration dnReg = Mockito.mock(DatanodeRegistration.class); + Mockito.when(dm.getDatanode(dnReg)).thenReturn(nodeInfo); + + DatanodeCommand[] cmds = dm.handleHeartbeat( + dnReg, new StorageReport[1], "bp-123", 0, 0, 10, maxTransfers, 0, null, + SlowPeerReports.EMPTY_REPORT, SlowDiskReports.EMPTY_REPORT); + + assertEquals(2, cmds.length); + assertTrue(cmds[0] instanceof BlockCommand); + BlockCommand replicaCmd = (BlockCommand) cmds[0]; + assertEquals(numPendingTasks, replicaCmd.getBlocks().length); + assertEquals(numPendingTasks, replicaCmd.getTargets().length); + assertTrue(cmds[1] instanceof BlockECReconstructionCommand); + BlockECReconstructionCommand ecRecoveryCmd = + (BlockECReconstructionCommand) cmds[1]; + assertEquals(numECTasks, ecRecoveryCmd.getECTasks().size()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org