Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id AB71D187BA for ; Thu, 8 Oct 2015 21:17:30 +0000 (UTC) Received: (qmail 69457 invoked by uid 500); 8 Oct 2015 21:17:16 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 69277 invoked by uid 500); 8 Oct 2015 21:17:16 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 67352 invoked by uid 99); 8 Oct 2015 21:17:14 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 08 Oct 2015 21:17:14 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3D386E0C5D; Thu, 8 Oct 2015 21:17:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jing9@apache.org To: common-commits@hadoop.apache.org Date: Thu, 08 Oct 2015 21:17:45 -0000 Message-Id: <27b2b8286a5a40f99cd36d978442f39a@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [34/35] hadoop git commit: HDFS-9204. DatanodeDescriptor#PendingReplicationWithoutTargets is wrongly calculated. Contributed by Mingliang Liu. HDFS-9204. DatanodeDescriptor#PendingReplicationWithoutTargets is wrongly calculated. Contributed by Mingliang Liu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/118a35bc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/118a35bc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/118a35bc Branch: refs/heads/HDFS-8966 Commit: 118a35bc2eabe3918b4797a1b626e9a39d77754b Parents: 0841940 Author: Jing Zhao Authored: Thu Oct 8 14:11:02 2015 -0700 Committer: Jing Zhao Committed: Thu Oct 8 14:11:02 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/blockmanagement/DatanodeDescriptor.java | 8 ++++---- .../hadoop/hdfs/server/blockmanagement/ReplicationWork.java | 3 +++ .../server/blockmanagement/TestUnderReplicatedBlocks.java | 5 +++++ 4 files changed, 15 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/118a35bc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1116f80..95351a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -356,6 +356,9 @@ Trunk (Unreleased) HDFS-8800. hdfs --daemon stop namenode corrupts logs (John Smith via aw) + HDFS-9204. DatanodeDescriptor#PendingReplicationWithoutTargets is wrongly + calculated. (Mingliang Liu via jing9) + BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS HDFS-7347. Configurable erasure coding policy for individual files and http://git-wip-us.apache.org/repos/asf/hadoop/blob/118a35bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index b258f06..fde645e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -256,7 +256,7 @@ public class DatanodeDescriptor extends DatanodeInfo { private boolean disallowed = false; // The number of replication work pending before targets are determined - private int PendingReplicationWithoutTargets = 0; + private int pendingReplicationWithoutTargets = 0; // HB processing can use it to tell if it is the first HB since DN restarted private boolean heartbeatedSinceRegistration = false; @@ -594,11 +594,11 @@ public class DatanodeDescriptor extends DatanodeInfo { } void incrementPendingReplicationWithoutTargets() { - PendingReplicationWithoutTargets++; + pendingReplicationWithoutTargets++; } void decrementPendingReplicationWithoutTargets() { - PendingReplicationWithoutTargets--; + pendingReplicationWithoutTargets--; } /** @@ -651,7 +651,7 @@ public class DatanodeDescriptor extends DatanodeInfo { * The number of work items that are pending to be replicated */ int getNumberOfBlocksToBeReplicated() { - return PendingReplicationWithoutTargets + replicateBlocks.size(); + return pendingReplicationWithoutTargets + replicateBlocks.size(); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/118a35bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java index 8266f45..8a3900c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java @@ -30,6 +30,9 @@ class ReplicationWork extends BlockRecoveryWork { int priority) { super(block, bc, srcNodes, containingNodes, liveReplicaStorages, additionalReplRequired, priority); + assert getSrcNodes().length == 1 : + "There should be exactly 1 source node that have been selected"; + getSrcNodes()[0].incrementPendingReplicationWithoutTargets(); BlockManager.LOG.debug("Creating a ReplicationWork to recover " + block); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/118a35bc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java index 27b35f0..c0b54b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; import java.util.Iterator; @@ -135,6 +136,10 @@ public class TestUnderReplicatedBlocks { assertEquals(NUM_OF_BLOCKS, bm.getUnderReplicatedNotMissingBlocks()); bm.computeDatanodeWork(); + assertTrue("The number of replication work pending before targets are " + + "determined should be non-negative.", + (Integer)Whitebox.getInternalState(secondDn, + "pendingReplicationWithoutTargets") >= 0); assertTrue("The number of blocks to be replicated should be less than " + "or equal to " + bm.replicationStreamsHardLimit,