Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 171AADB89 for ; Tue, 6 Nov 2012 20:01:33 +0000 (UTC) Received: (qmail 67600 invoked by uid 500); 6 Nov 2012 20:01:32 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 67563 invoked by uid 500); 6 Nov 2012 20:01:32 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 67555 invoked by uid 99); 6 Nov 2012 20:01:32 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Nov 2012 20:01:32 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Nov 2012 20:01:30 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0190623888CD; Tue, 6 Nov 2012 20:01:09 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1406290 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ Date: Tue, 06 Nov 2012 20:01:08 -0000 To: hdfs-commits@hadoop.apache.org From: daryn@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121106200109.0190623888CD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: daryn Date: Tue Nov 6 20:01:08 2012 New Revision: 1406290 URL: http://svn.apache.org/viewvc?rev=1406290&view=rev Log: svn merge -c 1406278 FIXES: HDFS-4075. Reduce recommissioning overhead (Kihwal Lee via daryn) Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1406290&r1=1406289&r2=1406290&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Nov 6 20:01:08 2012 @@ -16,6 +16,8 @@ Release 0.23.5 - UNRELEASED HDFS-3483. Better error message when hdfs fsck is run against a ViewFS config. (Stephen Fritz via atm) + HDFS-4075. Reduce recommissioning overhead (Kihwal Lee via daryn) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1406290&r1=1406289&r2=1406290&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Tue Nov 6 20:01:08 2012 @@ -2311,6 +2311,7 @@ public class BlockManager { void processOverReplicatedBlocksOnReCommission( final DatanodeDescriptor srcNode) { final Iterator it = srcNode.getBlockIterator(); + int numOverReplicated = 0; while(it.hasNext()) { final Block block = it.next(); INodeFile fileINode = blocksMap.getINode(block); @@ -2320,8 +2321,11 @@ public class BlockManager { if (numCurrentReplica > expectedReplication) { // over-replicated block processOverReplicatedBlock(block, expectedReplication, null, null); + numOverReplicated++; } } + LOG.info("Invalidated " + numOverReplicated + " over-replicated blocks on " + + srcNode + " during recommissioning"); } /** Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1406290&r1=1406289&r2=1406290&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Tue Nov 6 20:01:08 2012 @@ -522,7 +522,11 @@ public class DatanodeManager { if (node.isDecommissionInProgress() || node.isDecommissioned()) { LOG.info("Stop Decommissioning node " + node.getName()); heartbeatManager.stopDecommission(node); - blockManager.processOverReplicatedBlocksOnReCommission(node); + // Over-replicated blocks will be detected and processed when + // the dead node comes back and send in its full block report. + if (node.isAlive) { + blockManager.processOverReplicatedBlocksOnReCommission(node); + } } }