Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B9D5DDAC9 for ; Wed, 5 Sep 2012 00:04:58 +0000 (UTC) Received: (qmail 50632 invoked by uid 500); 5 Sep 2012 00:04:58 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 50596 invoked by uid 500); 5 Sep 2012 00:04:58 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 50588 invoked by uid 99); 5 Sep 2012 00:04:58 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 05 Sep 2012 00:04:58 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 05 Sep 2012 00:04:55 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id BB1AF2388900; Wed, 5 Sep 2012 00:04:11 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1380940 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./ CHANGES.txt src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Date: Wed, 05 Sep 2012 00:04:11 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120905000411.BB1AF2388900@eris.apache.org> Author: szetszwo Date: Wed Sep 5 00:04:11 2012 New Revision: 1380940 URL: http://svn.apache.org/viewvc?rev=1380940&view=rev Log: svn merge -c 1380939 from trunk for HDFS-3888. Clean up BlockPlacementPolicyDefault. Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1380939 Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1380940&r1=1380939&r2=1380940&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Sep 5 00:04:11 2012 @@ -242,6 +242,8 @@ Release 2.0.1-alpha - UNRELEASED HDFS-3887. Remove redundant chooseTarget methods in BlockPlacementPolicy. (Jing Zhao via szetszwo) + HDFS-3888. Clean up BlockPlacementPolicyDefault. (Jing Zhao via szetszwo) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1380939 Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1380940&r1=1380939&r2=1380940&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Wed Sep 5 00:04:11 2012 @@ -27,8 +27,6 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -55,9 +53,6 @@ import com.google.common.annotations.Vis @InterfaceAudience.Private public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { - private static final Log LOG = - LogFactory.getLog(BlockPlacementPolicyDefault.class.getName()); - private static final String enableDebugLogging = "For more information, please enable DEBUG log level on " + LOG.getClass().getName(); @@ -124,7 +119,6 @@ public class BlockPlacementPolicyDefault excludedNodes, blocksize); } - /** This is the implementation. */ DatanodeDescriptor[] chooseTarget(int numOfReplicas, DatanodeDescriptor writer, @@ -161,7 +155,8 @@ public class BlockPlacementPolicyDefault } DatanodeDescriptor localNode = chooseTarget(numOfReplicas, writer, - excludedNodes, blocksize, maxNodesPerRack, results); + excludedNodes, blocksize, + maxNodesPerRack, results); if (!returnChosenNodes) { results.removeAll(chosenNodes); } @@ -438,14 +433,29 @@ public class BlockPlacementPolicyDefault * does not have too much load, and the rack does not have too many nodes */ private boolean isGoodTarget(DatanodeDescriptor node, - long blockSize, int maxTargetPerLoc, + long blockSize, int maxTargetPerRack, List results) { - return isGoodTarget(node, blockSize, maxTargetPerLoc, + return isGoodTarget(node, blockSize, maxTargetPerRack, this.considerLoad, results); } - + + /** + * Determine if a node is a good target. + * + * @param node The target node + * @param blockSize Size of block + * @param maxTargetPerRack Maximum number of targets per rack. The value of + * this parameter depends on the number of racks in + * the cluster and total number of replicas for a block + * @param considerLoad whether or not to consider load of the target node + * @param results A list containing currently chosen nodes. Used to check if + * too many nodes has been chosen in the target rack. + * @return Return true if node has enough space, + * does not have too much load, + * and the rack does not have too many nodes. + */ private boolean isGoodTarget(DatanodeDescriptor node, - long blockSize, int maxTargetPerLoc, + long blockSize, int maxTargetPerRack, boolean considerLoad, List results) { // check if the node is (being) decommissed @@ -497,7 +507,7 @@ public class BlockPlacementPolicyDefault counter++; } } - if (counter>maxTargetPerLoc) { + if (counter>maxTargetPerRack) { if(LOG.isDebugEnabled()) { threadLocalBuilder.get().append(node.toString()).append(": ") .append("Node ").append(NodeBase.getPath(node))