Return-Path: Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: (qmail 95152 invoked from network); 4 Mar 2011 04:51:35 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 4 Mar 2011 04:51:35 -0000 Received: (qmail 41095 invoked by uid 500); 4 Mar 2011 04:51:35 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 41055 invoked by uid 500); 4 Mar 2011 04:51:34 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 41034 invoked by uid 99); 4 Mar 2011 04:51:34 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Mar 2011 04:51:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Mar 2011 04:51:32 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 1269A238899C; Fri, 4 Mar 2011 04:51:12 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1077748 - in /hadoop/common/branches/branch-0.20-security-patches/src: hdfs/org/apache/hadoop/hdfs/ hdfs/org/apache/hadoop/hdfs/protocol/ hdfs/org/apache/hadoop/hdfs/server/namenode/ test/org/apache/hadoop/hdfs/server/namenode/ Date: Fri, 04 Mar 2011 04:51:11 -0000 To: common-commits@hadoop.apache.org From: omalley@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110304045112.1269A238899C@eris.apache.org> Author: omalley Date: Fri Mar 4 04:51:11 2011 New Revision: 1077748 URL: http://svn.apache.org/viewvc?rev=1077748&view=rev Log: commit f8cfbf27caaad5a6987ee05bf84355bfa808fa0b Author: Tsz Wo Wo Sze Date: Thu Nov 25 00:34:37 2010 +0000 . Make BLOCK_INVALIDATE_LIMIT configurable. +++ b/YAHOO-CHANGES.txt + . Make BLOCK_INVALIDATE_LIMIT configurable. (szetszwo) + Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1077748&r1=1077747&r2=1077748&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java Fri Mar 4 04:51:11 2011 @@ -189,6 +189,8 @@ public class DFSConfigKeys extends Commo public static final long DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT = 21600000; public static final String DFS_BLOCKREPORT_INITIAL_DELAY_KEY = "dfs.blockreport.initialDelay"; public static final int DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT = 0; + public static final String DFS_BLOCK_INVALIDATE_LIMIT_KEY = "dfs.block.invalidate.limit"; + public static final int DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT = 100; //Keys with no defaults public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1077748&r1=1077747&r2=1077748&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java Fri Mar 4 04:51:11 2011 @@ -26,9 +26,6 @@ import org.apache.hadoop.conf.Configurat public interface FSConstants { public static int MIN_BLOCKS_FOR_WRITE = 5; - // Chunk the block Invalidate message - public static final int BLOCK_INVALIDATE_CHUNK = 100; - // Long that indicates "leave current quota unchanged" public static final long QUOTA_DONT_SET = Long.MAX_VALUE; public static final long QUOTA_RESET = -1L; Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1077748&r1=1077747&r2=1077748&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Mar 4 04:51:11 2011 @@ -337,7 +337,7 @@ public class FSNamesystem implements FSC private final GenerationStamp generationStamp = new GenerationStamp(); // Ask Datanode only up to this many blocks to delete. - private int blockInvalidateLimit = FSConstants.BLOCK_INVALIDATE_CHUNK; + private int blockInvalidateLimit = DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT; // precision of access times. private long accessTimePrecision = 0; @@ -504,8 +504,15 @@ public class FSNamesystem implements FSC conf.getInt("dfs.replication.interval", 3) * 1000L; this.defaultBlockSize = conf.getLong("dfs.block.size", DEFAULT_BLOCK_SIZE); this.maxFsObjects = conf.getLong("dfs.max.objects", 0); + + //default limit this.blockInvalidateLimit = Math.max(this.blockInvalidateLimit, 20*(int)(heartbeatInterval/1000)); + //use conf value if it is set. + this.blockInvalidateLimit = conf.getInt( + DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY, this.blockInvalidateLimit); + LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY + "=" + this.blockInvalidateLimit); + this.accessTimePrecision = conf.getLong("dfs.access.time.precision", 0); this.supportAppends = conf.getBoolean("dfs.support.append", false); this.isAccessTokenEnabled = conf.getBoolean( Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java?rev=1077748&r1=1077747&r2=1077748&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java Fri Mar 4 04:51:11 2011 @@ -2,7 +2,10 @@ package org.apache.hadoop.hdfs.server.na import java.util.ArrayList; +import junit.framework.TestCase; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.GenerationStamp; @@ -11,8 +14,6 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; -import junit.framework.TestCase; - /** * Test if FSNamesystem handles heartbeat right */ @@ -33,7 +34,7 @@ public class TestHeartbeatHandling exten final int REMAINING_BLOCKS = 1; final int MAX_REPLICATE_LIMIT = conf.getInt("dfs.max-repl-streams", 2); - final int MAX_INVALIDATE_LIMIT = FSNamesystem.BLOCK_INVALIDATE_CHUNK; + final int MAX_INVALIDATE_LIMIT = DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT; final int MAX_INVALIDATE_BLOCKS = 2*MAX_INVALIDATE_LIMIT+REMAINING_BLOCKS; final int MAX_REPLICATE_BLOCKS = 2*MAX_REPLICATE_LIMIT+REMAINING_BLOCKS; final DatanodeDescriptor[] ONE_TARGET = new DatanodeDescriptor[1];