Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0850A18111 for ; Thu, 18 Feb 2016 19:28:39 +0000 (UTC) Received: (qmail 5403 invoked by uid 500); 18 Feb 2016 19:28:30 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 5340 invoked by uid 500); 18 Feb 2016 19:28:30 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 5330 invoked by uid 99); 18 Feb 2016 19:28:30 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 18 Feb 2016 19:28:30 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7EA34DFF67; Thu, 18 Feb 2016 19:28:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kihwal@apache.org To: common-commits@hadoop.apache.org Message-Id: <4b8d5c5923744ac99dc47253078f3cb8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be configurable. Contributed by James Kinley and Nathan Roberts. Date: Thu, 18 Feb 2016 19:28:30 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.8 e5c8a3444 -> bb8079e55 HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be configurable. Contributed by James Kinley and Nathan Roberts. (cherry picked from commit ba6d5ed955aa34cdcb7aaa4b1025da6ee5e91355) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb8079e5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb8079e5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb8079e5 Branch: refs/heads/branch-2.8 Commit: bb8079e5538c04e0de044ac1769655dc20d69004 Parents: e5c8a34 Author: Kihwal Lee Authored: Thu Feb 18 13:28:17 2016 -0600 Committer: Kihwal Lee Committed: Thu Feb 18 13:28:17 2016 -0600 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +++ .../blockmanagement/BlockPlacementPolicyDefault.java | 7 ++++++- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 12 ++++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8079e5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e806e0a..e402030 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1803,6 +1803,9 @@ Release 2.7.3 - UNRELEASED HDFS-9669. TcpPeerServer should respect ipc.server.listen.queue.size (Elliot Clark via cmccabe) + HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be + configurable (James Kinley and Nathan Roberts via kihwal) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8079e5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index a4539d0..69c1dc2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -631,6 +631,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction"; public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; + public static final String DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY = + "dfs.namenode.block-placement-policy.default.prefer-local-node"; + public static final boolean DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT = true; public static final String DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY = "dfs.block.local-path-access.user"; public static final String DFS_DOMAIN_SOCKET_PATH_KEY = http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8079e5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 14439e7..ee891a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -59,7 +59,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { protected boolean considerLoad; protected double considerLoadFactor; - private boolean preferLocalNode = true; + private boolean preferLocalNode; protected NetworkTopology clusterMap; protected Host2NodesMap host2datanodeMap; private FSClusterStats stats; @@ -95,6 +95,11 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { this.staleInterval = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT); + this.preferLocalNode = conf.getBoolean( + DFSConfigKeys. + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY, + DFSConfigKeys. + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8079e5/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index f7ec36a..aa4eda9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -922,6 +922,18 @@ + dfs.namenode.block-placement-policy.default.prefer-local-node + true + Controls how the default block placement policy places + the first replica of a block. When true, it will prefer the node where + the client is running. When false, it will prefer a node in the same rack + as the client. Setting to false avoids situations where entire copies of + large files end up on a single node, thus creating hotspots. + + + + + dfs.stream-buffer-size 4096 The size of buffer to stream files.