From common-commits-return-82198-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Wed May 2 21:21:41 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 0FBFE180677 for ; Wed, 2 May 2018 21:21:40 +0200 (CEST) Received: (qmail 2913 invoked by uid 500); 2 May 2018 19:21:37 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 2518 invoked by uid 99); 2 May 2018 19:21:37 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 May 2018 19:21:37 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B7925E061B; Wed, 2 May 2018 19:21:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shv@apache.org To: common-commits@hadoop.apache.org Date: Wed, 02 May 2018 19:21:44 -0000 Message-Id: <158da3a6b1bf4a6e8451f75fd7d0d82e@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [09/50] [abbrv] hadoop git commit: HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie. HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7721082 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7721082 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7721082 Branch: refs/heads/YARN-8200 Commit: a7721082dc3d608dc8a3ac7e1ffa0de21781f00b Parents: 31d061e Author: Yiqun Lin Authored: Fri Apr 13 17:59:35 2018 +0800 Committer: Yiqun Lin Committed: Fri Apr 13 17:59:35 2018 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +++ .../hadoop/hdfs/net/DFSNetworkTopology.java | 9 +++- .../src/main/resources/hdfs-default.xml | 14 ++++++ .../blockmanagement/TestDatanodeManager.java | 52 ++++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7721082/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 42ce05b..c9fef06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker; @@ -1003,6 +1004,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.use.dfs.network.topology"; public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true; + public static final String DFS_NET_TOPOLOGY_IMPL_KEY = + "dfs.net.topology.impl"; + + public static final Class DFS_NET_TOPOLOGY_IMPL_DEFAULT = + DFSNetworkTopology.class; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7721082/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index e74cdec..f3074d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -22,11 +22,13 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; +import org.apache.hadoop.util.ReflectionUtils; import java.util.ArrayList; import java.util.Collection; @@ -44,8 +46,11 @@ public class DFSNetworkTopology extends NetworkTopology { private static final Random RANDOM = new Random(); public static DFSNetworkTopology getInstance(Configuration conf) { - DFSNetworkTopology nt = new DFSNetworkTopology(); - return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); + DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass( + DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, + DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT, + DFSNetworkTopology.class), conf); + return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7721082/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 282514a..66f9718 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4360,6 +4360,20 @@ true Enables DFSNetworkTopology to choose nodes for placing replicas. + When enabled, NetworkTopology will be instantiated as class defined in + property dfs.net.topology.impl, otherwise NetworkTopology will be + instantiated as class defined in property net.topology.impl. + + + + + dfs.net.topology.impl + org.apache.hadoop.hdfs.net.DFSNetworkTopology + + The implementation class of NetworkTopology used in HDFS. By default, + the class org.apache.hadoop.hdfs.net.DFSNetworkTopology is specified and + used in block placement. + This property only works when dfs.use.dfs.network.topology is true. http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7721082/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java index 30e2aaf..6e1685b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java @@ -40,6 +40,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -52,11 +54,13 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.net.DNSToSwitchMapping; +import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.util.Shell; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; + import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; @@ -491,4 +495,52 @@ public class TestDatanodeManager { Assert.assertEquals("Unexpected host or host in unexpected position", "127.0.0.1:23456", bothAgain.get(1).getInfoAddr()); } + + @Test + public void testNetworkTopologyInstantiation() throws Exception { + // case 1, dfs.use.dfs.network.topology=true, use the default + // DFSNetworkTopology impl. + Configuration conf1 = new HdfsConfiguration(); + FSNamesystem fsn = Mockito.mock(FSNamesystem.class); + DatanodeManager dm1 = mockDatanodeManager(fsn, conf1); + assertEquals(DFSNetworkTopology.class, dm1.getNetworkTopology().getClass()); + + // case 2, dfs.use.dfs.network.topology=false, use the default + // NetworkTopology impl. + Configuration conf2 = new HdfsConfiguration(); + conf2.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false); + DatanodeManager dm2 = mockDatanodeManager(fsn, conf2); + assertEquals(NetworkTopology.class, dm2.getNetworkTopology() + .getClass()); + + // case 3, dfs.use.dfs.network.topology=false, and specify the + // net.topology.impl property. + Configuration conf3 = new HdfsConfiguration(); + conf3.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, + MockDfsNetworkTopology.class, NetworkTopology.class); + conf3.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false); + DatanodeManager dm3 = mockDatanodeManager(fsn, conf3); + assertEquals(MockDfsNetworkTopology.class, dm3.getNetworkTopology() + .getClass()); + + // case 4, dfs.use.dfs.network.topology=true, and specify the + // dfs.net.topology.impl property. + Configuration conf4 = new HdfsConfiguration(); + conf4.setClass(DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, + MockDfsNetworkTopology.class, NetworkTopology.class); + conf4.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true); + DatanodeManager dm4 = mockDatanodeManager(fsn, conf4); + assertEquals(MockDfsNetworkTopology.class, dm4.getNetworkTopology() + .getClass()); + } + + /** + * A NetworkTopology implementation for test. + * + */ + public static class MockDfsNetworkTopology extends DFSNetworkTopology { + public MockDfsNetworkTopology(){ + super(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org