Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 19C86200B6D for ; Sat, 16 Jul 2016 05:21:27 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 18A35160A8A; Sat, 16 Jul 2016 03:21:27 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3898F160A79 for ; Sat, 16 Jul 2016 05:21:26 +0200 (CEST) Received: (qmail 87883 invoked by uid 500); 16 Jul 2016 03:21:24 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 85669 invoked by uid 99); 16 Jul 2016 03:21:22 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 16 Jul 2016 03:21:22 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8639FED978; Sat, 16 Jul 2016 03:21:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sjlee@apache.org To: common-commits@hadoop.apache.org Date: Sat, 16 Jul 2016 03:21:35 -0000 Message-Id: In-Reply-To: <75c79aea7cb54f018b55882ecee4c183@git.apache.org> References: <75c79aea7cb54f018b55882ecee4c183@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [14/50] hadoop git commit: HDFS-10544. Balancer doesn't work with IPFailoverProxyProvider. archived-at: Sat, 16 Jul 2016 03:21:27 -0000 HDFS-10544. Balancer doesn't work with IPFailoverProxyProvider. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/087290e6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/087290e6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/087290e6 Branch: refs/heads/YARN-5355 Commit: 087290e6b1cb1082646d966b65494082712ebe3e Parents: 06c56ff Author: Zhe Zhang Authored: Tue Jul 12 23:18:37 2016 -0700 Committer: Zhe Zhang Committed: Tue Jul 12 23:18:37 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 42 ++++++++++++------ .../org/apache/hadoop/hdfs/TestDFSUtil.java | 45 +++++++++++++++++++- 2 files changed, 73 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/087290e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 5236ee5..f1a6de7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -718,10 +718,11 @@ public class DFSUtil { /** * Get a URI for each internal nameservice. If a nameservice is - * HA-enabled, then the logical URI of the nameservice is returned. If the - * nameservice is not HA-enabled, then a URI corresponding to an RPC address - * of the single NN for that nameservice is returned, preferring the service - * RPC address over the client RPC address. + * HA-enabled, and the configured failover proxy provider supports logical + * URIs, then the logical URI of the nameservice is returned. + * Otherwise, a URI corresponding to an RPC address of the single NN for that + * nameservice is returned, preferring the service RPC address over the + * client RPC address. * * @param conf configuration * @return a collection of all configured NN URIs, preferring service @@ -735,9 +736,10 @@ public class DFSUtil { /** * Get a URI for each configured nameservice. If a nameservice is - * HA-enabled, then the logical URI of the nameservice is returned. If the - * nameservice is not HA-enabled, then a URI corresponding to the address of - * the single NN for that nameservice is returned. + * HA-enabled, and the configured failover proxy provider supports logical + * URIs, then the logical URI of the nameservice is returned. + * Otherwise, a URI corresponding to the address of the single NN for that + * nameservice is returned. * * @param conf configuration * @param keys configuration keys to try in order to get the URI for non-HA @@ -756,13 +758,27 @@ public class DFSUtil { Set nonPreferredUris = new HashSet(); for (String nsId : nameServices) { - if (HAUtil.isHAEnabled(conf, nsId)) { + URI nsUri; + try { + nsUri = new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId); + } catch (URISyntaxException ue) { + throw new IllegalArgumentException(ue); + } + /** + * Determine whether the logical URI of the name service can be resolved + * by the configured failover proxy provider. If not, we should try to + * resolve the URI here + */ + boolean useLogicalUri = false; + try { + useLogicalUri = HAUtil.useLogicalUri(conf, nsUri); + } catch (IOException e){ + LOG.warn("Getting exception while trying to determine if nameservice " + + nsId + " can use logical URI: " + e); + } + if (HAUtil.isHAEnabled(conf, nsId) && useLogicalUri) { // Add the logical URI of the nameservice. - try { - ret.add(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId)); - } catch (URISyntaxException ue) { - throw new IllegalArgumentException(ue); - } + ret.add(nsUri); } else { // Add the URI corresponding to the address of the NN. boolean uriFound = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/087290e6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 448ef6a..b139845 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -62,6 +62,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -533,7 +534,11 @@ public class TestDFSUtil { // Ditto for nameservice IDs, if multiple are defined assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf)); assertEquals(null, DFSUtil.getSecondaryNameServiceId(conf)); - + + String proxyProviderKey = HdfsClientConfigKeys.Failover. + PROXY_PROVIDER_KEY_PREFIX + ".ns2"; + conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha." + + "ConfiguredFailoverProxyProvider"); Collection uris = getInternalNameServiceUris(conf, DFS_NAMENODE_RPC_ADDRESS_KEY); assertEquals(2, uris.size()); assertTrue(uris.contains(new URI("hdfs://ns1"))); @@ -633,6 +638,7 @@ public class TestDFSUtil { public void testGetNNUris() throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); + final String NS1_NN_ADDR = "ns1-nn.example.com:9820"; final String NS1_NN1_ADDR = "ns1-nn1.example.com:9820"; final String NS1_NN2_ADDR = "ns1-nn2.example.com:9820"; final String NS2_NN_ADDR = "ns2-nn.example.com:9820"; @@ -665,12 +671,49 @@ public class TestDFSUtil { DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR); conf.set(DFSUtil.addKeySuffixes( + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), NS1_NN_ADDR); + + conf.set(DFSUtil.addKeySuffixes( DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns2"), NS2_NN_ADDR); conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, "hdfs://" + NN1_ADDR); conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN2_ADDR); + /** + * {@link DFSUtil#getInternalNsRpcUris} decides whether to resolve a logical + * URI based on whether the failover proxy provider supports logical URIs. + * We will test both cases. + * + * First configure ns1 to use {@link IPFailoverProxyProvider} which doesn't + * support logical Uris. So {@link DFSUtil#getInternalNsRpcUris} will + * resolve the logical URI of ns1 based on the configured value at + * dfs.namenode.servicerpc-address.ns1, which is {@link NS1_NN_ADDR} + */ + String proxyProviderKey = HdfsClientConfigKeys.Failover. + PROXY_PROVIDER_KEY_PREFIX + ".ns1"; + conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha." + + "IPFailoverProxyProvider"); + + uris = DFSUtil.getInternalNsRpcUris(conf); + assertEquals("Incorrect number of URIs returned", 3, uris.size()); + assertTrue("Missing URI for RPC address", + uris.contains(new URI("hdfs://" + NN1_ADDR))); + assertTrue("Missing URI for name service ns2", + uris.contains(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + + NS1_NN_ADDR))); + assertTrue("Missing URI for name service ns2", + uris.contains(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + + NS2_NN_ADDR))); + + /** + * Second, test ns1 with {@link ConfiguredFailoverProxyProvider} which does + * support logical URIs. So instead of {@link NS1_NN_ADDR}, the logical URI + * of ns1, hdfs://ns1, will be returned. + */ + conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha." + + "ConfiguredFailoverProxyProvider"); + uris = DFSUtil.getInternalNsRpcUris(conf); assertEquals("Incorrect number of URIs returned", 3, uris.size()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org