Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1DAC512E7E for ; Sun, 11 May 2014 01:59:26 +0000 (UTC) Received: (qmail 36625 invoked by uid 500); 10 May 2014 23:28:58 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 95046 invoked by uid 500); 10 May 2014 23:18:52 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 12462 invoked by uid 99); 10 May 2014 23:04:28 -0000 Received: from Unknown (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 10 May 2014 23:04:28 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 09 May 2014 02:08:46 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0E91E23888E2; Fri, 9 May 2014 02:08:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1593477 - in /hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/web/ src/test/java/org/apache/hadoop/hdfs/ src/test/java/org/apache/hadoop/hdfs/... Date: Fri, 09 May 2014 02:08:21 -0000 To: hdfs-commits@hadoop.apache.org From: kihwal@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140509020822.0E91E23888E2@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kihwal Date: Fri May 9 02:08:21 2014 New Revision: 1593477 URL: http://svn.apache.org/r1593477 Log: HDFS-6313. WebHdfs may use the wrong NN when configured for multiple HA NNs. Contributed by Kihwal Lee. Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1593477&r1=1593476&r2=1593477&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri May 9 02:08:21 2014 @@ -63,6 +63,9 @@ Release 2.4.1 - UNRELEASED HDFS-6329. WebHdfs does not work if HA is enabled on NN but logical URI is not configured. (kihwal) + HDFS-6313. WebHdfs may use the wrong NN when configured for multiple HA NNs. + (kihwal) + Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1593477&r1=1593476&r2=1593477&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Fri May 9 02:08:21 2014 @@ -759,11 +759,10 @@ public class DFSUtil { } else { Map> addresses = DFSUtil .getHaNnWebHdfsAddresses(conf, scheme); - - for (Map addrs : addresses.values()) { - for (InetSocketAddress addr : addrs.values()) { - ret.add(addr); - } + // Extract the entry corresponding to the logical name. + Map addrs = addresses.get(uri.getHost()); + for (InetSocketAddress addr : addrs.values()) { + ret.add(addr); } } Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1593477&r1=1593476&r2=1593477&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Fri May 9 02:08:21 2014 @@ -100,6 +100,7 @@ import org.apache.hadoop.security.token. import org.apache.hadoop.util.Progressable; import org.mortbay.util.ajax.JSON; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; import com.google.common.collect.Lists; @@ -1078,4 +1079,9 @@ public class WebHdfsFileSystem extends F return tokenServiceName == null ? super.getCanonicalServiceName() : tokenServiceName.toString(); } + + @VisibleForTesting + InetSocketAddress[] getResolvedNNAddr() { + return nnAddrs; + } } Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1593477&r1=1593476&r2=1593477&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Fri May 9 02:08:21 2014 @@ -147,16 +147,40 @@ public class DFSTestUtil { */ public static Configuration newHAConfiguration(final String logicalName) { Configuration conf = new Configuration(); - conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName); + addHAConfiguration(conf, logicalName); + return conf; + } + + /** + * Add a new HA configuration. + */ + public static void addHAConfiguration(Configuration conf, + final String logicalName) { + String nsIds = conf.get(DFSConfigKeys.DFS_NAMESERVICES); + if (nsIds == null) { + conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName); + } else { // append the nsid + conf.set(DFSConfigKeys.DFS_NAMESERVICES, nsIds + "," + logicalName); + } conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, logicalName), "nn1,nn2"); conf.set(DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "" + "." + logicalName, ConfiguredFailoverProxyProvider.class.getName()); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); - return conf; } + public static void setFakeHttpAddresses(Configuration conf, + final String logicalName) { + conf.set(DFSUtil.addKeySuffixes( + DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, + logicalName, "nn1"), "127.0.0.1:12345"); + conf.set(DFSUtil.addKeySuffixes( + DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, + logicalName, "nn2"), "127.0.0.1:12346"); + } + + /** class MyFile contains enough information to recreate the contents of * a single file. */ Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java?rev=1593477&r1=1593476&r2=1593477&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java Fri May 9 02:08:21 2014 @@ -156,4 +156,30 @@ public class TestWebHDFSForHA { } } } + + @Test + public void testMultipleNamespacesConfigured() throws Exception { + Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME); + MiniDFSCluster cluster = null; + WebHdfsFileSystem fs = null; + + try { + cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo) + .numDataNodes(1).build(); + + HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME); + + cluster.waitActive(); + DFSTestUtil.addHAConfiguration(conf, LOGICAL_NAME + "remote"); + DFSTestUtil.setFakeHttpAddresses(conf, LOGICAL_NAME + "remote"); + + fs = (WebHdfsFileSystem)FileSystem.get(WEBHDFS_URI, conf); + Assert.assertEquals(2, fs.getResolvedNNAddr().length); + } finally { + IOUtils.cleanup(null, fs); + if (cluster != null) { + cluster.shutdown(); + } + } + } }