hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From whe...@apache.org
Subject [37/50] [abbrv] hadoop git commit: HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies. Contributed by Xiao Chen.
Date Thu, 12 Nov 2015 18:22:49 GMT
HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies. Contributed by
Xiao Chen.

Change-Id: I9e42f724f27924cf73891425a832de37ce014a1e


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73b94d78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73b94d78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73b94d78

Branch: refs/heads/HDFS-8707
Commit: 73b94d789969354bb9a6872d99976763ca8470d7
Parents: 0eb9c60
Author: Zhe Zhang <zhz@apache.org>
Authored: Tue Nov 10 09:55:29 2015 -0800
Committer: Zhe Zhang <zhz@apache.org>
Committed: Tue Nov 10 09:55:29 2015 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hdfs/DFSUtilClient.java   | 22 +++++++++++-
 .../hadoop/hdfs/NameNodeProxiesClient.java      |  2 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../hadoop/hdfs/TestDFSClientFailover.java      | 35 ++++++++++++++++++++
 4 files changed, 60 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index c14d5d3..62c5d81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -615,7 +615,7 @@ public class DFSUtilClient {
 
   public static InetSocketAddress getNNAddress(Configuration conf) {
     URI filesystemURI = FileSystem.getDefaultUri(conf);
-    return getNNAddress(filesystemURI);
+    return getNNAddressCheckLogical(conf, filesystemURI);
   }
 
   /**
@@ -638,6 +638,26 @@ public class DFSUtilClient {
     return getNNAddress(authority);
   }
 
+  /**
+   * Get the NN address from the URI. If the uri is logical, default address is
+   * returned. Otherwise return the DNS-resolved address of the URI.
+   *
+   * @param conf configuration
+   * @param filesystemURI URI of the file system
+   * @return address of file system
+   */
+  public static InetSocketAddress getNNAddressCheckLogical(Configuration conf,
+      URI filesystemURI) {
+    InetSocketAddress retAddr;
+    if (HAUtilClient.isLogicalUri(conf, filesystemURI)) {
+      retAddr = InetSocketAddress.createUnresolved(filesystemURI.getAuthority(),
+          HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
+    } else {
+      retAddr = getNNAddress(filesystemURI);
+    }
+    return retAddr;
+  }
+
   public static URI getNNUri(InetSocketAddress namenode) {
     int port = namenode.getPort();
     String portString =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
index 39b188f..5ca7030 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
@@ -320,7 +320,7 @@ public class NameNodeProxiesClient {
           DFSUtilClient.getNNAddress(nameNodeUri));
     }
     return new ProxyAndInfo<>(proxy, dtService,
-        DFSUtilClient.getNNAddress(nameNodeUri));
+        DFSUtilClient.getNNAddressCheckLogical(conf, nameNodeUri));
   }
 
   public static ClientProtocol createNonHAProxyWithClientProtocol(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8625ecf..336db03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2285,6 +2285,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-9401. Fix findbugs warnings in BlockRecoveryWorker.
     (Brahma Reddy Battula via waltersu4549)
 
+    HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies.
+    (Xiao Chen via zhz)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
index b098711..60ffe30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
 import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.hdfs.server.namenode.ha.IPFailoverProxyProvider;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.retry.FailoverProxyProvider;
 import org.apache.hadoop.net.ConnectTimeoutException;
@@ -291,6 +292,40 @@ public class TestDFSClientFailover {
     Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost));
   }
 
+  /**
+   * Test that creating proxy doesn't ever try to DNS-resolve the logical URI.
+   * Regression test for HDFS-9364.
+   */
+  @Test(timeout=60000)
+  public void testCreateProxyDoesntDnsResolveLogicalURI() throws IOException {
+    final NameService spyNS = spyOnNameService();
+    final Configuration conf = new HdfsConfiguration();
+    final String service = "nameservice1";
+    final String namenode = "namenode113";
+    conf.set(DFSConfigKeys.DFS_NAMESERVICES, service);
+    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://" + service);
+    conf.set(
+        HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + service,
+        "org.apache.hadoop.hdfs.server.namenode.ha."
+        + "ConfiguredFailoverProxyProvider");
+    conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + service,
+        namenode);
+    conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + service + "."
+        + namenode, "localhost:8020");
+
+    // call createProxy implicitly and explicitly
+    Path p = new Path("/");
+    p.getFileSystem(conf);
+    NameNodeProxiesClient.createProxyWithClientProtocol(conf,
+        FileSystem.getDefaultUri(conf), null);
+    NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf),
+        NamenodeProtocol.class, null);
+
+    // Ensure that the logical hostname was never resolved.
+    Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(
+        Mockito.eq(service));
+  }
+
   /** Dummy implementation of plain FailoverProxyProvider */
   public static class DummyLegacyFailoverProxyProvider<T>
       implements FailoverProxyProvider<T> {


Mime
View raw message