hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iwasak...@apache.org
Subject hadoop git commit: HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets for DataTransferProtocol (Gary Helmling via iwasakims)
Date Fri, 12 Feb 2016 18:38:25 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk e6a7044b8 -> 372d1302c


HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets for DataTransferProtocol
(Gary Helmling via iwasakims)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/372d1302
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/372d1302
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/372d1302

Branch: refs/heads/trunk
Commit: 372d1302c63c6f49f99be5766c5da9647ebd9ca6
Parents: e6a7044
Author: Masatake Iwasaki <iwasakims@apache.org>
Authored: Sat Feb 13 03:31:15 2016 +0900
Committer: Masatake Iwasaki <iwasakims@apache.org>
Committed: Sat Feb 13 03:31:15 2016 +0900

----------------------------------------------------------------------
 .../main/java/org/apache/hadoop/hdfs/DFSClient.java    |  1 +
 .../main/java/org/apache/hadoop/hdfs/DataStreamer.java |  1 +
 .../hadoop/hdfs/client/HdfsClientConfigKeys.java       |  4 ++++
 .../apache/hadoop/hdfs/client/impl/DfsClientConf.java  | 13 +++++++++++++
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt            |  3 +++
 5 files changed, 22 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 9c00ea7..15a49f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1890,6 +1890,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
       String dnAddr = dn.getXferAddr(getConf().isConnectToDnViaHostname());
       LOG.debug("Connecting to datanode {}", dnAddr);
       NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
+      sock.setTcpNoDelay(dfsClientConf.getDataTransferTcpNoDelay());
       sock.setSoTimeout(timeout);
 
       OutputStream unbufOut = NetUtils.getOutputStream(sock);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index e3843de..9d3cb55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -131,6 +131,7 @@ class DataStreamer extends Daemon {
     final int timeout = client.getDatanodeReadTimeout(length);
     NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(),
         conf.getSocketTimeout());
+    sock.setTcpNoDelay(conf.getDataTransferTcpNoDelay());
     sock.setSoTimeout(timeout);
     sock.setKeepAlive(true);
     if (conf.getSocketSendBufferSize() > 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index fcfd49c..775907e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -160,6 +160,10 @@ public interface HdfsClientConfigKeys {
   String  DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix";
   String  DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user";
 
+  String DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY =
+      "dfs.data.transfer.client.tcpnodelay";
+  boolean DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT = true;
+
   /**
    * These are deprecated config keys to client code.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index 194f3ba..eae3c8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -58,6 +58,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCK
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
@@ -135,6 +137,7 @@ public class DfsClientConf {
 
   private final int stripedReadThreadpoolSize;
 
+  private final boolean dataTransferTcpNoDelay;
 
   public DfsClientConf(Configuration conf) {
     // The hdfsTimeout is currently the same as the ipc timeout
@@ -173,6 +176,9 @@ public class DfsClientConf {
         CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
         CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
     defaultChecksumOpt = getChecksumOptFromConf(conf);
+    dataTransferTcpNoDelay = conf.getBoolean(
+        DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY,
+        DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT);
     socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,
         HdfsConstants.READ_TIMEOUT);
     socketSendBufferSize = conf.getInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY,
@@ -406,6 +412,13 @@ public class DfsClientConf {
   }
 
   /**
+   * @return whether TCP_NODELAY should be set on client sockets
+   */
+  public boolean getDataTransferTcpNoDelay() {
+    return dataTransferTcpNoDelay;
+  }
+
+  /**
    * @return the socketTimeout
    */
   public int getSocketTimeout() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 27c4c0f..8d60d5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1908,6 +1908,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-9777. Fix typos in DFSAdmin command line and documentation.
     (Wei-Chiu Chuang via umamahesh)
 
+    HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets
+    for DataTransferProtocol (Gary Helmling via iwasakims)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than


Mime
View raw message