hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iwasak...@apache.org
Subject hadoop git commit: HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets for DataTransferProtocol (Gary Helmling via iwasakims)
Date Fri, 12 Feb 2016 18:43:24 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 df23f9c49 -> 2dd80938a


HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets for DataTransferProtocol
(Gary Helmling via iwasakims)

(cherry picked from commit 372d1302c63c6f49f99be5766c5da9647ebd9ca6)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java

(cherry picked from commit f71fa500f9a43d928fbd27c873544898bff50dd4)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2dd80938
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2dd80938
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2dd80938

Branch: refs/heads/branch-2.8
Commit: 2dd80938a287416807af29238cb9a831085574e1
Parents: df23f9c
Author: Masatake Iwasaki <iwasakims@apache.org>
Authored: Sat Feb 13 03:31:15 2016 +0900
Committer: Masatake Iwasaki <iwasakims@apache.org>
Committed: Sat Feb 13 03:34:07 2016 +0900

----------------------------------------------------------------------
 .../main/java/org/apache/hadoop/hdfs/DFSClient.java   |  1 +
 .../java/org/apache/hadoop/hdfs/DataStreamer.java     |  1 +
 .../hadoop/hdfs/client/HdfsClientConfigKeys.java      |  4 ++++
 .../apache/hadoop/hdfs/client/impl/DfsClientConf.java | 14 ++++++++++++++
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt           |  3 +++
 5 files changed, 23 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dd80938/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 6271da2..e0f00ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1950,6 +1950,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
       String dnAddr = dn.getXferAddr(getConf().isConnectToDnViaHostname());
       LOG.debug("Connecting to datanode {}", dnAddr);
       NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
+      sock.setTcpNoDelay(dfsClientConf.getDataTransferTcpNoDelay());
       sock.setSoTimeout(timeout);
 
       OutputStream unbufOut = NetUtils.getOutputStream(sock);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dd80938/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 5daf2ac..9ccb89a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -139,6 +139,7 @@ class DataStreamer extends Daemon {
     final int timeout = client.getDatanodeReadTimeout(length);
     NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(),
         conf.getSocketTimeout());
+    sock.setTcpNoDelay(conf.getDataTransferTcpNoDelay());
     sock.setSoTimeout(timeout);
     sock.setKeepAlive(true);
     if (conf.getSocketSendBufferSize() > 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dd80938/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 992cf3a..fb823e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -164,6 +164,10 @@ public interface HdfsClientConfigKeys {
   String  DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix";
   String  DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user";
 
+  String DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY =
+      "dfs.data.transfer.client.tcpnodelay";
+  boolean DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT = true;
+
   /**
    * These are deprecated config keys to client code.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dd80938/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index 7f3ae04..ed6cd23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -57,6 +57,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCK
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
@@ -135,6 +137,8 @@ public class DfsClientConf {
   private final List<Class<? extends ReplicaAccessorBuilder>>
       replicaAccessorBuilderClasses;
 
+  private final boolean dataTransferTcpNoDelay;
+
   public DfsClientConf(Configuration conf) {
     // The hdfsTimeout is currently the same as the ipc timeout
     hdfsTimeout = Client.getTimeout(conf);
@@ -172,6 +176,9 @@ public class DfsClientConf {
         CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
         CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
     defaultChecksumOpt = getChecksumOptFromConf(conf);
+    dataTransferTcpNoDelay = conf.getBoolean(
+        DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY,
+        DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT);
     socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,
         HdfsConstants.READ_TIMEOUT);
     socketSendBufferSize = conf.getInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY,
@@ -408,6 +415,13 @@ public class DfsClientConf {
   }
 
   /**
+   * @return whether TCP_NODELAY should be set on client sockets
+   */
+  public boolean getDataTransferTcpNoDelay() {
+    return dataTransferTcpNoDelay;
+  }
+
+  /**
    * @return the socketTimeout
    */
   public int getSocketTimeout() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dd80938/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index dba28c3..586632f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -905,6 +905,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-9777. Fix typos in DFSAdmin command line and documentation.
     (Wei-Chiu Chuang via umamahesh)
 
+    HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets
+    for DataTransferProtocol (Gary Helmling via iwasakims)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than


Mime
View raw message