hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oz...@apache.org
Subject hadoop git commit: HADOOP-11226. Add a configuration to set ipc.Client's traffic class with IPTOS_LOWDELAY|IPTOS_RELIABILITY. Contributed by Gopal V.
Date Tue, 10 Mar 2015 04:16:47 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 402145174 -> 5efee5efd


HADOOP-11226. Add a configuration to set ipc.Client's traffic class with IPTOS_LOWDELAY|IPTOS_RELIABILITY.
Contributed by Gopal V.

(cherry picked from commit 54639c7d7a34f4a46e8df50d57c79bab34b1ac07)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5efee5ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5efee5ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5efee5ef

Branch: refs/heads/branch-2
Commit: 5efee5efd3d21cef66e19e0a44d6ed6809658bdd
Parents: 4021451
Author: Tsuyoshi Ozawa <ozawa@apache.org>
Authored: Tue Mar 10 13:08:29 2015 +0900
Committer: Tsuyoshi Ozawa <ozawa@apache.org>
Committed: Tue Mar 10 13:09:05 2015 +0900

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 ++
 .../fs/CommonConfigurationKeysPublic.java       |  6 +++-
 .../main/java/org/apache/hadoop/ipc/Client.java | 33 ++++++++++++++++++--
 .../src/main/resources/core-default.xml         | 14 +++++++++
 4 files changed, 52 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5efee5ef/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a8f5313..d5b3418 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -6,6 +6,9 @@ Release 2.8.0 - UNRELEASED
 
   NEW FEATURES
 
+    HADOOP-11226. Add a configuration to set ipc.Client's traffic class with
+    IPTOS_LOWDELAY|IPTOS_RELIABILITY. (Gopal V via ozawa)
+
   IMPROVEMENTS
 
   OPTIMIZATIONS

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5efee5ef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 459b984..2a2cc24 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -206,8 +206,12 @@ public class CommonConfigurationKeysPublic {
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String  IPC_CLIENT_TCPNODELAY_KEY =
     "ipc.client.tcpnodelay";
-  /** Defalt value for IPC_CLIENT_TCPNODELAY_KEY */
+  /** Default value for IPC_CLIENT_TCPNODELAY_KEY */
   public static final boolean IPC_CLIENT_TCPNODELAY_DEFAULT = true;
+  /** Enable low-latency connections from the client */
+  public static final String   IPC_CLIENT_LOW_LATENCY = "ipc.client.low-latency";
+  /** Default value of IPC_CLIENT_LOW_LATENCY */
+  public static final boolean  IPC_CLIENT_LOW_LATENCY_DEFAULT = false;
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String  IPC_SERVER_LISTEN_QUEUE_SIZE_KEY =
     "ipc.server.listen.queue.size";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5efee5ef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 0b66cd6..30ccdda 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -384,7 +384,8 @@ public class Client {
     private final RetryPolicy connectionRetryPolicy;
     private final int maxRetriesOnSasl;
     private int maxRetriesOnSocketTimeouts;
-    private boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+    private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+    private final boolean tcpLowLatency; // if T then use low-delay QoS
     private boolean doPing; //do we need to send ping message
     private int pingInterval; // how often sends ping to the server in msecs
     private ByteArrayOutputStream pingRequest; // ping message
@@ -413,6 +414,7 @@ public class Client {
       this.maxRetriesOnSasl = remoteId.getMaxRetriesOnSasl();
       this.maxRetriesOnSocketTimeouts = remoteId.getMaxRetriesOnSocketTimeouts();
       this.tcpNoDelay = remoteId.getTcpNoDelay();
+      this.tcpLowLatency = remoteId.getTcpLowLatency();
       this.doPing = remoteId.getDoPing();
       if (doPing) {
         // construct a RPC header with the callId as the ping callId
@@ -585,6 +587,20 @@ public class Client {
           this.socket.setTcpNoDelay(tcpNoDelay);
           this.socket.setKeepAlive(true);
           
+          if (tcpLowLatency) {
+            /*
+             * This allows intermediate switches to shape IPC traffic
+             * differently from Shuffle/HDFS DataStreamer traffic.
+             *
+             * IPTOS_RELIABILITY (0x04) | IPTOS_LOWDELAY (0x10)
+             *
+             * Prefer to optimize connect() speed & response latency over net
+             * throughput.
+             */
+            this.socket.setTrafficClass(0x04 | 0x10);
+            this.socket.setPerformancePreferences(1, 2, 0);
+          }
+
           /*
            * Bind the socket to the host specified in the principal name of the
            * client, to ensure Server matching address of the client connection
@@ -1549,6 +1565,7 @@ public class Client {
     // the max. no. of retries for socket connections on time out exceptions
     private final int maxRetriesOnSocketTimeouts;
     private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+    private final boolean tcpLowLatency; // if T then use low-delay QoS
     private final boolean doPing; //do we need to send ping message
     private final int pingInterval; // how often sends ping to the server in msecs
     private String saslQop; // here for testing
@@ -1575,6 +1592,10 @@ public class Client {
       this.tcpNoDelay = conf.getBoolean(
           CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_KEY,
           CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_DEFAULT);
+      this.tcpLowLatency = conf.getBoolean(
+          CommonConfigurationKeysPublic.IPC_CLIENT_LOW_LATENCY,
+          CommonConfigurationKeysPublic.IPC_CLIENT_LOW_LATENCY_DEFAULT
+          );
       this.doPing = conf.getBoolean(
           CommonConfigurationKeys.IPC_CLIENT_PING_KEY,
           CommonConfigurationKeys.IPC_CLIENT_PING_DEFAULT);
@@ -1610,11 +1631,17 @@ public class Client {
     public int getMaxRetriesOnSocketTimeouts() {
       return maxRetriesOnSocketTimeouts;
     }
-    
+
+    /** disable nagle's algorithm */
     boolean getTcpNoDelay() {
       return tcpNoDelay;
     }
-    
+
+    /** use low-latency QoS bits over TCP */
+    boolean getTcpLowLatency() {
+      return tcpLowLatency;
+    }
+
     boolean getDoPing() {
       return doPing;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5efee5ef/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 690a22c..76b3e2f 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -970,6 +970,20 @@ for ldap providers in the same way as above does.
 </property>
 
 <property>
+  <name>ipc.client.tcpnodelay</name>
+  <value>true</value>
+  <description>Use TCP_NODELAY flag to bypass Nagle's algorithm transmission delays.
+  </description>
+</property>
+
+<property>
+  <name>ipc.client.low-latency</name>
+  <value>false</value>
+  <description>Use low-latency QoS markers for IPC connections.
+  </description>
+</property>
+
+<property>
   <name>ipc.server.listen.queue.size</name>
   <value>128</value>
   <description>Indicates the length of the listen queue for servers accepting


Mime
View raw message