hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject hadoop git commit: HADOOP-13212 Provide an option to set the socket buffers in S3AFileSystem (Rajesh Balamohan)
Date Wed, 20 Jul 2016 13:03:54 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0.0-alpha1 cc55b18ad -> 1b1e3c10e


HADOOP-13212 Provide an option to set the socket buffers in S3AFileSystem (Rajesh Balamohan)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b1e3c10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b1e3c10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b1e3c10

Branch: refs/heads/branch-3.0.0-alpha1
Commit: 1b1e3c10e8fb4267883e9410fabd19cc0d62d685
Parents: cc55b18
Author: Steve Loughran <stevel@apache.org>
Authored: Wed Jul 20 13:42:19 2016 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Wed Jul 20 14:03:44 2016 +0100

----------------------------------------------------------------------
 .../hadoop-common/src/main/resources/core-default.xml   | 12 ++++++++++++
 .../main/java/org/apache/hadoop/fs/s3a/Constants.java   |  8 ++++++++
 .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java    |  5 +++++
 .../src/site/markdown/tools/hadoop-aws/index.md         | 12 ++++++++++++
 .../fs/s3a/scale/TestS3AInputStreamPerformance.java     |  3 +++
 5 files changed, 40 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 3018884..0d751bb 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -908,6 +908,18 @@
 </property>
 
 <property>
+  <name>fs.s3a.socket.send.buffer</name>
+  <value>8192</value>
+  <description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
+</property>
+
+<property>
+  <name>fs.s3a.socket.recv.buffer</name>
+  <value>8192</value>
+  <description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
+</property>
+
+<property>
   <name>fs.s3a.paging.maximum</name>
   <value>5000</value>
   <description>How many keys to request from S3 when doing 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 3f6f347..612b648 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -82,6 +82,14 @@ public final class Constants {
   public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
   public static final int DEFAULT_SOCKET_TIMEOUT = 200000;
 
+  // socket send buffer to be used in Amazon client
+  public static final String SOCKET_SEND_BUFFER = "fs.s3a.socket.send.buffer";
+  public static final int DEFAULT_SOCKET_SEND_BUFFER = 8 * 1024;
+
+  // socket send buffer to be used in Amazon client
+  public static final String SOCKET_RECV_BUFFER = "fs.s3a.socket.recv.buffer";
+  public static final int DEFAULT_SOCKET_RECV_BUFFER = 8 * 1024;
+
   // number of records to get while paging through a directory listing
   public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
   public static final int DEFAULT_MAX_PAGING_KEYS = 5000;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 513b57c..10595e2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -163,6 +163,11 @@ public class S3AFileSystem extends FileSystem {
           DEFAULT_ESTABLISH_TIMEOUT, 0));
       awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
           DEFAULT_SOCKET_TIMEOUT, 0));
+      int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
+          DEFAULT_SOCKET_SEND_BUFFER, 2048);
+      int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
+          DEFAULT_SOCKET_RECV_BUFFER, 2048);
+      awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
       String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
       if (!signerOverride.isEmpty()) {
         LOG.debug("Signer override = {}", signerOverride);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 2b64754..01a2bae 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -504,6 +504,18 @@ this capability.
     </property>
 
     <property>
+      <name>fs.s3a.socket.send.buffer</name>
+      <value>8192</value>
+      <description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
+    </property>
+
+    <property>
+      <name>fs.s3a.socket.recv.buffer</name>
+      <value>8192</value>
+      <description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
+    </property>
+
+    <property>
       <name>fs.s3a.threads.keepalivetime</name>
       <value>60</value>
       <description>Number of seconds a thread can be idle before being

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
index 752e374..bddd8e2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
@@ -44,6 +44,7 @@ import java.io.EOFException;
 import java.io.IOException;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
 
 /**
  * Look at the performance of S3a operations.
@@ -71,6 +72,8 @@ public class TestS3AInputStreamPerformance extends S3AScaleTestBase {
   @Before
   public void openFS() throws IOException {
     Configuration conf = getConf();
+    conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024);
+    conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024);
     String testFile =  conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
     if (testFile.isEmpty()) {
       assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message