Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 1A206200B4A for ; Wed, 20 Jul 2016 15:03:56 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 11CDE160A73; Wed, 20 Jul 2016 13:03:56 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3426E160A64 for ; Wed, 20 Jul 2016 15:03:55 +0200 (CEST) Received: (qmail 68941 invoked by uid 500); 20 Jul 2016 13:03:54 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 68932 invoked by uid 99); 20 Jul 2016 13:03:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 20 Jul 2016 13:03:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3312CE04BA; Wed, 20 Jul 2016 13:03:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: stevel@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HADOOP-13212 Provide an option to set the socket buffers in S3AFileSystem (Rajesh Balamohan) Date: Wed, 20 Jul 2016 13:03:54 +0000 (UTC) archived-at: Wed, 20 Jul 2016 13:03:56 -0000 Repository: hadoop Updated Branches: refs/heads/branch-3.0.0-alpha1 cc55b18ad -> 1b1e3c10e HADOOP-13212 Provide an option to set the socket buffers in S3AFileSystem (Rajesh Balamohan) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b1e3c10 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b1e3c10 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b1e3c10 Branch: refs/heads/branch-3.0.0-alpha1 Commit: 1b1e3c10e8fb4267883e9410fabd19cc0d62d685 Parents: cc55b18 Author: Steve Loughran Authored: Wed Jul 20 13:42:19 2016 +0100 Committer: Steve Loughran Committed: Wed Jul 20 14:03:44 2016 +0100 ---------------------------------------------------------------------- .../hadoop-common/src/main/resources/core-default.xml | 12 ++++++++++++ .../main/java/org/apache/hadoop/fs/s3a/Constants.java | 8 ++++++++ .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 5 +++++ .../src/site/markdown/tools/hadoop-aws/index.md | 12 ++++++++++++ .../fs/s3a/scale/TestS3AInputStreamPerformance.java | 3 +++ 5 files changed, 40 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 3018884..0d751bb 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -908,6 +908,18 @@ + fs.s3a.socket.send.buffer + 8192 + Socket send buffer hint to amazon connector. Represented in bytes. + + + + fs.s3a.socket.recv.buffer + 8192 + Socket receive buffer hint to amazon connector. Represented in bytes. + + + fs.s3a.paging.maximum 5000 How many keys to request from S3 when doing http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 3f6f347..612b648 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -82,6 +82,14 @@ public final class Constants { public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout"; public static final int DEFAULT_SOCKET_TIMEOUT = 200000; + // socket send buffer to be used in Amazon client + public static final String SOCKET_SEND_BUFFER = "fs.s3a.socket.send.buffer"; + public static final int DEFAULT_SOCKET_SEND_BUFFER = 8 * 1024; + + // socket send buffer to be used in Amazon client + public static final String SOCKET_RECV_BUFFER = "fs.s3a.socket.recv.buffer"; + public static final int DEFAULT_SOCKET_RECV_BUFFER = 8 * 1024; + // number of records to get while paging through a directory listing public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum"; public static final int DEFAULT_MAX_PAGING_KEYS = 5000; http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 513b57c..10595e2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -163,6 +163,11 @@ public class S3AFileSystem extends FileSystem { DEFAULT_ESTABLISH_TIMEOUT, 0)); awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0)); + int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER, + DEFAULT_SOCKET_SEND_BUFFER, 2048); + int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER, + DEFAULT_SOCKET_RECV_BUFFER, 2048); + awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer); String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, ""); if (!signerOverride.isEmpty()) { LOG.debug("Signer override = {}", signerOverride); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 2b64754..01a2bae 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -504,6 +504,18 @@ this capability. + fs.s3a.socket.send.buffer + 8192 + Socket send buffer hint to amazon connector. Represented in bytes. + + + + fs.s3a.socket.recv.buffer + 8192 + Socket receive buffer hint to amazon connector. Represented in bytes. + + + fs.s3a.threads.keepalivetime 60 Number of seconds a thread can be idle before being http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1e3c10/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java index 752e374..bddd8e2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java @@ -44,6 +44,7 @@ import java.io.EOFException; import java.io.IOException; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; +import static org.apache.hadoop.fs.s3a.Constants.*; /** * Look at the performance of S3a operations. @@ -71,6 +72,8 @@ public class TestS3AInputStreamPerformance extends S3AScaleTestBase { @Before public void openFS() throws IOException { Configuration conf = getConf(); + conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024); + conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024); String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE); if (testFile.isEmpty()) { assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org