hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject [33/50] [abbrv] hadoop git commit: HADOOP-11521. Make connection timeout configurable in s3a. (Thomas Demoor via stevel)
Date Wed, 18 Feb 2015 21:19:31 GMT
HADOOP-11521. Make connection timeout configurable in s3a. (Thomas Demoor via stevel)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00b80958
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00b80958
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00b80958

Branch: refs/heads/YARN-2928
Commit: 00b80958d862dbcc08d6f186c09963d3351ba0fd
Parents: 6be5670
Author: Steve Loughran <stevel@apache.org>
Authored: Tue Feb 17 20:00:00 2015 +0000
Committer: Steve Loughran <stevel@apache.org>
Committed: Tue Feb 17 20:06:27 2015 +0000

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt                | 3 +++
 .../hadoop-common/src/main/resources/core-default.xml          | 6 ++++++
 .../src/main/java/org/apache/hadoop/fs/s3a/Constants.java      | 4 ++++
 .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java  | 2 ++
 .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md     | 6 ++++++
 5 files changed, 21 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 405e821..c1caf5f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -615,6 +615,9 @@ Release 2.7.0 - UNRELEASED
 
     HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel)
 
+    HADOOP-11521. Make connection timeout configurable in s3a.
+    (Thomas Demoor via stevel)
+
   OPTIMIZATIONS
 
     HADOOP-11323. WritableComparator#compare keeps reference to byte array.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 5eb301a..41121f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -758,6 +758,12 @@ for ldap providers in the same way as above does.
 </property>
 
 <property>
+  <name>fs.s3a.connection.establish.timeout</name>
+  <value>5000</value>
+  <description>Socket connection setup timeout in seconds.</description>
+</property>
+
+<property>
   <name>fs.s3a.connection.timeout</name>
   <value>50000</value>
   <description>Socket connection timeout in seconds.</description>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index b6863bb..1d4f67b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -42,6 +42,10 @@ public class Constants {
   // number of times we should retry errors
   public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
   public static final int DEFAULT_MAX_ERROR_RETRIES = 10;
+
+  // seconds until we give up trying to establish a connection to s3
+  public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout";
+  public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000;
   
   // seconds until we give up on a connection to s3
   public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 22350bc..4de5c13 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -174,6 +174,8 @@ public class S3AFileSystem extends FileSystem {
     awsConf.setProtocol(secureConnections ?  Protocol.HTTPS : Protocol.HTTP);
     awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, 
       DEFAULT_MAX_ERROR_RETRIES));
+    awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT,
+        DEFAULT_ESTABLISH_TIMEOUT));
     awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, 
       DEFAULT_SOCKET_TIMEOUT));
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 8f511dd..1e44864 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -211,6 +211,12 @@ If you do any of these: change your credentials immediately!
     </property>
 
     <property>
+      <name>fs.s3a.connection.establish.timeout</name>
+      <value>5000</value>
+      <description>Socket connection setup timeout in seconds.</description>
+    </property>
+
+    <property>
       <name>fs.s3a.connection.timeout</name>
       <value>50000</value>
       <description>Socket connection timeout in seconds.</description>


Mime
View raw message