hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject hadoop git commit: HADOOP-12891. S3AFileSystem should configure Multipart Copy threshold and chunk size. (Andrew Olson via stevel)
Date Fri, 22 Apr 2016 10:41:34 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 a0605ff00 -> edcb2a841


HADOOP-12891. S3AFileSystem should configure Multipart Copy threshold and chunk size. (Andrew
Olson via stevel)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edcb2a84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edcb2a84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edcb2a84

Branch: refs/heads/branch-2
Commit: edcb2a8410ec12e1b7fd43ec3f66f4878bbddc3a
Parents: a0605ff
Author: Steve Loughran <stevel@apache.org>
Authored: Fri Apr 22 11:24:24 2016 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Fri Apr 22 11:24:50 2016 +0100

----------------------------------------------------------------------
 .../hadoop-common/src/main/resources/core-default.xml           | 5 ++++-
 .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java   | 2 ++
 .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md      | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/edcb2a84/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 746541a..abdb0cf 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -860,7 +860,10 @@
 <property>
   <name>fs.s3a.multipart.threshold</name>
   <value>2147483647</value>
-  <description>Threshold before uploads or copies use parallel multipart operations.</description>
+  <description>How big (in bytes) to split upload or copy operations up into.
+    This also controls the partition size in renamed files, as rename() involves
+    copying the source file(s)
+  </description>
 </property>
 
 <property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/edcb2a84/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 83d1d53..4cd2783 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -318,6 +318,8 @@ public class S3AFileSystem extends FileSystem {
     TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
     transferConfiguration.setMinimumUploadPartSize(partSize);
     transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
+    transferConfiguration.setMultipartCopyPartSize(partSize);
+    transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);
 
     transfers = new TransferManager(s3, threadPoolExecutor);
     transfers.setConfiguration(transferConfiguration);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/edcb2a84/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 15b9837..9016264 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -315,7 +315,9 @@ this capability.
     <property>
       <name>fs.s3a.multipart.size</name>
       <value>104857600</value>
-      <description>How big (in bytes) to split upload or copy operations up into.</description>
+      <description>How big (in bytes) to split upload or copy operations up into.
+      This also controls the partition size in renamed files, as rename() involves
+      copying the source file(s)</description>
     </property>
 
     <property>


Mime
View raw message