hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [1/2] hadoop git commit: HADOOP-15267. S3A multipart upload fails when SSE-C encryption is enabled. Contributed by Anis Elleuch.
Date Wed, 07 Mar 2018 18:51:25 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 c0986b1b7 -> 1dedc68f9
  refs/heads/trunk d69b31f7f -> e0307e53e


HADOOP-15267. S3A multipart upload fails when SSE-C encryption is enabled.
Contributed by Anis Elleuch.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1dedc68f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1dedc68f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1dedc68f

Branch: refs/heads/branch-3.1
Commit: 1dedc68f9d8d8544d715e67ee77cd3f017c21699
Parents: c0986b1
Author: Steve Loughran <stevel@apache.org>
Authored: Wed Mar 7 18:50:27 2018 +0000
Committer: Steve Loughran <stevel@apache.org>
Committed: Wed Mar 7 18:50:27 2018 +0000

----------------------------------------------------------------------
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 18 ++++++
 .../apache/hadoop/fs/s3a/MockS3AFileSystem.java |  6 ++
 .../scale/ITestS3AHugeFilesSSECDiskBlocks.java  | 58 ++++++++++++++++++++
 3 files changed, 82 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1dedc68f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 4424eba..eb65cfa 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1572,6 +1572,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities
{
     long len = request.getPartSize();
     incrementPutStartStatistics(len);
     try {
+      setOptionalUploadPartRequestParameters(request);
       UploadPartResult uploadPartResult = s3.uploadPart(request);
       incrementPutCompletedStatistics(true, len);
       return uploadPartResult;
@@ -2564,6 +2565,23 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities
{
   }
 
   /**
+   * Sets server side encryption parameters to the part upload
+   * request when encryption is enabled.
+   * @param request upload part request
+   */
+  protected void setOptionalUploadPartRequestParameters(
+      UploadPartRequest request) {
+    switch (serverSideEncryptionAlgorithm) {
+    case SSE_C:
+      if (isNotBlank(getServerSideEncryptionKey(bucket, getConf()))) {
+        request.setSSECustomerKey(generateSSECustomerKey());
+      }
+      break;
+    default:
+    }
+  }
+
+  /**
    * Initiate a multipart upload from the preconfigured request.
    * Retry policy: none + untranslated.
    * @param request request to initiate

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1dedc68f/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
index 4952580..1062a12 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
@@ -24,6 +24,7 @@ import java.net.URI;
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.UploadPartRequest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -278,6 +279,11 @@ public class MockS3AFileSystem extends S3AFileSystem {
   }
 
   @Override
+  protected void setOptionalUploadPartRequestParameters(
+      UploadPartRequest request) {
+  }
+
+  @Override
   @SuppressWarnings("deprecation")
   public long getDefaultBlockSize() {
     return mock.getDefaultBlockSize();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1dedc68f/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
new file mode 100644
index 0000000..2e5185b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled;
+
+/**
+ * Concrete class that extends {@link ITestS3AHugeFilesDiskBlocks}
+ * and tests huge files operations with SSE-C encryption enabled.
+ * Skipped if the SSE tests are disabled.
+ */
+public class ITestS3AHugeFilesSSECDiskBlocks
+    extends ITestS3AHugeFilesDiskBlocks {
+
+  private static final String KEY_1
+      = "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=";
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    skipIfEncryptionTestsDisabled(getConfiguration());
+  }
+
+  @Override
+  protected Configuration createScaleConfiguration() {
+    Configuration conf = super.createScaleConfiguration();
+    S3ATestUtils.disableFilesystemCaching(conf);
+    conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+        getSSEAlgorithm().getMethod());
+    conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, KEY_1);
+    return conf;
+  }
+
+  private S3AEncryptionMethods getSSEAlgorithm() {
+    return S3AEncryptionMethods.SSE_C;
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message