hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [20/50] [abbrv] hadoop git commit: HADOOP-12963 Allow using path style addressing for accessing the s3 endpoint. (Stephen Montgomery via stevel)
Date Sat, 16 Apr 2016 00:08:00 GMT
HADOOP-12963 Allow using path style addressing for accessing the s3 endpoint. (Stephen Montgomery
via stevel)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df18b6e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df18b6e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df18b6e9

Branch: refs/heads/YARN-3368
Commit: df18b6e9849c53c51a3d317f1254298edd8b17d1
Parents: 40211d1
Author: Steve Loughran <stevel@apache.org>
Authored: Thu Apr 14 12:44:55 2016 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Thu Apr 14 12:44:55 2016 +0100

----------------------------------------------------------------------
 .../src/main/resources/core-default.xml         |  7 +++
 .../org/apache/hadoop/fs/s3a/Constants.java     | 10 +++--
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 10 +++++
 .../src/site/markdown/tools/hadoop-aws/index.md |  7 +++
 .../hadoop/fs/s3a/TestS3AConfiguration.java     | 47 ++++++++++++++++++--
 5 files changed, 75 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/df18b6e9/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b3436da..96b108f 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -766,6 +766,13 @@
 </property>
 
 <property>
+  <name>fs.s3a.path.style.access</name>
+  <description>Enable S3 path style access ie disabling the default virtual hosting
behaviour.
+    Useful for S3A-compliant storage providers as it removes the need to set up DNS for virtual
hosting.
+  </description>
+</property>
+
+<property>
   <name>fs.s3a.proxy.host</name>
   <description>Hostname of the (optional) proxy server for S3 connections.</description>
 </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df18b6e9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index f10f3db..9d79623 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -28,13 +28,17 @@ public class Constants {
   // number of simultaneous connections to s3
   public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";
   public static final int DEFAULT_MAXIMUM_CONNECTIONS = 15;
-  
+
   // connect to s3 over ssl?
   public static final String SECURE_CONNECTIONS = "fs.s3a.connection.ssl.enabled";
   public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
 
   //use a custom endpoint?
   public static final String ENDPOINT = "fs.s3a.endpoint";
+
+  //Enable path style access? Overrides default virtual hosting
+  public static final String PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
+
   //connect to s3 through a proxy server?
   public static final String PROXY_HOST = "fs.s3a.proxy.host";
   public static final String PROXY_PORT = "fs.s3a.proxy.port";
@@ -50,7 +54,7 @@ public class Constants {
   // seconds until we give up trying to establish a connection to s3
   public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout";
   public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000;
-  
+
   // seconds until we give up on a connection to s3
   public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
   public static final int DEFAULT_SOCKET_TIMEOUT = 200000;
@@ -74,7 +78,7 @@ public class Constants {
   // size of each of or multipart pieces in bytes
   public static final String MULTIPART_SIZE = "fs.s3a.multipart.size";
   public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
-  
+
   // minimum size in bytes before we start a multipart uploads or copy
   public static final String MIN_MULTIPART_THRESHOLD = "fs.s3a.multipart.threshold";
   public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = Integer.MAX_VALUE;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df18b6e9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index fe705ce..97092ac 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -38,6 +38,7 @@ import com.amazonaws.auth.AWSCredentialsProviderChain;
 
 import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
 import com.amazonaws.services.s3.model.CannedAccessControlList;
 import com.amazonaws.services.s3.model.DeleteObjectRequest;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
@@ -244,6 +245,15 @@ public class S3AFileSystem extends FileSystem {
         throw new IllegalArgumentException(msg, e);
       }
     }
+    enablePathStyleAccessIfRequired(conf);
+  }
+
+  private void enablePathStyleAccessIfRequired(Configuration conf) {
+    final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
+    if (pathStyleAccess) {
+      LOG.debug("Enabling path style access!");
+      s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+    }
   }
 
   private void initTransferManager() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df18b6e9/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 7382029..e87b884 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -225,6 +225,13 @@ this capability.
     </property>
 
     <property>
+      <name>fs.s3a.path.style.access</name>
+      <description>Enable S3 path style access ie disabling the default virtual hosting
behaviour.
+        Useful for S3A-compliant storage providers as it removes the need to set up DNS for
virtual hosting.
+      </description>
+    </property>
+
+    <property>
       <name>fs.s3a.proxy.host</name>
       <description>Hostname of the (optional) proxy server for S3 connections.</description>
     </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df18b6e9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
index ae1539d..4a0bfbb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
@@ -19,10 +19,14 @@
 package org.apache.hadoop.fs.s3a;
 
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+
 import org.apache.commons.lang.StringUtils;
 import com.amazonaws.AmazonClientException;
 import org.apache.hadoop.conf.Configuration;
-
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
@@ -30,17 +34,19 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.net.URI;
-import java.io.IOException;
+import java.lang.reflect.Field;
 
 import org.apache.hadoop.security.ProviderUtils;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
-
+import org.apache.http.HttpStatus;
 import org.junit.rules.TemporaryFolder;
 
 public class TestS3AConfiguration {
@@ -354,4 +360,39 @@ public class TestS3AConfiguration {
     assertEquals("SecretKey incorrect.", "456", creds.getAccessSecret());
 
   }
+
+  @Test
+  public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() throws Exception
{
+
+    conf = new Configuration();
+    conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true));
+    assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false));
+
+    try {
+      fs = S3ATestUtils.createTestFileSystem(conf);
+      final Object object = getClientOptionsField(fs.getAmazonS3Client(), "clientOptions");
+      assertNotNull(object);
+      assertTrue("Unexpected type found for clientOptions!", object instanceof S3ClientOptions);
+      assertTrue("Expected to find path style access to be switched on!", ((S3ClientOptions)
object).isPathStyleAccess());
+      byte[] file = ContractTestUtils.toAsciiByteArray("test file");
+      ContractTestUtils.writeAndRead(fs, new Path("/path/style/access/testFile"), file, file.length,
conf.getInt(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true);
+    } catch (final AmazonS3Exception e) {
+      LOG.error("Caught exception: ", e);
+      // Catch/pass standard path style access behaviour when live bucket
+      // isn't in the same region as the s3 client default. See
+      // http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
+      assertEquals(e.getStatusCode(), HttpStatus.SC_MOVED_PERMANENTLY);
+    }
+  }
+
+  private Object getClientOptionsField(AmazonS3Client s3client, String field)
+      throws NoSuchFieldException, IllegalAccessException {
+    final Field clientOptionsProps = s3client.getClass().getDeclaredField(field);
+    assertNotNull(clientOptionsProps);
+    if (!clientOptionsProps.isAccessible()) {
+      clientOptionsProps.setAccessible(true);
+    }
+    final Object object = clientOptionsProps.get(s3client);
+    return object;
+  }
 }


Mime
View raw message