From common-commits-return-92311-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Sat Jan 5 09:49:33 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id A7538180645 for ; Sat, 5 Jan 2019 09:49:32 +0100 (CET) Received: (qmail 77384 invoked by uid 500); 5 Jan 2019 08:49:31 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 77375 invoked by uid 99); 5 Jan 2019 08:49:31 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 05 Jan 2019 08:49:31 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id E14F5821BA; Sat, 5 Jan 2019 08:49:30 +0000 (UTC) Date: Sat, 05 Jan 2019 08:49:30 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] branch branch-3.2.0 updated: Revert "HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu." MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <154667817034.4805.2796778841402290986@gitbox.apache.org> From: aajisaka@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/branch-3.2.0 X-Git-Reftype: branch X-Git-Oldrev: d0699f48475ab6ddfe06b54a37779b21cce8ae46 X-Git-Newrev: 83ef50fdedfe11bf4ed25cde0be7f028816dd40e X-Git-Rev: 83ef50fdedfe11bf4ed25cde0be7f028816dd40e X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a commit to branch branch-3.2.0 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2.0 by this push: new 83ef50f Revert "HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu." 83ef50f is described below commit 83ef50fdedfe11bf4ed25cde0be7f028816dd40e Author: Sunil G AuthorDate: Fri Jan 4 20:35:17 2019 +0530 Revert "HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu." This reverts commit e4fca6aae46a3c04fc56897986a4ab4e5aa98503. Revert "HADOOP-15671. AliyunOSS: Support Assume Roles in AliyunOSS. Contributed by Jinhu Wu." This reverts commit 2b635125fb059fc204ed35bc0e264c42dd3a9fe9. (cherry picked from commit 1f425271a73fff1fdbe3fbfdb71e906fd1ac0574) (cherry picked from commit 71bee053397dba3c2144d47e3606a4487085a029) --- hadoop-project/pom.xml | 2 +- .../fs/aliyun/oss/AliyunOSSBlockOutputStream.java | 5 +- .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 5 +- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 8 +- .../aliyun/oss/AssumedRoleCredentialProvider.java | 115 --------------------- .../org/apache/hadoop/fs/aliyun/oss/Constants.java | 22 ---- .../src/site/markdown/tools/hadoop-aliyun/index.md | 50 --------- .../fs/aliyun/oss/TestAliyunCredentials.java | 55 +--------- 8 files changed, 13 insertions(+), 249 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e6bf605..08d93a1 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1261,7 +1261,7 @@ com.aliyun.oss aliyun-sdk-oss - 3.0.0 + 2.8.3 org.apache.httpcomponents diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java index 17f21cb..0a833b2 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java @@ -120,8 +120,7 @@ public class AliyunOSSBlockOutputStream extends OutputStream { if (null == partETags) { throw new IOException("Failed to multipart upload to oss, abort it."); } - store.completeMultipartUpload(key, uploadId, - new ArrayList<>(partETags)); + store.completeMultipartUpload(key, uploadId, partETags); } } finally { removePartFiles(); @@ -130,7 +129,7 @@ public class AliyunOSSBlockOutputStream extends OutputStream { } @Override - public synchronized void write(int b) throws IOException { + public void write(int b) throws IOException { singleByte[0] = (byte)b; write(singleByte, 0, 1); } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java index 7639eb3..dc5f99ee 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java @@ -149,7 +149,7 @@ public class AliyunOSSFileSystemStore { "null or empty. Please set proper endpoint with 'fs.oss.endpoint'."); } CredentialsProvider provider = - AliyunOSSUtils.getCredentialsProvider(uri, conf); + AliyunOSSUtils.getCredentialsProvider(conf); ossClient = new OSSClient(endPoint, provider, clientConf); uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(conf, MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT); @@ -168,8 +168,6 @@ public class AliyunOSSFileSystemStore { multipartThreshold = 1024 * 1024 * 1024; } - bucketName = uri.getHost(); - String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT); if (StringUtils.isNotEmpty(cannedACLName)) { CannedAccessControlList cannedACL = @@ -178,6 +176,7 @@ public class AliyunOSSFileSystemStore { } maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); + bucketName = uri.getHost(); } /** diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index 3e02d7f..a7536d6 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -20,7 +20,6 @@ package org.apache.hadoop.fs.aliyun.oss; import java.io.File; import java.io.IOException; -import java.net.URI; import com.aliyun.oss.common.auth.CredentialsProvider; import com.google.common.base.Preconditions; @@ -96,14 +95,13 @@ final public class AliyunOSSUtils { * Create credential provider specified by configuration, or create default * credential provider if not specified. * - * @param uri uri passed by caller * @param conf configuration * @return a credential provider * @throws IOException on any problem. Class construction issues may be * nested inside the IOE. */ - public static CredentialsProvider getCredentialsProvider( - URI uri, Configuration conf) throws IOException { + public static CredentialsProvider getCredentialsProvider(Configuration conf) + throws IOException { CredentialsProvider credentials; String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY); @@ -119,7 +117,7 @@ final public class AliyunOSSUtils { try { credentials = (CredentialsProvider)credClass.getDeclaredConstructor( - URI.class, Configuration.class).newInstance(uri, conf); + Configuration.class).newInstance(conf); } catch (NoSuchMethodException | SecurityException e) { credentials = (CredentialsProvider)credClass.getDeclaredConstructor() diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AssumedRoleCredentialProvider.java deleted file mode 100644 index 1c93a74..0000000 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AssumedRoleCredentialProvider.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.aliyun.oss; - -import com.aliyun.oss.common.auth.Credentials; -import com.aliyun.oss.common.auth.CredentialsProvider; -import com.aliyun.oss.common.auth.InvalidCredentialsException; -import com.aliyun.oss.common.auth.STSAssumeRoleSessionCredentialsProvider; -import com.aliyuncs.exceptions.ClientException; -import com.aliyuncs.profile.DefaultProfile; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; - -import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID; -import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET; - -/** - * Support assumed role credentials for authenticating with Aliyun. - * roleArn is configured in core-site.xml - */ -public class AssumedRoleCredentialProvider implements CredentialsProvider { - private static final Logger LOG = - LoggerFactory.getLogger(AssumedRoleCredentialProvider.class); - public static final String NAME - = "org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider"; - private Credentials credentials; - private String roleArn; - private long duration; - private String stsEndpoint; - private String sessionName; - private double expiredFactor; - private STSAssumeRoleSessionCredentialsProvider stsCredentialsProvider; - - public AssumedRoleCredentialProvider(URI uri, Configuration conf) { - roleArn = conf.getTrimmed(Constants.ROLE_ARN, ""); - if (StringUtils.isEmpty(roleArn)) { - throw new InvalidCredentialsException( - "fs.oss.assumed.role.arn is empty"); - } - - duration = conf.getLong(Constants.ASSUMED_ROLE_DURATION, - Constants.ASSUMED_ROLE_DURATION_DEFAULT); - - expiredFactor = conf.getDouble(Constants.ASSUMED_ROLE_STS_EXPIRED_FACTOR, - Constants.ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT); - - stsEndpoint = conf.getTrimmed(Constants.ASSUMED_ROLE_STS_ENDPOINT, ""); - if (StringUtils.isEmpty(stsEndpoint)) { - throw new InvalidCredentialsException( - "fs.oss.assumed.role.sts.endpoint is empty"); - } - - sessionName = conf.getTrimmed(Constants.ASSUMED_ROLE_SESSION_NAME, ""); - - String accessKeyId; - String accessKeySecret; - try { - accessKeyId = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_ID); - accessKeySecret = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_SECRET); - } catch (IOException e) { - throw new InvalidCredentialsException(e); - } - - try { - DefaultProfile.addEndpoint("", "", "Sts", stsEndpoint); - } catch (ClientException e) { - throw new InvalidCredentialsException(e); - } - - stsCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider( - new com.aliyuncs.auth.BasicCredentials(accessKeyId, accessKeySecret), - roleArn, DefaultProfile.getProfile("", accessKeyId, accessKeySecret)) - .withExpiredDuration(duration).withExpiredFactor(expiredFactor); - - if (!StringUtils.isEmpty(sessionName)) { - stsCredentialsProvider.withRoleSessionName(sessionName); - } - } - - @Override - public void setCredentials(Credentials creds) { - throw new InvalidCredentialsException( - "Should not set credentials from external call"); - } - - @Override - public Credentials getCredentials() { - credentials = stsCredentialsProvider.getCredentials(); - if (credentials == null) { - throw new InvalidCredentialsException("Invalid credentials"); - } - return credentials; - } -} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java index 24e35d9..ecbd749 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs.aliyun.oss; -import com.aliyun.oss.common.utils.AuthUtils; import com.aliyun.oss.common.utils.VersionInfoUtils; /** @@ -43,27 +42,6 @@ public final class Constants { public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret"; public static final String SECURITY_TOKEN = "fs.oss.securityToken"; - // Assume role configurations - public static final String ROLE_ARN = "fs.oss.assumed.role.arn"; - public static final String ASSUMED_ROLE_DURATION = - "fs.oss.assumed.role.session.duration"; - // Default session duration(in seconds) - public static final long ASSUMED_ROLE_DURATION_DEFAULT = 30 * 60; - - // Expired factor of sts token - // For example, if session duration is 900s and expiredFactor is 0.8 - // sts token will be refreshed after 900 * 0.8s - public static final String ASSUMED_ROLE_STS_EXPIRED_FACTOR = - "fs.oss.assumed.role.sts.expiredFactor"; - - public static final double ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT = - AuthUtils.DEFAULT_EXPIRED_FACTOR; - - public static final String ASSUMED_ROLE_STS_ENDPOINT = - "fs.oss.assumed.role.sts.endpoint"; - public static final String ASSUMED_ROLE_SESSION_NAME = - "fs.oss.assumed.role.session.name"; - // Number of simultaneous connections to oss public static final String MAXIMUM_CONNECTIONS_KEY = "fs.oss.connection.maximum"; diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md index 0703790..9f24ce6 100644 --- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md +++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md @@ -118,56 +118,6 @@ please raise your issues with them. - fs.oss.assumed.role.arn - - Role ARN for the role to be assumed. - Required if the fs.oss.credentials.provider is - org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider. - - - - - fs.oss.assumed.role.sts.endpoint - - STS Token Service endpoint. - Required if the fs.oss.credentials.provider is - org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider. - - - - - fs.oss.assumed.role.session.name - - - Session name for the assumed role, must be valid characters - according to Aliyun API. It is optional, will be generated by - oss java sdk if it is empty. - Only used if the fs.oss.credentials.provider is - org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider. - - - - - fs.oss.assumed.role.session.duration - - - Duration of assumed roles before it is expired. Default is 30 minutes. - Only used if the fs.oss.credentials.provider is - org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider. - - - - - fs.oss.assumed.role.sts.expiredFactor - - - Sts token will be refreshed after (expiredFactor * duration) seconds. - Only used if the fs.oss.credentials.provider is - org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider. - - - - fs.oss.proxy.host Hostname of the (optinal) proxy server for Aliyun OSS connection diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java index 7f7421d..e08a4dc 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java @@ -19,7 +19,6 @@ package org.apache.hadoop.fs.aliyun.oss; import com.aliyun.oss.common.auth.Credentials; -import com.aliyun.oss.common.auth.CredentialsProvider; import com.aliyun.oss.common.auth.InvalidCredentialsException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract; @@ -28,15 +27,9 @@ import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; import org.junit.Test; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.net.URI; import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID; import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET; -import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_SESSION_NAME; -import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_STS_ENDPOINT; -import static org.apache.hadoop.fs.aliyun.oss.Constants.CREDENTIALS_PROVIDER_KEY; -import static org.apache.hadoop.fs.aliyun.oss.Constants.ROLE_ARN; import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN; /** @@ -70,54 +63,16 @@ public class TestAliyunCredentials extends AbstractFSContractTestBase { validateCredential(conf); } - @Test - public void testCredentialMissingRoleArn() throws Throwable { - Configuration conf = new Configuration(); - conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME); - conf.set(ROLE_ARN, ""); - validateCredential(conf); - } - - @Test - public void testCredentialMissingStsEndpoint() throws Throwable { - Configuration conf = new Configuration(); - conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME); - conf.set(ASSUMED_ROLE_STS_ENDPOINT, ""); - validateCredential(conf); - } - - @Test - public void testCredentialInvalidSessionName() throws Throwable { - Configuration conf = new Configuration(); - conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME); - conf.set(ASSUMED_ROLE_SESSION_NAME, "hadoop oss"); - validateCredential(conf); - } - - private void validateCredential(URI uri, Configuration conf) { + private void validateCredential(Configuration conf) { try { - CredentialsProvider provider = - AliyunOSSUtils.getCredentialsProvider(uri, conf); + AliyunCredentialsProvider provider + = new AliyunCredentialsProvider(conf); Credentials credentials = provider.getCredentials(); fail("Expected a CredentialInitializationException, got " + credentials); } catch (InvalidCredentialsException expected) { // expected } catch (IOException e) { - Throwable cause = e.getCause(); - if (cause instanceof InvocationTargetException) { - boolean isInstance = - ((InvocationTargetException)cause).getTargetException() - instanceof InvalidCredentialsException; - if (!isInstance) { - fail("Unexpected exception."); - } - } else { - fail("Unexpected exception."); - } + fail("Unexpected exception."); } } - - private void validateCredential(Configuration conf) { - validateCredential(null, conf); - } -} \ No newline at end of file +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org