Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 2BADC200D1F for ; Fri, 29 Sep 2017 00:29:23 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 29D041609EC; Thu, 28 Sep 2017 22:29:23 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 21FE71609CD for ; Fri, 29 Sep 2017 00:29:21 +0200 (CEST) Received: (qmail 33762 invoked by uid 500); 28 Sep 2017 22:29:21 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 33753 invoked by uid 99); 28 Sep 2017 22:29:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Sep 2017 22:29:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1D660F5B5D; Thu, 28 Sep 2017 22:29:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctrezzo@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-7250. Update Shared cache client api to use URLs. Date: Thu, 28 Sep 2017 22:29:19 +0000 (UTC) archived-at: Thu, 28 Sep 2017 22:29:23 -0000 Repository: hadoop Updated Branches: refs/heads/trunk 6f789fe05 -> c114da5e6 YARN-7250. Update Shared cache client api to use URLs. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c114da5e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c114da5e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c114da5e Branch: refs/heads/trunk Commit: c114da5e64d14b1d9e614081c4171ea0391cb1aa Parents: 6f789fe Author: Chris Trezzo Authored: Thu Sep 28 15:28:06 2017 -0700 Committer: Chris Trezzo Committed: Thu Sep 28 15:28:06 2017 -0700 ---------------------------------------------------------------------- .../yarn/client/api/SharedCacheClient.java | 22 ++++-------- .../client/api/impl/SharedCacheClientImpl.java | 36 +++++--------------- .../api/impl/TestSharedCacheClientImpl.java | 31 +++++------------ 3 files changed, 23 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java index 60c1bd98..a9c1a07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.client.api.impl.SharedCacheClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -58,34 +59,25 @@ public abstract class SharedCacheClient extends AbstractService { * *

* The SharedCacheManager responds with whether or not the - * resource exists in the cache. If the resource exists, a Path - * to the resource in the shared cache is returned. If the resource does not + * resource exists in the cache. If the resource exists, a URL to + * the resource in the shared cache is returned. If the resource does not * exist, null is returned instead. *

* *

- * Once a path has been returned for a resource, that path is safe to use for + * Once a URL has been returned for a resource, that URL is safe to use for * the lifetime of the application that corresponds to the provided * ApplicationId. *

* - *

- * Additionally, a name for the resource should be specified. A fragment will - * be added to the path with the desired name if the desired name is different - * than the name of the provided path from the shared cache. This ensures that - * if the returned path is used to create a LocalResource, then the symlink - * created during YARN localization will match the name specified. - *

- * * @param applicationId ApplicationId of the application using the resource * @param resourceKey the key (i.e. checksum) that identifies the resource - * @param resourceName the desired name of the resource - * @return Path to the resource, or null if it does not exist + * @return URL to the resource, or null if it does not exist */ @Public @Unstable - public abstract Path use(ApplicationId applicationId, String resourceKey, - String resourceName) throws YarnException; + public abstract URL use(ApplicationId applicationId, String resourceKey) + throws YarnException; /** *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java index b910c28..3191d36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java @@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -38,6 +36,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequ import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.client.api.SharedCacheClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -113,8 +112,8 @@ public class SharedCacheClientImpl extends SharedCacheClient { } @Override - public Path use(ApplicationId applicationId, String resourceKey, - String resourceName) throws YarnException { + public URL use(ApplicationId applicationId, String resourceKey) + throws YarnException { Path resourcePath = null; UseSharedCacheResourceRequest request = Records.newRecord( UseSharedCacheResourceRequest.class); @@ -132,31 +131,12 @@ public class SharedCacheClientImpl extends SharedCacheClient { throw new YarnException(e); } if (resourcePath != null) { - if (resourcePath.getName().equals(resourceName)) { - // The preferred name is the same as the name of the item in the cache, - // so we skip generating the fragment to save space in the MRconfig. - return resourcePath; - } else { - // We are using the shared cache, and a preferred name has been - // specified that is different than the name of the resource in the - // shared cache. We need to set the fragment portion of the URI to - // preserve the desired name. - URI pathURI = resourcePath.toUri(); - try { - // We assume that there is no existing fragment in the URI since the - // shared cache manager does not use fragments. - pathURI = - new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(), - resourceName); - resourcePath = new Path(pathURI); - } catch (URISyntaxException e) { - throw new YarnException( - "Could not create a new URI due to syntax errors: " - + pathURI.toString(), e); - } - } + URL pathURL = URL.fromPath(resourcePath); + return pathURL; + } else { + // The resource was not in the cache. + return null; } - return resourcePath; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java index 16b32db..6a8edd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java @@ -27,7 +27,6 @@ import static org.mockito.Mockito.when; import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; -import java.net.URI; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,6 +39,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.exceptions.YarnException; import org.junit.After; import org.junit.AfterClass; @@ -114,36 +114,21 @@ public class TestSharedCacheClientImpl { response.setPath(null); when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn( response); - Path newPath = client.use(mock(ApplicationId.class), "key", null); - assertNull("The path is not null!", newPath); + URL newURL = client.use(mock(ApplicationId.class), "key"); + assertNull("The path is not null!", newURL); } @Test - public void testUseWithResourceName() throws Exception { + public void testUseCacheHit() throws Exception { Path file = new Path("viewfs://test/path"); - URI useUri = new URI("viewfs://test/path#linkName"); - Path usePath = new Path(useUri); + URL useUrl = URL.fromPath(new Path("viewfs://test/path")); UseSharedCacheResourceResponse response = new UseSharedCacheResourceResponsePBImpl(); response.setPath(file.toString()); when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn( response); - Path newPath = client.use(mock(ApplicationId.class), "key", "linkName"); - assertEquals("The paths are not equal!", usePath, newPath); - } - - @Test - public void testUseWithSameResourceName() throws Exception { - Path file = new Path("viewfs://test/path"); - URI useUri = new URI("viewfs://test/path"); - Path usePath = new Path(useUri); - UseSharedCacheResourceResponse response = - new UseSharedCacheResourceResponsePBImpl(); - response.setPath(file.toString()); - when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn( - response); - Path newPath = client.use(mock(ApplicationId.class), "key", "path"); - assertEquals("The paths are not equal!", usePath, newPath); + URL newURL = client.use(mock(ApplicationId.class), "key"); + assertEquals("The paths are not equal!", useUrl, newURL); } @Test(expected = YarnException.class) @@ -151,7 +136,7 @@ public class TestSharedCacheClientImpl { String message = "Mock IOExcepiton!"; when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow( new IOException(message)); - client.use(mock(ApplicationId.class), "key", null); + client.use(mock(ApplicationId.class), "key"); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org