hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sj...@apache.org
Subject [2/2] hadoop git commit: YARN-3637. Handle localization sym-linking correctly at the YARN level. Contributed by Chris Trezzo.
Date Wed, 25 Jan 2017 23:52:34 GMT
YARN-3637. Handle localization sym-linking correctly at the YARN level. Contributed by Chris
Trezzo.

(cherry picked from commit 425a7e502869c4250aba927ecc3c6f3c561c6ff2)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b799ea76
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b799ea76
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b799ea76

Branch: refs/heads/branch-2
Commit: b799ea7641535408c80735b71b6711d87d212800
Parents: c4d3b28
Author: Sangjin Lee <sjlee@apache.org>
Authored: Wed Jan 25 15:51:36 2017 -0800
Committer: Sangjin Lee <sjlee@apache.org>
Committed: Wed Jan 25 15:52:05 2017 -0800

----------------------------------------------------------------------
 .../yarn/client/api/SharedCacheClient.java      | 23 +++++++++---
 .../client/api/impl/SharedCacheClientImpl.java  | 31 ++++++++++++++--
 .../api/impl/TestSharedCacheClientImpl.java     | 37 +++++++++++++++++---
 3 files changed, 81 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b799ea76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
index 7cbe0e1..60c1bd98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
@@ -55,22 +55,37 @@ public abstract class SharedCacheClient extends AbstractService {
    * {@link ApplicationId} to identify which application will be using the
    * resource.
    * </p>
-   * 
+   *
    * <p>
    * The <code>SharedCacheManager</code> responds with whether or not the
    * resource exists in the cache. If the resource exists, a <code>Path</code>
    * to the resource in the shared cache is returned. If the resource does not
    * exist, null is returned instead.
    * </p>
-   * 
+   *
+   * <p>
+   * Once a path has been returned for a resource, that path is safe to use for
+   * the lifetime of the application that corresponds to the provided
+   * ApplicationId.
+   * </p>
+   *
+   * <p>
+   * Additionally, a name for the resource should be specified. A fragment will
+   * be added to the path with the desired name if the desired name is different
+   * than the name of the provided path from the shared cache. This ensures that
+   * if the returned path is used to create a LocalResource, then the symlink
+   * created during YARN localization will match the name specified.
+   * </p>
+   *
    * @param applicationId ApplicationId of the application using the resource
    * @param resourceKey the key (i.e. checksum) that identifies the resource
+   * @param resourceName the desired name of the resource
    * @return Path to the resource, or null if it does not exist
    */
   @Public
   @Unstable
-  public abstract Path use(ApplicationId applicationId, String resourceKey)
-      throws YarnException;
+  public abstract Path use(ApplicationId applicationId, String resourceKey,
+      String resourceName) throws YarnException;
 
   /**
    * <p>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b799ea76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
index 0a61ee0..b910c28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.client.api.impl;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -111,8 +113,8 @@ public class SharedCacheClientImpl extends SharedCacheClient {
   }
 
   @Override
-  public Path use(ApplicationId applicationId, String resourceKey)
-      throws YarnException {
+  public Path use(ApplicationId applicationId, String resourceKey,
+      String resourceName) throws YarnException {
     Path resourcePath = null;
     UseSharedCacheResourceRequest request = Records.newRecord(
         UseSharedCacheResourceRequest.class);
@@ -129,6 +131,31 @@ public class SharedCacheClientImpl extends SharedCacheClient {
       // We don't handle different exceptions separately at this point.
       throw new YarnException(e);
     }
+    if (resourcePath != null) {
+      if (resourcePath.getName().equals(resourceName)) {
+        // The preferred name is the same as the name of the item in the cache,
+        // so we skip generating the fragment to save space in the MRconfig.
+        return resourcePath;
+      } else {
+        // We are using the shared cache, and a preferred name has been
+        // specified that is different than the name of the resource in the
+        // shared cache. We need to set the fragment portion of the URI to
+        // preserve the desired name.
+        URI pathURI = resourcePath.toUri();
+        try {
+          // We assume that there is no existing fragment in the URI since the
+          // shared cache manager does not use fragments.
+          pathURI =
+              new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(),
+                  resourceName);
+          resourcePath = new Path(pathURI);
+        } catch (URISyntaxException e) {
+          throw new YarnException(
+              "Could not create a new URI due to syntax errors: "
+                  + pathURI.toString(), e);
+        }
+      }
+    }
     return resourcePath;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b799ea76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
index 3985e54..16b32db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.client.api.impl;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.mockito.Matchers.isA;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -26,6 +27,7 @@ import static org.mockito.Mockito.when;
 import java.io.DataOutputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.URI;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -106,15 +108,42 @@ public class TestSharedCacheClientImpl {
   }
 
   @Test
-  public void testUse() throws Exception {
+  public void testUseCacheMiss() throws Exception {
+    UseSharedCacheResourceResponse response =
+        new UseSharedCacheResourceResponsePBImpl();
+    response.setPath(null);
+    when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
+        response);
+    Path newPath = client.use(mock(ApplicationId.class), "key", null);
+    assertNull("The path is not null!", newPath);
+  }
+
+  @Test
+  public void testUseWithResourceName() throws Exception {
+    Path file = new Path("viewfs://test/path");
+    URI useUri = new URI("viewfs://test/path#linkName");
+    Path usePath = new Path(useUri);
+    UseSharedCacheResourceResponse response =
+        new UseSharedCacheResourceResponsePBImpl();
+    response.setPath(file.toString());
+    when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
+        response);
+    Path newPath = client.use(mock(ApplicationId.class), "key", "linkName");
+    assertEquals("The paths are not equal!", usePath, newPath);
+  }
+
+  @Test
+  public void testUseWithSameResourceName() throws Exception {
     Path file = new Path("viewfs://test/path");
+    URI useUri = new URI("viewfs://test/path");
+    Path usePath = new Path(useUri);
     UseSharedCacheResourceResponse response =
         new UseSharedCacheResourceResponsePBImpl();
     response.setPath(file.toString());
     when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
         response);
-    Path newPath = client.use(mock(ApplicationId.class), "key");
-    assertEquals(file, newPath);
+    Path newPath = client.use(mock(ApplicationId.class), "key", "path");
+    assertEquals("The paths are not equal!", usePath, newPath);
   }
 
   @Test(expected = YarnException.class)
@@ -122,7 +151,7 @@ public class TestSharedCacheClientImpl {
     String message = "Mock IOExcepiton!";
     when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow(
         new IOException(message));
-    client.use(mock(ApplicationId.class), "key");
+    client.use(mock(ApplicationId.class), "key", null);
   }
 
   @Test


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message