hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject hadoop git commit: YARN-7799. Improved YARN service jar file handling. Contributed by Billie Rinaldi
Date Tue, 01 May 2018 20:48:58 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 9e2cfb2d3 -> 24eeea8b1


YARN-7799. Improved YARN service jar file handling.
           Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24eeea8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24eeea8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24eeea8b

Branch: refs/heads/trunk
Commit: 24eeea8b18749e02ea05b03eb18e3cf0455077c5
Parents: 9e2cfb2
Author: Eric Yang <eyang@apache.org>
Authored: Tue May 1 16:46:34 2018 -0400
Committer: Eric Yang <eyang@apache.org>
Committed: Tue May 1 16:46:34 2018 -0400

----------------------------------------------------------------------
 .../hadoop-yarn-services-core/pom.xml           |   5 +
 .../yarn/service/client/ServiceClient.java      |  95 +++++++++++----
 .../yarn/service/utils/CoreFileSystem.java      |   7 ++
 .../yarn/service/client/TestServiceCLI.java     | 118 +++++++++++++++++--
 .../markdown/yarn-service/Configurations.md     |   2 +-
 5 files changed, 195 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/24eeea8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 3ce8876..7efe8bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -173,6 +173,11 @@
     </dependency>
 
     <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+    </dependency>
+
+    <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24eeea8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 8dd5342..67306d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -28,7 +28,9 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.registry.client.api.RegistryConstants;
@@ -37,8 +39,8 @@ import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
 import org.apache.hadoop.registry.client.binding.RegistryUtils;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
@@ -896,13 +898,13 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
 
   protected Path addJarResource(String serviceName,
       Map<String, LocalResource> localResources)
-      throws IOException, SliderException {
+      throws IOException, YarnException {
     Path libPath = fs.buildClusterDirPath(serviceName);
     ProviderUtils
         .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
             libPath, "lib", false);
     Path dependencyLibTarGzip = fs.getDependencyTarGzip();
-    if (fs.isFile(dependencyLibTarGzip)) {
+    if (actionDependency(null, false) == EXIT_SUCCESS) {
       LOG.info("Loading lib tar from " + dependencyLibTarGzip);
       fs.submitTarGzipAndUpdate(localResources);
     } else {
@@ -1223,18 +1225,18 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
     return actionDependency(destinationFolder, true);
   }
 
-  public int actionDependency(String destinationFolder, boolean overwrite)
-      throws IOException, YarnException {
+  public int actionDependency(String destinationFolder, boolean overwrite) {
     String currentUser = RegistryUtils.currentUser();
     LOG.info("Running command as user {}", currentUser);
 
+    Path dependencyLibTarGzip;
     if (destinationFolder == null) {
-      destinationFolder = String.format(YarnServiceConstants.DEPENDENCY_DIR,
-          VersionInfo.getVersion());
+      dependencyLibTarGzip = fs.getDependencyTarGzip();
+    } else {
+      dependencyLibTarGzip = new Path(destinationFolder,
+          YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+              + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
     }
-    Path dependencyLibTarGzip = new Path(destinationFolder,
-        YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
-            + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
 
     // Check if dependency has already been uploaded, in which case log
     // appropriately and exit success (unless overwrite has been requested)
@@ -1247,24 +1249,71 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
 
     String[] libDirs = ServiceUtils.getLibDirs();
     if (libDirs.length > 0) {
-      File tempLibTarGzipFile = File.createTempFile(
-          YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
-          YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
-      // copy all jars
-      tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
-
-      LOG.info("Version Info: " + VersionInfo.getBuildVersion());
-      fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
-          new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
-      LOG.info("To let apps use this tarball, in yarn-site set config property "
-          + "{} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH,
-          dependencyLibTarGzip);
-      return EXIT_SUCCESS;
+      File tempLibTarGzipFile = null;
+      try {
+        if (!checkPermissions(dependencyLibTarGzip)) {
+          return EXIT_UNAUTHORIZED;
+        }
+
+        tempLibTarGzipFile = File.createTempFile(
+            YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
+            YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+        // copy all jars
+        tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
+
+        fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
+            new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
+        LOG.info("To let apps use this tarball, in yarn-site set config " +
+                "property {} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH,
+            dependencyLibTarGzip);
+        return EXIT_SUCCESS;
+      } catch (IOException e) {
+        LOG.error("Got exception creating tarball and uploading to HDFS", e);
+        return EXIT_EXCEPTION_THROWN;
+      } finally {
+        if (tempLibTarGzipFile != null) {
+          if (!tempLibTarGzipFile.delete()) {
+            LOG.warn("Failed to delete tmp file {}", tempLibTarGzipFile);
+          }
+        }
+      }
     } else {
       return EXIT_FALSE;
     }
   }
 
+  private boolean checkPermissions(Path dependencyLibTarGzip) throws
+      IOException {
+    AccessControlList yarnAdminAcl = new AccessControlList(getConfig().get(
+        YarnConfiguration.YARN_ADMIN_ACL,
+        YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+    AccessControlList dfsAdminAcl = new AccessControlList(
+        getConfig().get(DFSConfigKeys.DFS_ADMIN, " "));
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    if (!yarnAdminAcl.isUserAllowed(ugi) && !dfsAdminAcl.isUserAllowed(ugi)) {
+      LOG.error("User must be on the {} or {} list to have permission to " +
+          "upload AM dependency tarball", YarnConfiguration.YARN_ADMIN_ACL,
+          DFSConfigKeys.DFS_ADMIN);
+      return false;
+    }
+
+    Path parent = dependencyLibTarGzip.getParent();
+    while (parent != null) {
+      if (fs.getFileSystem().exists(parent)) {
+        FsPermission perm = fs.getFileSystem().getFileStatus(parent)
+            .getPermission();
+        if (!perm.getOtherAction().implies(FsAction.READ_EXECUTE)) {
+          LOG.error("Parent directory {} of {} tarball location {} does not " +
+              "have world read/execute permission", parent, YarnServiceConf
+              .DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip);
+          return false;
+        }
+      }
+      parent = parent.getParent();
+    }
+    return true;
+  }
+
   protected ClientAMProtocol createAMProxy(String serviceName,
       ApplicationReport appReport) throws IOException, YarnException {
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24eeea8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 5c2bac6..50b22e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -363,6 +364,12 @@ public class CoreFileSystem {
     if (configuredDependencyTarballPath != null) {
       dependencyLibTarGzip = new Path(configuredDependencyTarballPath);
     }
+    if (dependencyLibTarGzip == null) {
+      dependencyLibTarGzip = new Path(String.format(YarnServiceConstants
+          .DEPENDENCY_DIR, VersionInfo.getVersion()),
+          YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+              + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+    }
     return dependencyLibTarGzip;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24eeea8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index 7290962..c40a39d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.service.client;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -27,12 +31,15 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,21 +47,33 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.util.Arrays;
 import java.util.List;
 
 import static org.apache.hadoop.yarn.client.api.AppAdminClient.YARN_APP_ADMIN_CLIENT_PREFIX;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEPENDENCY_TARBALL_PATH;
 import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_SUCCESS;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_UNAUTHORIZED;
 import static org.mockito.Mockito.spy;
 
 public class TestServiceCLI {
   private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI
       .class);
 
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
+
   private Configuration conf = new YarnConfiguration();
-  private File basedir;
   private SliderFileSystem fs;
-  private String basedirProp;
   private ApplicationCLI cli;
+  private File basedir;
+  private String basedirProp;
+  private File dependencyTarGzBaseDir;
+  private Path dependencyTarGz;
+  private String dependencyTarGzProp;
+  private String yarnAdminNoneAclProp;
+  private String dfsAdminAclProp;
 
   private void createCLI() {
     cli = new ApplicationCLI();
@@ -67,12 +86,17 @@ public class TestServiceCLI {
     cli.setConf(conf);
   }
 
+  private int runCLI(String[] args) throws Exception {
+    LOG.info("running CLI: yarn {}", Arrays.asList(args));
+    return ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+  }
+
   private void buildApp(String serviceName, String appDef) throws Throwable {
     String[] args = {"app",
         "-D", basedirProp, "-save", serviceName,
         ExampleAppJson.resourceName(appDef),
         "-appTypes", DUMMY_APP_TYPE};
-    ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+    Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
   }
 
   private void buildApp(String serviceName, String appDef,
@@ -83,7 +107,13 @@ public class TestServiceCLI {
         "-appTypes", DUMMY_APP_TYPE,
         "-updateLifetime", lifetime,
         "-changeQueue", queue};
-    ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+    Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
+  }
+
+  private static Path getDependencyTarGz(File dir) {
+    return new Path(new File(dir, YarnServiceConstants
+        .DEPENDENCY_TAR_GZ_FILE_NAME + YarnServiceConstants
+        .DEPENDENCY_TAR_GZ_FILE_EXT).getAbsolutePath());
   }
 
   @Before
@@ -91,12 +121,22 @@ public class TestServiceCLI {
     basedir = new File("target", "apps");
     basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
     conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+    dependencyTarGzBaseDir = tmpFolder.getRoot();
+    dependencyTarGz = getDependencyTarGz(dependencyTarGzBaseDir);
+    dependencyTarGzProp = DEPENDENCY_TARBALL_PATH + "=" + dependencyTarGz
+        .toString();
+    conf.set(DEPENDENCY_TARBALL_PATH, dependencyTarGz.toString());
     fs = new SliderFileSystem(conf);
     if (basedir.exists()) {
       FileUtils.deleteDirectory(basedir);
     } else {
       basedir.mkdirs();
     }
+    yarnAdminNoneAclProp = YarnConfiguration.YARN_ADMIN_ACL + "=none";
+    dfsAdminAclProp = DFSConfigKeys.DFS_ADMIN + "=" +
+        UserGroupInformation.getCurrentUser();
+    System.setProperty(YarnServiceConstants.PROPERTY_LIB_DIR, basedir
+        .getAbsolutePath());
     createCLI();
   }
 
@@ -108,7 +148,7 @@ public class TestServiceCLI {
     cli.stop();
   }
 
-  @Test
+  @Test (timeout = 180000)
   public void testFlexComponents() throws Throwable {
     // currently can only test building apps, since that is the only
     // operation that doesn't require an RM
@@ -122,7 +162,7 @@ public class TestServiceCLI {
     checkApp(serviceName, "master", 1L, 1000L, "qname");
   }
 
-  @Test
+  @Test (timeout = 180000)
   public void testInitiateServiceUpgrade() throws Exception {
     String[] args = {"app", "-upgrade", "app-1",
         "-initiate", ExampleAppJson.resourceName(ExampleAppJson.APP_JSON),
@@ -131,7 +171,7 @@ public class TestServiceCLI {
     Assert.assertEquals(result, 0);
   }
 
-  @Test
+  @Test (timeout = 180000)
   public void testInitiateAutoFinalizeServiceUpgrade() throws Exception {
     String[] args =  {"app", "-upgrade", "app-1",
         "-initiate", ExampleAppJson.resourceName(ExampleAppJson.APP_JSON),
@@ -141,7 +181,7 @@ public class TestServiceCLI {
     Assert.assertEquals(result, 0);
   }
 
-  @Test
+  @Test (timeout = 180000)
   public void testUpgradeInstances() throws Exception {
     conf.set(YARN_APP_ADMIN_CLIENT_PREFIX + DUMMY_APP_TYPE,
         DummyServiceClient.class.getName());
@@ -153,6 +193,68 @@ public class TestServiceCLI {
     Assert.assertEquals(result, 0);
   }
 
+  @Test (timeout = 180000)
+  public void testEnableFastLaunch() throws Exception {
+    fs.getFileSystem().create(new Path(basedir.getAbsolutePath(), "test.jar"))
+        .close();
+
+    Path defaultPath = new Path(dependencyTarGz.toString());
+    Assert.assertFalse("Dependency tarball should not exist before the test",
+        fs.isFile(defaultPath));
+    String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
+        "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
+    Assert.assertTrue("Dependency tarball did not exist after the test",
+        fs.isFile(defaultPath));
+
+    File secondBaseDir = new File(dependencyTarGzBaseDir, "2");
+    Path secondTarGz = getDependencyTarGz(secondBaseDir);
+    Assert.assertFalse("Dependency tarball should not exist before the test",
+        fs.isFile(secondTarGz));
+    String[] args2 = {"app", "-D", yarnAdminNoneAclProp, "-D",
+        dfsAdminAclProp, "-D", dependencyTarGzProp, "-enableFastLaunch",
+        secondBaseDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_SUCCESS, runCLI(args2));
+    Assert.assertTrue("Dependency tarball did not exist after the test",
+        fs.isFile(secondTarGz));
+  }
+
+  @Test (timeout = 180000)
+  public void testEnableFastLaunchUserPermissions() throws Exception {
+    String[] args = {"app", "-D", yarnAdminNoneAclProp, "-D",
+        dependencyTarGzProp, "-enableFastLaunch", "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args));
+  }
+
+  @Test (timeout = 180000)
+  public void testEnableFastLaunchFilePermissions() throws Exception {
+    File badDir = new File(dependencyTarGzBaseDir, "bad");
+    badDir.mkdir();
+    fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
+        new FsPermission("751"));
+
+    String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
+        badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args));
+
+    badDir = new File(badDir, "child");
+    badDir.mkdir();
+    fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
+        new FsPermission("755"));
+
+    String[] args2 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
+        badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args2));
+
+    badDir = new File(dependencyTarGzBaseDir, "badx");
+    badDir.mkdir();
+    fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
+        new FsPermission("754"));
+
+    String[] args3 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
+        badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args3));
+  }
 
   private void checkApp(String serviceName, String compName, long count, Long
       lifetime, String queue) throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/24eeea8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
index 75186dc..524cfb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -100,7 +100,7 @@ System-wide service AM properties can only be configured in the cluster
`yarn-si
 
 | System-Level Config Name | Description |
 | ------------ | ------------- |
-|yarn.service.framework.path | HDFS parent directory where the service AM dependency tarball
can be found.|
+|yarn.service.framework.path | HDFS path of the service AM dependency tarball. When no file
exists at this location, AM dependencies will be uploaded by the RM the first time a service
is started or launched. If the RM user does not have permission to upload the file to this
location or the location is not world readable, the AM dependency jars will be uploaded each
time a service is started or launched. If unspecified, value will be assumed to be /yarn-services/${hadoop.version}/service-dep.tar.gz.|
 |yarn.service.base.path | HDFS parent directory where service artifacts will be stored (default
${user_home_dir}/.yarn/).
 |yarn.service.client-am.retry.max-wait-ms | Max retry time in milliseconds for the service
client to talk to the service AM (default 900000, i.e. 15 minutes).|
 |yarn.service.client-am.retry-interval-ms | Retry interval in milliseconds for the service
client to talk to the service AM (default 2000, i.e. 2 seconds).|


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message