From common-commits-return-82160-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue May 1 22:50:00 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id DAB73180675 for ; Tue, 1 May 2018 22:49:59 +0200 (CEST) Received: (qmail 83205 invoked by uid 500); 1 May 2018 20:49:58 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 83196 invoked by uid 99); 1 May 2018 20:49:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 May 2018 20:49:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B528BE17C3; Tue, 1 May 2018 20:49:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: eyang@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-7799. Improved YARN service jar file handling. Contributed by Billie Rinaldi (cherry picked from commit 24eeea8b18749e02ea05b03eb18e3cf0455077c5) Date: Tue, 1 May 2018 20:49:58 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-3.1 4aadfb3c9 -> a6f3d6ffc YARN-7799. Improved YARN service jar file handling. Contributed by Billie Rinaldi (cherry picked from commit 24eeea8b18749e02ea05b03eb18e3cf0455077c5) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6f3d6ff Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6f3d6ff Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6f3d6ff Branch: refs/heads/branch-3.1 Commit: a6f3d6ffcb20c4536d8da5e1c334df0ffd18ab0e Parents: 4aadfb3 Author: Eric Yang Authored: Tue May 1 16:46:34 2018 -0400 Committer: Eric Yang Committed: Tue May 1 16:49:29 2018 -0400 ---------------------------------------------------------------------- .../hadoop-yarn-services-core/pom.xml | 5 + .../yarn/service/client/ServiceClient.java | 95 +++++++++++---- .../yarn/service/utils/CoreFileSystem.java | 7 ++ .../yarn/service/client/TestServiceCLI.java | 118 +++++++++++++++++-- .../markdown/yarn-service/Configurations.md | 2 +- 5 files changed, 195 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6f3d6ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index fce070d..38aa4fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -173,6 +173,11 @@ + org.apache.hadoop + hadoop-hdfs + + + com.google.protobuf protobuf-java http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6f3d6ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 8dd5342..67306d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -28,7 +28,9 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.registry.client.api.RegistryConstants; @@ -37,8 +39,8 @@ import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -896,13 +898,13 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, protected Path addJarResource(String serviceName, Map localResources) - throws IOException, SliderException { + throws IOException, YarnException { Path libPath = fs.buildClusterDirPath(serviceName); ProviderUtils .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs, libPath, "lib", false); Path dependencyLibTarGzip = fs.getDependencyTarGzip(); - if (fs.isFile(dependencyLibTarGzip)) { + if (actionDependency(null, false) == EXIT_SUCCESS) { LOG.info("Loading lib tar from " + dependencyLibTarGzip); fs.submitTarGzipAndUpdate(localResources); } else { @@ -1223,18 +1225,18 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, return actionDependency(destinationFolder, true); } - public int actionDependency(String destinationFolder, boolean overwrite) - throws IOException, YarnException { + public int actionDependency(String destinationFolder, boolean overwrite) { String currentUser = RegistryUtils.currentUser(); LOG.info("Running command as user {}", currentUser); + Path dependencyLibTarGzip; if (destinationFolder == null) { - destinationFolder = String.format(YarnServiceConstants.DEPENDENCY_DIR, - VersionInfo.getVersion()); + dependencyLibTarGzip = fs.getDependencyTarGzip(); + } else { + dependencyLibTarGzip = new Path(destinationFolder, + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); } - Path dependencyLibTarGzip = new Path(destinationFolder, - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME - + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); // Check if dependency has already been uploaded, in which case log // appropriately and exit success (unless overwrite has been requested) @@ -1247,24 +1249,71 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, String[] libDirs = ServiceUtils.getLibDirs(); if (libDirs.length > 0) { - File tempLibTarGzipFile = File.createTempFile( - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_", - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); - // copy all jars - tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter()); - - LOG.info("Version Info: " + VersionInfo.getBuildVersion()); - fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip, - new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS)); - LOG.info("To let apps use this tarball, in yarn-site set config property " - + "{} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH, - dependencyLibTarGzip); - return EXIT_SUCCESS; + File tempLibTarGzipFile = null; + try { + if (!checkPermissions(dependencyLibTarGzip)) { + return EXIT_UNAUTHORIZED; + } + + tempLibTarGzipFile = File.createTempFile( + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_", + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); + // copy all jars + tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter()); + + fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip, + new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS)); + LOG.info("To let apps use this tarball, in yarn-site set config " + + "property {} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH, + dependencyLibTarGzip); + return EXIT_SUCCESS; + } catch (IOException e) { + LOG.error("Got exception creating tarball and uploading to HDFS", e); + return EXIT_EXCEPTION_THROWN; + } finally { + if (tempLibTarGzipFile != null) { + if (!tempLibTarGzipFile.delete()) { + LOG.warn("Failed to delete tmp file {}", tempLibTarGzipFile); + } + } + } } else { return EXIT_FALSE; } } + private boolean checkPermissions(Path dependencyLibTarGzip) throws + IOException { + AccessControlList yarnAdminAcl = new AccessControlList(getConfig().get( + YarnConfiguration.YARN_ADMIN_ACL, + YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); + AccessControlList dfsAdminAcl = new AccessControlList( + getConfig().get(DFSConfigKeys.DFS_ADMIN, " ")); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + if (!yarnAdminAcl.isUserAllowed(ugi) && !dfsAdminAcl.isUserAllowed(ugi)) { + LOG.error("User must be on the {} or {} list to have permission to " + + "upload AM dependency tarball", YarnConfiguration.YARN_ADMIN_ACL, + DFSConfigKeys.DFS_ADMIN); + return false; + } + + Path parent = dependencyLibTarGzip.getParent(); + while (parent != null) { + if (fs.getFileSystem().exists(parent)) { + FsPermission perm = fs.getFileSystem().getFileStatus(parent) + .getPermission(); + if (!perm.getOtherAction().implies(FsAction.READ_EXECUTE)) { + LOG.error("Parent directory {} of {} tarball location {} does not " + + "have world read/execute permission", parent, YarnServiceConf + .DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip); + return false; + } + } + parent = parent.getParent(); + } + return true; + } + protected ClientAMProtocol createAMProxy(String serviceName, ApplicationReport appReport) throws IOException, YarnException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6f3d6ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java index 5c2bac6..50b22e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -363,6 +364,12 @@ public class CoreFileSystem { if (configuredDependencyTarballPath != null) { dependencyLibTarGzip = new Path(configuredDependencyTarballPath); } + if (dependencyLibTarGzip == null) { + dependencyLibTarGzip = new Path(String.format(YarnServiceConstants + .DEPENDENCY_DIR, VersionInfo.getVersion()), + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); + } return dependencyLibTarGzip; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6f3d6ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index 7290962..c40a39d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.service.client; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.client.cli.ApplicationCLI; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -27,12 +31,15 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.conf.ExampleAppJson; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,21 +47,33 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.util.Arrays; import java.util.List; import static org.apache.hadoop.yarn.client.api.AppAdminClient.YARN_APP_ADMIN_CLIENT_PREFIX; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEPENDENCY_TARBALL_PATH; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; +import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_SUCCESS; +import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_UNAUTHORIZED; import static org.mockito.Mockito.spy; public class TestServiceCLI { private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI .class); + @Rule + public TemporaryFolder tmpFolder = new TemporaryFolder(); + private Configuration conf = new YarnConfiguration(); - private File basedir; private SliderFileSystem fs; - private String basedirProp; private ApplicationCLI cli; + private File basedir; + private String basedirProp; + private File dependencyTarGzBaseDir; + private Path dependencyTarGz; + private String dependencyTarGzProp; + private String yarnAdminNoneAclProp; + private String dfsAdminAclProp; private void createCLI() { cli = new ApplicationCLI(); @@ -67,12 +86,17 @@ public class TestServiceCLI { cli.setConf(conf); } + private int runCLI(String[] args) throws Exception { + LOG.info("running CLI: yarn {}", Arrays.asList(args)); + return ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args)); + } + private void buildApp(String serviceName, String appDef) throws Throwable { String[] args = {"app", "-D", basedirProp, "-save", serviceName, ExampleAppJson.resourceName(appDef), "-appTypes", DUMMY_APP_TYPE}; - ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args)); + Assert.assertEquals(EXIT_SUCCESS, runCLI(args)); } private void buildApp(String serviceName, String appDef, @@ -83,7 +107,13 @@ public class TestServiceCLI { "-appTypes", DUMMY_APP_TYPE, "-updateLifetime", lifetime, "-changeQueue", queue}; - ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args)); + Assert.assertEquals(EXIT_SUCCESS, runCLI(args)); + } + + private static Path getDependencyTarGz(File dir) { + return new Path(new File(dir, YarnServiceConstants + .DEPENDENCY_TAR_GZ_FILE_NAME + YarnServiceConstants + .DEPENDENCY_TAR_GZ_FILE_EXT).getAbsolutePath()); } @Before @@ -91,12 +121,22 @@ public class TestServiceCLI { basedir = new File("target", "apps"); basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath(); conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); + dependencyTarGzBaseDir = tmpFolder.getRoot(); + dependencyTarGz = getDependencyTarGz(dependencyTarGzBaseDir); + dependencyTarGzProp = DEPENDENCY_TARBALL_PATH + "=" + dependencyTarGz + .toString(); + conf.set(DEPENDENCY_TARBALL_PATH, dependencyTarGz.toString()); fs = new SliderFileSystem(conf); if (basedir.exists()) { FileUtils.deleteDirectory(basedir); } else { basedir.mkdirs(); } + yarnAdminNoneAclProp = YarnConfiguration.YARN_ADMIN_ACL + "=none"; + dfsAdminAclProp = DFSConfigKeys.DFS_ADMIN + "=" + + UserGroupInformation.getCurrentUser(); + System.setProperty(YarnServiceConstants.PROPERTY_LIB_DIR, basedir + .getAbsolutePath()); createCLI(); } @@ -108,7 +148,7 @@ public class TestServiceCLI { cli.stop(); } - @Test + @Test (timeout = 180000) public void testFlexComponents() throws Throwable { // currently can only test building apps, since that is the only // operation that doesn't require an RM @@ -122,7 +162,7 @@ public class TestServiceCLI { checkApp(serviceName, "master", 1L, 1000L, "qname"); } - @Test + @Test (timeout = 180000) public void testInitiateServiceUpgrade() throws Exception { String[] args = {"app", "-upgrade", "app-1", "-initiate", ExampleAppJson.resourceName(ExampleAppJson.APP_JSON), @@ -131,7 +171,7 @@ public class TestServiceCLI { Assert.assertEquals(result, 0); } - @Test + @Test (timeout = 180000) public void testInitiateAutoFinalizeServiceUpgrade() throws Exception { String[] args = {"app", "-upgrade", "app-1", "-initiate", ExampleAppJson.resourceName(ExampleAppJson.APP_JSON), @@ -141,7 +181,7 @@ public class TestServiceCLI { Assert.assertEquals(result, 0); } - @Test + @Test (timeout = 180000) public void testUpgradeInstances() throws Exception { conf.set(YARN_APP_ADMIN_CLIENT_PREFIX + DUMMY_APP_TYPE, DummyServiceClient.class.getName()); @@ -153,6 +193,68 @@ public class TestServiceCLI { Assert.assertEquals(result, 0); } + @Test (timeout = 180000) + public void testEnableFastLaunch() throws Exception { + fs.getFileSystem().create(new Path(basedir.getAbsolutePath(), "test.jar")) + .close(); + + Path defaultPath = new Path(dependencyTarGz.toString()); + Assert.assertFalse("Dependency tarball should not exist before the test", + fs.isFile(defaultPath)); + String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch", + "-appTypes", DUMMY_APP_TYPE}; + Assert.assertEquals(EXIT_SUCCESS, runCLI(args)); + Assert.assertTrue("Dependency tarball did not exist after the test", + fs.isFile(defaultPath)); + + File secondBaseDir = new File(dependencyTarGzBaseDir, "2"); + Path secondTarGz = getDependencyTarGz(secondBaseDir); + Assert.assertFalse("Dependency tarball should not exist before the test", + fs.isFile(secondTarGz)); + String[] args2 = {"app", "-D", yarnAdminNoneAclProp, "-D", + dfsAdminAclProp, "-D", dependencyTarGzProp, "-enableFastLaunch", + secondBaseDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE}; + Assert.assertEquals(EXIT_SUCCESS, runCLI(args2)); + Assert.assertTrue("Dependency tarball did not exist after the test", + fs.isFile(secondTarGz)); + } + + @Test (timeout = 180000) + public void testEnableFastLaunchUserPermissions() throws Exception { + String[] args = {"app", "-D", yarnAdminNoneAclProp, "-D", + dependencyTarGzProp, "-enableFastLaunch", "-appTypes", DUMMY_APP_TYPE}; + Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args)); + } + + @Test (timeout = 180000) + public void testEnableFastLaunchFilePermissions() throws Exception { + File badDir = new File(dependencyTarGzBaseDir, "bad"); + badDir.mkdir(); + fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()), + new FsPermission("751")); + + String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch", + badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE}; + Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args)); + + badDir = new File(badDir, "child"); + badDir.mkdir(); + fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()), + new FsPermission("755")); + + String[] args2 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch", + badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE}; + Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args2)); + + badDir = new File(dependencyTarGzBaseDir, "badx"); + badDir.mkdir(); + fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()), + new FsPermission("754")); + + String[] args3 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch", + badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE}; + Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args3)); + } private void checkApp(String serviceName, String compName, long count, Long lifetime, String queue) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6f3d6ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md index 75186dc..524cfb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md @@ -100,7 +100,7 @@ System-wide service AM properties can only be configured in the cluster `yarn-si | System-Level Config Name | Description | | ------------ | ------------- | -|yarn.service.framework.path | HDFS parent directory where the service AM dependency tarball can be found.| +|yarn.service.framework.path | HDFS path of the service AM dependency tarball. When no file exists at this location, AM dependencies will be uploaded by the RM the first time a service is started or launched. If the RM user does not have permission to upload the file to this location or the location is not world readable, the AM dependency jars will be uploaded each time a service is started or launched. If unspecified, value will be assumed to be /yarn-services/${hadoop.version}/service-dep.tar.gz.| |yarn.service.base.path | HDFS parent directory where service artifacts will be stored (default ${user_home_dir}/.yarn/). |yarn.service.client-am.retry.max-wait-ms | Max retry time in milliseconds for the service client to talk to the service AM (default 900000, i.e. 15 minutes).| |yarn.service.client-am.retry-interval-ms | Retry interval in milliseconds for the service client to talk to the service AM (default 2000, i.e. 2 seconds).| --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org