From common-commits-return-77966-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Wed Jan 24 23:35:47 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id 145A4180676 for ; Wed, 24 Jan 2018 23:35:47 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 04D79160C4E; Wed, 24 Jan 2018 22:35:47 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C5455160C3C for ; Wed, 24 Jan 2018 23:35:45 +0100 (CET) Received: (qmail 87199 invoked by uid 500); 24 Jan 2018 22:35:32 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 86768 invoked by uid 99); 24 Jan 2018 22:35:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Jan 2018 22:35:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BC4F5F4DFE; Wed, 24 Jan 2018 22:35:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Wed, 24 Jan 2018 22:36:08 -0000 Message-Id: In-Reply-To: <449ff7436d854b3e8bafac8efec3974b@git.apache.org> References: <449ff7436d854b3e8bafac8efec3974b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [39/50] [abbrv] hadoop git commit: YARN-7766. Introduce a new config property for YARN Service dependency tarball location. Contributed by Gour Saha YARN-7766. Introduce a new config property for YARN Service dependency tarball location. Contributed by Gour Saha Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a72cdcc4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a72cdcc4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a72cdcc4 Branch: refs/heads/HDFS-7240 Commit: a72cdcc47a1b3a4fcadbc9e12bd1bae2bbf04d83 Parents: f63d13f Author: Jian He Authored: Tue Jan 23 10:53:27 2018 -0800 Committer: Jian He Committed: Tue Jan 23 10:53:27 2018 -0800 ---------------------------------------------------------------------- .../yarn/service/client/ServiceClient.java | 35 +++++++++++++++----- .../yarn/service/conf/YarnServiceConf.java | 6 ++++ .../tarball/TarballProviderService.java | 4 +-- .../yarn/service/utils/CoreFileSystem.java | 32 ++++++++---------- .../hadoop/yarn/client/api/AppAdminClient.java | 12 ++++--- .../hadoop/yarn/client/cli/ApplicationCLI.java | 13 +++++--- .../hadoop/yarn/client/cli/TestYarnCLI.java | 5 ++- 7 files changed, 68 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index c224089..b8034b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.client.cli.ApplicationCLI; import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -718,14 +719,20 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, libPath, "lib", false); Path dependencyLibTarGzip = fs.getDependencyTarGzip(); if (fs.isFile(dependencyLibTarGzip)) { - LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/" - + dependencyLibTarGzip); + LOG.info("Loading lib tar from " + dependencyLibTarGzip); fs.submitTarGzipAndUpdate(localResources); } else { + if (dependencyLibTarGzip != null) { + LOG.warn("Property {} has a value {}, but is not a valid file", + YarnServiceConf.DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip); + } String[] libs = ServiceUtils.getLibDirs(); - LOG.info("Uploading all dependency jars to HDFS. For faster submission of" + - " apps, pre-upload dependency jars to HDFS " - + "using command: yarn app -enableFastLaunch"); + LOG.info("Uploading all dependency jars to HDFS. For faster submission of" + + " apps, set config property {} to the dependency tarball location." + + " Dependency tarball can be uploaded to any HDFS path directly" + + " or by using command: yarn app -{} []", + YarnServiceConf.DEPENDENCY_TARBALL_PATH, + ApplicationCLI.ENABLE_FAST_LAUNCH); for (String libDirProp : libs) { ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib", libDirProp); @@ -988,16 +995,23 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, return this.yarnClient; } - public int enableFastLaunch() throws IOException, YarnException { - return actionDependency(true); + public int enableFastLaunch(String destinationFolder) + throws IOException, YarnException { + return actionDependency(destinationFolder, true); } - public int actionDependency(boolean overwrite) + public int actionDependency(String destinationFolder, boolean overwrite) throws IOException, YarnException { String currentUser = RegistryUtils.currentUser(); LOG.info("Running command as user {}", currentUser); - Path dependencyLibTarGzip = fs.getDependencyTarGzip(); + if (destinationFolder == null) { + destinationFolder = String.format(YarnServiceConstants.DEPENDENCY_DIR, + VersionInfo.getVersion()); + } + Path dependencyLibTarGzip = new Path(destinationFolder, + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); // Check if dependency has already been uploaded, in which case log // appropriately and exit success (unless overwrite has been requested) @@ -1019,6 +1033,9 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, LOG.info("Version Info: " + VersionInfo.getBuildVersion()); fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip, new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS)); + LOG.info("To let apps use this tarball, in yarn-site set config property " + + "{} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH, + dependencyLibTarGzip); return EXIT_SUCCESS; } else { return EXIT_FALSE; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java index 22926e2..21470d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java @@ -94,6 +94,12 @@ public class YarnServiceConf { public static final int DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS = 120000; /** + * The dependency tarball file location. + */ + public static final String DEPENDENCY_TARBALL_PATH = YARN_SERVICE_PREFIX + + "framework.path"; + + /** * Get long value for the property. First get from the userConf, if not * present, get from systemConf. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java index 9f29c8b..7f29d1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java @@ -38,9 +38,9 @@ public class TarballProviderService extends AbstractProviderService { Path artifact = new Path(instance.getCompSpec().getArtifact().getId()); if (!fileSystem.isFile(artifact)) { throw new IOException( - "Package doesn't exist as a resource: " + artifact.toString()); + "Package doesn't exist as a resource: " + artifact); } - log.info("Adding resource {}", artifact.toString()); + log.info("Adding resource {}", artifact); LocalResourceType type = LocalResourceType.ARCHIVE; LocalResource packageResource = fileSystem.createAmResource(artifact, type); launcher.addLocalResource(APP_LIB_DIR, packageResource); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java index 6b9d4d5..284825e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java @@ -295,6 +295,9 @@ public class CoreFileSystem { * reasons including if file check throws IOException */ public boolean isFile(Path path) { + if (path == null) { + return false; + } boolean isFile = false; try { FileStatus status = fileSystem.getFileStatus(path); @@ -321,26 +324,18 @@ public class CoreFileSystem { } /** - * Get slider dependency parent dir in HDFS + * Get service dependency absolute filepath in HDFS used for application + * submission. * - * @return the parent dir path of slider.tar.gz in HDFS - */ - public Path getDependencyPath() { - String parentDir = YarnServiceConstants.DEPENDENCY_DIR; - return new Path(String.format(parentDir, VersionInfo.getVersion())); - } - - /** - * Get slider.tar.gz absolute filepath in HDFS - * - * @return the absolute path to slider.tar.gz in HDFS + * @return the absolute path to service dependency tarball in HDFS */ public Path getDependencyTarGzip() { - Path dependencyLibAmPath = getDependencyPath(); - Path dependencyLibTarGzip = new Path( - dependencyLibAmPath.toUri().toString(), - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME - + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); + Path dependencyLibTarGzip = null; + String configuredDependencyTarballPath = configuration + .get(YarnServiceConf.DEPENDENCY_TARBALL_PATH); + if (configuredDependencyTarballPath != null) { + dependencyLibTarGzip = new Path(configuredDependencyTarballPath); + } return dependencyLibTarGzip; } @@ -467,8 +462,7 @@ public class CoreFileSystem { fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); fileSystem.mkdirs(destPath.getParent(), fp); - log.info("Copying file {} to {}", localPath.toURI(), - fileSystem.getScheme() + ":/" + destPath.toUri()); + log.info("Copying file {} to {}", localPath.toURI(), destPath); fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()), destPath); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java index 6aba91a..55be13b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java @@ -196,14 +196,18 @@ public abstract class AppAdminClient extends CompositeService { * faster since the dependencies do not have to be uploaded on each launch. *

* + * @param destinationFolder + * an optional HDFS folder where dependency tarball will be uploaded * @return exit code - * @throws IOException IOException - * @throws YarnException exception in client or server + * @throws IOException + * IOException + * @throws YarnException + * exception in client or server */ @Public @Unstable - public abstract int enableFastLaunch() throws IOException, - YarnException; + public abstract int enableFastLaunch(String destinationFolder) + throws IOException, YarnException; /** *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index c751f79..471b4d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -232,9 +232,10 @@ public class ApplicationCLI extends YarnCLI { "the number of components/containers running for an application / " + "long-running service. Supports absolute or relative changes, such " + "as +1, 2, or -3."); - opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " + - "to HDFS to make future launches faster. Supports -appTypes option" + - " to specify which client implementation to use."); + opts.addOption(ENABLE_FAST_LAUNCH, true, "Uploads AM dependencies " + + "to HDFS to make future launches faster. Supports -appTypes option " + + "to specify which client implementation to use. Optionally a " + + "destination folder for the tarball can be specified."); opts.getOption(LAUNCH_CMD).setArgName("Application Name> Uploads AM dependencies to HDFS"); pw.println(" to make future launches faster."); pw.println(" Supports -appTypes option to"); pw.println(" specify which client"); pw.println(" implementation to use."); + pw.println(" Optionally a destination folder"); + pw.println(" for the tarball can be"); + pw.println(" specified."); pw.println(" -flex Changes number of running"); pw.println(" containers for a component of an"); pw.println(" application / long-running"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org