Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B3AD3200D36 for ; Mon, 6 Nov 2017 23:10:41 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id B23B8160C02; Mon, 6 Nov 2017 22:10:41 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D2370160BEC for ; Mon, 6 Nov 2017 23:10:39 +0100 (CET) Received: (qmail 40126 invoked by uid 500); 6 Nov 2017 22:10:22 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 37454 invoked by uid 99); 6 Nov 2017 22:10:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Nov 2017 22:10:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0CD40E038F; Mon, 6 Nov 2017 22:10:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Mon, 06 Nov 2017 22:11:09 -0000 Message-Id: In-Reply-To: <46e6bce734a34ff69f6f6ed15936f905@git.apache.org> References: <46e6bce734a34ff69f6f6ed15936f905@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [51/60] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi archived-at: Mon, 06 Nov 2017 22:10:41 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index f22d487..df4b1df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -20,88 +20,71 @@ package org.apache.hadoop.yarn.service.client; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.client.cli.ApplicationCLI; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.ClientAMProtocol; import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.client.params.ClientArgs; +import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.conf.ExampleAppJson; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS; -import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; -import static org.mockito.Mockito.*; public class TestServiceCLI { + private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI + .class); - protected Configuration conf = new YarnConfiguration(); + private Configuration conf = new YarnConfiguration(); private File basedir; - private ServiceCLI cli; private SliderFileSystem fs; + private String basedirProp; - private void buildApp(String appDef) throws Throwable { - String[] args = - { "build", ARG_FILE, ExampleAppJson.resourceName(appDef)}; - ClientArgs clientArgs = new ClientArgs(args); - clientArgs.parse(); - cli.exec(clientArgs); + private void runCLI(String[] args) throws Exception { + LOG.info("running CLI: yarn {}", Arrays.asList(args)); + ApplicationCLI cli = new ApplicationCLI(); + cli.setSysOutPrintStream(System.out); + cli.setSysErrPrintStream(System.err); + int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args)); + cli.stop(); + } + + private void buildApp(String serviceName, String appDef) throws Throwable { + String[] args = {"app", "-D", basedirProp, "-save", serviceName, + ExampleAppJson.resourceName(appDef)}; + runCLI(args); + } + + private void buildApp(String serviceName, String appDef, String lifetime, + String queue) throws Throwable { + String[] args = {"app", "-D", basedirProp, "-save", serviceName, + ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime, + "-changeQueue", queue}; + runCLI(args); } @Before public void setup() throws Throwable { basedir = new File("target", "apps"); + basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath(); conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); - conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0); - conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1); - conf.setInt( - CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); - conf.setInt(CommonConfigurationKeysPublic. - IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0); fs = new SliderFileSystem(conf); if (basedir.exists()) { FileUtils.deleteDirectory(basedir); } else { basedir.mkdirs(); } - - // create a CLI and skip connection to AM - cli = new ServiceCLI() { - @Override protected void createServiceClient() { - client = new ServiceClient() { - @Override - protected void serviceInit(Configuration configuration) - throws Exception { - super.serviceInit(conf); - yarnClient = spy(yarnClient); - ApplicationReport report = Records.newRecord(ApplicationReport.class); - report.setYarnApplicationState(YarnApplicationState.RUNNING); - report.setHost("localhost"); - doReturn(report).when(yarnClient).getApplicationReport(anyObject()); - } - @Override - protected ClientAMProtocol createAMProxy(String host, int port) - throws IOException { - return mock(ClientAMProtocol.class); - } - }; - client.init(conf); - client.start(); - } - }; } @After @@ -111,41 +94,31 @@ public class TestServiceCLI { } } - // Test flex components count are persisted. @Test public void testFlexComponents() throws Throwable { + // currently can only test building apps, since that is the only + // operation that doesn't require an RM + // TODO: expand CLI test to try other commands String serviceName = "app-1"; - buildApp(ExampleAppJson.APP_JSON); - checkCompCount("master",serviceName, 1L); - - // increase by 2 - String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"}; - ClientArgs clientArgs = new ClientArgs(flexUpArgs); - clientArgs.parse(); - cli.exec(clientArgs); - checkCompCount("master", serviceName, 3L); - - // decrease by 1 - String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"}; - clientArgs = new ClientArgs(flexDownArgs); - clientArgs.parse(); - cli.exec(clientArgs); - checkCompCount("master", serviceName, 2L); + buildApp(serviceName, ExampleAppJson.APP_JSON); + checkApp(serviceName, "master", 1L, 3600L, null); - String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"}; - clientArgs = new ClientArgs(flexAbsoluteArgs); - clientArgs.parse(); - cli.exec(clientArgs); - checkCompCount("master", serviceName, 10L); + serviceName = "app-2"; + buildApp(serviceName, ExampleAppJson.APP_JSON, "1000", "qname"); + checkApp(serviceName, "master", 1L, 1000L, "qname"); } - private void checkCompCount(String compName, String serviceName, long count) - throws IOException { - List components = - ServiceApiUtil.getComponents(fs, serviceName); + private void checkApp(String serviceName, String compName, long count, Long + lifetime, String queue) throws IOException { + Service service = ServiceApiUtil.loadService(fs, serviceName); + Assert.assertEquals(serviceName, service.getName()); + Assert.assertEquals(lifetime, service.getLifetime()); + Assert.assertEquals(queue, service.getQueue()); + List components = service.getComponents(); for (Component component : components) { if (component.getName().equals(compName)) { - Assert.assertEquals(count, component.getNumberOfContainers().longValue()); + Assert.assertEquals(count, component.getNumberOfContainers() + .longValue()); return; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java new file mode 100644 index 0000000..6310178 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.client.api; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +import java.io.IOException; +import java.util.Map; + +/** + * Client for managing applications. + */ +@Public +@Unstable +public abstract class AppAdminClient extends CompositeService { + public static final String YARN_APP_ADMIN_CLIENT_PREFIX = "yarn" + + ".application.admin.client.class."; + public static final String DEFAULT_TYPE = "yarn-service"; + public static final String DEFAULT_CLASS_NAME = "org.apache.hadoop.yarn" + + ".service.client.ServiceClient"; + + @Private + protected AppAdminClient() { + super(AppAdminClient.class.getName()); + } + + /** + *

+ * Create a new instance of AppAdminClient. + *

+ * + * @param appType application type + * @param conf configuration + * @return app admin client + */ + @Public + @Unstable + public static AppAdminClient createAppAdminClient(String appType, + Configuration conf) { + Map clientClassMap = + conf.getPropsWithPrefix(YARN_APP_ADMIN_CLIENT_PREFIX); + if (!clientClassMap.containsKey(DEFAULT_TYPE)) { + clientClassMap.put(DEFAULT_TYPE, DEFAULT_CLASS_NAME); + } + if (!clientClassMap.containsKey(appType)) { + throw new IllegalArgumentException("App admin client class name not " + + "specified for type " + appType); + } + String clientClassName = clientClassMap.get(appType); + Class clientClass; + try { + clientClass = (Class) Class.forName( + clientClassName); + } catch (ClassNotFoundException e) { + throw new YarnRuntimeException("Invalid app admin client class", e); + } + + AppAdminClient appAdminClient = ReflectionUtils.newInstance(clientClass, + conf); + appAdminClient.init(conf); + appAdminClient.start(); + return appAdminClient; + } + + /** + *

+ * Launch a new YARN application. + *

+ * + * @param fileName specification of application + * @param appName name of the application + * @param lifetime lifetime of the application + * @param queue queue of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionLaunch(String fileName, String appName, Long + lifetime, String queue) throws IOException, YarnException; + + /** + *

+ * Stop a YARN application (attempt to stop gracefully before killing the + * application). In the case of a long-running service, the service may be + * restarted later. + *

+ * + * @param appName the name of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionStop(String appName) throws IOException, + YarnException; + + /** + *

+ * Start a YARN application from a previously saved specification. In the + * case of a long-running service, the service must have been previously + * launched/started and then stopped, or previously saved but not started. + *

+ * + * @param appName the name of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionStart(String appName) throws IOException, + YarnException; + + /** + *

+ * Save the specification for a YARN application / long-running service. + * The application may be started later. + *

+ * + * @param fileName specification of application to save + * @param appName name of the application + * @param lifetime lifetime of the application + * @param queue queue of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionSave(String fileName, String appName, Long + lifetime, String queue) throws IOException, YarnException; + + /** + *

+ * Remove the specification and all application data for a YARN application. + * The application cannot be running. + *

+ * + * @param appName the name of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionDestroy(String appName) throws IOException, + YarnException; + + /** + *

+ * Change the number of running containers for a component of a YARN + * application / long-running service. + *

+ * + * @param appName the name of the application + * @param componentCounts map of component name to new component count or + * amount to change existing component count (e.g. + * 5, +5, -5) + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionFlex(String appName, Map + componentCounts) throws IOException, YarnException; + + /** + *

+ * Upload AM dependencies to HDFS. This makes future application launches + * faster since the dependencies do not have to be uploaded on each launch. + *

+ * + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int enableFastLaunch() throws IOException, + YarnException; + + /** + *

+ * Get detailed status string for a YARN application. + *

+ * + * @param applicationId application id + * @return status string + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract String getStatusString(String applicationId) throws + IOException, YarnException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 2a9b3bc..fb08fcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -23,12 +23,7 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.Charset; import java.text.DecimalFormat; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -54,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; @@ -85,6 +81,7 @@ public class ApplicationCLI extends YarnCLI { "%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s" + System.getProperty("line.separator"); + public static final String APP = "app"; public static final String APPLICATION = "application"; public static final String APPLICATION_ATTEMPT = "applicationattempt"; public static final String CONTAINER = "container"; @@ -93,22 +90,52 @@ public class ApplicationCLI extends YarnCLI { public static final String UPDATE_LIFETIME = "updateLifetime"; public static final String CHANGE_APPLICATION_QUEUE = "changeQueue"; + // app admin options + public static final String LAUNCH_CMD = "launch"; + public static final String STOP_CMD = "stop"; + public static final String START_CMD = "start"; + public static final String SAVE_CMD = "save"; + public static final String DESTROY_CMD = "destroy"; + public static final String FLEX_CMD = "flex"; + public static final String COMPONENT = "component"; + public static final String ENABLE_FAST_LAUNCH = "enableFastLaunch"; + + private static String firstArg = null; + private boolean allAppStates; public static void main(String[] args) throws Exception { ApplicationCLI cli = new ApplicationCLI(); cli.setSysOutPrintStream(System.out); cli.setSysErrPrintStream(System.err); - int res = ToolRunner.run(cli, args); + int res = ToolRunner.run(cli, preProcessArgs(args)); cli.stop(); System.exit(res); } + @VisibleForTesting + public static String[] preProcessArgs(String[] args) { + if (args.length > 0) { + // first argument (app|application|applicationattempt|container) must + // be stripped off for GenericOptionsParser to work + firstArg = args[0]; + return Arrays.copyOfRange(args, 1, args.length); + } else { + return args; + } + } + @Override public int run(String[] args) throws Exception { Options opts = new Options(); String title = null; - if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION)) { + if (firstArg != null) { + title = firstArg; + } else if (args.length > 0) { + title = args[0]; + } + if (title != null && (title.equalsIgnoreCase(APPLICATION) || title + .equalsIgnoreCase(APP))) { title = APPLICATION; opts.addOption(STATUS_CMD, true, "Prints the status of the application."); @@ -168,8 +195,52 @@ public class ApplicationCLI extends YarnCLI { opts.getOption(UPDATE_PRIORITY).setArgName("Priority"); opts.getOption(UPDATE_LIFETIME).setArgName("Timeout"); opts.getOption(CHANGE_APPLICATION_QUEUE).setArgName("Queue Name"); - } else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { - title = APPLICATION_ATTEMPT; + opts.addOption(LAUNCH_CMD, true, "Launches application from " + + "specification file (saves specification and starts application). " + + "Options -updateLifetime and -changeQueue can be specified to alter" + + " the values provided in the file. Supports -appTypes option to " + + "specify which client implementation to use."); + opts.addOption(STOP_CMD, true, "Stops application gracefully (may be " + + "started again later). If name is provided, appType must be " + + "provided unless it is the default yarn-service. If ID is provided," + + " the appType will be looked up. Supports -appTypes option to " + + "specify which client implementation to use."); + opts.addOption(START_CMD, true, "Starts a previously saved " + + "application. Supports -appTypes option to specify which client " + + "implementation to use."); + opts.addOption(SAVE_CMD, true, "Saves specification file for " + + "an application. Options -updateLifetime and -changeQueue can be " + + "specified to alter the values provided in the file. Supports " + + "-appTypes option to specify which client implementation to use."); + opts.addOption(DESTROY_CMD, true, "Destroys a saved application " + + "specification and removes all application data permanently. " + + "Supports -appTypes option to specify which client implementation " + + "to use."); + opts.addOption(FLEX_CMD, true, "Changes number of " + + "running containers for a component of an application / " + + "long-running service. Requires -component option. If name is " + + "provided, appType must be provided unless it is the default " + + "yarn-service. If ID is provided, the appType will be looked up. " + + "Supports -appTypes option to specify which client implementation " + + "to use."); + opts.addOption(COMPONENT, true, "Works with -flex option to change " + + "the number of components/containers running for an application / " + + "long-running service. Supports absolute or relative changes, such " + + "as +1, 2, or -3."); + opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " + + "to HDFS to make future launches faster. Supports -appTypes option" + + " to specify which client implementation to use."); + opts.getOption(LAUNCH_CMD).setArgName("Application Name> 0 && args[0].equalsIgnoreCase(CONTAINER)) { - title = CONTAINER; + } else if (title != null && title.equalsIgnoreCase(CONTAINER)) { opts.addOption(STATUS_CMD, true, "Prints the status of the container."); opts.addOption(LIST_CMD, true, @@ -205,23 +275,53 @@ public class ApplicationCLI extends YarnCLI { printUsage(title, opts); return exitCode; } + String[] unparsedArgs = cliParser.getArgs(); + if (firstArg == null) { + if (unparsedArgs.length != 1) { + printUsage(title, opts); + return exitCode; + } + } else { + if (unparsedArgs.length != 0) { + printUsage(title, opts); + return exitCode; + } + } if (cliParser.hasOption(STATUS_CMD)) { - if (args.length != 3) { + if (hasAnyOtherCLIOptions(cliParser, opts, STATUS_CMD)) { printUsage(title, opts); return exitCode; } - if (args[0].equalsIgnoreCase(APPLICATION)) { - exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); - } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { + if (title.equalsIgnoreCase(APPLICATION) || + title.equalsIgnoreCase(APP)) { + ApplicationReport report = printApplicationReport(cliParser + .getOptionValue(STATUS_CMD)); + if (report == null) { + exitCode = -1; + } else { + exitCode = 0; + String appType = report.getApplicationType(); + try { + AppAdminClient client = AppAdminClient.createAppAdminClient(appType, + getConf()); + sysout.println("Detailed Application Status :"); + sysout.println(client.getStatusString(cliParser.getOptionValue( + STATUS_CMD))); + } catch (IllegalArgumentException e) { + // app type does not have app admin client implementation + } + } + } else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) { exitCode = printApplicationAttemptReport(cliParser .getOptionValue(STATUS_CMD)); - } else if (args[0].equalsIgnoreCase(CONTAINER)) { + } else if (title.equalsIgnoreCase(CONTAINER)) { exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD)); } return exitCode; } else if (cliParser.hasOption(LIST_CMD)) { - if (args[0].equalsIgnoreCase(APPLICATION)) { + if (title.equalsIgnoreCase(APPLICATION) || + title.equalsIgnoreCase(APP)) { allAppStates = false; Set appTypes = new HashSet(); if (cliParser.hasOption(APP_TYPE_CMD)) { @@ -272,21 +372,21 @@ public class ApplicationCLI extends YarnCLI { } } listApplications(appTypes, appStates, appTags); - } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { - if (args.length != 3) { + } else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) { + if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) { printUsage(title, opts); return exitCode; } listApplicationAttempts(cliParser.getOptionValue(LIST_CMD)); - } else if (args[0].equalsIgnoreCase(CONTAINER)) { - if (args.length != 3) { + } else if (title.equalsIgnoreCase(CONTAINER)) { + if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) { printUsage(title, opts); return exitCode; } listContainers(cliParser.getOptionValue(LIST_CMD)); } } else if (cliParser.hasOption(KILL_CMD)) { - if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) { printUsage(title, opts); return exitCode; } @@ -299,7 +399,7 @@ public class ApplicationCLI extends YarnCLI { moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD), cliParser.getOptionValue(QUEUE_CMD)); } else if (cliParser.hasOption(FAIL_CMD)) { - if (!args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { + if (!title.equalsIgnoreCase(APPLICATION_ATTEMPT)) { printUsage(title, opts); return exitCode; } @@ -314,6 +414,103 @@ public class ApplicationCLI extends YarnCLI { } updateApplicationPriority(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(UPDATE_PRIORITY)); + } else if (cliParser.hasOption(SIGNAL_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, SIGNAL_CMD)) { + printUsage(title, opts); + return exitCode; + } + final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD); + final String containerId = signalArgs[0]; + SignalContainerCommand command = + SignalContainerCommand.OUTPUT_THREAD_DUMP; + if (signalArgs.length == 2) { + command = SignalContainerCommand.valueOf(signalArgs[1]); + } + signalToContainer(containerId, command); + } else if (cliParser.hasOption(LAUNCH_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, LAUNCH_CMD, APP_TYPE_CMD, + UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) { + printUsage(title, opts); + return exitCode; + } + String appType = getSingleAppTypeFromCLI(cliParser); + Long lifetime = null; + if (cliParser.hasOption(UPDATE_LIFETIME)) { + lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME)); + } + String queue = null; + if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) { + queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE); + } + String[] nameAndFile = cliParser.getOptionValues(LAUNCH_CMD); + return AppAdminClient.createAppAdminClient(appType, getConf()) + .actionLaunch(nameAndFile[1], nameAndFile[0], lifetime, queue); + } else if (cliParser.hasOption(STOP_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, STOP_CMD, APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + String[] appNameAndType = getAppNameAndType(cliParser, STOP_CMD); + return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf()) + .actionStop(appNameAndType[0]); + } else if (cliParser.hasOption(START_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, START_CMD, APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + String appType = getSingleAppTypeFromCLI(cliParser); + return AppAdminClient.createAppAdminClient(appType, getConf()) + .actionStart(cliParser.getOptionValue(START_CMD)); + } else if (cliParser.hasOption(SAVE_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, SAVE_CMD, APP_TYPE_CMD, + UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) { + printUsage(title, opts); + return exitCode; + } + String appType = getSingleAppTypeFromCLI(cliParser); + Long lifetime = null; + if (cliParser.hasOption(UPDATE_LIFETIME)) { + lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME)); + } + String queue = null; + if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) { + queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE); + } + String[] nameAndFile = cliParser.getOptionValues(SAVE_CMD); + return AppAdminClient.createAppAdminClient(appType, getConf()) + .actionSave(nameAndFile[1], nameAndFile[0], lifetime, queue); + } else if (cliParser.hasOption(DESTROY_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, DESTROY_CMD, APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + String appType = getSingleAppTypeFromCLI(cliParser); + return AppAdminClient.createAppAdminClient(appType, getConf()) + .actionDestroy(cliParser.getOptionValue(DESTROY_CMD)); + } else if (cliParser.hasOption(FLEX_CMD)) { + if (!cliParser.hasOption(COMPONENT) || + hasAnyOtherCLIOptions(cliParser, opts, FLEX_CMD, COMPONENT, + APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + String[] rawCounts = cliParser.getOptionValues(COMPONENT); + Map counts = new HashMap<>(rawCounts.length/2); + for (int i = 0; i < rawCounts.length - 1; i+=2) { + counts.put(rawCounts[i], rawCounts[i+1]); + } + String[] appNameAndType = getAppNameAndType(cliParser, FLEX_CMD); + return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf()) + .actionFlex(appNameAndType[0], counts); + } else if (cliParser.hasOption(ENABLE_FAST_LAUNCH)) { + String appType = getSingleAppTypeFromCLI(cliParser); + if (hasAnyOtherCLIOptions(cliParser, opts, ENABLE_FAST_LAUNCH, + APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + return AppAdminClient.createAppAdminClient(appType, getConf()) + .enableFastLaunch(); } else if (cliParser.hasOption(UPDATE_LIFETIME)) { if (!cliParser.hasOption(APP_ID)) { printUsage(title, opts); @@ -332,19 +529,6 @@ public class ApplicationCLI extends YarnCLI { } moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE)); - } else if (cliParser.hasOption(SIGNAL_CMD)) { - if (args.length < 3 || args.length > 4) { - printUsage(title, opts); - return exitCode; - } - final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD); - final String containerId = signalArgs[0]; - SignalContainerCommand command = - SignalContainerCommand.OUTPUT_THREAD_DUMP; - if (signalArgs.length == 2) { - command = SignalContainerCommand.valueOf(signalArgs[1]); - } - signalToContainer(containerId, command); } else { syserr.println("Invalid Command Usage : "); printUsage(title, opts); @@ -352,6 +536,47 @@ public class ApplicationCLI extends YarnCLI { return 0; } + private ApplicationReport getApplicationReport(ApplicationId applicationId) + throws IOException, YarnException { + ApplicationReport appReport = null; + try { + appReport = client.getApplicationReport(applicationId); + } catch (ApplicationNotFoundException e) { + throw new YarnException("Application with id '" + applicationId + + "' doesn't exist in RM or Timeline Server."); + } + return appReport; + } + + private String[] getAppNameAndType(CommandLine cliParser, String option) + throws IOException, YarnException { + String applicationIdOrName = cliParser.getOptionValue(option); + try { + ApplicationId id = ApplicationId.fromString(applicationIdOrName); + ApplicationReport report = getApplicationReport(id); + return new String[]{report.getName(), report.getApplicationType()}; + } catch (IllegalArgumentException e) { + // assume CLI option provided the app name + // and read appType from command line since id wasn't provided + String appType = getSingleAppTypeFromCLI(cliParser); + return new String[]{applicationIdOrName, appType}; + } + } + + private static String getSingleAppTypeFromCLI(CommandLine cliParser) { + if (cliParser.hasOption(APP_TYPE_CMD)) { + String[] types = cliParser.getOptionValues(APP_TYPE_CMD); + if (types != null) { + for (String type : types) { + if (!type.trim().isEmpty()) { + return StringUtils.toLowerCase(type).trim(); + } + } + } + } + return AppAdminClient.DEFAULT_TYPE; + } + private void updateApplicationTimeout(String applicationId, ApplicationTimeoutType timeoutType, long timeoutInSec) throws YarnException, IOException { @@ -572,7 +797,7 @@ public class ApplicationCLI extends YarnCLI { /** * Kills applications with the application id as appId * - * @param Array of applicationIds + * @param applicationIds Array of applicationIds * @return errorCode * @throws YarnException * @throws IOException @@ -663,10 +888,10 @@ public class ApplicationCLI extends YarnCLI { * Prints the application report for an application id. * * @param applicationId - * @return exitCode + * @return ApplicationReport * @throws YarnException */ - private int printApplicationReport(String applicationId) + private ApplicationReport printApplicationReport(String applicationId) throws YarnException, IOException { ApplicationReport appReport = null; try { @@ -675,7 +900,7 @@ public class ApplicationCLI extends YarnCLI { } catch (ApplicationNotFoundException e) { sysout.println("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server."); - return -1; + return null; } // Use PrintWriter.println, which uses correct platform line ending. ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -739,11 +964,11 @@ public class ApplicationCLI extends YarnCLI { + "' doesn't exist in RM."); appReportStr.close(); sysout.println(baos.toString("UTF-8")); - return -1; + return null; } appReportStr.close(); sysout.println(baos.toString("UTF-8")); - return 0; + return appReport; } private void printResourceUsage(PrintWriter appReportStr, @@ -856,11 +1081,12 @@ public class ApplicationCLI extends YarnCLI { @SuppressWarnings("unchecked") private boolean hasAnyOtherCLIOptions(CommandLine cliParser, Options opts, - String excludeOption) { + String... excludeOptions) { Collection