Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 52553200C5E for ; Sat, 22 Apr 2017 09:39:49 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 50EDC160BAD; Sat, 22 Apr 2017 07:39:49 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id F1CA7160BB3 for ; Sat, 22 Apr 2017 09:39:46 +0200 (CEST) Received: (qmail 54112 invoked by uid 500); 22 Apr 2017 07:39:28 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 47350 invoked by uid 99); 22 Apr 2017 07:39:24 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 22 Apr 2017 07:39:24 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7FA7EE8F07; Sat, 22 Apr 2017 07:39:24 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Sat, 22 Apr 2017 07:40:11 -0000 Message-Id: <9dc10a188c9141e7872d45414acff30a@git.apache.org> In-Reply-To: <275ccf679f8f42eea5a919fe59101a1d@git.apache.org> References: <275ccf679f8f42eea5a919fe59101a1d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [49/50] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy unit tests to yarn native services. Contributed by Billie Rinaldi archived-at: Sat, 22 Apr 2017 07:39:49 -0000 YARN-6335. Port slider's groovy unit tests to yarn native services. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/633eb661 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/633eb661 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/633eb661 Branch: refs/heads/yarn-native-services Commit: 633eb661f7dab6f75252a2499d59f6803d8a358d Parents: 8769775 Author: Jian He Authored: Thu Apr 20 23:53:04 2017 +0800 Committer: Jian He Committed: Sat Apr 22 15:31:51 2017 +0800 ---------------------------------------------------------------------- .../api/impl/ApplicationApiService.java | 11 +- .../hadoop-yarn-slider-core/pom.xml | 31 +- .../org/apache/slider/api/InternalKeys.java | 6 +- .../org/apache/slider/api/ResourceKeys.java | 15 +- .../slider/api/SliderClusterProtocol.java | 4 +- .../slider/api/resource/Configuration.java | 22 + .../apache/slider/api/types/RoleStatistics.java | 6 - .../org/apache/slider/client/SliderClient.java | 512 ++------- .../apache/slider/client/SliderClientAPI.java | 62 +- .../client/ipc/SliderClusterOperations.java | 20 +- .../apache/slider/common/SliderXmlConfKeys.java | 19 - .../AbstractClusterBuildingActionArgs.java | 53 + .../slider/common/params/ActionBuildArgs.java | 31 + .../slider/common/params/ActionCreateArgs.java | 12 - .../slider/common/params/ActionEchoArgs.java | 33 - .../slider/common/params/ActionFlexArgs.java | 33 +- .../common/params/ActionInstallKeytabArgs.java | 57 - .../common/params/ActionInstallPackageArgs.java | 58 - .../slider/common/params/ActionKeytabArgs.java | 7 - .../slider/common/params/ActionPackageArgs.java | 81 -- .../slider/common/params/ActionUpgradeArgs.java | 2 +- .../params/AppAndResouceOptionArgsDelegate.java | 111 -- .../apache/slider/common/params/Arguments.java | 26 - .../apache/slider/common/params/ClientArgs.java | 34 +- .../common/params/ComponentArgsDelegate.java | 2 +- .../common/params/OptionArgsDelegate.java | 66 ++ .../slider/common/params/SliderActions.java | 7 - .../slider/common/tools/ConfigHelper.java | 8 - .../apache/slider/common/tools/SliderUtils.java | 118 +- .../slider/core/launch/AbstractLauncher.java | 1 - .../apache/slider/core/zk/ZKIntegration.java | 9 +- .../apache/slider/providers/ProviderRole.java | 12 +- .../server/appmaster/SliderAppMaster.java | 63 +- .../appmaster/actions/ActionFlexCluster.java | 4 +- .../server/appmaster/metrics/SliderMetrics.java | 12 +- .../rpc/SliderClusterProtocolPBImpl.java | 6 +- .../rpc/SliderClusterProtocolProxy.java | 6 +- .../server/appmaster/rpc/SliderIPCService.java | 6 +- .../security/SecurityConfiguration.java | 235 ++-- .../slider/server/appmaster/state/AppState.java | 257 +++-- .../appmaster/state/ContainerOutcome.java | 4 +- .../server/appmaster/state/NodeEntry.java | 2 +- .../server/appmaster/state/RoleStatus.java | 46 +- .../server/appmaster/web/view/IndexBlock.java | 4 +- .../src/main/proto/SliderClusterMessages.proto | 10 +- .../src/main/proto/SliderClusterProtocol.proto | 2 +- .../org/apache/slider/api/TestRPCBinding.java | 50 + .../apache/slider/client/TestClientBadArgs.java | 229 ++++ .../slider/client/TestClientBasicArgs.java | 81 ++ .../slider/client/TestCommonArgParsing.java | 522 +++++++++ .../slider/client/TestKeytabCommandOptions.java | 405 +++++++ .../slider/client/TestSliderClientMethods.java | 142 +++ .../slider/client/TestSliderTokensCommand.java | 124 ++ .../slider/common/tools/TestClusterNames.java | 122 ++ .../slider/common/tools/TestConfigHelper.java | 57 + .../common/tools/TestConfigHelperHDFS.java | 57 + .../common/tools/TestExecutionEnvironment.java | 67 ++ .../common/tools/TestMiscSliderUtils.java | 49 + .../slider/common/tools/TestPortScan.java | 184 +++ .../common/tools/TestSliderFileSystem.java | 62 + .../common/tools/TestSliderTestUtils.java | 97 ++ .../slider/common/tools/TestSliderUtils.java | 25 - .../slider/common/tools/TestWindowsSupport.java | 177 +++ .../slider/common/tools/TestZKIntegration.java | 187 +++ .../slider/core/conf/ExampleConfResources.java | 58 + .../core/conf/TestConfTreeLoadExamples.java | 64 ++ .../core/conf/TestConfigurationResolve.java | 118 ++ .../slider/other/TestFilesystemPermissions.java | 263 +++++ .../apache/slider/other/TestLocalDirStatus.java | 166 +++ .../slider/providers/TestProviderFactory.java | 54 + .../slider/registry/TestConfigSetNaming.java | 85 ++ .../slider/registry/TestRegistryPaths.java | 74 ++ .../server/appmaster/actions/TestActions.java | 246 ++++ .../model/appstate/BaseMockAppStateAATest.java | 73 ++ .../TestMockAppStateAAOvercapacity.java | 112 ++ .../appstate/TestMockAppStateAAPlacement.java | 380 +++++++ .../TestMockAppStateContainerFailure.java | 387 +++++++ .../TestMockAppStateDynamicHistory.java | 212 ++++ .../appstate/TestMockAppStateDynamicRoles.java | 243 ++++ .../TestMockAppStateFlexDynamicRoles.java | 160 +++ .../model/appstate/TestMockAppStateFlexing.java | 201 ++++ .../appstate/TestMockAppStateRMOperations.java | 382 +++++++ .../TestMockAppStateRebuildOnAMRestart.java | 117 ++ .../appstate/TestMockAppStateRolePlacement.java | 122 ++ .../appstate/TestMockAppStateRoleRelease.java | 82 ++ .../appstate/TestMockAppStateUniqueNames.java | 111 ++ .../TestMockContainerResourceAllocations.java | 89 ++ .../appstate/TestMockLabelledAAPlacement.java | 156 +++ .../TestOutstandingRequestValidation.java | 110 ++ .../model/history/TestRoleHistoryAA.java | 269 +++++ .../history/TestRoleHistoryContainerEvents.java | 447 ++++++++ ...TestRoleHistoryFindNodesForNewInstances.java | 177 +++ .../history/TestRoleHistoryNIComparators.java | 133 +++ ...estRoleHistoryOutstandingRequestTracker.java | 385 +++++++ .../model/history/TestRoleHistoryRW.java | 371 ++++++ .../history/TestRoleHistoryRWOrdering.java | 162 +++ .../history/TestRoleHistoryRequestTracking.java | 298 +++++ .../history/TestRoleHistoryUpdateBlacklist.java | 117 ++ .../server/appmaster/model/mock/Allocator.java | 123 ++ .../model/mock/BaseMockAppStateTest.java | 524 +++++++++ .../server/appmaster/model/mock/MockAM.java | 26 + .../appmaster/model/mock/MockAppState.java | 82 ++ .../model/mock/MockApplicationAttemptId.java | 61 + .../appmaster/model/mock/MockApplicationId.java | 67 ++ .../model/mock/MockClusterServices.java | 38 + .../appmaster/model/mock/MockContainer.java | 131 +++ .../appmaster/model/mock/MockContainerId.java | 104 ++ .../appmaster/model/mock/MockFactory.java | 270 +++++ .../appmaster/model/mock/MockFileSystem.java | 32 + .../server/appmaster/model/mock/MockNodeId.java | 62 + .../appmaster/model/mock/MockPriority.java | 46 + .../model/mock/MockProviderService.java | 140 +++ .../model/mock/MockRMOperationHandler.java | 120 ++ .../appmaster/model/mock/MockRecordFactory.java | 27 + .../model/mock/MockRegistryOperations.java | 83 ++ .../appmaster/model/mock/MockResource.java | 75 ++ .../appmaster/model/mock/MockRoleHistory.java | 53 + .../server/appmaster/model/mock/MockRoles.java | 30 + .../appmaster/model/mock/MockYarnCluster.java | 342 ++++++ .../appmaster/model/mock/MockYarnEngine.java | 188 ++++ .../appmaster/model/monkey/TestMockMonkey.java | 208 ++++ .../security/TestSecurityConfiguration.java | 215 ++++ .../TestServiceTimelinePublisher.java | 24 +- .../web/rest/registry/PathEntryMarshalling.java | 28 + .../registry/TestRegistryRestMarshalling.java | 51 + .../web/view/TestClusterSpecificationBlock.java | 78 ++ .../web/view/TestContainerStatsBlock.java | 255 +++++ .../appmaster/web/view/TestIndexBlock.java | 175 +++ .../slider/server/management/TestGauges.java | 55 + .../server/servicemonitor/TestPortProbe.java | 2 +- .../apache/slider/test/ContractTestUtils.java | 901 --------------- .../org/apache/slider/tools/TestUtility.java | 181 --- .../apache/slider/utils/ContractTestUtils.java | 901 +++++++++++++++ .../org/apache/slider/utils/KeysForTests.java | 38 + .../org/apache/slider/utils/MicroZKCluster.java | 87 ++ .../java/org/apache/slider/utils/Outcome.java | 46 + .../org/apache/slider/utils/SliderTestBase.java | 60 + .../apache/slider/utils/SliderTestUtils.java | 1065 ++++++++++++++++++ .../org/apache/slider/utils/TestAssertions.java | 60 + .../org/apache/slider/utils/TestUtility.java | 181 +++ .../slider/utils/YarnMiniClusterTestBase.java | 832 ++++++++++++++ .../slider/utils/YarnZKMiniClusterTestBase.java | 179 +++ .../src/test/resources/log4j.properties | 66 ++ .../slider/common/tools/test/metainfo.txt | 16 - .../slider/common/tools/test/metainfo.xml | 98 -- .../slider/common/tools/test/someOtherFile.txt | 16 - .../slider/common/tools/test/someOtherFile.xml | 17 - .../conf/examples/app-override-resolved.json | 49 + .../slider/core/conf/examples/app-override.json | 43 + .../slider/core/conf/examples/app-resolved.json | 81 ++ .../apache/slider/core/conf/examples/app.json | 54 + .../agent/application/metadata/metainfo.xml | 180 --- .../appmaster/web/rest/registry/sample.json | 9 + .../slider/server/avro/history-v01-3-role.json | 6 + .../slider/server/avro/history-v01-6-role.json | 8 + .../slider/server/avro/history_v01b_1_role.json | 38 + 156 files changed, 17407 insertions(+), 2707 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index b4f6a2e..5a4de0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -50,6 +50,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -251,10 +252,12 @@ public class ApplicationApiService { .getNumberOfContainers()).build(); } try { - long original = SLIDER_CLIENT.flex(appName, component); - return Response.ok().entity( - "Updating " + componentName + " size from " + original + " to " - + component.getNumberOfContainers()).build(); + Map original = SLIDER_CLIENT.flex(appName, Collections + .singletonMap(component.getName(), + component.getNumberOfContainers())); + return Response.ok().entity("Updating " + componentName + " size from " + + original.get(componentName) + " to " + + component.getNumberOfContainers()).build(); } catch (YarnException | IOException e) { ApplicationStatus status = new ApplicationStatus(); status.setDiagnostics(e.getMessage()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index 16a2bb2..05c4d88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -327,16 +327,38 @@ easymock 3.1 test + + + org.objenesis + objenesis + + org.powermock powermock-api-easymock - 1.5 + 1.6.5 test + org.powermock + powermock-module-junit4 + 1.6.5 + + + org.javassist + javassist + + + org.objenesis + objenesis + + + + + javax.servlet.jsp jsp-api runtime @@ -359,6 +381,13 @@ swagger-annotations 1.5.4 + + + org.apache.hadoop + hadoop-minicluster + test + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java index fcaaf0e..f690f5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java @@ -126,9 +126,9 @@ public interface InternalKeys { String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes"; String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds"; - int DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0; - int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0; - int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0; + long DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0; + long DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0; + long DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0; String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay"; String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java index 92890be..2f71004 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java @@ -72,6 +72,15 @@ public interface ResourceKeys { String YARN_CORES = "yarn.vcores"; /** + * If normalization is set to false, then if the resource (memory and/or + * vcore) requested by a role is higher than YARN limits, then the resource + * request is not normalized. If this causes failures at the YARN level then + * applications are expecting that to happen. Default value is true. + */ + String YARN_RESOURCE_NORMALIZATION_ENABLED = + "yarn.resource.normalization.enabled"; + + /** * Number of disks per instance to ask YARN for * {@value} */ @@ -140,9 +149,9 @@ public interface ResourceKeys { - int DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0; - int DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6; - int DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES = 0; + long DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0; + long DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6; + long DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES = 0; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java index 7f768b9..448d4ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java @@ -53,8 +53,8 @@ public interface SliderClusterProtocol extends VersionedProtocol { YarnException; - Messages.FlexComponentResponseProto flexComponent( - Messages.FlexComponentRequestProto request) throws IOException; + Messages.FlexComponentsResponseProto flexComponents( + Messages.FlexComponentsRequestProto request) throws IOException; /** * Get the current cluster status http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java index c43bd64..7b3b93e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java @@ -116,6 +116,28 @@ public class Configuration implements Serializable { return Long.parseLong(value); } + public int getPropertyInt(String name, int defaultValue) { + if (name == null) { + return defaultValue; + } + String value = properties.get(name.trim()); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } + return Integer.parseInt(value); + } + + public boolean getPropertyBool(String name, boolean defaultValue) { + if (name == null) { + return defaultValue; + } + String value = properties.get(name.trim()); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } + return Boolean.parseBoolean(value); + } + public String getProperty(String name, String defaultValue) { if (name == null) { return defaultValue; http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java index c926600..25f4d9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java @@ -35,11 +35,8 @@ public class RoleStatistics { public long limitsExceeded = 0L; public long nodeFailed = 0L; public long preempted = 0L; - public long releasing = 0L; public long requested = 0L; public long started = 0L; - public long startFailed = 0L; - public long totalRequested = 0L; /** * Add another statistics instance @@ -56,11 +53,8 @@ public class RoleStatistics { limitsExceeded += that.limitsExceeded; nodeFailed += that.nodeFailed; preempted += that.preempted; - releasing += that.releasing; requested += that.requested; started += that.started; - startFailed += that.totalRequested; - totalRequested += that.totalRequested; return this; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 8bceddf..2b0982f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -20,12 +20,9 @@ package org.apache.slider.client; import com.google.common.annotations.VisibleForTesting; import com.google.common.io.Files; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; @@ -67,7 +64,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Times; -import org.apache.slider.api.ClusterNode; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.proto.Messages; import org.apache.slider.api.resource.Application; @@ -83,22 +79,17 @@ import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; -import org.apache.slider.common.params.ActionCreateArgs; import org.apache.slider.common.params.ActionDependencyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; -import org.apache.slider.common.params.ActionEchoArgs; import org.apache.slider.common.params.ActionExistsArgs; import org.apache.slider.common.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; -import org.apache.slider.common.params.ActionInstallKeytabArgs; -import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionKDiagArgs; import org.apache.slider.common.params.ActionKeytabArgs; import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionLookupArgs; import org.apache.slider.common.params.ActionNodesArgs; -import org.apache.slider.common.params.ActionPackageArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; import org.apache.slider.common.params.ActionResourceArgs; @@ -122,7 +113,6 @@ import org.apache.slider.core.exceptions.NotFoundException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; import org.apache.slider.core.exceptions.UsageException; -import org.apache.slider.core.exceptions.WaitTimeoutException; import org.apache.slider.core.launch.ClasspathConstructor; import org.apache.slider.core.launch.CredentialUtils; import org.apache.slider.core.launch.JavaCommandLineBuilder; @@ -144,7 +134,6 @@ import org.apache.slider.core.zk.ZKIntegration; import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.SliderProviderFactory; -import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; @@ -160,11 +149,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; -import java.io.Console; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.io.InterruptedIOException; import java.io.OutputStreamWriter; import java.io.PrintStream; @@ -177,8 +164,6 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; @@ -226,7 +211,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private ClientArgs serviceArgs; public ApplicationId applicationId; - + private String deployedClusterName; /** * Cluster operations against the deployed cluster -will be null @@ -334,23 +319,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe exitCode = actionAmSuicide(clusterName, serviceArgs.getActionAMSuicideArgs()); break; - + + case ACTION_BUILD: + exitCode = actionBuild(getApplicationFromArgs(clusterName, + serviceArgs.getActionBuildArgs())); + break; + case ACTION_CLIENT: exitCode = actionClient(serviceArgs.getActionClientArgs()); break; case ACTION_CREATE: - ActionCreateArgs args = serviceArgs.getActionCreateArgs(); - File file = args.getAppDef(); - Path filePath = new Path(file.getAbsolutePath()); - log.info("Loading app definition from: " + filePath); - Application application = - jsonSerDeser.load(FileSystem.getLocal(getConfig()), filePath); - if(args.lifetime > 0) { - application.setLifetime(args.lifetime); - } - application.setName(clusterName); - actionCreate(application); + actionCreate(getApplicationFromArgs(clusterName, + serviceArgs.getActionCreateArgs())); break; case ACTION_DEPENDENCY: @@ -391,14 +372,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe serviceArgs.getActionKillContainerArgs()); break; - case ACTION_INSTALL_KEYTAB: - exitCode = actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs()); - break; - - case ACTION_INSTALL_PACKAGE: - exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs()); - break; - case ACTION_KEYTAB: exitCode = actionKeytab(serviceArgs.getActionKeytabArgs()); break; @@ -415,10 +388,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe exitCode = actionNodes("", serviceArgs.getActionNodesArgs()); break; - case ACTION_PACKAGE: - exitCode = actionPackage(serviceArgs.getActionPackageArgs()); - break; - case ACTION_REGISTRY: exitCode = actionRegistry(serviceArgs.getActionRegistryArgs()); break; @@ -605,15 +574,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * force=true by default. */ @Override - public void actionDestroy(String appName) + public int actionDestroy(String appName) throws YarnException, IOException { validateClusterName(appName); + verifyNoLiveApp(appName, "Destroy"); Path appDir = sliderFileSystem.buildClusterDirPath(appName); FileSystem fs = sliderFileSystem.getFileSystem(); if (fs.exists(appDir)) { if (fs.delete(appDir, true)) { - log.info("Successfully deleted application + " + appName); - return; + log.info("Successfully deleted application dir for " + appName); } else { String message = "Failed to delete application + " + appName + " at: " + appDir; @@ -627,7 +596,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.warn(message); throw new YarnException(message); } - //TODO clean registry + + //TODO clean registry? + String registryPath = SliderRegistryUtils.registryPathForInstance( + appName); + try { + getRegistryOperations().delete(registryPath, true); + } catch (IOException e) { + log.warn("Error deleting registry entry {}: {} ", registryPath, e, e); + } catch (SliderException e) { + log.warn("Error binding to registry {} ", e, e); + } + + log.info("Destroyed cluster {}", appName); + return EXIT_SUCCESS; } @@ -648,6 +630,26 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return factory.createClientProvider(); } + private Application getApplicationFromArgs(String clusterName, + AbstractClusterBuildingActionArgs args) throws IOException { + File file = args.getAppDef(); + Path filePath = new Path(file.getAbsolutePath()); + log.info("Loading app definition from: " + filePath); + Application application = + jsonSerDeser.load(FileSystem.getLocal(getConfig()), filePath); + if(args.lifetime > 0) { + application.setLifetime(args.lifetime); + } + application.setName(clusterName); + return application; + } + + public int actionBuild(Application application) throws YarnException, + IOException { + Path appDir = checkAppNotExistOnHdfs(application); + persistApp(appDir, application); + return EXIT_SUCCESS; + } public ApplicationId actionCreate(Application application) throws IOException, YarnException { @@ -684,8 +686,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } submissionContext.setMaxAppAttempts(conf.getInt(KEY_AM_RESTART_LIMIT, 2)); - Map localResources = - new HashMap(); + Map localResources = new HashMap<>(); // copy local slideram-log4j.properties to hdfs and add to localResources boolean hasSliderAMLog4j = @@ -724,10 +725,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe amLaunchContext.setLocalResources(localResources); addCredentialsIfSecure(conf, amLaunchContext); submissionContext.setAMContainerSpec(amLaunchContext); - yarnClient.submitApplication(submissionContext); + submitApplication(submissionContext); return submissionContext.getApplicationId(); } + @VisibleForTesting + public ApplicationId submitApplication(ApplicationSubmissionContext context) + throws IOException, YarnException { + return yarnClient.submitApplication(context); + } + private void printLocalResources(Map map) { log.info("Added LocalResource for localization: "); StringBuilder builder = new StringBuilder(); @@ -800,7 +807,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private Map addAMEnv(Configuration conf, Path tempPath) throws IOException { - Map env = new HashMap(); + Map env = new HashMap<>(); ClasspathConstructor classpath = buildClasspath(SliderKeys.SUBMITTED_CONF_DIR, "lib", sliderFileSystem, getUsingMiniMRCluster()); @@ -929,69 +936,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return 0; } -// protected static void checkForCredentials(Configuration conf, -// ConfTree tree, String clusterName) throws IOException { -// if (tree.credentials == null || tree.credentials.isEmpty()) { -// log.info("No credentials requested"); -// return; -// } -// -// Console console = System.console(); -// for (Entry> cred : tree.credentials.entrySet()) { -// String provider = cred.getKey() -// .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName) -// .replaceAll(Pattern.quote("${CLUSTER}"), clusterName); -// List aliases = cred.getValue(); -// if (aliases == null || aliases.isEmpty()) { -// continue; -// } -// Configuration c = new Configuration(conf); -// c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider); -// CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0); -// Set existingAliases = new HashSet<>(credentialProvider.getAliases()); -// for (String alias : aliases) { -// if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) { -// log.info("Credentials for " + alias + " found in " + provider); -// } else { -// if (console == null) { -// throw new IOException("Unable to input password for " + alias + -// " because System.console() is null; provider " + provider + -// " must be populated manually"); -// } -// char[] pass = readPassword(alias, console); -// credentialProvider.createCredentialEntry(alias, pass); -// credentialProvider.flush(); -// Arrays.fill(pass, ' '); -// } -// } -// } -// } - - private static char[] readPassword(String alias, Console console) - throws IOException { - char[] cred = null; - - boolean noMatch; - do { - console.printf("%s %s: \n", PASSWORD_PROMPT, alias); - char[] newPassword1 = console.readPassword(); - console.printf("%s %s again: \n", PASSWORD_PROMPT, alias); - char[] newPassword2 = console.readPassword(); - noMatch = !Arrays.equals(newPassword1, newPassword2); - if (noMatch) { - if (newPassword1 != null) Arrays.fill(newPassword1, ' '); - log.info(String.format("Passwords don't match. Try again.")); - } else { - cred = newPassword1; - } - if (newPassword2 != null) Arrays.fill(newPassword2, ' '); - } while (noMatch); - if (cred == null) - throw new IOException("Could not read credentials for " + alias + - " from stdin"); - return cred; - } - @Override public int actionKeytab(ActionKeytabArgs keytabInfo) throws YarnException, IOException { @@ -1078,43 +1022,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } @Override - public int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo) - throws YarnException, IOException { - log.warn("The 'install-keytab' option has been deprecated. Please use 'keytab --install'."); - return actionKeytab(new ActionKeytabArgs(installKeytabInfo)); - } - - @Override - public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws - YarnException, - IOException { - log.warn("The " + ACTION_INSTALL_PACKAGE - + " option has been deprecated. Please use '" - + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'."); - if (StringUtils.isEmpty(installPkgInfo.name)) { - throw new BadCommandArgumentsException( - E_INVALID_APPLICATION_TYPE_NAME + "\n" - + CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE)); - } - Path srcFile = extractPackagePath(installPkgInfo.packageURI); - - // Do not provide new options to install-package command as it is in - // deprecated mode. So version is kept null here. Use package --install. - Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name, - null); - FileSystem sfs = sliderFileSystem.getFileSystem(); - sfs.mkdirs(pkgPath); - - Path fileInFs = new Path(pkgPath, srcFile.getName()); - log.info("Installing package {} at {} and overwrite is {}.", - srcFile, fileInFs, installPkgInfo.replacePkg); - require(!(sfs.exists(fileInFs) && !installPkgInfo.replacePkg), - "Package exists at %s. : %s", fileInFs.toUri(), E_USE_REPLACEPKG_TO_OVERWRITE); - sfs.copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs); - return EXIT_SUCCESS; - } - - @Override public int actionResource(ActionResourceArgs resourceInfo) throws YarnException, IOException { if (resourceInfo.help) { @@ -1287,236 +1194,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } - - @Override - public int actionPackage(ActionPackageArgs actionPackageInfo) - throws YarnException, IOException { - initializeOutputStream(actionPackageInfo.out); - int exitCode = -1; - if (actionPackageInfo.help) { - exitCode = actionHelp(ACTION_PACKAGE); - } - if (actionPackageInfo.install) { - exitCode = actionPackageInstall(actionPackageInfo); - } - if (actionPackageInfo.delete) { - exitCode = actionPackageDelete(actionPackageInfo); - } - if (actionPackageInfo.list) { - exitCode = actionPackageList(); - } - if (actionPackageInfo.instances) { - exitCode = actionPackageInstances(); - } - finalizeOutputStream(actionPackageInfo.out); - if (exitCode != -1) { - return exitCode; - } - throw new BadCommandArgumentsException( - "Select valid package operation option"); - } - - private void initializeOutputStream(String outFile) - throws IOException { - if (outFile != null) { - clientOutputStream = new PrintStream(outFile, "UTF-8"); - } else { - clientOutputStream = System.out; - } - } - - private void finalizeOutputStream(String outFile) { - if (outFile != null && clientOutputStream != null) { - clientOutputStream.flush(); - clientOutputStream.close(); - } - clientOutputStream = System.out; - } - - private int actionPackageInstances() throws YarnException, IOException { -// Map persistentInstances = sliderFileSystem -// .listPersistentInstances(); -// if (persistentInstances.isEmpty()) { -// log.info("No slider cluster specification available"); -// return EXIT_SUCCESS; -// } -// String pkgPathValue = sliderFileSystem -// .buildPackageDirPath(StringUtils.EMPTY, StringUtils.EMPTY).toUri() -// .getPath(); -// FileSystem fs = sliderFileSystem.getFileSystem(); -// Iterator> instanceItr = persistentInstances -// .entrySet().iterator(); -// log.info("List of applications with its package name and path"); -// println("%-25s %15s %30s %s", "Cluster Name", "Package Name", -// "Package Version", "Application Location"); - //TODO deal with packages -// while(instanceItr.hasNext()) { -// Map.Entry entry = instanceItr.next(); -// String clusterName = entry.getKey(); -// Path clusterPath = entry.getValue(); -// AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( -// clusterName, clusterPath); -// Path appDefPath = null; -// try { -// appDefPath = new Path( -// getApplicationDefinitionPath(instanceDefinition -// .getAppConfOperations())); -// } catch (BadConfigException e) { -// // Invalid cluster state, so move on to next. No need to log anything -// // as this is just listing of instances. -// continue; -// } -// if (!appDefPath.isUriPathAbsolute()) { -// appDefPath = new Path(fs.getHomeDirectory(), appDefPath); -// } -// String appDefPathStr = appDefPath.toUri().toString(); -// try { -// if (appDefPathStr.contains(pkgPathValue) && fs.isFile(appDefPath)) { -// String packageName = appDefPath.getParent().getName(); -// String packageVersion = StringUtils.EMPTY; -// if (instanceDefinition.isVersioned()) { -// packageVersion = packageName; -// packageName = appDefPath.getParent().getParent().getName(); -// } -// println("%-25s %15s %30s %s", clusterName, packageName, -// packageVersion, appDefPathStr); -// } -// } catch (IOException e) { -// log.debug("{} application definition path {} is not found.", clusterName, appDefPathStr); -// } -// } - return EXIT_SUCCESS; - } - - private int actionPackageList() throws IOException { - Path pkgPath = sliderFileSystem.buildPackageDirPath(StringUtils.EMPTY, - StringUtils.EMPTY); - log.info("Package install path : {}", pkgPath); - FileSystem sfs = sliderFileSystem.getFileSystem(); - if (!sfs.isDirectory(pkgPath)) { - log.info("No package(s) installed"); - return EXIT_SUCCESS; - } - FileStatus[] fileStatus = sfs.listStatus(pkgPath); - boolean hasPackage = false; - StringBuilder sb = new StringBuilder(); - sb.append("List of installed packages:\n"); - for (FileStatus fstat : fileStatus) { - if (fstat.isDirectory()) { - sb.append("\t").append(fstat.getPath().getName()); - sb.append("\n"); - hasPackage = true; - } - } - if (hasPackage) { - println(sb.toString()); - } else { - log.info("No package(s) installed"); - } - return EXIT_SUCCESS; - } - - private void createSummaryMetainfoFile(Path srcFile, Path destFile, - boolean overwrite) throws IOException { - FileSystem srcFs = srcFile.getFileSystem(getConfig()); - try (InputStream inputStreamJson = SliderUtils - .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.json"); - InputStream inputStreamXml = SliderUtils - .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.xml");) { - InputStream inputStream = null; - Path summaryFileInFs = null; - if (inputStreamJson != null) { - inputStream = inputStreamJson; - summaryFileInFs = new Path(destFile.getParent(), destFile.getName() - + ".metainfo.json"); - log.info("Found JSON metainfo file in package"); - } else if (inputStreamXml != null) { - inputStream = inputStreamXml; - summaryFileInFs = new Path(destFile.getParent(), destFile.getName() - + ".metainfo.xml"); - log.info("Found XML metainfo file in package"); - } - if (inputStream != null) { - try (FSDataOutputStream dataOutputStream = sliderFileSystem - .getFileSystem().create(summaryFileInFs, overwrite)) { - log.info("Creating summary metainfo file"); - IOUtils.copy(inputStream, dataOutputStream); - } - } - } - } - - private int actionPackageInstall(ActionPackageArgs actionPackageArgs) - throws YarnException, IOException { - requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); - - Path srcFile = extractPackagePath(actionPackageArgs.packageURI); - - Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, - actionPackageArgs.version); - FileSystem fs = sliderFileSystem.getFileSystem(); - if (!fs.exists(pkgPath)) { - fs.mkdirs(pkgPath); - } - - Path fileInFs = new Path(pkgPath, srcFile.getName()); - require(actionPackageArgs.replacePkg || !fs.exists(fileInFs), - E_PACKAGE_EXISTS +" at %s. Use --replacepkg to overwrite.", fileInFs.toUri()); - - log.info("Installing package {} to {} (overwrite set to {})", srcFile, - fileInFs, actionPackageArgs.replacePkg); - fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs); - createSummaryMetainfoFile(srcFile, fileInFs, actionPackageArgs.replacePkg); - - String destPathWithHomeDir = Path - .getPathWithoutSchemeAndAuthority(fileInFs).toString(); - String destHomeDir = Path.getPathWithoutSchemeAndAuthority( - fs.getHomeDirectory()).toString(); - // a somewhat contrived approach to stripping out the home directory and any trailing - // separator; designed to work on windows and unix - String destPathWithoutHomeDir; - if (destPathWithHomeDir.startsWith(destHomeDir)) { - destPathWithoutHomeDir = destPathWithHomeDir.substring(destHomeDir.length()); - if (destPathWithoutHomeDir.startsWith("/") || destPathWithoutHomeDir.startsWith("\\")) { - destPathWithoutHomeDir = destPathWithoutHomeDir.substring(1); - } - } else { - destPathWithoutHomeDir = destPathWithHomeDir; - } - log.info("Set " + AgentKeys.APP_DEF + " in your app config JSON to {}", - destPathWithoutHomeDir); - - return EXIT_SUCCESS; - } - - private Path extractPackagePath(String packageURI) - throws BadCommandArgumentsException { - require(isSet(packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION); - File pkgFile = new File(packageURI); - require(pkgFile.isFile(), - E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + ": " + pkgFile.getAbsolutePath()); - return new Path(pkgFile.toURI()); - } - - private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws - YarnException, IOException { - requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); - - Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, - actionPackageArgs.version); - FileSystem fs = sliderFileSystem.getFileSystem(); - require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri()); - log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath); - - if(fs.delete(pkgPath, true)) { - log.info("Deleted package {} " + actionPackageArgs.name); - return EXIT_SUCCESS; - } else { - log.warn("Package deletion failed."); - return EXIT_NOT_FOUND; - } - } - @Override public int actionUpdate(String clustername, AbstractClusterBuildingActionArgs buildInfo) throws @@ -1594,7 +1271,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe String PLACEHOLDER_PATTERN = "\\$\\{[^{]+\\}"; Pattern placeholderPattern = Pattern.compile(PLACEHOLDER_PATTERN); Matcher placeholderMatcher = placeholderPattern.matcher(env); - Map placeholderKeyValueMap = new HashMap(); + Map placeholderKeyValueMap = new HashMap<>(); if (placeholderMatcher.find()) { String placeholderKey = placeholderMatcher.group(); String systemKey = placeholderKey @@ -1865,16 +1542,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @Override @VisibleForTesting - public void actionFlex(String appName, ActionFlexArgs args) + public int actionFlex(String appName, ActionFlexArgs args) throws YarnException, IOException { - Component component = new Component(); - component.setNumberOfContainers(args.getNumberOfContainers()); - if (StringUtils.isEmpty(args.getComponent())) { - component.setName("DEFAULT"); - } else { - component.setName(args.getComponent()); + Map componentCounts = new HashMap<>(args.getComponentMap() + .size()); + for (Entry entry : args.getComponentMap().entrySet()) { + long numberOfContainers = Long.parseLong(entry.getValue()); + componentCounts.put(entry.getKey(), numberOfContainers); } - flex(appName, component); + // throw usage exception if no changes proposed + if (componentCounts.size() == 0) { + actionHelp(ACTION_FLEX); + } + flex(appName, componentCounts); + return EXIT_SUCCESS; } @Override @@ -1966,19 +1647,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } - @Override - public String actionEcho(String name, ActionEchoArgs args) throws - YarnException, - IOException { - String message = args.message; - if (message == null) { - throw new BadCommandArgumentsException("missing message"); - } - SliderClusterOperations clusterOps = - new SliderClusterOperations(bondToCluster(name)); - return clusterOps.echo(message); - } - /** * Find an instance of an application belonging to the current user. * @param appname application name @@ -2099,7 +1767,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } @Override - public void actionStop(String appName, ActionFreezeArgs freezeArgs) + public int actionStop(String appName, ActionFreezeArgs freezeArgs) throws YarnException, IOException { validateClusterName(appName); ApplicationReport app = findInstance(appName); @@ -2112,7 +1780,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe .ordinal()) { log.info("Application {} is in a terminated state {}", appName, app.getYarnApplicationState()); - return; + return EXIT_SUCCESS; } try { @@ -2127,6 +1795,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe + " gracefully, forcefully kill the app."); yarnClient.killApplication(app.getApplicationId(), freezeArgs.message); } + return EXIT_SUCCESS; } @Override @@ -2143,30 +1812,30 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return 0; } - public long flex(String appName, Component component) - throws YarnException, IOException { + public Map flex(String appName, Map + componentCounts) throws YarnException, IOException { validateClusterName(appName); Path appDir = sliderFileSystem.buildClusterDirPath(appName); Path appJson = new Path(appDir, appName + ".json"); Application persistedApp = jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); - long original = 0; - boolean foundComponent = false; + Map original = new HashMap<>(componentCounts.size()); for (Component persistedComp : persistedApp.getComponents()) { - if (persistedComp.getName().equals(component.getName())) { - original = persistedComp.getNumberOfContainers(); - persistedComp.setNumberOfContainers(component.getNumberOfContainers()); - foundComponent = true; - break; + String name = persistedComp.getName(); + if (componentCounts.containsKey(persistedComp.getName())) { + original.put(name, persistedComp.getNumberOfContainers()); + persistedComp.setNumberOfContainers(componentCounts.get(name)); } } - if (!foundComponent) { - throw new YarnException("Component " + component.getName() - + " does not exist in app definition."); + if (original.size() < componentCounts.size()) { + componentCounts.keySet().removeAll(original.keySet()); + throw new YarnException("Components " + componentCounts.keySet() + + " do not exist in app definition."); } jsonSerDeser .save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true); - log.info("Updated app definition file for component " + component); + log.info("Updated app definition file for components " + componentCounts + .keySet()); ApplicationReport instance = findInstance(appName); if (instance != null) { @@ -2174,11 +1843,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe SliderClusterProtocol appMaster = connect(instance); SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster); - clusterOps.flex(component); - log.info( - "Application name = " + appName + ", Component name = " + component - .getName() + ", number of containers updated from " + original - + " to " + component.getNumberOfContainers()); + clusterOps.flex(componentCounts); + for (Entry componentCount : componentCounts.entrySet()) { + log.info( + "Application name = " + appName + ", Component name = " + + componentCount.getKey() + ", number of containers updated " + + "from " + original.get(componentCount.getKey()) + " to " + + componentCount.getValue()); + } } else { String message = "Application " + appName + "does not exist in RM. "; throw new YarnException(message); http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 2bb224b..197a564 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -20,26 +20,18 @@ package org.apache.slider.client; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; import org.apache.slider.common.params.ActionDependencyArgs; -import org.apache.slider.common.params.ActionDestroyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; -import org.apache.slider.common.params.ActionEchoArgs; import org.apache.slider.common.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; -import org.apache.slider.common.params.ActionInstallKeytabArgs; -import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionKeytabArgs; import org.apache.slider.common.params.ActionNodesArgs; -import org.apache.slider.common.params.ActionPackageArgs; import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionRegistryArgs; @@ -53,7 +45,6 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.providers.AbstractClientProvider; import java.io.IOException; -import java.util.Map; /** * Interface of those method calls in the slider API that are intended @@ -63,8 +54,7 @@ import java.util.Map; */ public interface SliderClientAPI extends Service { - void actionDestroy(String clustername) throws YarnException, - IOException; + int actionDestroy(String clustername) throws YarnException, IOException; /** * AM to commit an asynchronous suicide @@ -82,18 +72,6 @@ public interface SliderClientAPI extends Service { throws SliderException; /** - * Upload keytab to a designated sub-directory of the user home directory - * - * @param installKeytabInfo the arguments needed to upload the keytab - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - * @deprecated use #actionKeytab - */ - int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo) - throws YarnException, IOException; - - /** * Manage keytabs leveraged by slider * * @param keytabInfo the arguments needed to manage the keytab @@ -105,17 +83,6 @@ public interface SliderClientAPI extends Service { throws YarnException, IOException; /** - * Upload application package to user home directory - * - * @param installPkgInfo the arguments needed to upload the package - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) - throws YarnException, IOException; - - /** * Manage file resources leveraged by slider * * @param resourceInfo the arguments needed to manage the resource @@ -138,17 +105,6 @@ public interface SliderClientAPI extends Service { throws IOException, YarnException; /** - * Managing slider application package - * - * @param pkgInfo the arguments needed to upload, delete or list the package - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - int actionPackage(ActionPackageArgs pkgInfo) - throws YarnException, IOException; - - /** * Update the cluster specification * * @param clustername cluster name @@ -179,7 +135,8 @@ public interface SliderClientAPI extends Service { int actionList(String clustername, ActionListArgs args) throws IOException, YarnException; - void actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException; + int actionFlex(String name, ActionFlexArgs args) throws YarnException, + IOException; /** * Test for a cluster existing probe for a cluster of the given name existing @@ -200,17 +157,6 @@ public interface SliderClientAPI extends Service { throws YarnException, IOException; /** - * Echo operation (not currently wired up to command line) - * @param name cluster name - * @param args arguments - * @return the echoed text - * @throws YarnException - * @throws IOException - */ - String actionEcho(String name, ActionEchoArgs args) - throws YarnException, IOException; - - /** * Status operation * * @param clustername cluster name @@ -246,7 +192,7 @@ public interface SliderClientAPI extends Service { * @param freezeArgs arguments to the stop * @return EXIT_SUCCESS if the cluster was not running by the end of the operation */ - void actionStop(String clustername, ActionFreezeArgs freezeArgs) + int actionStop(String clustername, ActionFreezeArgs freezeArgs) throws YarnException, IOException; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java index 3bb2af6..e89a660 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java @@ -25,7 +25,6 @@ import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.api.types.NodeInformation; import org.apache.slider.api.types.NodeInformationList; @@ -44,6 +43,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import static org.apache.slider.api.types.RestTypeMarshalling.unmarshall; @@ -283,12 +284,17 @@ public class SliderClusterOperations { return state; } - public void flex(Component component) throws IOException{ - Messages.FlexComponentRequestProto request = - Messages.FlexComponentRequestProto.newBuilder() - .setNumberOfContainers(component.getNumberOfContainers().intValue()) - .setName(component.getName()).build(); - appMaster.flexComponent(request); + public void flex(Map componentCounts) throws IOException{ + Messages.FlexComponentsRequestProto.Builder builder = + Messages.FlexComponentsRequestProto.newBuilder(); + for (Entry componentCount : componentCounts.entrySet()) { + Messages.ComponentCountProto componentProto = + Messages.ComponentCountProto.newBuilder() + .setName(componentCount.getKey()) + .setNumberOfContainers(componentCount.getValue()).build(); + builder.addComponents(componentProto); + } + appMaster.flexComponents(builder.build()); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index b666834..e881edf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -18,8 +18,6 @@ package org.apache.slider.common; -import org.apache.hadoop.registry.client.api.RegistryConstants; - /** * These are the keys that can be added to conf/slider-client.xml. */ @@ -105,23 +103,6 @@ public interface SliderXmlConfKeys { */ String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750"; - /** - * - * Use {@link RegistryConstants#KEY_REGISTRY_ZK_ROOT} - * - */ - @Deprecated - String REGISTRY_PATH = "slider.registry.path"; - - /** - * - * @Deprecated use {@link RegistryConstants#KEY_REGISTRY_ZK_QUORUM} - * - */ - @Deprecated - String REGISTRY_ZK_QUORUM = "slider.zookeeper.quorum"; - - String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH = "ipc.client.fallback-to-simple-auth-allowed"; String HADOOP_HTTP_FILTER_INITIALIZERS = http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index a2d4e38..c983a63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -19,6 +19,13 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; +import com.google.common.annotations.VisibleForTesting; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; + +import java.io.File; +import java.util.List; +import java.util.Map; /** * Abstract Action to build things; shares args across build and @@ -26,6 +33,14 @@ import com.beust.jcommander.Parameter; */ public abstract class AbstractClusterBuildingActionArgs extends AbstractActionArgs { + @Parameter(names = {ARG_APPDEF}, + description = "Template application definition file in JSON format.") + public File appDef; + + public File getAppDef() { + return appDef; + } + @Parameter(names = { ARG_QUEUE }, description = "Queue to submit the application") public String queue; @@ -33,4 +48,42 @@ public abstract class AbstractClusterBuildingActionArgs @Parameter(names = { ARG_LIFETIME }, description = "Lifetime of the application from the time of request") public long lifetime; + + @ParametersDelegate + public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); + + @ParametersDelegate + public OptionArgsDelegate optionsDelegate = + new OptionArgsDelegate(); + + + public Map getOptionsMap() throws + BadCommandArgumentsException { + return optionsDelegate.getOptionsMap(); + } + + /** + * Get the role heap mapping (may be empty, but never null). + * @return role heap mapping + * @throws BadCommandArgumentsException parse problem + */ + public Map> getCompOptionMap() throws + BadCommandArgumentsException { + return optionsDelegate.getCompOptionMap(); + } + + @VisibleForTesting + public List getComponentTuples() { + return componentDelegate.getComponentTuples(); + } + + /** + * Get the role mapping (may be empty, but never null). + * @return role mapping + * @throws BadCommandArgumentsException parse problem + */ + public Map getComponentMap() throws + BadCommandArgumentsException { + return componentDelegate.getComponentMap(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java new file mode 100644 index 0000000..57e4b02 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = {SliderActions.ACTION_BUILD}, + commandDescription = SliderActions.DESCRIBE_ACTION_BUILD) + +public class ActionBuildArgs extends AbstractClusterBuildingActionArgs { + + @Override + public String getActionName() { + return SliderActions.ACTION_BUILD; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java index c8cac65..4cc1077 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java @@ -18,25 +18,13 @@ package org.apache.slider.common.params; -import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import com.beust.jcommander.ParametersDelegate; - -import java.io.File; @Parameters(commandNames = {SliderActions.ACTION_CREATE}, commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) public class ActionCreateArgs extends AbstractClusterBuildingActionArgs { - @Parameter(names = {ARG_APPDEF}, - description = "Template application definition file in JSON format.") - public File appDef; - - public File getAppDef() { - return appDef; - } - @Override public String getActionName() { return SliderActions.ACTION_CREATE; http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java deleted file mode 100644 index d05f10b..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; - -public class ActionEchoArgs extends AbstractActionArgs { - @Override - public String getActionName() { - return SliderActions.ACTION_ECHO; - } - - @Parameter(names = {ARG_MESSAGE}, - description = "message to echo") - public String message; - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java index c565484..21cb609 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java @@ -18,31 +18,38 @@ package org.apache.slider.common.params; -import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.beust.jcommander.ParametersDelegate; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; + +import java.util.List; +import java.util.Map; @Parameters(commandNames = {SliderActions.ACTION_FLEX}, commandDescription = SliderActions.DESCRIBE_ACTION_FLEX) public class ActionFlexArgs extends AbstractActionArgs { - @Parameter(names = {ARG_COMPONENT}, - description = "component name") - String componentName; - - @Parameter(names = {ARG_COUNT}, - description = "number of containers>") - long numberOfContainers; - @Override public String getActionName() { return SliderActions.ACTION_FLEX; } - public String getComponent() { - return componentName; + @ParametersDelegate + public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); + + /** + * Get the component mapping (may be empty, but never null) + * @return mapping + * @throws BadCommandArgumentsException parse problem + */ + public Map getComponentMap() throws + BadCommandArgumentsException { + return componentDelegate.getComponentMap(); } - public long getNumberOfContainers() { - return numberOfContainers; + + public List getComponentTuples() { + return componentDelegate.getComponentTuples(); } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java deleted file mode 100644 index 4cfb889..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -@Parameters(commandNames = {SliderActions.ACTION_INSTALL_KEYTAB}, - commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_KEYTAB) - -public class ActionInstallKeytabArgs extends AbstractActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_INSTALL_KEYTAB; - } - - @Parameter(names = {ARG_KEYTAB}, - description = "Path to keytab on local disk") - public String keytabUri; - - @Parameter(names = {ARG_FOLDER}, - description = "The name of the folder in which to store the keytab") - public String folder; - - @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab") - public boolean overwrite = false; - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - public int getMinParams() { - return 0; - } - - @Override - public int getMaxParams() { - return 3; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java deleted file mode 100644 index 646e795..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.ParametersDelegate; - -@Parameters(commandNames = {SliderActions.ACTION_INSTALL_PACKAGE}, - commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE) - -public class ActionInstallPackageArgs extends AbstractActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_INSTALL_PACKAGE; - } - - @Parameter(names = {ARG_PACKAGE}, - description = "Path to app package on local disk") - public String packageURI; - - @Parameter(names = {ARG_NAME}, - description = "The type of the package") - public String name; - - @Parameter(names = {ARG_REPLACE_PKG}, description = "Overwrite existing package") - public boolean replacePkg = false; - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - public int getMinParams() { - return 0; - } - - @Override - public int getMaxParams() { - return 1; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java index 32b1d2b..9a708ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java @@ -26,13 +26,6 @@ import com.beust.jcommander.Parameters; public class ActionKeytabArgs extends AbstractActionArgs { - public ActionKeytabArgs(ActionInstallKeytabArgs installKeytabInfo) { - this.install = true; - this.overwrite = installKeytabInfo.overwrite; - this.keytab = installKeytabInfo.keytabUri; - this.folder = installKeytabInfo.folder; - } - public ActionKeytabArgs() { super(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/633eb661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java deleted file mode 100644 index 4833934..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -@Parameters(commandNames = {SliderActions.ACTION_PACKAGE}, - commandDescription = SliderActions.DESCRIBE_ACTION_PACKAGE) - -public class ActionPackageArgs extends AbstractActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_PACKAGE; - } - - @Parameter(names = {ARG_INSTALL}, - description = "Install package in the sub-folder 'package' of the user's Slider base directory") - public boolean install; - - @Parameter(names = {ARG_PKGDELETE}, - description = "Delete package operation") - public boolean delete; - - @Parameter(names = {ARG_PKGLIST}, - description = "List of package(s) installed") - public boolean list; - - @Parameter(names = {ARG_PKGINSTANCES}, - description = "Lists all application instances referring to package") - public boolean instances; - - @Parameter(names = {ARG_PACKAGE}, - description = "Path to app package on local disk") - public String packageURI; - - @Parameter(names = {ARG_NAME}, - description = "Package name") - public String name; - - @Parameter(names = {ARG_VERSION}, description = "Package version") - public String version; - - @Parameter(names = {ARG_REPLACE_PKG}, - description = "Overwrite existing package") - public boolean replacePkg = false; - - @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, - description = "Output file for package data") - public String out; - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - public int getMinParams() { - return 0; - } - - @Override - public int getMaxParams() { - return 1; - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org