Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 44AA3200CA7 for ; Tue, 30 May 2017 19:10:35 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 432DF160BE2; Tue, 30 May 2017 17:10:35 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3CE67160BB1 for ; Tue, 30 May 2017 19:10:33 +0200 (CEST) Received: (qmail 44846 invoked by uid 500); 30 May 2017 17:10:26 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 41320 invoked by uid 99); 30 May 2017 17:10:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 30 May 2017 17:10:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 20426E0892; Tue, 30 May 2017 17:10:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sunilg@apache.org To: common-commits@hadoop.apache.org Date: Tue, 30 May 2017 17:10:40 -0000 Message-Id: <0d7a8dfc61c24923bc05340c2d4520df@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [20/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He archived-at: Tue, 30 May 2017 17:10:35 -0000 YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99c1074c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99c1074c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99c1074c Branch: refs/heads/yarn-native-services Commit: 99c1074c1ad3abb841c8b14a1932d0671b9b2f4b Parents: 48e0e12 Author: Jian He Authored: Sun Mar 26 21:42:14 2017 +0800 Committer: Sunil G Committed: Tue May 30 20:37:32 2017 +0530 ---------------------------------------------------------------------- .../yarn/services/api/ApplicationApi.java | 2 - .../api/impl/ApplicationApiService.java | 1477 ++---------- .../yarn/services/utils/RestApiConstants.java | 63 - .../services/utils/RestApiErrorMessages.java | 83 - .../services/webapp/ApplicationApiWebApp.java | 2 +- .../api/impl/TestApplicationApiService.java | 65 +- .../apache/slider/api/SliderApplicationApi.java | 8 - .../slider/api/SliderClusterProtocol.java | 14 +- .../apache/slider/api/resource/Application.java | 16 +- .../apache/slider/api/resource/Component.java | 38 +- .../apache/slider/api/resource/ConfigFile.java | 26 +- .../slider/api/resource/Configuration.java | 30 + .../apache/slider/api/resource/Resource.java | 2 +- .../org/apache/slider/client/SliderClient.java | 2139 ++++-------------- .../apache/slider/client/SliderClientAPI.java | 71 +- .../slider/client/SliderYarnClientImpl.java | 200 +- .../client/ipc/SliderApplicationIpcClient.java | 11 - .../client/ipc/SliderClusterOperations.java | 59 +- .../rest/SliderApplicationApiRestClient.java | 23 - .../org/apache/slider/common/SliderKeys.java | 10 +- .../apache/slider/common/SliderXmlConfKeys.java | 3 + .../common/params/AbstractActionArgs.java | 2 +- .../AbstractClusterBuildingActionArgs.java | 185 +- .../slider/common/params/ActionBuildArgs.java | 32 - .../slider/common/params/ActionCreateArgs.java | 34 +- .../slider/common/params/ActionFlexArgs.java | 34 +- .../slider/common/params/ActionFreezeArgs.java | 4 +- .../slider/common/params/ActionThawArgs.java | 4 +- .../slider/common/params/ActionUpgradeArgs.java | 59 +- .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 16 +- .../slider/common/params/SliderAMArgs.java | 2 +- .../slider/common/params/SliderActions.java | 6 +- .../slider/common/tools/CoreFileSystem.java | 1 + .../slider/common/tools/SliderFileSystem.java | 9 + .../apache/slider/common/tools/SliderUtils.java | 32 +- .../slider/core/launch/AppMasterLauncher.java | 233 -- .../slider/core/launch/LaunchedApplication.java | 108 - .../slider/core/launch/RunningApplication.java | 76 - .../core/persist/AppDefinitionPersister.java | 263 --- .../slider/core/persist/JsonSerDeser.java | 6 + .../providers/AbstractClientProvider.java | 15 +- .../providers/AbstractProviderService.java | 438 ---- .../apache/slider/providers/ProviderCore.java | 12 - .../apache/slider/providers/ProviderRole.java | 19 +- .../slider/providers/ProviderService.java | 157 +- .../apache/slider/providers/ProviderUtils.java | 929 +------- .../providers/docker/DockerClientProvider.java | 35 +- .../providers/docker/DockerProviderService.java | 423 +--- .../slideram/SliderAMClientProvider.java | 305 --- .../slideram/SliderAMProviderService.java | 185 -- .../server/appmaster/RoleLaunchService.java | 122 +- .../server/appmaster/SliderAppMaster.java | 540 ++--- .../appmaster/actions/ActionFlexCluster.java | 14 +- .../actions/ActionRegisterServiceInstance.java | 17 +- .../server/appmaster/metrics/SliderMetrics.java | 80 + .../ProviderNotifyingOperationHandler.java | 63 - .../rpc/SliderClusterProtocolPBImpl.java | 22 +- .../rpc/SliderClusterProtocolProxy.java | 19 +- .../server/appmaster/rpc/SliderIPCService.java | 55 +- .../security/SecurityConfiguration.java | 1 + .../slider/server/appmaster/state/AppState.java | 1031 ++------- .../appmaster/state/AppStateBindingInfo.java | 10 +- .../appmaster/state/ProviderAppState.java | 48 +- .../server/appmaster/state/RoleHistory.java | 15 +- .../server/appmaster/state/RoleInstance.java | 2 + .../server/appmaster/state/RoleStatus.java | 324 +-- .../state/StateAccessForProviders.java | 61 +- .../server/appmaster/web/SliderAMWebApp.java | 15 +- .../slider/server/appmaster/web/WebAppApi.java | 7 - .../server/appmaster/web/WebAppApiImpl.java | 15 +- .../appmaster/web/rest/AMWebServices.java | 33 +- .../server/appmaster/web/rest/RestPaths.java | 23 +- .../ApplicationResouceContentCacheFactory.java | 27 - .../rest/application/ApplicationResource.java | 516 ----- .../resources/AggregateModelRefresher.java | 6 +- .../application/resources/AppconfRefresher.java | 5 +- .../resources/LiveResourcesRefresher.java | 68 - .../resources/LiveStatisticsRefresher.java | 39 - .../resources/ResourceSnapshotRefresher.java | 40 - .../web/rest/management/ManagementResource.java | 3 +- .../web/view/ClusterSpecificationBlock.java | 2 +- .../appmaster/web/view/ContainerStatsBlock.java | 16 +- .../server/appmaster/web/view/IndexBlock.java | 71 +- .../servicemonitor/YarnApplicationProbe.java | 86 - .../YarnRegistryViewForProviders.java | 2 - .../apache/slider/util/RestApiConstants.java | 63 + .../slider/util/RestApiErrorMessages.java | 83 + .../org/apache/slider/util/ServiceApiUtil.java | 203 ++ .../src/main/proto/SliderClusterMessages.proto | 16 +- .../src/main/proto/SliderClusterProtocol.proto | 14 +- .../main/resources/org/apache/slider/slider.xml | 4 - .../core/launch/TestAppMasterLauncher.java | 157 -- .../TestAppMasterLauncherWithAmReset.java | 92 - .../appmaster/TestServiceRecordAttributes.java | 68 - 95 files changed, 2027 insertions(+), 10038 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java index 0fb6402..0f4bdae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java @@ -30,8 +30,6 @@ public interface ApplicationApi { Response getApplications(String state); - Response getApplication(String appName); - Response deleteApplication(String appName); Response updateApplication(String appName, Application updateAppData); http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index 5a4726e..b4f6a2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -17,103 +17,55 @@ package org.apache.hadoop.yarn.services.api.impl; -import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; -import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.lang.reflect.UndeclaredThrowableException; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.regex.Pattern; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; - -import org.apache.commons.lang.SerializationUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.security.UserGroupInformation; +import com.google.inject.Singleton; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.services.api.ApplicationApi; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.ApplicationState; import org.apache.slider.api.resource.ApplicationStatus; -import org.apache.slider.api.resource.Artifact; import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.api.resource.Configuration; -import org.apache.slider.api.resource.Container; -import org.apache.slider.api.resource.ContainerState; -import org.apache.slider.api.resource.Resource; -import org.apache.slider.api.OptionKeys; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.StateValues; +import org.apache.slider.util.ServiceApiUtil; import org.apache.slider.client.SliderClient; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.params.ActionCreateArgs; -import org.apache.slider.common.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; -import org.apache.slider.common.params.ActionListArgs; -import org.apache.slider.common.params.ActionRegistryArgs; -import org.apache.slider.common.params.ActionThawArgs; -import org.apache.slider.common.params.ActionUpdateArgs; -import org.apache.slider.common.params.ComponentArgsDelegate; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; import org.apache.slider.core.buildutils.BuildHelper; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.NotFoundException; import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.providers.docker.DockerKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.inject.Singleton; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.slider.util.RestApiConstants.*; @Singleton @Path(APPLICATIONS_API_RESOURCE_PATH) @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) -public class ApplicationApiService implements ApplicationApi { - private static final Logger logger = LoggerFactory - .getLogger(ApplicationApiService.class); - private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG; - private static UserGroupInformation SLIDER_USER; +public class ApplicationApiService { + private static final Logger logger = + LoggerFactory.getLogger(ApplicationApiService.class); + private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG = + new YarnConfiguration(); private static SliderClient SLIDER_CLIENT; private static Response SLIDER_VERSION; - private static final JsonParser JSON_PARSER = new JsonParser(); - private static final JsonObject EMPTY_JSON_OBJECT = new JsonObject(); - private static final ActionListArgs ACTION_LIST_ARGS = new ActionListArgs(); private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs(); static { @@ -122,8 +74,6 @@ public class ApplicationApiService implements ApplicationApi { // initialize all the common resources - order is important protected static void init() { - SLIDER_CONFIG = getSliderClientConfiguration(); - SLIDER_USER = getSliderUser(); SLIDER_CLIENT = createSliderClient(); SLIDER_VERSION = initSliderVersion(); } @@ -131,8 +81,7 @@ public class ApplicationApiService implements ApplicationApi { @GET @Path("/versions/slider-version") @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getSliderVersion() { + @Produces({ MediaType.APPLICATION_JSON }) public Response getSliderVersion() { logger.info("GET: getSliderVersion"); return SLIDER_VERSION; } @@ -148,580 +97,45 @@ public class ApplicationApiService implements ApplicationApi { + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build(); } - @POST - @Consumes({ MediaType.APPLICATION_JSON }) + @POST @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response createApplication(Application application) { - logger.info("POST: createApplication for app = {}", application); + logger.info("POST: createApplication = {}", application); ApplicationStatus applicationStatus = new ApplicationStatus(); - - Map compNameArtifactIdMap = new HashMap<>(); - // post payload validation try { - validateApplicationPostPayload(application, compNameArtifactIdMap); + ApplicationId applicationId = SLIDER_CLIENT.actionCreate(application); + logger.info("Successfully created application " + application.getName() + + " applicationId = " + applicationId); + applicationStatus.setState(ApplicationState.ACCEPTED); + applicationStatus.setUri( + CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/" + application + .getName()); + return Response.status(Status.CREATED).entity(applicationStatus).build(); } catch (IllegalArgumentException e) { applicationStatus.setDiagnostics(e.getMessage()); return Response.status(Status.BAD_REQUEST).entity(applicationStatus) .build(); - } - String applicationId = null; - try { - applicationId = createSliderApp(application, compNameArtifactIdMap); - applicationStatus.setState(ApplicationState.ACCEPTED); - } catch (SliderException se) { - logger.error("Create application failed", se); - if (se.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE) { - applicationStatus.setDiagnostics(ERROR_APPLICATION_IN_USE); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } else if (se.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) { - applicationStatus.setDiagnostics(ERROR_APPLICATION_INSTANCE_EXISTS); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } else { - applicationStatus.setDiagnostics(se.getMessage()); - } } catch (Exception e) { - logger.error("Create application failed", e); - applicationStatus.setDiagnostics(e.getMessage()); - } - - if (StringUtils.isNotEmpty(applicationId)) { - applicationStatus.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH - + "/" + application.getName()); - // 202 = ACCEPTED - return Response.status(HTTP_STATUS_CODE_ACCEPTED) - .entity(applicationStatus).build(); - } else { + String message = "Failed to create application " + application.getName(); + logger.error(message, e); + applicationStatus.setDiagnostics(message + ": " + e.getMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(applicationStatus).build(); } } - @VisibleForTesting - protected void validateApplicationPostPayload(Application application, - Map compNameArtifactIdMap) { - if (StringUtils.isEmpty(application.getName())) { - throw new IllegalArgumentException(ERROR_APPLICATION_NAME_INVALID); - } - if (!SliderUtils.isClusternameValid(application.getName())) { - throw new IllegalArgumentException(ERROR_APPLICATION_NAME_INVALID_FORMAT); - } - - // If the application has no components do top-level checks - if (application.getComponents() == null - || application.getComponents().size() == 0) { - // artifact - if (application.getArtifact() == null) { - throw new IllegalArgumentException(ERROR_ARTIFACT_INVALID); - } - if (StringUtils.isEmpty(application.getArtifact().getId())) { - throw new IllegalArgumentException(ERROR_ARTIFACT_ID_INVALID); - } - - // If artifact is of type APPLICATION, add a slider specific property - if (application.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - if (application.getConfiguration() == null) { - application.setConfiguration(new Configuration()); - } - addPropertyToConfiguration(application.getConfiguration(), - SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); - } - // resource - validateApplicationResource(application.getResource(), null, application - .getArtifact().getType()); - - // container size - if (application.getNumberOfContainers() == null) { - throw new IllegalArgumentException(ERROR_CONTAINERS_COUNT_INVALID); - } - - // Since it is a simple app with no components, create a default component - application.setComponents(getDefaultComponentAsList(application)); - } else { - // If the application has components, then run checks for each component. - // Let global values take effect if component level values are not - // provided. - Artifact globalArtifact = application.getArtifact(); - Resource globalResource = application.getResource(); - Long globalNumberOfContainers = application.getNumberOfContainers(); - for (Component comp : application.getComponents()) { - // artifact - if (comp.getArtifact() == null) { - comp.setArtifact(globalArtifact); - } - // If still null raise validation exception - if (comp.getArtifact() == null) { - throw new IllegalArgumentException(String.format( - ERROR_ARTIFACT_FOR_COMP_INVALID, comp.getName())); - } - if (StringUtils.isEmpty(comp.getArtifact().getId())) { - throw new IllegalArgumentException(String.format( - ERROR_ARTIFACT_ID_FOR_COMP_INVALID, comp.getName())); - } - - // If artifact is of type APPLICATION, add a slider specific property - if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - if (comp.getConfiguration() == null) { - comp.setConfiguration(new Configuration()); - } - addPropertyToConfiguration(comp.getConfiguration(), - SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); - compNameArtifactIdMap.put(comp.getName(), comp.getArtifact().getId()); - comp.setName(comp.getArtifact().getId()); - } - - // resource - if (comp.getResource() == null) { - comp.setResource(globalResource); - } - validateApplicationResource(comp.getResource(), comp, comp - .getArtifact().getType()); - - // container count - if (comp.getNumberOfContainers() == null) { - comp.setNumberOfContainers(globalNumberOfContainers); - } - if (comp.getNumberOfContainers() == null) { - throw new IllegalArgumentException(String.format( - ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID, comp.getName())); - } - } - } - - // Application lifetime if not specified, is set to unlimited lifetime - if (application.getLifetime() == null) { - application.setLifetime(DEFAULT_UNLIMITED_LIFETIME); - } - } - - private void validateApplicationResource(Resource resource, Component comp, - Artifact.TypeEnum artifactType) { - // Only apps/components of type APPLICATION can skip resource requirement - if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) { - return; - } - if (resource == null) { - throw new IllegalArgumentException(comp == null ? ERROR_RESOURCE_INVALID - : String.format(ERROR_RESOURCE_FOR_COMP_INVALID, comp.getName())); - } - // One and only one of profile OR cpus & memory can be specified. Specifying - // both raises validation error. - if (StringUtils.isNotEmpty(resource.getProfile()) - && (resource.getCpus() != null - || StringUtils.isNotEmpty(resource.getMemory()))) { - throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED - : String.format( - ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, - comp.getName())); - } - // Currently resource profile is not supported yet, so we will raise - // validation error if only resource profile is specified - if (StringUtils.isNotEmpty(resource.getProfile())) { - throw new IllegalArgumentException( - ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET); - } - - String memory = resource.getMemory(); - Integer cpus = resource.getCpus(); - if (StringUtils.isEmpty(memory)) { - throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_MEMORY_INVALID : String.format( - ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, comp.getName())); - } - if (cpus == null) { - throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_CPUS_INVALID : String.format( - ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, comp.getName())); - } - if (cpus <= 0) { - throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_CPUS_INVALID_RANGE : String.format( - ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, comp.getName())); - } - } - - private String createSliderApp(Application application, - Map compNameArtifactIdMap) throws IOException, - YarnException, InterruptedException { - final String appName = application.getName(); - final String queueName = application.getQueue(); - - final ActionCreateArgs createArgs = new ActionCreateArgs(); - addAppConfOptions(createArgs, application, compNameArtifactIdMap); - addResourceOptions(createArgs, application); - - createArgs.provider = DockerKeys.PROVIDER_DOCKER; - - if (queueName != null && queueName.trim().length() > 0) { - createArgs.queue = queueName.trim(); - } - createArgs.lifetime = application.getLifetime(); - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public String run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - sliderClient.actionCreate(appName, createArgs); - ApplicationId applicationId = sliderClient.applicationId; - if (applicationId != null) { - return applicationId.toString(); - // return getApplicationIdString(applicationId); - } - return null; - } - }); - } - - private void addAppConfOptions(ActionCreateArgs createArgs, - Application application, Map compNameArtifactIdMap) throws IOException { - List appCompOptionTriples = createArgs.optionsDelegate.compOptTriples; // TODO: optionTuples instead of compOptTriples - logger.info("Initial appCompOptionTriples = {}", - Arrays.toString(appCompOptionTriples.toArray())); - List appOptions = createArgs.optionsDelegate.optionTuples; - logger.info("Initial appOptions = {}", - Arrays.toString(appOptions.toArray())); - // TODO: Set Slider-AM memory and vcores here - // appCompOptionTriples.addAll(Arrays.asList(SLIDER_APPMASTER_COMPONENT_NAME, - // "", "")); - - // Global configuration - for override purpose - // TODO: add it to yaml - Configuration globalConfig = null; - // Configuration globalConfig = (Configuration) SerializationUtils - // .clone(application.getConfiguration()); - - // TODO: Add the below into globalConfig - // if (application.getConfigurations() != null) { - // for (Entry entry : application.getConfigurations() - // .entrySet()) { - // globalConf.addProperty(entry.getKey(), entry.getValue()); - // } - // } - - Set uniqueGlobalPropertyCache = new HashSet<>(); - if (application.getConfiguration() != null) { - if (application.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : application - .getConfiguration().getProperties().entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - propEntry.getKey(), propEntry.getValue()); - } - } - List configFiles = application.getConfiguration().getFiles(); - if (configFiles != null && !configFiles.isEmpty()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - SliderKeys.AM_CONFIG_GENERATION, "true"); - for (ConfigFile configFile : configFiles) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.NAME_SUFFIX, configFile.getDestFile()); - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.TYPE_SUFFIX, configFile.getType().toString()); - } - } - } - if (application.getComponents() != null) { - - Map appQuicklinks = application.getQuicklinks(); - if (appQuicklinks != null) { - for (Map.Entry quicklink : appQuicklinks.entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.EXPORT_PREFIX + quicklink.getKey(), - quicklink.getValue()); - } - } - - Map placeholders = new HashMap<>(); - placeholders.put(PLACEHOLDER_APP_NAME, application.getName()); - for (Component comp : application.getComponents()) { - placeholders.put(PLACEHOLDER_APP_COMPONENT_NAME, comp.getName()); - if (comp.getArtifact().getType() == Artifact.TypeEnum.DOCKER) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_IMAGE, comp.getArtifact().getId() == null ? - application.getArtifact().getId() : comp.getArtifact().getId())); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_START_COMMAND, comp.getLaunchCommand() == null ? - replacePlaceholders(application.getLaunchCommand(), placeholders) - : replacePlaceholders(comp.getLaunchCommand(), placeholders))); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_NETWORK, DockerKeys.DEFAULT_DOCKER_NETWORK)); - if (comp.getRunPrivilegedContainer() != null) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_USE_PRIVILEGED, - comp.getRunPrivilegedContainer().toString())); - } - } - - if (comp.getConfiguration() != null) { - List configFiles = comp.getConfiguration().getFiles(); - if (configFiles != null && !configFiles.isEmpty()) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - SliderKeys.AM_CONFIG_GENERATION, "true")); - for (ConfigFile configFile : configFiles) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.NAME_SUFFIX, configFile.getDestFile())); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.TYPE_SUFFIX, configFile.getType().toString())); - } - } - } - - if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) { - for (int i = 1; i <= comp.getNumberOfContainers(); i++) { - placeholders.put(PLACEHOLDER_COMPONENT_ID, Integer.toString(i)); - appCompOptionTriples.addAll(createAppConfigComponent( - comp.getName() + i, comp, comp.getName() + i, globalConfig, - placeholders, compNameArtifactIdMap)); - } - } else { - appCompOptionTriples.addAll(createAppConfigComponent(comp.getName(), - comp, comp.getName(), globalConfig, null, compNameArtifactIdMap)); - } - } - } - - logger.info("Updated appCompOptionTriples = {}", - Arrays.toString(appCompOptionTriples.toArray())); - logger.info("Updated appOptions = {}", - Arrays.toString(appOptions.toArray())); - } - - private void addOptionsIfNotPresent(List options, - Set uniqueGlobalPropertyCache, String key, String value) { - if (uniqueGlobalPropertyCache == null) { - options.addAll(Arrays.asList(key, value)); - } else if (!uniqueGlobalPropertyCache.contains(key)) { - options.addAll(Arrays.asList(key, value)); - uniqueGlobalPropertyCache.add(key); - } - } - - private void addPropertyToConfiguration(Configuration conf, String key, - String value) { - if (conf == null) { - return; - } - if (conf.getProperties() == null) { - conf.setProperties(new HashMap()); - } - conf.getProperties().put(key, value); - } - - private List createAppConfigComponent(String compName, - Component component, String configPrefix, Configuration globalConf, - Map placeholders, - Map compNameArtifactIdMap) { - List appConfOptTriples = new ArrayList<>(); - - if (component.getConfiguration() != null - && component.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : component.getConfiguration() - .getProperties().entrySet()) { - appConfOptTriples.addAll(Arrays.asList(compName, propEntry.getKey(), - replacePlaceholders(propEntry.getValue(), placeholders))); - } - } - - // If artifact is of type APPLICATION, then in the POST JSON there will - // be no component definition for that artifact. Hence it's corresponding id - // field is added. Every external APPLICATION has a unique id field. - List convertedDeps = new ArrayList<>(); - for (String dep : component.getDependencies()) { - if (compNameArtifactIdMap.containsKey(dep)) { - convertedDeps.add(compNameArtifactIdMap.get(dep)); - } else { - convertedDeps.add(dep); - } - } - // If the DNS dependency property is set to true for a component, it means - // that it is ensured that DNS entry has been added for all the containers - // of this component, before moving on to the next component in the DAG. - if (hasPropertyWithValue(component, PROPERTY_DNS_DEPENDENCY, "true")) { - if (component.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - convertedDeps.add(component.getArtifact().getId()); - } else { - convertedDeps.add(compName); - } - } - if (convertedDeps.size() > 0) { - appConfOptTriples.addAll(Arrays.asList(compName, "requires", - StringUtils.join(convertedDeps, ","))); - } - return appConfOptTriples; - } - - private String replacePlaceholders(String value, - Map placeholders) { - if (StringUtils.isEmpty(value) || placeholders == null) { - return value; - } - for (Map.Entry placeholder : placeholders.entrySet()) { - value = value.replaceAll(Pattern.quote(placeholder.getKey()), - placeholder.getValue()); - } - return value; - } - - private List createAppConfigGlobal(Component component, - Configuration globalConf, Set uniqueGlobalPropertyCache) { - List appOptions = new ArrayList<>(); - if (component.getConfiguration() != null - && component.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : component.getConfiguration() - .getProperties().entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - propEntry.getKey(), propEntry.getValue()); - } - } - return appOptions; - } - - private void addResourceOptions(ActionCreateArgs createArgs, - Application application) throws IOException { - List resCompOptionTriples = createArgs.optionsDelegate.resCompOptTriples; - logger.info("Initial resCompOptTriples = {}", - Arrays.toString(resCompOptionTriples.toArray())); - // TODO: Add any Slider AM resource specific props here like jvm.heapsize - // resCompOptionTriples.addAll(Arrays.asList(SLIDER_APPMASTER_COMPONENT_NAME, - // "", "")); - - // Global resource - for override purpose - Resource globalResource = (Resource) SerializationUtils.clone(application - .getResource()); - // Priority seeded with 1, expecting every new component will increase it by - // 1 making it ready for the next component to use. - if (application.getComponents() != null) { - int priority = 1; - for (Component comp : application.getComponents()) { - if (hasPropertyWithValue(comp, SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP)) { - continue; - } - if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) { - for (int i = 1; i <= comp.getNumberOfContainers(); i++) { - resCompOptionTriples.addAll(createResourcesComponent(comp.getName() - + i, comp, priority, 1, globalResource)); - priority++; - } - } else { - resCompOptionTriples.addAll(createResourcesComponent(comp.getName(), - comp, priority, comp.getNumberOfContainers(), globalResource)); - priority++; - } - } - } - - logger.info("Updated resCompOptTriples = {}", - Arrays.toString(resCompOptionTriples.toArray())); - } - - private boolean hasPropertyWithValue(Component comp, String key, String value) { - if (comp == null || key == null) { - return false; - } - if (comp.getConfiguration() == null - || comp.getConfiguration().getProperties() == null) { - return false; - } - Map props = comp.getConfiguration().getProperties(); - if (props.containsKey(key)) { - if (value == null) { - return props.get(key) == null; - } else { - if (value.equals(props.get(key))) { - return true; - } - } - } - return false; - } - - private List createResourcesComponent(String compName, - Component component, int priority, long numInstances, - Resource globalResource) { - String memory = component.getResource() == null ? globalResource - .getMemory() : component.getResource().getMemory(); - Integer cpus = component.getResource() == null ? globalResource.getCpus() - : component.getResource().getCpus(); - - List resCompOptTriples = new ArrayList(); - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_PRIORITY, Integer.toString(priority))); - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_INSTANCES, Long.toString(numInstances))); - resCompOptTriples.addAll(Arrays.asList(compName, ResourceKeys.YARN_MEMORY, - memory)); - resCompOptTriples.addAll(Arrays.asList(compName, ResourceKeys.YARN_CORES, - cpus.toString())); - if (component.getPlacementPolicy() != null) { - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_PLACEMENT_POLICY, - component.getPlacementPolicy().getLabel())); - } - - return resCompOptTriples; - } - - private static UserGroupInformation getSliderUser() { - if (SLIDER_USER != null) { - return SLIDER_USER; - } - UserGroupInformation sliderUser = null; - UserGroupInformation.setConfiguration(SLIDER_CONFIG); - String loggedInUser = getUserToRunAs(); - try { - sliderUser = UserGroupInformation.getBestUGI(null, loggedInUser); - // TODO: Once plugged into RM process we should remove the previous call - // and replace it with getCurrentUser as commented below. - // sliderUser = UserGroupInformation.getCurrentUser(); - } catch (IOException e) { - throw new RuntimeException("Unable to create UGI (slider user)", e); - } - return sliderUser; - } - - private T invokeSliderClientRunnable( - final SliderClientContextRunnable runnable) - throws IOException, InterruptedException, YarnException { - try { - T value = SLIDER_USER.doAs(new PrivilegedExceptionAction() { - @Override - public T run() throws Exception { - return runnable.run(SLIDER_CLIENT); - } - }); - return value; - } catch (UndeclaredThrowableException e) { - Throwable cause = e.getCause(); - if (cause instanceof YarnException) { - YarnException ye = (YarnException) cause; - throw ye; - } - throw e; - } - } - protected static SliderClient createSliderClient() { if (SLIDER_CLIENT != null) { return SLIDER_CLIENT; } - org.apache.hadoop.conf.Configuration sliderClientConfiguration = SLIDER_CONFIG; + org.apache.hadoop.conf.Configuration sliderClientConfiguration = + SLIDER_CONFIG; SliderClient client = new SliderClient() { - @Override - public void init(org.apache.hadoop.conf.Configuration conf) { + @Override public void init(org.apache.hadoop.conf.Configuration conf) { super.init(conf); try { initHadoopBinding(); - } catch (SliderException e) { - throw new RuntimeException( - "Unable to automatically init Hadoop binding", e); - } catch (IOException e) { + } catch (SliderException | IOException e) { throw new RuntimeException( "Unable to automatically init Hadoop binding", e); } @@ -730,8 +144,7 @@ public class ApplicationApiService implements ApplicationApi { try { logger .debug("Slider Client configuration: {}", sliderClientConfiguration); - sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, - new String[] { "help" }); + sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, new String[] { "help" }); client.init(sliderClientConfiguration); client.start(); } catch (Exception e) { @@ -741,608 +154,116 @@ public class ApplicationApiService implements ApplicationApi { return client; } - private static String getUserToRunAs() { - String user = System.getenv(PROPERTY_APP_RUNAS_USER); - if (StringUtils.isEmpty(user)) { - user = "root"; - } - return user; - } - - private static org.apache.hadoop.conf.Configuration getSliderClientConfiguration() { - if (SLIDER_CONFIG != null) { - return SLIDER_CONFIG; - } - YarnConfiguration yarnConfig = new YarnConfiguration(); - logger.info("prop yarn.resourcemanager.address = {}", - yarnConfig.get("yarn.resourcemanager.address")); - - return yarnConfig; - } - - private interface SliderClientContextRunnable { - T run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException; - } - - @GET - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getApplications(@QueryParam("state") String state) { - logger.info("GET: getApplications with param state = {}", state); - - // Get all applications in a specific state - lighter projection. For full - // detail, call getApplication on a specific app. - Set applications; - try { - if (StringUtils.isNotEmpty(state)) { - ApplicationStatus appStatus = new ApplicationStatus(); - try { - ApplicationState.valueOf(state); - } catch (IllegalArgumentException e) { - appStatus.setDiagnostics("Invalid value for param state - " + state); - return Response.status(Status.BAD_REQUEST).entity(appStatus).build(); - } - applications = getSliderApplications(state); - } else { - applications = getSliderApplications(true); - } - } catch (Exception e) { - logger.error("Get applications failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - - Set apps = new HashSet(); - if (applications.size() > 0) { - try { - for (ApplicationReport app : applications) { - Application application = new Application(); - application.setLifetime(app.getApplicationTimeouts().get( - ApplicationTimeoutType.LIFETIME).getRemainingTime()); - application.setLaunchTime(new Date(app.getStartTime())); - application.setName(app.getName()); - // Containers not required, setting to null to avoid empty list - application.setContainers(null); - apps.add(application); - } - } catch (Exception e) { - logger.error("Get applications failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } + // The information this REST endpoint currently returned can be retrieved from + // RM web services + // Probably the data from AM is more important. Do that later. +// @GET @Consumes({ MediaType.APPLICATION_JSON }) +// @Produces({ MediaType.APPLICATION_JSON }) +// public Response getApplications(@QueryParam("state") String state) { +// logger.info("GET: getApplications with param state = {}", state); +// return null; +// } - return Response.ok().entity(apps).build(); - } - - @GET - @Path("/{app_name}") + @GET @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response getApplication(@PathParam("app_name") String appName) { logger.info("GET: getApplication for appName = {}", appName); + ApplicationStatus applicationStatus = new ApplicationStatus(); // app name validation if (!SliderUtils.isClusternameValid(appName)) { - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Invalid application name"); + applicationStatus.setDiagnostics("Invalid application name: " + appName); applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID); return Response.status(Status.NOT_FOUND).entity(applicationStatus) .build(); } - // Check if app exists try { - int livenessCheck = getSliderList(appName); - if (livenessCheck < 0) { - logger.info("Application not running"); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_NOT_RUNNING); - applicationStatus.setCode(ERROR_CODE_APP_IS_NOT_RUNNING); + Application app = SLIDER_CLIENT.actionStatus(appName); + ApplicationReport report = SLIDER_CLIENT.findInstance(appName); + if (app != null && report != null) { + app.setLifetime( + report.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME) + .getRemainingTime()); + logger.info("Application = {}", app); + return Response.ok(app).build(); + } else { + String message = "Application " + appName + " does not exist."; + logger.info(message); + applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST); + applicationStatus.setDiagnostics(message); return Response.status(Status.NOT_FOUND).entity(applicationStatus) .build(); } - } catch (UnknownApplicationInstanceException e) { - logger.error("Get application failed, application not found", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_DOES_NOT_EXIST); - applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); - } catch (Exception e) { - logger.error("Get application failed, application not running", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_NOT_RUNNING); - applicationStatus.setCode(ERROR_CODE_APP_IS_NOT_RUNNING); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); - } - - Application app = new Application(); - app.setName(appName); - app.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/" - + appName); - // TODO: add status - app.setState(ApplicationState.ACCEPTED); - JsonObject appStatus = null; - JsonObject appRegistryQuicklinks = null; - try { - appStatus = getSliderApplicationStatus(appName); - appRegistryQuicklinks = getSliderApplicationRegistry(appName, - "quicklinks"); - return populateAppData(app, appStatus, appRegistryQuicklinks); - } catch (BadClusterStateException | NotFoundException e) { - logger.error( - "Get application failed, application not in running state yet", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application not running yet"); - applicationStatus.setCode(ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); } catch (Exception e) { logger.error("Get application failed", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Failed to retrieve application: " - + e.getMessage()); + applicationStatus + .setDiagnostics("Failed to retrieve application: " + e.getMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(applicationStatus).build(); } } - private Response populateAppData(Application app, JsonObject appStatus, - JsonObject appRegistryQuicklinks) { - String appName = jsonGetAsString(appStatus, "name"); - Long totalNumberOfRunningContainers = 0L; - Long totalExpectedNumberOfRunningContainers = 0L; - Long totalNumberOfIpAssignedContainers = 0L; - - // info - JsonObject applicationInfo = jsonGetAsObject(appStatus, "info"); - if (applicationInfo != null) { - String applicationId = jsonGetAsString(applicationInfo, "info.am.app.id"); - if (applicationId != null) { - app.setId(applicationId); - } - } - - // state - String appState = jsonGetAsString(appStatus, "state"); - if (appState == null) { - // consider that app is still in ACCEPTED state - appState = String.valueOf(StateValues.STATE_INCOMPLETE); - } - switch (Integer.parseInt(appState)) { - case StateValues.STATE_LIVE: - app.setState(ApplicationState.STARTED); - break; - case StateValues.STATE_CREATED: - case StateValues.STATE_INCOMPLETE: - case StateValues.STATE_SUBMITTED: - app.setState(ApplicationState.ACCEPTED); - return Response.ok(app).build(); - case StateValues.STATE_DESTROYED: - case StateValues.STATE_STOPPED: - app.setState(ApplicationState.STOPPED); - return Response.ok(app).build(); - default: - break; - } - - // start time - app.setLaunchTime(appStatus.get("createTime") == null ? null - : new Date(appStatus.get("createTime").getAsLong())); - - app.setLifetime(queryLifetime(appName)); - - // Quicklinks - Map appQuicklinks = new HashMap<>(); - for (Map.Entry quicklink : appRegistryQuicklinks - .entrySet()) { - appQuicklinks.put(quicklink.getKey(), quicklink.getValue() == null ? null - : quicklink.getValue().getAsString()); - } - if (!appQuicklinks.isEmpty()) { - app.setQuicklinks(appQuicklinks); - } - - ArrayList componentNames = new ArrayList<>(); - - // status.live - JsonObject applicationStatus = jsonGetAsObject(appStatus, "status"); - // roles - JsonObject applicationRoles = jsonGetAsObject(appStatus, "roles"); - // statistics - JsonObject applicationStatistics = jsonGetAsObject(appStatus, "statistics"); - if (applicationRoles == null) { - // initialize to empty object to avoid too many null checks - applicationRoles = EMPTY_JSON_OBJECT; - } - if (applicationStatus != null) { - JsonObject applicationLive = jsonGetAsObject(applicationStatus, "live"); - if (applicationLive != null) { - for (Entry entry : applicationLive.entrySet()) { - if (entry.getKey().equals(SLIDER_APPMASTER_COMPONENT_NAME)) { - continue; - } - componentNames.add(entry.getKey()); - JsonObject componentRole = applicationRoles - .get(entry.getKey()) == null ? EMPTY_JSON_OBJECT - : applicationRoles.get(entry.getKey()).getAsJsonObject(); - JsonObject liveContainers = entry.getValue().getAsJsonObject(); - if (liveContainers != null) { - for (Map.Entry liveContainerEntry : liveContainers - .entrySet()) { - String containerId = liveContainerEntry.getKey(); - Container container = new Container(); - container.setId(containerId); - JsonObject liveContainer = (JsonObject) liveContainerEntry - .getValue(); - container - .setLaunchTime(liveContainer.get("startTime") == null ? null - : new Date(liveContainer.get("startTime").getAsLong())); - container - .setComponentName(jsonGetAsString(liveContainer, "role")); - container.setIp(jsonGetAsString(liveContainer, "ip")); - // If ip is non-null increment count - if (container.getIp() != null) { - totalNumberOfIpAssignedContainers++; - } - container.setHostname(jsonGetAsString(liveContainer, "hostname")); - container.setState(ContainerState.INIT); - if (StringUtils.isNotEmpty(container.getIp()) - && StringUtils.isNotEmpty(container.getHostname())) { - container.setState(ContainerState.READY); - } - container.setBareHost(jsonGetAsString(liveContainer, "host")); - container.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH - + "/" + appName + CONTAINERS_API_RESOURCE_PATH + "/" - + containerId); - Resource resource = new Resource(); - resource.setCpus(jsonGetAsInt(componentRole, "yarn.vcores")); - resource.setMemory(jsonGetAsString(componentRole, "yarn.memory")); - container.setResource(resource); - Artifact artifact = new Artifact(); - String dockerImageName = jsonGetAsString(componentRole, - "docker.image"); - if (StringUtils.isNotEmpty(dockerImageName)) { - artifact.setId(dockerImageName); - artifact.setType(Artifact.TypeEnum.DOCKER); - } else { - // Might have to handle tarballs here - artifact.setType(null); - } - container.setArtifact(artifact); - container.setPrivilegedContainer( - jsonGetAsBoolean(componentRole, "docker.usePrivileged")); - // TODO: add container property - for response only? - app.addContainer(container); - } - } - } - } - } - - // application info - if (applicationRoles != null && !componentNames.isEmpty()) { - JsonObject applicationRole = jsonGetAsObject(applicationRoles, - componentNames.get(0)); - if (applicationRole != null) { - Artifact artifact = new Artifact(); - // how to get artifact id - docker image name?? - artifact.setId(null); - } - } - - // actual and expected number of containers - if (applicationStatistics != null) { - for (Entry entry : applicationStatistics.entrySet()) { - if (entry.getKey().equals(SLIDER_APPMASTER_COMPONENT_NAME)) { - continue; - } - JsonObject containerStats = (JsonObject) entry.getValue(); - totalNumberOfRunningContainers += jsonGetAsInt(containerStats, - "containers.live"); - totalExpectedNumberOfRunningContainers += jsonGetAsInt(containerStats, - "containers.desired"); - } - app.setNumberOfContainers(totalExpectedNumberOfRunningContainers); - app.setNumberOfRunningContainers(totalNumberOfRunningContainers); - } - - // If all containers of the app has IP assigned, then according to the REST - // API it is considered to be READY. Note, application readiness from - // end-users point of view, is out of scope of the REST API. Also, this - // readiness has nothing to do with readiness-check defined at the component - // level (which is used for dependency resolution of component DAG). - if (totalNumberOfIpAssignedContainers - .longValue() == totalExpectedNumberOfRunningContainers.longValue()) { - app.setState(ApplicationState.READY); - } - logger.info("Application = {}", app); - return Response.ok(app).build(); - } - - private String jsonGetAsString(JsonObject object, String key) { - return object.get(key) == null ? null : object.get(key).getAsString(); - } - - private Integer jsonGetAsInt(JsonObject object, String key) { - return object.get(key) == null ? null - : object.get(key).isJsonNull() ? null : object.get(key).getAsInt(); - } - - private Boolean jsonGetAsBoolean(JsonObject object, String key) { - return object.get(key) == null ? null - : object.get(key).isJsonNull() ? null : object.get(key).getAsBoolean(); - } - - private JsonObject jsonGetAsObject(JsonObject object, String key) { - return object.get(key) == null ? null : object.get(key).getAsJsonObject(); - } - - private long queryLifetime(String appName) { - try { - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public Long run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - ApplicationReport report = sliderClient.findInstance(appName); - return report.getApplicationTimeouts() - .get(ApplicationTimeoutType.LIFETIME).getRemainingTime(); - } - }); - } catch (Exception e) { - logger.error("Error when querying lifetime for " + appName, e); - return DEFAULT_UNLIMITED_LIFETIME; - } - } - - private JsonObject getSliderApplicationStatus(final String appName) - throws IOException, YarnException, InterruptedException { - - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public JsonObject run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - String status = null; - try { - status = sliderClient.actionStatus(appName); - } catch (BadClusterStateException e) { - logger.warn("Application not running yet", e); - return EMPTY_JSON_OBJECT; - } catch (Exception e) { - logger.error("Exception calling slider.actionStatus", e); - return EMPTY_JSON_OBJECT; - } - JsonElement statusElement = JSON_PARSER.parse(status); - return (statusElement == null || statusElement instanceof JsonNull) - ? EMPTY_JSON_OBJECT : (JsonObject) statusElement; - } - }); - } - - private JsonObject getSliderApplicationRegistry(final String appName, - final String registryName) - throws IOException, YarnException, InterruptedException { - final ActionRegistryArgs registryArgs = new ActionRegistryArgs(); - registryArgs.name = appName; - registryArgs.getConf = registryName; - registryArgs.format = ConfigFormat.JSON.toString(); - - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public JsonObject run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - String registry = null; - try { - registry = sliderClient.actionRegistryGetConfig(registryArgs) - .asJson(); - } catch (FileNotFoundException | NotFoundException e) { - // ignore and return empty object - return EMPTY_JSON_OBJECT; - } catch (Exception e) { - logger.error("Exception calling slider.actionRegistryGetConfig", - e); - return EMPTY_JSON_OBJECT; - } - JsonElement registryElement = JSON_PARSER.parse(registry); - return (registryElement == null - || registryElement instanceof JsonNull) ? EMPTY_JSON_OBJECT - : (JsonObject) registryElement; - } - }); - } - - private Integer getSliderList(final String appName) - throws IOException, YarnException, InterruptedException { - return getSliderList(appName, true); - } - - private Integer getSliderList(final String appName, final boolean liveOnly) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Integer run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - int status = 0; - if (liveOnly) { - status = sliderClient.actionList(appName); - } else { - status = sliderClient.actionList(appName, ACTION_LIST_ARGS); - } - return status; - } - }); - } - - private Set getSliderApplications(final String state) - throws IOException, YarnException, InterruptedException { - return getSliderApplications(false, state); - } - - private Set getSliderApplications(final boolean liveOnly) - throws IOException, YarnException, InterruptedException { - return getSliderApplications(liveOnly, null); - } - - private Set getSliderApplications(final boolean liveOnly, - final String state) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable( - new SliderClientContextRunnable>() { - @Override - public Set run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - Set apps; - ActionListArgs listArgs = new ActionListArgs(); - if (liveOnly) { - apps = sliderClient.getApplicationList(null); - } else if (StringUtils.isNotEmpty(state)) { - listArgs.state = state; - apps = sliderClient.getApplicationList(null, listArgs); - } else { - apps = sliderClient.getApplicationList(null, listArgs); - } - return apps; - } - }); - } - @DELETE @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response deleteApplication(@PathParam("app_name") String appName) { logger.info("DELETE: deleteApplication for appName = {}", appName); + return stopApplication(appName, true); + } + private Response stopApplication(String appName, boolean destroy) { try { - Response stopResponse = stopSliderApplication(appName); - if (stopResponse.getStatus() == Status.INTERNAL_SERVER_ERROR - .getStatusCode()) { - return Response.status(Status.NOT_FOUND).build(); + SLIDER_CLIENT.actionStop(appName, ACTION_FREEZE_ARGS); + if (destroy) { + SLIDER_CLIENT.actionDestroy(appName); + logger.info("Successfully deleted application {}", appName); + } else { + logger.info("Successfully stopped application {}", appName); } - } catch (UnknownApplicationInstanceException e) { - logger.error("Application does not exist", e); - return Response.status(Status.NOT_FOUND).build(); + return Response.status(Status.NO_CONTENT).build(); + } catch (ApplicationNotFoundException e) { + ApplicationStatus applicationStatus = new ApplicationStatus(); + applicationStatus.setDiagnostics( + "Application " + appName + " not found " + e.getMessage()); + return Response.status(Status.NOT_FOUND).entity(applicationStatus) + .build(); } catch (Exception e) { - logger.error("Delete application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - - // Although slider client stop returns immediately, it usually takes a - // little longer for it to stop from YARN point of view. Slider destroy - // fails if the application is not completely stopped. Hence the need to - // call destroy in a controlled loop few times (only if exit code is - // EXIT_APPLICATION_IN_USE or EXIT_INSTANCE_EXISTS), before giving up. - boolean keepTrying = true; - int maxDeleteAttempts = 5; - int deleteAttempts = 0; - int sleepIntervalInMillis = 500; - while (keepTrying && deleteAttempts < maxDeleteAttempts) { - try { - destroySliderApplication(appName); - keepTrying = false; - } catch (SliderException e) { - if (e.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE - || e.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) { - deleteAttempts++; - // If we used up all the allowed delete attempts, let's log it as - // error before giving up. Otherwise log as warn. - if (deleteAttempts < maxDeleteAttempts) { - logger.warn("Application not in stopped state, waiting for {}ms" - + " before trying delete again", sleepIntervalInMillis); - } else { - logger.error("Delete application failed", e); - } - try { - Thread.sleep(sleepIntervalInMillis); - } catch (InterruptedException e1) { - } - } else { - logger.error("Delete application threw exception", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } catch (Exception e) { - logger.error("Delete application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + ApplicationStatus applicationStatus = new ApplicationStatus(); + applicationStatus.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(applicationStatus).build(); } - return Response.status(Status.NO_CONTENT).build(); } - private Response stopSliderApplication(final String appName) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Response run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - int returnCode = sliderClient.actionFreeze(appName, ACTION_FREEZE_ARGS); - if (returnCode == 0) { - logger.info("Successfully stopped application {}", appName); - return Response.status(Status.NO_CONTENT).build(); - } else { - logger.error("Stop of application {} failed with return code ", - appName, returnCode); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Stop of application " + appName - + " failed"); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - }); - } - - private Response startSliderApplication(final String appName, Application app) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Response run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - ActionThawArgs thawArgs = new ActionThawArgs(); - if (app.getLifetime() == null) { - app.setLifetime(DEFAULT_UNLIMITED_LIFETIME); - } - thawArgs.lifetime = app.getLifetime(); - int returnCode = sliderClient.actionThaw(appName, thawArgs); - if (returnCode == 0) { - logger.info("Successfully started application {}", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setState(ApplicationState.ACCEPTED); - applicationStatus.setUri(CONTEXT_ROOT - + APPLICATIONS_API_RESOURCE_PATH + "/" + appName); - // 202 = ACCEPTED - return Response.status(HTTP_STATUS_CODE_ACCEPTED) - .entity(applicationStatus).build(); - } else { - logger.error("Start of application {} failed with returnCode ", - appName, returnCode); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Start of application " + appName - + " failed"); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - }); - } + @PUT @Path("/{app_name}/components/{component_name}") + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response updateComponent(@PathParam("app_name") String appName, + @PathParam("component_name") String componentName, Component component) { - private Void destroySliderApplication(final String appName) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Void run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - sliderClient.actionDestroy(appName); - return null; - } - }); + if (component.getNumberOfContainers() < 0) { + return Response.status(Status.BAD_REQUEST).entity( + "Application = " + appName + ", Component = " + component.getName() + + ": Invalid number of containers specified " + component + .getNumberOfContainers()).build(); + } + try { + long original = SLIDER_CLIENT.flex(appName, component); + return Response.ok().entity( + "Updating " + componentName + " size from " + original + " to " + + component.getNumberOfContainers()).build(); + } catch (YarnException | IOException e) { + ApplicationStatus status = new ApplicationStatus(); + status.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) + .build(); + } } - @PUT - @Path("/{app_name}") + @PUT @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response updateApplication(@PathParam("app_name") String appName, @@ -1354,158 +275,70 @@ public class ApplicationApiService implements ApplicationApi { // path param updateAppData.setName(appName); - // Adding support for stop and start // For STOP the app should be running. If already stopped then this // operation will be a no-op. For START it should be in stopped state. // If already running then this operation will be a no-op. - - // Check if app exists in any state - try { - int appsFound = getSliderList(appName, false); - if (appsFound < 0) { - return Response.status(Status.NOT_FOUND).build(); - } - } catch (Exception e) { - logger.error("Update application failed", e); - return Response.status(Status.NOT_FOUND).build(); - } - - // If a STOP is requested if (updateAppData.getState() != null && updateAppData.getState() == ApplicationState.STOPPED) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck == 0) { - return stopSliderApplication(appName); - } else { - logger.info("Application {} is already stopped", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application " + appName - + " is already stopped"); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } - } catch (Exception e) { - logger.error("Stop application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return stopApplication(appName, false); } // If a START is requested if (updateAppData.getState() != null && updateAppData.getState() == ApplicationState.STARTED) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck != 0) { - return startSliderApplication(appName, updateAppData); - } else { - logger.info("Application {} is already running", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application " + appName - + " is already running"); - applicationStatus.setUri(CONTEXT_ROOT - + APPLICATIONS_API_RESOURCE_PATH + "/" + appName); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } - } catch (Exception e) { - logger.error("Start application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } - - // If no of instances specified then treat it as a flex - if (updateAppData.getNumberOfContainers() != null - && updateAppData.getComponents() == null) { - updateAppData.setComponents(getDefaultComponentAsList()); - } - - // At this point if there are components then it is a flex - if (updateAppData.getComponents() != null) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck == 0) { - flexSliderApplication(appName, updateAppData); - } - return Response.status(Status.NO_CONTENT).build(); - } catch (Exception e) { - logger.error("Update application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return startApplication(appName); } // If new lifetime value specified then update it if (updateAppData.getLifetime() != null && updateAppData.getLifetime() > 0) { - try { - updateAppLifetime(appName, updateAppData.getLifetime()); - } catch (Exception e) { - logger.error("Failed to update application (" + appName + ") lifetime (" - + updateAppData.getLifetime() + ")", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return updateLifetime(appName, updateAppData); + } + + // flex a single component app + if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil + .hasComponent( + updateAppData)) { + Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData); + return updateComponent(updateAppData.getName(), defaultComp.getName(), + defaultComp); } // If nothing happens consider it a no-op return Response.status(Status.NO_CONTENT).build(); } - private Void updateAppLifetime(String appName, long lifetime) - throws InterruptedException, YarnException, IOException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override public Void run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - ActionUpdateArgs args = new ActionUpdateArgs(); - args.lifetime = lifetime; - sliderClient.actionUpdate(appName, args); - return null; - } - }); - } - - // create default component and initialize with app level global values - private List getDefaultComponentAsList(Application app) { - List comps = getDefaultComponentAsList(); - Component comp = comps.get(0); - comp.setArtifact(app.getArtifact()); - comp.setResource(app.getResource()); - comp.setNumberOfContainers(app.getNumberOfContainers()); - comp.setLaunchCommand(app.getLaunchCommand()); - return comps; - } - - private List getDefaultComponentAsList() { - Component comp = new Component(); - comp.setName(DEFAULT_COMPONENT_NAME); - List comps = new ArrayList<>(); - comps.add(comp); - return comps; + private Response updateLifetime(String appName, Application updateAppData) { + try { + String newLifeTime = + SLIDER_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); + return Response.ok("Application " + appName + " lifeTime is successfully updated to " + + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build(); + } catch (Exception e) { + String message = + "Failed to update application (" + appName + ") lifetime (" + + updateAppData.getLifetime() + ")"; + logger.error(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(message + " : " + e.getMessage()).build(); + } } - private Void flexSliderApplication(final String appName, - final Application updateAppData) throws IOException, YarnException, - InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Void run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - ActionFlexArgs flexArgs = new ActionFlexArgs(); - ComponentArgsDelegate compDelegate = new ComponentArgsDelegate(); - Long globalNumberOfContainers = updateAppData.getNumberOfContainers(); - for (Component comp : updateAppData.getComponents()) { - Long noOfContainers = comp.getNumberOfContainers() == null - ? globalNumberOfContainers : comp.getNumberOfContainers(); - if (noOfContainers != null) { - compDelegate.componentTuples.addAll( - Arrays.asList(comp.getName(), String.valueOf(noOfContainers))); - } - } - if (!compDelegate.componentTuples.isEmpty()) { - flexArgs.componentDelegate = compDelegate; - sliderClient.actionFlex(appName, flexArgs); - } - return null; - } - }); + private Response startApplication(String appName) { + try { + int ret = SLIDER_CLIENT.actionList(appName); + if (ret == 0) { + return Response.ok() + .entity("Application " + appName + " is already alive.").build(); + } + SLIDER_CLIENT.actionStart(appName, null); + logger.info("Successfully started application " + appName); + return Response.ok("Application " + appName + " is successfully started").build(); + } catch (Exception e) { + String message = "Failed to start application " + appName; + logger.info(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(message + ": " + e.getMessage()).build(); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java deleted file mode 100644 index 23b7ad4..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.services.utils; - -public interface RestApiConstants { - String CONTEXT_ROOT = "/services/v1"; - String APPLICATIONS_API_RESOURCE_PATH = "/applications"; - String CONTAINERS_API_RESOURCE_PATH = "/containers"; - String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster"; - String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0"; - String METAINFO_SCHEMA_VERSION = "2.1"; - String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker"; - - String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd"; - String DEFAULT_COMPONENT_NAME = "DEFAULT"; - String DEFAULT_IMAGE = "centos:centos6"; - String DEFAULT_NETWORK = "bridge"; - String DEFAULT_COMMAND_PATH = "/usr/bin/docker"; - String DEFAULT_USE_NETWORK_SCRIPT = "yes"; - - String PLACEHOLDER_APP_NAME = "${APP_NAME}"; - String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}"; - String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}"; - - String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST"; - String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT"; - String PROPERTY_APP_LIFETIME = "docker.lifetime"; - String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER"; - Long DEFAULT_UNLIMITED_LIFETIME = -1l; - - Integer HTTP_STATUS_CODE_ACCEPTED = 202; - String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip"; - - Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200; - - String PROPERTY_PYTHON_PATH = "python.path"; - String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency"; - - String COMMAND_ORDER_SUFFIX_START = "-START"; - String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED"; - String EXPORT_GROUP_NAME = "QuickLinks"; - - Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001; - Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002; - Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003; - Integer ERROR_CODE_APP_NAME_INVALID = 404004; - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java deleted file mode 100644 index 2d739a4..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.services.utils; - -public interface RestApiErrorMessages { - String ERROR_APPLICATION_NAME_INVALID = - "Application name is either empty or not provided"; - String ERROR_APPLICATION_NAME_INVALID_FORMAT = - "Application name is not valid - only lower case letters, digits," - + " underscore and hyphen are allowed"; - - String ERROR_APPLICATION_NOT_RUNNING = "Application not running"; - String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found"; - String ERROR_APPLICATION_IN_USE = "Application already exists in started" - + " state"; - String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in" - + " stopped/failed state (either restart with PUT or destroy with DELETE" - + " before creating a new one)"; - - String ERROR_SUFFIX_FOR_COMPONENT = - " for component %s (nor at the global level)"; - String ERROR_ARTIFACT_INVALID = "Artifact is not provided"; - String ERROR_ARTIFACT_FOR_COMP_INVALID = - ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT; - String ERROR_ARTIFACT_ID_INVALID = - "Artifact id (like docker image name) is either empty or not provided"; - String ERROR_ARTIFACT_ID_FOR_COMP_INVALID = - ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT; - - String ERROR_RESOURCE_INVALID = "Resource is not provided"; - String ERROR_RESOURCE_FOR_COMP_INVALID = - ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT; - String ERROR_RESOURCE_MEMORY_INVALID = - "Application resource or memory not provided"; - String ERROR_RESOURCE_CPUS_INVALID = - "Application resource or cpus not provided"; - String ERROR_RESOURCE_CPUS_INVALID_RANGE = - "Unacceptable no of cpus specified, either zero or negative"; - String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID = - ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT; - String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID = - ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT; - String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE = - ERROR_RESOURCE_CPUS_INVALID_RANGE - + " for component %s (or at the global level)"; - String ERROR_CONTAINERS_COUNT_INVALID = - "Required no of containers not specified"; - String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID = - ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT; - - String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED = - "Cannot specify" + " cpus/memory along with profile"; - String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED = - ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED - + " for component %s"; - String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET = - "Resource profile is not " + "supported yet. Please specify cpus/memory."; - - String ERROR_NULL_ARTIFACT_ID = - "Artifact Id can not be null if artifact type is none"; - String ERROR_ABSENT_NUM_OF_INSTANCE = - "Num of instances should appear either globally or per component"; - String ERROR_ABSENT_LAUNCH_COMMAND = - "launch command should appear if type is slider-zip or none"; - - String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at" - + " component level, needs corresponding values set at application level"; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java index e1bddb5..7fc01a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java @@ -17,7 +17,7 @@ package org.apache.hadoop.yarn.services.webapp; -import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; +import static org.apache.slider.util.RestApiConstants.*; import java.io.IOException; import java.net.InetAddress; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org