Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C2140200BD8 for ; Wed, 7 Dec 2016 22:10:11 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id C0C94160B2F; Wed, 7 Dec 2016 21:10:11 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 85D1A160B0C for ; Wed, 7 Dec 2016 22:10:10 +0100 (CET) Received: (qmail 38140 invoked by uid 500); 7 Dec 2016 21:10:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 37091 invoked by uid 99); 7 Dec 2016 21:10:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Dec 2016 21:10:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E4708F2142; Wed, 7 Dec 2016 21:10:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Wed, 07 Dec 2016 21:10:16 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [16/76] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe) archived-at: Wed, 07 Dec 2016 21:10:11 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java new file mode 100644 index 0000000..67d3647 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.slideram; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.tools.ConfigHelper; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.info.CustomRegistryConstants; +import org.apache.slider.providers.AbstractProviderService; +import org.apache.slider.providers.ProviderCore; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.providers.agent.AgentKeys; +import org.apache.slider.server.appmaster.PublishedArtifacts; +import org.apache.slider.server.appmaster.web.rest.RestPaths; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; + +/** + * Exists just to move some functionality out of AppMaster into a peer class + * of the actual service provider doing the real work + */ +public class SliderAMProviderService extends AbstractProviderService implements + ProviderCore, + AgentKeys, + SliderKeys { + + public SliderAMProviderService() { + super("SliderAMProviderService"); + } + + @Override + public String getHumanName() { + return "Slider Application"; + } + + @Override + public Configuration loadProviderConfigurationInformation(File confDir) throws + BadCommandArgumentsException, + IOException { + return null; + } + + @Override + public void buildContainerLaunchContext(ContainerLauncher containerLauncher, + AggregateConf instanceDefinition, + Container container, + ProviderRole role, + SliderFileSystem sliderFileSystem, + Path generatedConfPath, + MapOperations resourceComponent, + MapOperations appComponent, + Path containerTmpDirPath) throws IOException, SliderException { + } + + @Override + public List getRoles() { + return new ArrayList<>(0); + } + + @Override + public void validateInstanceDefinition(AggregateConf instanceDefinition) throws + SliderException { + + } + + @Override + public void applyInitialRegistryDefinitions(URL amWebURI, + URL agentOpsURI, + URL agentStatusURI, + ServiceRecord serviceRecord) + throws IOException { + super.applyInitialRegistryDefinitions(amWebURI, + agentOpsURI, + agentStatusURI, + serviceRecord); + // now publish site.xml files + YarnConfiguration defaultYarnConfig = new YarnConfiguration(); + amState.getPublishedSliderConfigurations().put( + PublishedArtifacts.COMPLETE_CONFIG, + new PublishedConfiguration( + "Complete slider application settings", + getConfig(), getConfig())); + amState.getPublishedSliderConfigurations().put( + PublishedArtifacts.YARN_SITE_CONFIG, + new PublishedConfiguration( + "YARN site settings", + ConfigHelper.loadFromResource("yarn-site.xml"), + defaultYarnConfig) ); + + amState.getPublishedSliderConfigurations().put( + PublishedArtifacts.CORE_SITE_CONFIG, + new PublishedConfiguration( + "Core site settings", + ConfigHelper.loadFromResource("core-site.xml"), + defaultYarnConfig) ); + amState.getPublishedSliderConfigurations().put( + PublishedArtifacts.HDFS_SITE_CONFIG, + new PublishedConfiguration( + "HDFS site settings", + ConfigHelper.loadFromResource("hdfs-site.xml"), + new HdfsConfiguration(true)) ); + + + try { + + URL managementAPI = new URL(amWebURI, RELATIVE_PATH_MANAGEMENT); + URL registryREST = new URL(amWebURI, RELATIVE_PATH_REGISTRY); + + URL publisherURL = new URL(amWebURI, RELATIVE_PATH_PUBLISHER); + + // Set the configurations URL. + + String configurationsURL = SliderUtils.appendToURL( + publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET); + String exportsURL = SliderUtils.appendToURL( + publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS); + + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.webEndpoint( + CustomRegistryConstants.WEB_UI, amWebURI.toURI())); + + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.webEndpoint( + CustomRegistryConstants.AM_REST_BASE, amWebURI.toURI())); + + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.MANAGEMENT_REST_API, + managementAPI.toURI())); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.PUBLISHER_REST_API, + publisherURL.toURI())); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.REGISTRY_REST_API, + registryREST.toURI())); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API, + new URI(configurationsURL))); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.PUBLISHER_EXPORTS_API, + new URI(exportsURL))); + + } catch (URISyntaxException e) { + throw new IOException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java new file mode 100644 index 0000000..288f25a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster; + +import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions; + +/** + * Interface of AM operations + */ +public interface AppMasterActionOperations extends RMOperationHandlerActions { + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java new file mode 100644 index 0000000..65b88cf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster; + + +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; + +import java.net.InetSocketAddress; +import java.security.PrivilegedAction; + +/** + * Implement privileged connection to the CM + * + */ +public class PrivilegedConnectToCM implements PrivilegedAction { + final SliderAppMaster appMaster; + final InetSocketAddress cmAddress; + + public PrivilegedConnectToCM(SliderAppMaster appMaster, + InetSocketAddress cmAddress) { + this.appMaster = appMaster; + this.cmAddress = cmAddress; + } + + + @Override //PrivilegedAction + public ContainerManagementProtocol run() { + return ((ContainerManagementProtocol) appMaster.getProxy( + ContainerManagementProtocol.class, + cmAddress)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java new file mode 100644 index 0000000..5d52441 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.slider.server.appmaster.state.AbstractClusterServices; + +public class ProtobufClusterServices extends AbstractClusterServices { + + public Resource newResource() { + return Records.newRecord(Resource.class); + } + + @Override + public Resource newResource(int memory, int cores) { + return Resources.createResource(memory, cores); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java new file mode 100644 index 0000000..fdc386f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster; + +/** + * This is the name of YARN artifacts that are published + */ +public interface PublishedArtifacts { + + String COMPLETE_CONFIG = "complete-config"; + String CORE_SITE_CONFIG = "core-site"; + String HDFS_SITE_CONFIG = "hdfs-site"; + String YARN_SITE_CONFIG = "yarn-site"; + String LOG4J = "log4j"; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java new file mode 100644 index 0000000..3cfe167 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.providers.ProviderService; +import org.apache.slider.providers.agent.AgentKeys; +import org.apache.slider.server.appmaster.actions.ActionStartContainer; +import org.apache.slider.server.appmaster.actions.QueueAccess; +import org.apache.slider.server.appmaster.state.ContainerAssignment; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.apache.slider.server.services.workflow.WorkflowExecutorService; +import org.apache.slider.server.services.workflow.ServiceThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * A service for launching containers + */ +public class RoleLaunchService + extends WorkflowExecutorService { + protected static final Logger log = + LoggerFactory.getLogger(RoleLaunchService.class); + + public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService"; + + + /** + * Queue submission API + */ + private final QueueAccess actionQueue; + + /** + * Provider building up the command + */ + private final ProviderService provider; + + /** + * Filesystem to use for the launch + */ + private final SliderFileSystem fs; + + /** + * Path in the launch filesystem that refers to a configuration directory + * -the interpretation of it is left to the Provider + */ + private final Path generatedConfDirPath; + /** + * Path in the launch filesystem that refers to a temp directory + * which will be cleaned up at (some) time in the future + */ + private final Path launcherTmpDirPath; + + private Map envVars; + + /** + * Construct an instance of the launcher + * @param queueAccess + * @param provider the provider + * @param fs filesystem + * @param generatedConfDirPath path in the FS for the generated dir + * @param envVars environment variables + * @param launcherTmpDirPath path for a temporary data in the launch process + */ + public RoleLaunchService(QueueAccess queueAccess, + ProviderService provider, + SliderFileSystem fs, + Path generatedConfDirPath, + Map envVars, + Path launcherTmpDirPath) { + super(ROLE_LAUNCH_SERVICE); + this.actionQueue = queueAccess; + this.fs = fs; + this.generatedConfDirPath = generatedConfDirPath; + this.launcherTmpDirPath = launcherTmpDirPath; + this.provider = provider; + this.envVars = envVars; + } + + @Override + public void init(Configuration conf) { + super.init(conf); + setExecutor(Executors.newCachedThreadPool( + new ServiceThreadFactory(ROLE_LAUNCH_SERVICE, true))); + } + + /** + * Start an asychronous launch operation + * @param assignment container assignment + * @param clusterSpec cluster spec to use for template + * @param credentials credentials to use + */ + public void launchRole(ContainerAssignment assignment, + AggregateConf clusterSpec, + Credentials credentials) { + RoleStatus role = assignment.role; + String roleName = role.getName(); + String roleGroup = role.getGroup(); + // prelaunch safety check + Preconditions.checkArgument(provider.isSupportedRole(roleName)); + RoleLaunchService.RoleLauncher launcher = + new RoleLaunchService.RoleLauncher(assignment, + clusterSpec, + clusterSpec.getResourceOperations().getOrAddComponent(roleGroup), + clusterSpec.getAppConfOperations().getOrAddComponent(roleGroup), + credentials); + execute(launcher); + } + + /** + * Thread that runs on the AM to launch a container + */ + private class RoleLauncher implements Runnable { + + private final ContainerAssignment assignment; + // Allocated container + public final Container container; + private final MapOperations resourceComponent; + private final MapOperations appComponent; + private final AggregateConf instanceDefinition; + public final ProviderRole role; + private final Credentials credentials; + private Exception raisedException; + + public RoleLauncher(ContainerAssignment assignment, + AggregateConf instanceDefinition, + MapOperations resourceComponent, + MapOperations appComponent, + Credentials credentials) { + this.assignment = assignment; + this.credentials = credentials; + this.container = assignment.container; + RoleStatus roleStatus = assignment.role; + + assert resourceComponent != null; + assert appComponent != null; + ProviderRole providerRole = roleStatus.getProviderRole(); + assert providerRole != null; + this.role = providerRole; + this.resourceComponent = resourceComponent; + this.appComponent = appComponent; + this.instanceDefinition = instanceDefinition; + } + + public Exception getRaisedException() { + return raisedException; + } + + @Override + public String toString() { + return "RoleLauncher{" + + "container=" + container.getId() + + ", containerRole='" + role.name + '\'' + + ", containerGroup='" + role.group + '\'' + + '}'; + } + + @Override + public void run() { + try { + ContainerLauncher containerLauncher = + new ContainerLauncher(getConfig(), fs, container, credentials); + containerLauncher.setupUGI(); + containerLauncher.putEnv(envVars); + + log.debug("Launching container {} into role {}", + container.getId(), + role.name); + + //now build up the configuration data + Path containerTmpDirPath = + new Path(launcherTmpDirPath, container.getId().toString()); + provider.buildContainerLaunchContext(containerLauncher, + instanceDefinition, + container, + role, + fs, + generatedConfDirPath, + resourceComponent, + appComponent, + containerTmpDirPath); + + RoleInstance instance = new RoleInstance(container); + String[] envDescription = containerLauncher.dumpEnvToString(); + + String commandsAsString = containerLauncher.getCommandsAsString(); + log.info("Starting container with command: {}", + commandsAsString); + + instance.command = commandsAsString; + instance.role = role.name; + instance.group = role.group; + instance.roleId = role.id; + instance.appVersion = instanceDefinition.getAppConfOperations() + .getGlobalOptions().get(SliderKeys.APP_VERSION); + instance.environment = envDescription; + int delay = appComponent.getOptionInt( + AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0); + int maxDelay = + getConfig().getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); + if (delay > maxDelay/1000) { + log.warn("Container launch delay of {} exceeds the maximum allowed of" + + " {} seconds. Delay will not be utilized.", + delay, maxDelay/1000); + delay = 0; + } + log.info("Container launch delay for {} set to {} seconds", + role.name, delay); + actionQueue.schedule(new ActionStartContainer("starting " + role.name, + container, + containerLauncher.completeContainerLaunch(), + instance, + delay, + TimeUnit.SECONDS)); + } catch (Exception e) { + log.error("Exception thrown while trying to start {}: {}", + role.name, e, e); + raisedException = e; + } + } + + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org