hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [16/76] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)
Date Wed, 07 Dec 2016 21:10:16 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
new file mode 100644
index 0000000..67d3647
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.slideram;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.server.appmaster.PublishedArtifacts;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
+
+/**
+ * Exists just to move some functionality out of AppMaster into a peer class
+ * of the actual service provider doing the real work
+ */
+public class SliderAMProviderService extends AbstractProviderService implements
+    ProviderCore,
+    AgentKeys,
+    SliderKeys {
+
+  public SliderAMProviderService() {
+    super("SliderAMProviderService");
+  }
+
+  @Override
+  public String getHumanName() {
+    return "Slider Application";
+  }
+  
+  @Override
+  public Configuration loadProviderConfigurationInformation(File confDir) throws
+      BadCommandArgumentsException,
+      IOException {
+    return null;
+  }
+
+  @Override
+  public void buildContainerLaunchContext(ContainerLauncher containerLauncher,
+      AggregateConf instanceDefinition,
+      Container container,
+      ProviderRole role,
+      SliderFileSystem sliderFileSystem,
+      Path generatedConfPath,
+      MapOperations resourceComponent,
+      MapOperations appComponent,
+      Path containerTmpDirPath) throws IOException, SliderException {
+  }
+
+  @Override
+  public List<ProviderRole> getRoles() {
+    return new ArrayList<>(0);
+  }
+
+  @Override
+  public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+      SliderException {
+
+  }
+
+  @Override
+  public void applyInitialRegistryDefinitions(URL amWebURI,
+      URL agentOpsURI,
+      URL agentStatusURI,
+      ServiceRecord serviceRecord)
+      throws IOException {
+    super.applyInitialRegistryDefinitions(amWebURI,
+        agentOpsURI,
+        agentStatusURI,
+        serviceRecord);
+    // now publish site.xml files
+    YarnConfiguration defaultYarnConfig = new YarnConfiguration();
+    amState.getPublishedSliderConfigurations().put(
+        PublishedArtifacts.COMPLETE_CONFIG,
+        new PublishedConfiguration(
+            "Complete slider application settings",
+            getConfig(), getConfig()));
+    amState.getPublishedSliderConfigurations().put(
+        PublishedArtifacts.YARN_SITE_CONFIG,
+        new PublishedConfiguration(
+            "YARN site settings",
+            ConfigHelper.loadFromResource("yarn-site.xml"),
+            defaultYarnConfig) );
+
+    amState.getPublishedSliderConfigurations().put(
+        PublishedArtifacts.CORE_SITE_CONFIG,
+        new PublishedConfiguration(
+            "Core site settings",
+            ConfigHelper.loadFromResource("core-site.xml"),
+            defaultYarnConfig) );
+    amState.getPublishedSliderConfigurations().put(
+        PublishedArtifacts.HDFS_SITE_CONFIG,
+        new PublishedConfiguration(
+            "HDFS site settings",
+            ConfigHelper.loadFromResource("hdfs-site.xml"),
+            new HdfsConfiguration(true)) );
+
+
+    try {
+
+      URL managementAPI = new URL(amWebURI, RELATIVE_PATH_MANAGEMENT);
+      URL registryREST = new URL(amWebURI, RELATIVE_PATH_REGISTRY);
+
+      URL publisherURL = new URL(amWebURI, RELATIVE_PATH_PUBLISHER);
+
+      // Set the configurations URL.
+
+      String configurationsURL = SliderUtils.appendToURL(
+          publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET);
+      String exportsURL = SliderUtils.appendToURL(
+          publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS);
+
+      serviceRecord.addExternalEndpoint(
+          RegistryTypeUtils.webEndpoint(
+              CustomRegistryConstants.WEB_UI, amWebURI.toURI()));
+      
+      serviceRecord.addExternalEndpoint(
+          RegistryTypeUtils.webEndpoint(
+              CustomRegistryConstants.AM_REST_BASE, amWebURI.toURI()));
+      
+      serviceRecord.addExternalEndpoint(
+          RegistryTypeUtils.restEndpoint(
+              CustomRegistryConstants.MANAGEMENT_REST_API,
+              managementAPI.toURI()));
+      serviceRecord.addExternalEndpoint(
+          RegistryTypeUtils.restEndpoint(
+              CustomRegistryConstants.PUBLISHER_REST_API,
+              publisherURL.toURI()));
+      serviceRecord.addExternalEndpoint(
+          RegistryTypeUtils.restEndpoint(
+              CustomRegistryConstants.REGISTRY_REST_API,
+              registryREST.toURI()));
+      serviceRecord.addExternalEndpoint(
+          RegistryTypeUtils.restEndpoint(
+              CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API,
+              new URI(configurationsURL)));
+      serviceRecord.addExternalEndpoint(
+          RegistryTypeUtils.restEndpoint(
+              CustomRegistryConstants.PUBLISHER_EXPORTS_API,
+              new URI(exportsURL)));
+
+    } catch (URISyntaxException e) {
+      throw new IOException(e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
new file mode 100644
index 0000000..288f25a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
+
+/**
+ * Interface of AM operations
+ */
+public interface AppMasterActionOperations extends RMOperationHandlerActions {
+
+  
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
new file mode 100644
index 0000000..65b88cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+
+/**
+ * Implement privileged connection to the CM
+ *
+ */
+public class PrivilegedConnectToCM implements PrivilegedAction<ContainerManagementProtocol>
{
+  final SliderAppMaster appMaster;
+  final InetSocketAddress cmAddress;
+
+  public PrivilegedConnectToCM(SliderAppMaster appMaster,
+                               InetSocketAddress cmAddress) {
+    this.appMaster = appMaster;
+    this.cmAddress = cmAddress;
+  }
+
+
+  @Override //PrivilegedAction
+  public ContainerManagementProtocol run() {
+    return ((ContainerManagementProtocol) appMaster.getProxy(
+      ContainerManagementProtocol.class,
+      cmAddress));
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
new file mode 100644
index 0000000..5d52441
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.slider.server.appmaster.state.AbstractClusterServices;
+
+public class ProtobufClusterServices extends AbstractClusterServices {
+
+  public Resource newResource() {
+    return Records.newRecord(Resource.class);
+  }
+
+  @Override
+  public Resource newResource(int memory, int cores) {
+    return Resources.createResource(memory, cores);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
new file mode 100644
index 0000000..fdc386f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/PublishedArtifacts.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+/**
+ * This is the name of YARN artifacts that are published
+ */
+public interface PublishedArtifacts {
+
+  String COMPLETE_CONFIG = "complete-config";
+  String CORE_SITE_CONFIG = "core-site";
+  String HDFS_SITE_CONFIG = "hdfs-site";
+  String YARN_SITE_CONFIG = "yarn-site";
+  String LOG4J = "log4j";
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
new file mode 100644
index 0000000..3cfe167
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.server.appmaster.actions.ActionStartContainer;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.services.workflow.WorkflowExecutorService;
+import org.apache.slider.server.services.workflow.ServiceThreadFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A service for launching containers
+ */
+public class RoleLaunchService
+    extends WorkflowExecutorService<ExecutorService> {
+  protected static final Logger log =
+    LoggerFactory.getLogger(RoleLaunchService.class);
+
+  public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService";
+
+
+  /**
+   * Queue submission API
+   */
+  private final QueueAccess actionQueue;
+
+  /**
+   * Provider building up the command
+   */
+  private final ProviderService provider;
+  
+  /**
+   * Filesystem to use for the launch
+   */
+  private final SliderFileSystem fs;
+
+  /**
+   * Path in the launch filesystem that refers to a configuration directory
+   * -the interpretation of it is left to the Provider
+   */
+  private final Path generatedConfDirPath;
+  /**
+   * Path in the launch filesystem that refers to a temp directory
+   * which will be cleaned up at (some) time in the future
+   */
+  private final Path launcherTmpDirPath;
+
+  private Map<String, String> envVars;
+
+  /**
+   * Construct an instance of the launcher
+   * @param queueAccess
+   * @param provider the provider
+   * @param fs filesystem
+   * @param generatedConfDirPath path in the FS for the generated dir
+   * @param envVars environment variables
+   * @param launcherTmpDirPath path for a temporary data in the launch process
+   */
+  public RoleLaunchService(QueueAccess queueAccess,
+      ProviderService provider,
+      SliderFileSystem fs,
+      Path generatedConfDirPath,
+      Map<String, String> envVars,
+      Path launcherTmpDirPath) {
+    super(ROLE_LAUNCH_SERVICE);
+    this.actionQueue = queueAccess;
+    this.fs = fs;
+    this.generatedConfDirPath = generatedConfDirPath;
+    this.launcherTmpDirPath = launcherTmpDirPath;
+    this.provider = provider;
+    this.envVars = envVars;
+  }
+
+  @Override
+  public void init(Configuration conf) {
+    super.init(conf);
+    setExecutor(Executors.newCachedThreadPool(
+        new ServiceThreadFactory(ROLE_LAUNCH_SERVICE, true)));
+  }
+
+  /**
+   * Start an asychronous launch operation
+   * @param assignment container assignment
+   * @param clusterSpec cluster spec to use for template
+   * @param credentials credentials to use
+   */
+  public void launchRole(ContainerAssignment assignment,
+      AggregateConf clusterSpec,
+      Credentials credentials) {
+    RoleStatus role = assignment.role;
+    String roleName = role.getName();
+    String roleGroup = role.getGroup();
+    // prelaunch safety check
+    Preconditions.checkArgument(provider.isSupportedRole(roleName));
+    RoleLaunchService.RoleLauncher launcher =
+      new RoleLaunchService.RoleLauncher(assignment,
+         clusterSpec,
+         clusterSpec.getResourceOperations().getOrAddComponent(roleGroup),
+         clusterSpec.getAppConfOperations().getOrAddComponent(roleGroup),
+         credentials);
+    execute(launcher);
+  }
+
+  /**
+   * Thread that runs on the AM to launch a container
+   */
+  private class RoleLauncher implements Runnable {
+
+    private final ContainerAssignment assignment;
+    // Allocated container
+    public final Container container;
+    private final MapOperations resourceComponent;
+    private final MapOperations appComponent;
+    private final AggregateConf instanceDefinition;
+    public final ProviderRole role;
+    private final Credentials credentials;
+    private Exception raisedException;
+
+    public RoleLauncher(ContainerAssignment assignment,
+        AggregateConf instanceDefinition,
+        MapOperations resourceComponent,
+        MapOperations appComponent,
+        Credentials credentials) {
+      this.assignment = assignment;
+      this.credentials = credentials;
+      this.container = assignment.container;
+      RoleStatus roleStatus = assignment.role;
+
+      assert resourceComponent != null;
+      assert appComponent != null;
+      ProviderRole providerRole = roleStatus.getProviderRole();
+      assert providerRole != null;
+      this.role = providerRole;
+      this.resourceComponent = resourceComponent;
+      this.appComponent = appComponent;
+      this.instanceDefinition = instanceDefinition;
+    }
+
+    public Exception getRaisedException() {
+      return raisedException;
+    }
+
+    @Override
+    public String toString() {
+      return "RoleLauncher{" +
+             "container=" + container.getId() +
+             ", containerRole='" + role.name + '\'' +
+             ", containerGroup='" + role.group + '\'' +
+             '}';
+    }
+
+    @Override
+    public void run() {
+      try {
+        ContainerLauncher containerLauncher =
+            new ContainerLauncher(getConfig(), fs, container, credentials);
+        containerLauncher.setupUGI();
+        containerLauncher.putEnv(envVars);
+
+        log.debug("Launching container {} into role {}",
+                  container.getId(),
+                  role.name);
+
+        //now build up the configuration data
+        Path containerTmpDirPath =
+          new Path(launcherTmpDirPath, container.getId().toString());
+        provider.buildContainerLaunchContext(containerLauncher,
+            instanceDefinition,
+            container,
+            role,
+            fs,
+            generatedConfDirPath,
+            resourceComponent,
+            appComponent,
+            containerTmpDirPath);
+
+        RoleInstance instance = new RoleInstance(container);
+        String[] envDescription = containerLauncher.dumpEnvToString();
+
+        String commandsAsString = containerLauncher.getCommandsAsString();
+        log.info("Starting container with command: {}",
+                 commandsAsString);
+
+        instance.command = commandsAsString;
+        instance.role = role.name;
+        instance.group = role.group;
+        instance.roleId = role.id;
+        instance.appVersion = instanceDefinition.getAppConfOperations()
+            .getGlobalOptions().get(SliderKeys.APP_VERSION);
+        instance.environment = envDescription;
+        int delay = appComponent.getOptionInt(
+            AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0);
+        int maxDelay =
+            getConfig().getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+                               YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
+        if (delay > maxDelay/1000) {
+          log.warn("Container launch delay of {} exceeds the maximum allowed of"
+                   + " {} seconds.  Delay will not be utilized.",
+                   delay, maxDelay/1000);
+          delay = 0;
+        }
+        log.info("Container launch delay for {} set to {} seconds",
+                 role.name, delay);
+        actionQueue.schedule(new ActionStartContainer("starting " + role.name,
+                                                      container,
+                                                      containerLauncher.completeContainerLaunch(),
+                                                      instance,
+                                                      delay,
+                                                      TimeUnit.SECONDS));
+      } catch (Exception e) {
+        log.error("Exception thrown while trying to start {}: {}",
+            role.name, e, e);
+        raisedException = e;
+      }
+    }
+
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message