hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [17/66] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)
Date Wed, 23 Nov 2016 23:39:43 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
new file mode 100644
index 0000000..78bb8c1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Component defined in master package metainfo.json
+ */
+public class Component extends AbstractComponent {
+
+  String category = CATEGORY_MASTER;
+  String publishConfig = Boolean.FALSE.toString();
+  String minInstanceCount = "0";
+  String maxInstanceCount;
+  String autoStartOnFailure = Boolean.FALSE.toString();
+  String appExports;
+  String compExports;
+  String type = TYPE_STANDARD;
+  List<ComponentExport> componentExports = new ArrayList<>();
+  List<DockerContainer> dockerContainers = new ArrayList<>();
+  List<ConfigFile> configFiles = new ArrayList<>();
+
+  public Component() {
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  public String getCategory() {
+    return category;
+  }
+
+  public void setCategory(String category) {
+    this.category = category;
+  }
+
+  public String getPublishConfig() {
+    return publishConfig;
+  }
+
+  public void setPublishConfig(String publishConfig) {
+    this.publishConfig = publishConfig;
+  }
+
+  public String getAutoStartOnFailure() {
+    return autoStartOnFailure;
+  }
+
+  public void setAutoStartOnFailure(String autoStartOnFailure) {
+    this.autoStartOnFailure = autoStartOnFailure;
+  }
+
+  public String getAppExports() {
+    return appExports;
+  }
+
+  public void setAppExports(String appExports) {
+    this.appExports = appExports;
+  }
+
+  public String getCompExports() {
+    return compExports;
+  }
+
+  public void setCompExports(String compExports) {
+    this.compExports = compExports;
+  }
+
+  public String getMinInstanceCount() {
+    return minInstanceCount;
+  }
+  
+  @JsonProperty("dockerContainers")
+  public List<DockerContainer> getDockerContainers() {
+     return this.dockerContainers;
+  }
+  
+  public Boolean getAutoStartOnFailureBoolean() {
+    if (SliderUtils.isUnset(getAutoStartOnFailure())) {
+      return Boolean.FALSE;
+    }
+
+    return Boolean.parseBoolean(getAutoStartOnFailure());
+  }
+
+  public int getMinInstanceCountInt() throws BadConfigException {
+    if (SliderUtils.isUnset(minInstanceCount)) {
+      return 0;
+    }
+
+    try {
+      return Integer.parseInt(minInstanceCount);
+    } catch (NumberFormatException nfe) {
+      throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name);
+    }
+  }
+
+  public int getMaxInstanceCountInt() throws BadConfigException {
+    if (SliderUtils.isUnset(maxInstanceCount)) {
+      return Integer.MAX_VALUE;
+    }
+
+    try {
+      return Integer.parseInt(maxInstanceCount);
+    } catch (NumberFormatException nfe) {
+      throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name);
+    }
+  }
+
+  public void setMinInstanceCount(String minInstanceCount) {
+    this.minInstanceCount = minInstanceCount;
+  }
+
+  public String getMaxInstanceCount() {
+    return maxInstanceCount;
+  }
+
+  public void setMaxInstanceCount(String maxInstanceCount) {
+    this.maxInstanceCount = maxInstanceCount;
+  }
+
+  public void addComponentExport(ComponentExport export) {
+    componentExports.add(export);
+  }
+
+  public List<ComponentExport> getComponentExports() {
+    return componentExports;
+  }
+
+  public Boolean getRequiresAutoRestart() {
+    return Boolean.parseBoolean(this.autoStartOnFailure);
+  }
+
+  public void addConfigFile(ConfigFile configFile) {
+    this.configFiles.add(configFile);
+  }
+
+  @JsonProperty("configFiles")
+  public List<ConfigFile> getConfigFiles() {
+    return configFiles;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+        new StringBuilder("{");
+    sb.append("\n\"name\": ").append(name);
+    sb.append(",\n\"category\": ").append(category);
+    sb.append(",\n\"commandScript\" :").append(commandScript);
+    for(DockerContainer dc : dockerContainers){
+      sb.append(",\n\"container\" :").append(dc.toString());
+    }    
+    sb.append('}');
+    return sb.toString();
+  }
+
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getName(), "name", "component");
+    Metainfo.checkNonNull(getCategory(), "category", "component");
+    if (!getCategory().equals(CATEGORY_MASTER)
+        && !getCategory().equals(CATEGORY_SLAVE)
+        && !getCategory().equals(CATEGORY_CLIENT)) {
+      throw new SliderException("Invalid category for the component " + getCategory());
+    }
+
+    Metainfo.checkNonNull(getType(), "type", "component");
+    if (!getType().equals(TYPE_DOCKER)
+        && !getType().equals(TYPE_STANDARD)) {
+      throw new SliderException("Invalid type for the component " + getType());
+    }
+
+    if (version.equals(Metainfo.VERSION_TWO_ZERO)) {
+      if (getType().equals(TYPE_DOCKER)) {
+        throw new SliderException(TYPE_DOCKER + " is not supported in version " + Metainfo.VERSION_TWO_ZERO);
+      }
+
+      if (getCommands().size() > 0) {
+        throw new SliderException("commands are not supported in version " + Metainfo.VERSION_TWO_ZERO);
+      }
+    }
+
+    if (commandScript != null) {
+      commandScript.validate(version);
+    }
+
+    if (version.equals(Metainfo.VERSION_TWO_ONE)) {
+      for (ComponentCommand cc : getCommands()) {
+        cc.validate(version);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
new file mode 100644
index 0000000..52117c5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents the metadata associated with the application.
+ */
+public class ComponentCommand implements Validate {
+  protected static final Logger
+      log = LoggerFactory.getLogger(ComponentCommand.class);
+
+
+  private String exec;
+  private String name = "START";
+  private String type = "SHELL";
+
+  /**
+   * Creator.
+   */
+  public ComponentCommand() {
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setExec(String exec) {
+    this.exec = exec;
+  }
+
+  public String getExec() {
+    return exec;
+  }
+
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getName(), "name", "componentCommand");
+
+    Metainfo.checkNonNull(getType(), "version", "application");
+  }
+
+  public static ComponentCommand getDefaultComponentCommand() {
+    ComponentCommand cc = new ComponentCommand();
+    cc.setExec("DEFAULT");
+    return cc;
+  }
+
+  public static ComponentCommand getDefaultComponentCommand(String commandName) {
+    ComponentCommand cc = new ComponentCommand();
+    cc.setExec("DEFAULT");
+    cc.setName(commandName);
+    return cc;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
new file mode 100644
index 0000000..a18854c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ *
+ */
+public class ComponentExport {
+  String name;
+  String value;
+
+  public ComponentExport() {
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getValue() {
+    return value;
+  }
+
+  public void setValue(String value) {
+    this.value = value;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+        new StringBuilder("{");
+    sb.append(",\n\"name\": ").append(name);
+    sb.append(",\n\"value\": ").append(value);
+    sb.append('}');
+    return sb.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
new file mode 100644
index 0000000..855e5b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+public class ComponentsInAddonPackage extends AbstractComponent {
+
+  @Override
+  public void validate(String version) throws SliderException {
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
new file mode 100644
index 0000000..cb47512
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ *
+ */
+public class ConfigFile implements Validate {
+  String type;
+  String fileName;
+  String dictionaryName;
+
+  public ConfigFile() {
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  public String getFileName() {
+    return fileName;
+  }
+
+  public void setFileName(String fileName) {
+    this.fileName = fileName;
+  }
+
+  public String getDictionaryName() {
+    return dictionaryName;
+  }
+
+  public void setDictionaryName(String dictionaryName) {
+    this.dictionaryName = dictionaryName;
+  }
+
+  public void validate(String version) throws SliderException {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
new file mode 100644
index 0000000..46c8836
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Application default config
+ */
+public class DefaultConfig {
+  List<PropertyInfo> propertyInfos;
+
+  public DefaultConfig() {
+    propertyInfos = new ArrayList<PropertyInfo>();
+  }
+
+  public void addPropertyInfo(PropertyInfo propertyInfo) {
+    propertyInfos.add(propertyInfo);
+  }
+
+  public List<PropertyInfo> getPropertyInfos() {
+    return propertyInfos;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
new file mode 100644
index 0000000..e136775
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.commons.digester.Digester;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class DefaultConfigParser {
+
+  public DefaultConfig parse(InputStream configFileStream) throws IOException {
+    Digester digester = new Digester();
+    digester.setValidating(false);
+
+    digester.addObjectCreate("configuration", DefaultConfig.class);
+
+    digester.addObjectCreate("*/property", PropertyInfo.class);
+    digester.addBeanPropertySetter("*/property/name");
+    digester.addBeanPropertySetter("*/property/value");
+    digester.addBeanPropertySetter("*/property/description");
+    digester.addSetNext("*/property", "addPropertyInfo");
+
+    try {
+      return (DefaultConfig) digester.parse(configFileStream);
+    } catch (IOException e) {
+
+    } catch (SAXException e) {
+
+    } finally {
+      configFileStream.close();
+    }
+
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
new file mode 100644
index 0000000..4c61e7f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents a docker container
+ */
+public class DockerContainer implements Validate {
+  protected static final Logger
+      log = LoggerFactory.getLogger(DockerContainer.class);
+
+  private String name;
+  private String image;
+  private String network;
+  private String useNetworkScript;
+  private String options;
+  private List<DockerContainerMount> mounts = new ArrayList<>();
+  private List<DockerContainerPort> ports = new ArrayList<>();
+  private String statusCommand;
+  private String startCommand;
+  private String commandPath;
+  private String additionalParam;
+  private String runPrivilegedContainer;
+  private List<DockerContainerInputFile> inputFiles = new ArrayList<>();
+  private List<ConfigFile> configFiles = new ArrayList<>();
+
+  public DockerContainer() {
+  }
+
+  @JsonProperty("mounts")
+  public List<DockerContainerMount> getMounts() { return this.mounts; }
+
+  @JsonProperty("ports")
+  public List<DockerContainerPort> getPorts() {
+    return this.ports;
+  }
+
+  @JsonProperty("inputFiles")
+  public List<DockerContainerInputFile> getInputFiles() {
+    return this.inputFiles;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getImage() {
+    return image;
+  }
+
+  public void setImage(String image) {
+    this.image = image;
+  }
+
+  public String getNetwork() {
+    return network;
+  }
+
+  public void setNetwork(String network) {
+    this.network = network;
+  }
+
+  public String getUseNetworkScript() {
+    return useNetworkScript;
+  }
+
+  public void setUseNetworkScript(String useNetworkScript) {
+    this.useNetworkScript = useNetworkScript;
+  }
+
+  public String getOptions() {
+    return options;
+  }
+
+  public void setOptions(String options) {
+    this.options = options;
+  }
+
+  @Override
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getName(), "name", "dockerContainer");
+    Metainfo.checkNonNull(getImage(), "image", "dockerContainer");
+    for (DockerContainerMount dcm : getMounts()) {
+      dcm.validate(version);
+    }
+    for (DockerContainerPort dcp : getPorts()) {
+      dcp.validate(version);
+    }
+  }
+
+  @JsonProperty("statusCommand")
+  public String getStatusCommand() {
+    return statusCommand;
+  }
+
+  @JsonProperty("statusCommand")
+  public void setStatusCommand(String statusCommand) {
+    this.statusCommand = statusCommand;
+  }
+
+  public String getCommandPath() {
+    return commandPath;
+  }
+
+  public void setCommandPath(String commandPath) {
+    this.commandPath = commandPath;
+  }
+
+  public String getAdditionalParam() {
+    return additionalParam;
+  }
+
+  public void setAdditionalParam(String additionalParam) {
+    this.additionalParam = additionalParam;
+  }
+
+  @JsonProperty("startCommand")
+  public String getStartCommand() {
+    return startCommand;
+  }
+
+  @JsonProperty("startCommand")
+  public void setStartCommand(String startCommand) {
+    this.startCommand = startCommand;
+  }
+
+  @JsonProperty("runPrivilegedContainer")
+  public String getRunPrivilegedContainer() {
+    return runPrivilegedContainer;
+  }
+
+  @JsonProperty("runPrivilegedContainer")
+  public void setRunPrivilegedContainer(String runPrivilegedContainer) {
+    this.runPrivilegedContainer = runPrivilegedContainer;
+  }
+
+  public List<ConfigFile> getConfigFiles() {
+    return configFiles;
+  }
+
+  public void setConfigFiles(List<ConfigFile> configFiles) {
+    this.configFiles = configFiles;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder result = new StringBuilder("DockerContainer [name=")
+        .append(name).append(", image=").append(image).append(", options=")
+        .append(options).append(", mounts=").append(mounts).append(", ports=")
+        .append(ports).append(", statusCommand=").append(statusCommand)
+        .append(", commandPath=").append(commandPath)
+        .append(", additionalParam=").append(additionalParam)
+        .append(", inputFiles=").append(inputFiles).append(", startCommand=")
+        .append(startCommand).append(", runPriviledgedContainer=")
+        .append(runPrivilegedContainer).append(", configFiles=")
+        .append(configFiles).append("]");
+    return result.toString();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
new file mode 100644
index 0000000..0faceb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DockerContainerInputFile {
+  protected static final Logger log = LoggerFactory
+      .getLogger(DockerContainerInputFile.class);
+
+  private String containerPath;
+  private String fileLocalPath;
+
+  public DockerContainerInputFile() {
+  }
+
+  public String getContainerMount() {
+    return containerPath;
+  }
+
+  public void setContainerMount(String containerMount) {
+    this.containerPath = containerMount;
+  }
+
+  public String getFileLocalPath() {
+    return fileLocalPath;
+  }
+
+  public void setFileLocalPath(String fileLocalPath) {
+    this.fileLocalPath = fileLocalPath;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
new file mode 100644
index 0000000..61f07f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents a docker container mount
+ */
+public class DockerContainerMount implements Validate {
+  protected static final Logger
+      log = LoggerFactory.getLogger(DockerContainerMount.class);
+
+
+  private String containerMount;
+  private String hostMount;
+
+  public DockerContainerMount() {
+  }
+
+  public String getContainerMount() {
+    return containerMount;
+  }
+
+  public void setContainerMount(String containerMount) {
+    this.containerMount = containerMount;
+  }
+
+  public String getHostMount() {
+    return hostMount;
+  }
+
+  public void setHostMount(String hostMount) {
+    this.hostMount = hostMount;
+  }
+
+  @Override
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getContainerMount(), "containerMount", "dockerContainerMount");
+    Metainfo.checkNonNull(getHostMount(), "hostMount", "dockerContainerMount");
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
new file mode 100644
index 0000000..0629d9d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents a docker container port
+ */
+public class DockerContainerPort implements Validate {
+  protected static final Logger
+      log = LoggerFactory.getLogger(DockerContainerPort.class);
+
+
+  private String containerPort;
+  private String hostPort;
+
+  public DockerContainerPort() {
+  }
+
+  public String getContainerPort() {
+    return containerPort;
+  }
+
+  public void setContainerPort(String containerPort) {
+    this.containerPort = containerPort;
+  }
+
+  public String getHostPort() {
+    return hostPort;
+  }
+
+  public void setHostPort(String hostPort) {
+    this.hostPort = hostPort;
+  }
+
+  @Override
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getContainerPort(), "containerPort", "dockerContainerPort");
+    Metainfo.checkNonNull(getHostPort(), "hostPort", "dockerContainerPort");
+  }
+
+  @Override
+  public String toString() {
+    return "DockerContainerPort [containerPort=" + containerPort
+         + ", hostPort=" + hostPort + "]";
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
new file mode 100644
index 0000000..5e0fb24
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ *
+ */
+public class Export implements Validate {
+  String name;
+  String value;
+
+  public Export() {
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getValue() {
+    return value;
+  }
+
+  public void setValue(String value) {
+    this.value = value;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+        new StringBuilder("{");
+    sb.append(",\n\"name\": ").append(name);
+    sb.append(",\n\"value\": ").append(value);
+    sb.append('}');
+    return sb.toString();
+  }
+
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getName(), "name", "export");
+    Metainfo.checkNonNull(getValue(), "value", "export");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
new file mode 100644
index 0000000..3d9f34c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class ExportGroup implements Validate {
+  String name;
+  List<Export> exports;
+
+  public ExportGroup() {
+    exports = new ArrayList<Export>();
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public void addExport(Export export) {
+    exports.add(export);
+  }
+
+  public List<Export> getExports() {
+    return exports;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+        new StringBuilder("{");
+    sb.append(",\n\"name\": ").append(name);
+    sb.append(",\n\"exports\" : {");
+    for (Export export : exports) {
+      sb.append("\n").append(export);
+    }
+    sb.append("\n},");
+    sb.append('}');
+    return sb.toString();
+  }
+
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getName(), "name", "exportGroup");
+    for(Export exp : getExports()) {
+      exp.validate(version);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
new file mode 100644
index 0000000..10c497f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Application metainfo uber class
+ */
+public class Metainfo {
+  protected static final Logger log =
+      LoggerFactory.getLogger(Metainfo.class);
+  public static String VERSION_TWO_ZERO = "2.0";
+  public static String VERSION_TWO_ONE = "2.1";
+
+  String schemaVersion;
+  ApplicationPackage applicationPackage;
+  Application application;
+
+  public String getSchemaVersion() {
+    return schemaVersion;
+  }
+
+  public void setSchemaVersion(String schemaVersion) {
+    this.schemaVersion = schemaVersion;
+  }
+
+  public ApplicationPackage getApplicationPackage() {
+    return applicationPackage;
+  }
+
+  public void setApplicationPackage(ApplicationPackage pkg) {
+    this.applicationPackage = pkg;
+  }
+
+  public Application getApplication() {
+    return application;
+  }
+
+  public void setApplication(Application application) {
+    this.application = application;
+  }
+
+  public Component getApplicationComponent(String roleGroup) {
+    if (application == null) {
+      log.error("Malformed app definition: Expect application as the top level element for metainfo");
+    } else {
+      for (Component component : application.getComponents()) {
+        if (component.getName().equals(roleGroup)) {
+          return component;
+        }
+      }
+    }
+    return null;
+  }
+
+  public List<ConfigFile> getComponentConfigFiles(String roleGroup) {
+    List<ConfigFile> componentConfigFiles = new ArrayList<>();
+    componentConfigFiles.addAll(application.getConfigFiles());
+    Component component = getApplicationComponent(roleGroup);
+    if (component != null) {
+      componentConfigFiles.addAll(component.getConfigFiles());
+    }
+    return componentConfigFiles;
+  }
+
+  public void validate() throws SliderException {
+    if (!VERSION_TWO_ONE.equals(schemaVersion) &&
+        !VERSION_TWO_ZERO.equals(schemaVersion)) {
+      throw new SliderException("Unsupported version " + getSchemaVersion());
+    }
+    if (application != null) {
+      application.validate(schemaVersion);
+    }
+    if (applicationPackage != null) {
+      applicationPackage.validate(schemaVersion);
+    }
+  }
+
+  public static void checkNonNull(String value, String field, String type) throws SliderException {
+    if (SliderUtils.isUnset(value)) {
+      throw new SliderException(type + "." + field + " cannot be null");
+    }
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder builder = new StringBuilder();
+    builder.append("Metainfo [schemaVersion=");
+    builder.append(schemaVersion);
+    builder.append(", applicationPackage=");
+    builder.append(applicationPackage);
+    builder.append(", application=");
+    builder.append(application);
+    builder.append("]");
+    return builder.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
new file mode 100644
index 0000000..8b520eb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.commons.digester.Digester;
+
+/**
+ *
+ */
+public class MetainfoParser extends AbstractMetainfoParser{
+  
+  protected void composeSchema(Digester digester){
+    digester.addObjectCreate("metainfo", Metainfo.class);
+    digester.addBeanPropertySetter("metainfo/schemaVersion");
+
+    digester.addObjectCreate("*/application", Application.class);
+    digester.addBeanPropertySetter("*/application/name");
+    digester.addBeanPropertySetter("*/application/comment");
+    digester.addBeanPropertySetter("*/application/version");
+    digester.addBeanPropertySetter("*/application/exportedConfigs");
+
+    digester.addObjectCreate("*/commandOrder", CommandOrder.class);
+    digester.addBeanPropertySetter("*/commandOrder/command");
+    digester.addBeanPropertySetter("*/commandOrder/requires");
+    digester.addSetNext("*/commandOrder", "addCommandOrder");
+
+    digester.addObjectCreate("*/exportGroup", ExportGroup.class);
+    digester.addBeanPropertySetter("*/exportGroup/name");
+    digester.addObjectCreate("*/export", Export.class);
+    digester.addBeanPropertySetter("*/export/name");
+    digester.addBeanPropertySetter("*/export/value");
+    digester.addSetNext("*/export", "addExport");
+    digester.addSetNext("*/exportGroup", "addExportGroup");
+
+    digester.addObjectCreate("*/component", Component.class);
+    digester.addBeanPropertySetter("*/component/name");
+    digester.addBeanPropertySetter("*/component/category");
+    digester.addBeanPropertySetter("*/component/publishConfig");
+    digester.addBeanPropertySetter("*/component/minInstanceCount");
+    digester.addBeanPropertySetter("*/component/maxInstanceCount");
+    digester.addBeanPropertySetter("*/component/autoStartOnFailure");
+    digester.addBeanPropertySetter("*/component/appExports");
+    digester.addBeanPropertySetter("*/component/compExports");
+    digester.addObjectCreate("*/componentExport", ComponentExport.class);
+    digester.addBeanPropertySetter("*/componentExport/name");
+    digester.addBeanPropertySetter("*/componentExport/value");
+    digester.addSetNext("*/componentExport", "addComponentExport");
+    digester.addSetNext("*/component", "addComponent");
+
+    digester.addObjectCreate("*/commandScript", CommandScript.class);
+    digester.addBeanPropertySetter("*/commandScript/script");
+    digester.addBeanPropertySetter("*/commandScript/scriptType");
+    digester.addBeanPropertySetter("*/commandScript/timeout");
+    digester.addSetNext("*/commandScript", "addCommandScript");
+
+    digester.addObjectCreate("*/command", ComponentCommand.class);
+    digester.addBeanPropertySetter("*/command/exec");
+    digester.addBeanPropertySetter("*/command/name");
+    digester.addBeanPropertySetter("*/command/type");
+    digester.addSetNext("*/command", "addCommand");
+
+    digester.addObjectCreate("*/osSpecific", OSSpecific.class);
+    digester.addBeanPropertySetter("*/osSpecific/osType");
+    digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class);
+    digester.addBeanPropertySetter("*/osSpecific/packages/package/type");
+    digester.addBeanPropertySetter("*/osSpecific/packages/package/name");
+    digester.addSetNext("*/osSpecific/packages/package", "addOSPackage");
+    digester.addSetNext("*/osSpecific", "addOSSpecific");
+
+    digester.addObjectCreate("*/application/packages/package", Package.class);
+    digester.addBeanPropertySetter("*/application/packages/package/type");
+    digester.addBeanPropertySetter("*/application/packages/package/name");
+    digester.addSetNext("*/application/packages/package", "addPackage");
+
+    digester.addObjectCreate("*/configFile", ConfigFile.class);
+    digester.addBeanPropertySetter("*/configFile/type");
+    digester.addBeanPropertySetter("*/configFile/fileName");
+    digester.addBeanPropertySetter("*/configFile/dictionaryName");
+    digester.addSetNext("*/configFile", "addConfigFile");
+
+    digester.addSetRoot("*/application", "setApplication");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
new file mode 100644
index 0000000..32b4890
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ *
+ */
+public class OSPackage implements Validate {
+  String type;
+  String name;
+
+  public OSPackage() {
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getName(), "name", "osPackage");
+    Metainfo.checkNonNull(getType(), "type", "osPackage");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
new file mode 100644
index 0000000..c06d498
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class OSSpecific implements Validate {
+  String osType;
+  List<OSPackage> packages;
+
+  public OSSpecific() {
+    packages = new ArrayList<OSPackage>();
+  }
+
+  public String getOsType() {
+    return osType;
+  }
+
+  public void setOsType(String osType) {
+    this.osType = osType;
+  }
+
+  public void addOSPackage(OSPackage osPackage) {
+    packages.add(osPackage);
+  }
+
+  public List<OSPackage> getPackages() {
+    return packages;
+  }
+
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getOsType(), "osType", "osSpecific");
+    for (OSPackage opkg : getPackages()) {
+      opkg.validate(version);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
new file mode 100644
index 0000000..b88f77d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents package description.
+ */
+public class Package implements Validate {
+  protected static final Logger
+      log = LoggerFactory.getLogger(Package.class);
+
+
+  private String name;
+  private String type;
+
+  public Package() {
+  }
+
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  public void validate(String version) throws SliderException {
+    Metainfo.checkNonNull(getName(), "name", "package");
+    Metainfo.checkNonNull(getType(), "type", "package");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
new file mode 100644
index 0000000..62ee0f5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ * Application config property info
+ */
+public class PropertyInfo {
+  String name;
+  String value;
+  String description;
+
+  public PropertyInfo() {
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getValue() {
+    return value;
+  }
+
+  public void setValue(String value) {
+    this.value = value;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
new file mode 100644
index 0000000..ef03dcd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ * Implementer provides a validate method
+ */
+public interface Validate {
+
+  public void validate(String version) throws SliderException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
new file mode 100644
index 0000000..dfd1373
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
@@ -0,0 +1,22 @@
+<!---
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+  
+# todo list
+
+* Retry on failure
+  * Agent can toleate a configurable number of failures (e.g. 3) before giving up
+* Agent should separate out hostname and label that is received for registration

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ee280b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
new file mode 100644
index 0000000..e5430f2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.slideram;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.launch.JavaCommandLineBuilder;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
+
+/**
+ * handles the setup of the Slider AM.
+ * This keeps aspects of role, cluster validation and Clusterspec setup
+ * out of the core slider client
+ */
+public class SliderAMClientProvider extends AbstractClientProvider
+    implements SliderKeys {
+
+
+  protected static final Logger log =
+    LoggerFactory.getLogger(SliderAMClientProvider.class);
+  protected static final String NAME = "SliderAM";
+  public static final String INSTANCE_RESOURCE_BASE = PROVIDER_RESOURCE_BASE_ROOT +
+                                                       "slideram/instance/";
+  public static final String INTERNAL_JSON =
+    INSTANCE_RESOURCE_BASE + "internal.json";
+  public static final String APPCONF_JSON =
+    INSTANCE_RESOURCE_BASE + "appconf.json";
+  public static final String RESOURCES_JSON =
+    INSTANCE_RESOURCE_BASE + "resources.json";
+
+  public SliderAMClientProvider(Configuration conf) {
+    super(conf);
+  }
+
+  /**
+   * List of roles
+   */
+  public static final List<ProviderRole> ROLES =
+    new ArrayList<ProviderRole>();
+
+  public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX;
+
+  public static final ProviderRole APPMASTER =
+      new ProviderRole(COMPONENT_AM, KEY_AM,
+          PlacementPolicy.EXCLUDE_FROM_FLEXING,
+          ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, 
+          0, "");
+
+  /**
+   * Initialize role list
+   */
+  static {
+    ROLES.add(APPMASTER);
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+
+  @Override
+  public List<ProviderRole> getRoles() {
+    return ROLES;
+  }
+
+
+  @Override //Client
+  public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
+                                                    String clustername,
+                                                    Configuration configuration,
+                                                    AggregateConf instanceDefinition,
+                                                    Path clusterDirPath,
+                                                    Path generatedConfDirPath,
+                                                    boolean secure)
+      throws SliderException, IOException {
+
+    super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure);
+    //add a check for the directory being writeable by the current user
+    String
+      dataPath = instanceDefinition.getInternalOperations()
+                                   .getGlobalOptions()
+                                   .getMandatoryOption(
+                                     InternalKeys.INTERNAL_DATA_DIR_PATH);
+
+    Path path = new Path(dataPath);
+    sliderFileSystem.verifyDirectoryWriteAccess(path);
+    Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME);
+    sliderFileSystem.verifyDirectoryWriteAccess(historyPath);
+  }
+
+  /**
+   * Verify that an instance definition is considered valid by the provider
+   * @param instanceDefinition instance definition
+   * @throws SliderException if the configuration is not valid
+   */
+  public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
+      SliderException {
+
+    super.validateInstanceDefinition(instanceDefinition, fs);
+    
+    // make sure there is no negative entry in the instance count
+    Map<String, Map<String, String>> instanceMap =
+        instanceDefinition.getResources().components;
+    for (Map.Entry<String, Map<String, String>> entry : instanceMap.entrySet()) {
+      MapOperations mapOperations = new MapOperations(entry);
+      int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0);
+      if (instances < 0) {
+        throw new BadClusterStateException(
+            "Component %s has negative instance count: %d",
+            mapOperations.name,
+            instances);
+      }
+    }
+  }
+  
+  /**
+   * The Slider AM sets up all the dependency JARs above slider.jar itself
+   * {@inheritDoc}
+   */
+  public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
+      Configuration serviceConf,
+      AbstractLauncher launcher,
+      AggregateConf instanceDescription,
+      Path snapshotConfDirPath,
+      Path generatedConfDirPath,
+      Configuration clientConfExtras,
+      String libdir,
+      Path tempPath, boolean miniClusterTestRun)
+    throws IOException, SliderException {
+
+    Map<String, LocalResource> providerResources = new HashMap<>();
+
+    ProviderUtils.addProviderJar(providerResources,
+        this,
+        SLIDER_JAR,
+        fileSystem,
+        tempPath,
+        libdir,
+        miniClusterTestRun);
+
+    String libDirProp =
+        System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
+    log.info("Loading all dependencies for AM.");
+    // If slider.tar.gz is available in hdfs use it, else upload all jars
+    Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip();
+    if (fileSystem.isFile(dependencyLibTarGzip)) {
+      SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem);
+    } else {
+      ProviderUtils.addAllDependencyJars(providerResources,
+                                         fileSystem,
+                                         tempPath,
+                                         libdir,
+                                         libDirProp);
+    }
+    addKeytabResourceIfNecessary(fileSystem,
+                                 instanceDescription,
+                                 providerResources);
+
+    launcher.addLocalResources(providerResources);
+
+    //also pick up all env variables from a map
+    launcher.copyEnvVars(
+      instanceDescription.getInternalOperations().getOrAddComponent(
+        SliderKeys.COMPONENT_AM));
+  }
+
+  /**
+   * If the cluster is secure, and an HDFS installed keytab is available for AM
+   * authentication, add this keytab as a local resource for the AM launch.
+   *
+   * @param fileSystem
+   * @param instanceDescription
+   * @param providerResources
+   * @throws IOException
+   * @throws BadConfigException if there's no keytab and it is explicitly required.
+   */
+  protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem,
+                                              AggregateConf instanceDescription,
+                                              Map<String, LocalResource> providerResources)
+    throws IOException, BadConfigException {
+    if (UserGroupInformation.isSecurityEnabled()) {
+      String keytabPathOnHost = instanceDescription.getAppConfOperations()
+          .getComponent(SliderKeys.COMPONENT_AM).get(
+              SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+      if (SliderUtils.isUnset(keytabPathOnHost)) {
+        String amKeytabName = instanceDescription.getAppConfOperations()
+            .getComponent(SliderKeys.COMPONENT_AM).get(
+                SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+        String keytabDir = instanceDescription.getAppConfOperations()
+            .getComponent(SliderKeys.COMPONENT_AM).get(
+                SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+        Path keytabPath = fileSystem.buildKeytabPath(keytabDir, amKeytabName,
+                                                     instanceDescription.getName());
+        if (fileSystem.getFileSystem().exists(keytabPath)) {
+          LocalResource keytabRes = fileSystem.createAmResource(keytabPath,
+                                                  LocalResourceType.FILE);
+
+          providerResources.put(SliderKeys.KEYTAB_DIR + "/" +
+                                 amKeytabName, keytabRes);
+        } else {
+          log.warn("No keytab file was found at {}.", keytabPath);
+          if (getConf().getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
+            throw new BadConfigException("No keytab file was found at %s.", keytabPath);
+
+          } else {
+            log.warn("The AM will be "
+              + "started without a kerberos authenticated identity. "
+              + "The application is therefore not guaranteed to remain "
+              + "operational beyond 24 hours.");
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Update the AM resource with any local needs
+   * @param capability capability to update
+   */
+  public void prepareAMResourceRequirements(MapOperations sliderAM,
+                                            Resource capability) {
+    capability.setMemory(sliderAM.getOptionInt(
+      ResourceKeys.YARN_MEMORY,
+      capability.getMemory()));
+    capability.setVirtualCores(
+        sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores()));
+  }
+  
+  /**
+   * Extract any JVM options from the cluster specification and
+   * add them to the command line
+   */
+  public void addJVMOptions(AggregateConf aggregateConf,
+                            JavaCommandLineBuilder cmdLine)
+      throws BadConfigException {
+
+    MapOperations sliderAM =
+        aggregateConf.getAppConfOperations().getMandatoryComponent(
+        SliderKeys.COMPONENT_AM);
+    cmdLine.forceIPv4().headless();
+    String heap = sliderAM.getOption(RoleKeys.JVM_HEAP,
+                                   DEFAULT_JVM_HEAP);
+    cmdLine.setJVMHeap(heap);
+    String jvmopts = sliderAM.getOption(RoleKeys.JVM_OPTS, "");
+    if (SliderUtils.isSet(jvmopts)) {
+      cmdLine.add(jvmopts);
+    }
+  }
+
+
+  @Override
+  public void prepareInstanceConfiguration(AggregateConf aggregateConf)
+      throws SliderException, IOException {
+    mergeTemplates(aggregateConf,
+        INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON
+                  );
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message