hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [62/93] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
Date Fri, 13 Oct 2017 18:22:08 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
new file mode 100644
index 0000000..684f655
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Formatter;
+import java.util.Locale;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+  protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
+      .class);
+
+  private MonitorUtils() {
+  }
+
+  public static String toPlural(int val) {
+    return val != 1 ? "s" : "";
+  }
+
+  /**
+   * Convert milliseconds to human time -the exact format is unspecified
+   * @param milliseconds a time in milliseconds
+   * @return a time that is converted to human intervals
+   */
+  public static String millisToHumanTime(long milliseconds) {
+    StringBuilder sb = new StringBuilder();
+    // Send all output to the Appendable object sb
+    Formatter formatter = new Formatter(sb, Locale.US);
+
+    long s = Math.abs(milliseconds / 1000);
+    long m = Math.abs(milliseconds % 1000);
+    if (milliseconds > 0) {
+      formatter.format("%d.%03ds", s, m);
+    } else if (milliseconds == 0) {
+      formatter.format("0");
+    } else {
+      formatter.format("-%d.%03ds", s, m);
+    }
+    return sb.toString();
+  }
+
+  public static Probe getProbe(ReadinessCheck readinessCheck) {
+    if (readinessCheck == null) {
+      return null;
+    }
+    if (readinessCheck.getType() == null) {
+      return null;
+    }
+    try {
+      switch (readinessCheck.getType()) {
+      case HTTP:
+        return HttpProbe.create(readinessCheck.getProps());
+      case PORT:
+        return PortProbe.create(readinessCheck.getProps());
+      default:
+        return null;
+      }
+    } catch (Throwable t) {
+      throw new IllegalArgumentException("Error creating readiness check " +
+          t);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
new file mode 100644
index 0000000..aba5859
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.Map;
+
+/**
+ * Probe for a port being open.
+ */
+public class PortProbe extends Probe {
+  protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
+  private final int port;
+  private final int timeout;
+
+  public PortProbe(int port, int timeout) {
+    super("Port probe of " + port + " for " + timeout + "ms", null);
+    this.port = port;
+    this.timeout = timeout;
+  }
+
+  public static PortProbe create(Map<String, String> props)
+      throws IOException {
+    int port = getPropertyInt(props, PORT_PROBE_PORT, null);
+
+    if (port >= 65536) {
+      throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
+          "range");
+    }
+
+    int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
+        PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
+
+    return new PortProbe(port, timeout);
+  }
+
+  /**
+   * Try to connect to the (host,port); a failure to connect within
+   * the specified timeout is a failure.
+   * @param instance role instance
+   * @return the outcome
+   */
+  @Override
+  public ProbeStatus ping(ComponentInstance instance) {
+    ProbeStatus status = new ProbeStatus();
+
+    if (instance.getContainerStatus() == null || SliderUtils
+        .isEmpty(instance.getContainerStatus().getIPs())) {
+      status.fail(this, new IOException(
+          instance.getCompInstanceName() + ": IP is not available yet"));
+      return status;
+    }
+
+    String ip = instance.getContainerStatus().getIPs().get(0);
+    InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
+    Socket socket = new Socket();
+    try {
+      if (log.isDebugEnabled()) {
+        log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
+            .toString() + ", timeout=" + MonitorUtils
+            .millisToHumanTime(timeout));
+      }
+      socket.connect(sockAddr, timeout);
+      status.succeed(this);
+    } catch (Throwable e) {
+      String error =
+          instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
+      log.debug(error, e);
+      status.fail(this, new IOException(error, e));
+    } finally {
+      IOUtils.closeSocket(socket);
+    }
+    return status;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
new file mode 100644
index 0000000..3237a2b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Base class of all probes.
+ */
+public abstract class Probe implements MonitorKeys {
+
+  protected final Configuration conf;
+  private String name;
+
+  /**
+   * Create a probe of a specific name
+   *
+   * @param name probe name
+   * @param conf configuration being stored.
+   */
+  public Probe(String name, Configuration conf) {
+    this.name = name;
+    this.conf = conf;
+  }
+
+
+  protected void setName(String name) {
+    this.name = name;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+
+  @Override
+  public String toString() {
+    return getName();
+  }
+
+  public static String getProperty(Map<String, String> props, String name,
+      String defaultValue) throws IOException {
+    String value = props.get(name);
+    if (StringUtils.isEmpty(value)) {
+      if (defaultValue == null) {
+        throw new IOException(name + " not specified");
+      }
+      return defaultValue;
+    }
+    return value;
+  }
+
+  public static int getPropertyInt(Map<String, String> props, String name,
+      Integer defaultValue) throws IOException {
+    String value = props.get(name);
+    if (StringUtils.isEmpty(value)) {
+      if (defaultValue == null) {
+        throw new IOException(name + " not specified");
+      }
+      return defaultValue;
+    }
+    return Integer.parseInt(value);
+  }
+
+  /**
+   * perform any prelaunch initialization
+   */
+  public void init() throws IOException {
+
+  }
+
+  /**
+   * Ping the endpoint. All exceptions must be caught and included in the
+   * (failure) status.
+   *
+   * @param instance instance to ping
+   * @return the status
+   */
+  public abstract ProbeStatus ping(ComponentInstance instance);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
new file mode 100644
index 0000000..bc62dcd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * Status message of a probe. This is designed to be sent over the wire, though the exception
+ * Had better be unserializable at the far end if that is to work.
+ */
+public final class ProbeStatus implements Serializable {
+  private static final long serialVersionUID = 165468L;
+
+  private long timestamp;
+  private String timestampText;
+  private boolean success;
+  private boolean realOutcome;
+  private String message;
+  private Throwable thrown;
+  private transient Probe originator;
+
+  public ProbeStatus() {
+  }
+
+  public ProbeStatus(long timestamp, String message, Throwable thrown) {
+    this.success = false;
+    this.message = message;
+    this.thrown = thrown;
+    setTimestamp(timestamp);
+  }
+
+  public ProbeStatus(long timestamp, String message) {
+    this.success = true;
+    setTimestamp(timestamp);
+    this.message = message;
+    this.thrown = null;
+  }
+
+  public long getTimestamp() {
+    return timestamp;
+  }
+
+  public void setTimestamp(long timestamp) {
+    this.timestamp = timestamp;
+    timestampText = new Date(timestamp).toString();
+  }
+
+  public boolean isSuccess() {
+    return success;
+  }
+
+  /**
+   * Set both the success and the real outcome bits to the same value
+   * @param success the new value
+   */
+  public void setSuccess(boolean success) {
+    this.success = success;
+    realOutcome = success;
+  }
+
+  public String getTimestampText() {
+    return timestampText;
+  }
+
+  public boolean getRealOutcome() {
+    return realOutcome;
+  }
+
+  public String getMessage() {
+    return message;
+  }
+
+  public void setMessage(String message) {
+    this.message = message;
+  }
+
+  public Throwable getThrown() {
+    return thrown;
+  }
+
+  public void setThrown(Throwable thrown) {
+    this.thrown = thrown;
+  }
+
+  /**
+   * Get the probe that generated this result. May be null
+   * @return a possibly null reference to a probe
+   */
+  public Probe getOriginator() {
+    return originator;
+  }
+
+  /**
+   * The probe has succeeded -capture the current timestamp, set
+   * success to true, and record any other data needed.
+   * @param probe probe
+   */
+  public void succeed(Probe probe) {
+    finish(probe, true, probe.getName(), null);
+  }
+
+  /**
+   * A probe has failed either because the test returned false, or an exception
+   * was thrown. The {@link #success} field is set to false, any exception 
+   * thrown is recorded.
+   * @param probe probe that failed
+   * @param thrown an exception that was thrown.
+   */
+  public void fail(Probe probe, Throwable thrown) {
+    finish(probe, false, "Failure in " + probe, thrown);
+  }
+
+  public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
+    setTimestamp(System.currentTimeMillis());
+    setSuccess(succeeded);
+    originator = probe;
+    message = text;
+    this.thrown = thrown;
+  }
+
+  @Override
+  public String toString() {
+    LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
+    builder.elt("time", timestampText)
+           .elt("outcome", (success ? "success" : "failure"));
+
+    if (success != realOutcome) {
+      builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
+    }
+    builder.elt("message", message);
+    if (thrown != null) {
+      builder.elt("exception", thrown);
+    }
+
+    return builder.toString();
+  }
+
+  /**
+   * Flip the success bit on while the real outcome bit is kept false
+   */
+  public void markAsSuccessful() {
+    success = true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
new file mode 100644
index 0000000..0d11be2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public abstract class AbstractClientProvider {
+
+  public AbstractClientProvider() {
+  }
+
+  /**
+   * Generates a fixed format of application tags given one or more of
+   * application name, version and description. This allows subsequent query for
+   * an application with a name only, version only or description only or any
+   * combination of those as filters.
+   *
+   * @param appName name of the application
+   * @param appVersion version of the application
+   * @param appDescription brief description of the application
+   * @return
+   */
+  public static final Set<String> createApplicationTags(String appName,
+      String appVersion, String appDescription) {
+    Set<String> tags = new HashSet<>();
+    tags.add(SliderUtils.createNameTag(appName));
+    if (appVersion != null) {
+      tags.add(SliderUtils.createVersionTag(appVersion));
+    }
+    if (appDescription != null) {
+      tags.add(SliderUtils.createDescriptionTag(appDescription));
+    }
+    return tags;
+  }
+
+  /**
+   * Validate the artifact.
+   * @param artifact
+   */
+  public abstract void validateArtifact(Artifact artifact, FileSystem
+      fileSystem) throws IOException;
+
+  protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException;
+
+  /**
+   * Validate the config files.
+   * @param configFiles config file list
+   * @param fs file system
+   */
+  public void validateConfigFiles(List<ConfigFile> configFiles,
+      FileSystem fs) throws IOException {
+    Set<String> destFileSet = new HashSet<>();
+
+    for (ConfigFile file : configFiles) {
+      if (file.getType() == null) {
+        throw new IllegalArgumentException("File type is empty");
+      }
+
+      if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
+          .isEmpty(file.getSrcFile())) {
+        throw new IllegalArgumentException(
+            "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+
+      }
+      if (!StringUtils.isEmpty(file.getSrcFile())) {
+        Path p = new Path(file.getSrcFile());
+        if (!fs.exists(p)) {
+          throw new IllegalArgumentException(
+              "Src_file does not exist for config file: " + file
+                  .getSrcFile());
+        }
+      }
+
+      if (StringUtils.isEmpty(file.getDestFile())) {
+        throw new IllegalArgumentException("Dest_file is empty.");
+      }
+
+      if (destFileSet.contains(file.getDestFile())) {
+        throw new IllegalArgumentException(
+            "Duplicated ConfigFile exists: " + file.getDestFile());
+      }
+      destFileSet.add(file.getDestFile());
+
+      java.nio.file.Path destPath = Paths.get(file.getDestFile());
+      if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
+        throw new IllegalArgumentException("Non-absolute dest_file has more " +
+            "than one path element");
+      }
+
+      // provider-specific validation
+      validateConfigFile(file, fs);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
new file mode 100644
index 0000000..8d607ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_MAX;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+public abstract class AbstractProviderService implements ProviderService,
+    YarnServiceConstants {
+
+  protected static final Logger log =
+      LoggerFactory.getLogger(AbstractProviderService.class);
+
+  public abstract void processArtifact(AbstractLauncher launcher,
+      ComponentInstance compInstance, SliderFileSystem fileSystem,
+      Service service)
+      throws IOException;
+
+  public void buildContainerLaunchContext(AbstractLauncher launcher,
+      Service service, ComponentInstance instance,
+      SliderFileSystem fileSystem, Configuration yarnConf)
+      throws IOException, SliderException {
+    Component component = instance.getComponent().getComponentSpec();;
+    processArtifact(launcher, instance, fileSystem, service);
+
+    ServiceContext context =
+        instance.getComponent().getScheduler().getContext();
+    // Generate tokens (key-value pair) for config substitution.
+    // Get pre-defined tokens
+    Map<String, String> globalTokens =
+        instance.getComponent().getScheduler().globalTokens;
+    Map<String, String> tokensForSubstitution = ProviderUtils
+        .initCompTokensForSubstitute(instance);
+    tokensForSubstitution.putAll(globalTokens);
+    // Set the environment variables in launcher
+    launcher.putEnv(SliderUtils
+        .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
+    launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
+    launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    if (System.getenv(HADOOP_USER_NAME) != null) {
+      launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
+    }
+    launcher.setEnv("LANG", "en_US.UTF-8");
+    launcher.setEnv("LC_ALL", "en_US.UTF-8");
+    launcher.setEnv("LANGUAGE", "en_US.UTF-8");
+
+    for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
+      tokensForSubstitution.put($(entry.getKey()), entry.getValue());
+    }
+    //TODO add component host tokens?
+//    ProviderUtils.addComponentHostTokens(tokensForSubstitution, amState);
+
+    // create config file on hdfs and add local resource
+    ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
+        component, tokensForSubstitution, instance, context);
+
+    // substitute launch command
+    String launchCommand = ProviderUtils
+        .substituteStrWithTokens(component.getLaunchCommand(),
+            tokensForSubstitution);
+    CommandLineBuilder operation = new CommandLineBuilder();
+    operation.add(launchCommand);
+    operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+    launcher.addCommand(operation.build());
+
+    // By default retry forever every 30 seconds
+    launcher.setRetryContext(YarnServiceConf
+        .getInt(CONTAINER_RETRY_MAX, -1, service.getConfiguration(),
+            yarnConf), YarnServiceConf
+        .getInt(CONTAINER_RETRY_INTERVAL, 30000, service.getConfiguration(),
+            yarnConf));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
new file mode 100644
index 0000000..0f949e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for factories.
+ */
+public abstract class ProviderFactory {
+  protected static final Logger LOG =
+      LoggerFactory.getLogger(ProviderFactory.class);
+
+  protected ProviderFactory() {}
+
+  public abstract AbstractClientProvider createClientProvider();
+
+  public abstract ProviderService createServerProvider();
+
+  public static synchronized ProviderService getProviderService(Artifact
+      artifact) {
+    return createServiceProviderFactory(artifact).createServerProvider();
+  }
+
+  public static synchronized AbstractClientProvider getClientProvider(Artifact
+      artifact) {
+    return createServiceProviderFactory(artifact).createClientProvider();
+  }
+
+  /**
+   * Create a provider for a specific service
+   * @param artifact artifact
+   * @return provider factory
+   */
+  public static synchronized ProviderFactory createServiceProviderFactory(
+      Artifact artifact) {
+    if (artifact == null || artifact.getType() == null) {
+      LOG.debug("Loading service provider type default");
+      return DefaultProviderFactory.getInstance();
+    }
+    LOG.debug("Loading service provider type {}", artifact.getType());
+    switch (artifact.getType()) {
+      // TODO add handling for custom types?
+      // TODO handle service
+      case DOCKER:
+        return DockerProviderFactory.getInstance();
+      case TARBALL:
+        return TarballProviderFactory.getInstance();
+      default:
+        throw new IllegalArgumentException(String.format("Resolution error, " +
+                "%s should not be passed to createServiceProviderFactory",
+            artifact.getType()));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
new file mode 100644
index 0000000..eb721b4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+import java.io.IOException;
+
+public interface ProviderService {
+
+  /**
+   * Set up the entire container launch context
+   */
+  void buildContainerLaunchContext(AbstractLauncher containerLauncher,
+      Service service, ComponentInstance instance,
+      SliderFileSystem sliderFileSystem, Configuration yarnConf)
+      throws IOException, SliderException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
new file mode 100644
index 0000000..ec0c2ca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
+import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
+
+/**
+ * This is a factoring out of methods handy for providers. It's bonded to a log
+ * at construction time.
+ */
+public class ProviderUtils implements YarnServiceConstants {
+
+  protected static final Logger log =
+      LoggerFactory.getLogger(ProviderUtils.class);
+
+
+  /**
+   * Add oneself to the classpath. This does not work
+   * on minicluster test runs where the JAR is not built up.
+   * @param providerResources map of provider resources to add these entries to
+   * @param providerClass provider to add
+   * @param jarName name of the jar to use
+   * @param sliderFileSystem target filesystem
+   * @param tempPath path in the cluster FS for temp files
+   * @param libdir relative directory to place resources
+   * @param miniClusterTestRun true if minicluster is being used
+   * @return true if the class was found in a JAR
+   * 
+   * @throws FileNotFoundException if the JAR was not found and this is NOT
+   * a mini cluster test run
+   * @throws IOException IO problems
+   * @throws SliderException any Slider problem
+   */
+  public static boolean addProviderJar(
+      Map<String, LocalResource> providerResources,
+      Class providerClass,
+      String jarName,
+      SliderFileSystem sliderFileSystem,
+      Path tempPath,
+      String libdir,
+      boolean miniClusterTestRun) throws
+      IOException,
+      SliderException {
+    try {
+      SliderUtils.putJar(providerResources,
+          sliderFileSystem,
+          providerClass,
+          tempPath,
+          libdir,
+          jarName);
+      return true;
+    } catch (FileNotFoundException e) {
+      if (miniClusterTestRun) {
+        return false;
+      } else {
+        throw e;
+      }
+    }
+  }
+  
+  /**
+   * Loads all dependency jars from the default path.
+   * @param providerResources map of provider resources to add these entries to
+   * @param sliderFileSystem target filesystem
+   * @param tempPath path in the cluster FS for temp files
+   * @param libDir relative directory to place resources
+   * @param libLocalSrcDir explicitly supplied local libs dir
+   * @throws IOException trouble copying to HDFS
+   * @throws SliderException trouble copying to HDFS
+   */
+  public static void addAllDependencyJars(
+      Map<String, LocalResource> providerResources,
+      SliderFileSystem sliderFileSystem,
+      Path tempPath,
+      String libDir,
+      String libLocalSrcDir)
+      throws IOException, SliderException {
+    if (SliderUtils.isSet(libLocalSrcDir)) {
+      File file = new File(libLocalSrcDir);
+      if (!file.exists() || !file.isDirectory()) {
+        throw new BadCommandArgumentsException(
+            "Supplied lib src dir %s is not valid", libLocalSrcDir);
+      }
+    }
+    SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
+        libDir, libLocalSrcDir);
+  }
+
+  public static String substituteStrWithTokens(String content,
+      Map<String, String> tokensForSubstitution) {
+    for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+      content =
+          content.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+    }
+    return content;
+  }
+
+  // configs will be substituted by corresponding env in tokenMap
+  public static void substituteMapWithTokens(Map<String, String> configs,
+      Map<String, String> tokenMap) {
+    for (Map.Entry<String, String> entry : configs.entrySet()) {
+      String value = entry.getValue();
+      if (tokenMap != null) {
+        for (Map.Entry<String, String> token : tokenMap.entrySet()) {
+          value =
+              value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+        }
+      }
+      entry.setValue(value);
+    }
+  }
+
+  /**
+   * Localize the service keytabs for the service.
+   * @param launcher container launcher
+   * @param fileSystem file system
+   * @throws IOException trouble uploading to HDFS
+   */
+  public void localizeServiceKeytabs(AbstractLauncher launcher,
+      SliderFileSystem fileSystem, Service service) throws IOException {
+
+    Configuration conf = service.getConfiguration();
+    String keytabPathOnHost =
+        conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
+    if (SliderUtils.isUnset(keytabPathOnHost)) {
+      String amKeytabName =
+          conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
+      String keytabDir =
+          conf.getProperty(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
+      // we need to localize the keytab files in the directory
+      Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
+          service.getName());
+      boolean serviceKeytabsDeployed = false;
+      if (fileSystem.getFileSystem().exists(keytabDirPath)) {
+        FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
+            keytabDirPath);
+        LocalResource keytabRes;
+        for (FileStatus keytab : keytabs) {
+          if (!amKeytabName.equals(keytab.getPath().getName())
+              && keytab.getPath().getName().endsWith(".keytab")) {
+            serviceKeytabsDeployed = true;
+            log.info("Localizing keytab {}", keytab.getPath().getName());
+            keytabRes = fileSystem.createAmResource(keytab.getPath(),
+                LocalResourceType.FILE);
+            launcher.addLocalResource(KEYTAB_DIR + "/" +
+                    keytab.getPath().getName(),
+                keytabRes);
+          }
+        }
+      }
+      if (!serviceKeytabsDeployed) {
+        log.warn("No service keytabs for the service have been localized.  "
+            + "If the service requires keytabs for secure operation, "
+            + "please ensure that the required keytabs have been uploaded "
+            + "to the folder {}", keytabDirPath);
+      }
+    }
+  }
+
+  // 1. Create all config files for a component on hdfs for localization
+  // 2. Add the config file to localResource
+  public static synchronized void createConfigFileAndAddLocalResource(
+      AbstractLauncher launcher, SliderFileSystem fs, Component component,
+      Map<String, String> tokensForSubstitution, ComponentInstance instance,
+      ServiceContext context) throws IOException {
+    Path compDir =
+        new Path(new Path(fs.getAppDir(), "components"), component.getName());
+    Path compInstanceDir =
+        new Path(compDir, instance.getCompInstanceName());
+    if (!fs.getFileSystem().exists(compInstanceDir)) {
+      log.info(instance.getCompInstanceId() + ": Creating dir on hdfs: " + compInstanceDir);
+      fs.getFileSystem().mkdirs(compInstanceDir,
+          new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+      instance.setCompInstanceDir(compInstanceDir);
+    } else {
+      log.info("Component instance conf dir already exists: " + compInstanceDir);
+    }
+
+    if (log.isDebugEnabled()) {
+      log.debug("Tokens substitution for component instance: " + instance
+          .getCompInstanceName() + System.lineSeparator()
+          + tokensForSubstitution);
+    }
+
+    for (ConfigFile originalFile : component.getConfiguration().getFiles()) {
+      ConfigFile configFile = originalFile.copy();
+      String fileName = new Path(configFile.getDestFile()).getName();
+
+      // substitute file name
+      for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+        configFile.setDestFile(configFile.getDestFile()
+            .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
+      }
+
+      Path remoteFile = new Path(compInstanceDir, fileName);
+      if (!fs.getFileSystem().exists(remoteFile)) {
+        log.info("Saving config file on hdfs for component " + instance
+            .getCompInstanceName() + ": " + configFile);
+
+        if (configFile.getSrcFile() != null) {
+          // Load config file template
+          switch (configFile.getType()) {
+          case HADOOP_XML:
+            // Hadoop_xml_template
+            resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
+                tokensForSubstitution, configFile, remoteFile, context);
+            break;
+          case TEMPLATE:
+            // plain-template
+            resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
+                tokensForSubstitution, configFile, remoteFile, context);
+            break;
+          default:
+            log.info("Not supporting loading src_file for " + configFile);
+            break;
+          }
+        } else {
+          // non-template
+          resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
+              instance, configFile, fileName, remoteFile);
+        }
+      }
+
+      // Add resource for localization
+      LocalResource configResource =
+          fs.createAmResource(remoteFile, LocalResourceType.FILE);
+      File destFile = new File(configFile.getDestFile());
+      String symlink = APP_CONF_DIR + "/" + fileName;
+      if (destFile.isAbsolute()) {
+        launcher.addLocalResource(symlink, configResource,
+            configFile.getDestFile());
+        log.info("Add config file for localization: " + symlink + " -> "
+            + configResource.getResource().getFile() + ", dest mount path: "
+            + configFile.getDestFile());
+      } else {
+        launcher.addLocalResource(symlink, configResource);
+        log.info("Add config file for localization: " + symlink + " -> "
+            + configResource.getResource().getFile());
+      }
+    }
+  }
+
+  private static void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
+      Map<String, String> tokensForSubstitution, ComponentInstance instance,
+      ConfigFile configFile, String fileName, Path remoteFile)
+      throws IOException {
+    // substitute non-template configs
+    substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
+
+    // write configs onto hdfs
+    PublishedConfiguration publishedConfiguration =
+        new PublishedConfiguration(fileName,
+            configFile.getProps().entrySet());
+    if (!fs.getFileSystem().exists(remoteFile)) {
+      PublishedConfigurationOutputter configurationOutputter =
+          PublishedConfigurationOutputter.createOutputter(
+              ConfigFormat.resolve(configFile.getType().toString()),
+              publishedConfiguration);
+      try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) {
+        configurationOutputter.save(os);
+        os.flush();
+      }
+    } else {
+      log.info("Component instance = " + instance.getCompInstanceName()
+              + ", config file already exists: " + remoteFile);
+    }
+  }
+
+  // 1. substitute config template - only handle hadoop_xml format
+  // 2. save on hdfs
+  @SuppressWarnings("unchecked")
+  private static void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
+      Map<String, String> tokensForSubstitution, ConfigFile configFile,
+      Path remoteFile, ServiceContext context) throws IOException {
+    Map<String, String> conf;
+    try {
+      conf = (Map<String, String>) context.configCache.get(configFile);
+    } catch (ExecutionException e) {
+      log.info("Failed to load config file: " + configFile, e);
+      return;
+    }
+    // make a copy for substitution
+    org.apache.hadoop.conf.Configuration confCopy =
+        new org.apache.hadoop.conf.Configuration(false);
+    for (Map.Entry<String, String> entry : conf.entrySet()) {
+      confCopy.set(entry.getKey(), entry.getValue());
+    }
+    // substitute properties
+    for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
+      confCopy.set(entry.getKey(), entry.getValue());
+    }
+    // substitute env variables
+    for (Map.Entry<String, String> entry : confCopy) {
+      String val = entry.getValue();
+      if (val != null) {
+        for (Map.Entry<String, String> token : tokensForSubstitution
+            .entrySet()) {
+          val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+          confCopy.set(entry.getKey(), val);
+        }
+      }
+    }
+    // save on hdfs
+    try (OutputStream output = fs.create(remoteFile)) {
+      confCopy.writeXml(output);
+      log.info("Reading config from: " + configFile.getSrcFile()
+          + ", writing to: " + remoteFile);
+    }
+  }
+
+  // 1) read the template as a string
+  // 2) do token substitution
+  // 3) save on hdfs
+  private static void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
+      Map<String, String> tokensForSubstitution, ConfigFile configFile,
+      Path remoteFile, ServiceContext context) {
+    String content;
+    try {
+      content = (String) context.configCache.get(configFile);
+    } catch (ExecutionException e) {
+      log.info("Failed to load config file: " + configFile, e);
+      return;
+    }
+    // substitute tokens
+    content = substituteStrWithTokens(content, tokensForSubstitution);
+
+    try (OutputStream output = fs.create(remoteFile)) {
+      org.apache.commons.io.IOUtils.write(content, output);
+    } catch (IOException e) {
+      log.info("Failed to create " + remoteFile);
+    }
+  }
+
+  /**
+   * Get initial component token map to be substituted into config values.
+   * @return tokens to replace
+   */
+  public static Map<String, String> initCompTokensForSubstitute(
+      ComponentInstance instance) {
+    Map<String, String> tokens = new HashMap<>();
+    tokens.put(COMPONENT_NAME, instance.getCompSpec().getName());
+    tokens
+        .put(COMPONENT_NAME_LC, instance.getCompSpec().getName().toLowerCase());
+    tokens.put(COMPONENT_INSTANCE_NAME, instance.getCompInstanceName());
+    tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
+    tokens.put(COMPONENT_ID,
+        String.valueOf(instance.getCompInstanceId().getId()));
+    return tokens;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
new file mode 100644
index 0000000..0920a9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class DefaultClientProvider extends AbstractClientProvider {
+
+  public DefaultClientProvider() {
+  }
+
+  @Override
+  public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+  }
+
+  @Override
+  protected void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException {
+    // validate dest_file is not absolute
+    if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+      throw new IllegalArgumentException(
+          "Dest_file must not be absolute path: " + configFile.getDestFile());
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
new file mode 100644
index 0000000..868bba8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public final class DefaultProviderFactory extends ProviderFactory {
+  private static final ProviderFactory FACTORY = new
+      DefaultProviderFactory();
+
+  private DefaultProviderFactory() {}
+
+  private static class Client {
+    static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
+  }
+
+  private static class Server {
+    static final ProviderService PROVIDER = new DefaultProviderService();
+  }
+
+  @Override
+  public AbstractClientProvider createClientProvider() {
+    return Client.PROVIDER;
+  }
+
+  @Override
+  public ProviderService createServerProvider() {
+    return Server.PROVIDER;
+  }
+
+  public static ProviderFactory getInstance() {
+    return FACTORY;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
new file mode 100644
index 0000000..a3a0c1f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+
+public class DefaultProviderService extends AbstractProviderService {
+
+  @Override
+  public void processArtifact(AbstractLauncher launcher,
+      ComponentInstance compInstance, SliderFileSystem fileSystem,
+      Service service)
+      throws IOException {
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
new file mode 100644
index 0000000..d4a2254
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+
+import java.io.IOException;
+
+public class DockerClientProvider extends AbstractClientProvider
+    implements YarnServiceConstants {
+
+  public DockerClientProvider() {
+    super();
+  }
+
+  @Override
+  public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+    if (artifact == null) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+    }
+    if (StringUtils.isEmpty(artifact.getId())) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+    }
+  }
+
+  @Override
+  protected void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException {
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
new file mode 100644
index 0000000..f30c002
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+public interface DockerKeys {
+  String PROVIDER_DOCKER = "docker";
+  String DOCKER_PREFIX = "docker.";
+  String DOCKER_IMAGE = DOCKER_PREFIX + "image";
+  String DOCKER_NETWORK = DOCKER_PREFIX + "network";
+  String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
+  String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
+
+  String DEFAULT_DOCKER_NETWORK = "bridge";
+  Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
new file mode 100644
index 0000000..57330ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public class DockerProviderFactory extends ProviderFactory {
+  private static final ProviderFactory FACTORY = new
+      DockerProviderFactory();
+
+  private DockerProviderFactory() {
+  }
+
+  private static class Client {
+    static final AbstractClientProvider PROVIDER = new DockerClientProvider();
+  }
+
+  private static class Server {
+    static final ProviderService PROVIDER = new DockerProviderService();
+  }
+
+  @Override
+  public AbstractClientProvider createClientProvider() {
+    return Client.PROVIDER;
+  }
+
+  @Override
+  public ProviderService createServerProvider() {
+    return Server.PROVIDER;
+  }
+
+  public static ProviderFactory getInstance() {
+    return FACTORY;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
new file mode 100644
index 0000000..0741947
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+
+public class DockerProviderService extends AbstractProviderService
+    implements DockerKeys {
+
+  public void processArtifact(AbstractLauncher launcher,
+      ComponentInstance compInstance, SliderFileSystem fileSystem,
+      Service service) throws IOException{
+    launcher.setYarnDockerMode(true);
+    launcher.setDockerImage(compInstance.getCompSpec().getArtifact().getId());
+    launcher.setDockerNetwork(compInstance.getCompSpec().getConfiguration()
+        .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
+    String domain = compInstance.getComponent().getScheduler().getConfig()
+        .get(RegistryConstants.KEY_DNS_DOMAIN);
+    String hostname;
+    if (domain == null || domain.isEmpty()) {
+      hostname = MessageFormat
+          .format("{0}.{1}.{2}", compInstance.getCompInstanceName(),
+              service.getName(), RegistryUtils.currentUser());
+    } else {
+      hostname = MessageFormat
+          .format("{0}.{1}.{2}.{3}", compInstance.getCompInstanceName(),
+              service.getName(), RegistryUtils.currentUser(), domain);
+    }
+    launcher.setDockerHostname(hostname);
+    launcher.setRunPrivilegedContainer(
+        compInstance.getCompSpec().getRunPrivilegedContainer());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
new file mode 100644
index 0000000..01f7b20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class TarballClientProvider extends AbstractClientProvider
+    implements YarnServiceConstants {
+
+  public TarballClientProvider() {
+  }
+
+  @Override
+  public void validateArtifact(Artifact artifact, FileSystem fs)
+      throws IOException {
+    if (artifact == null) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+    }
+    if (StringUtils.isEmpty(artifact.getId())) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+    }
+    Path p = new Path(artifact.getId());
+    if (!fs.exists(p)) {
+      throw new IllegalArgumentException( "Artifact tarball does not exist "
+          + artifact.getId());
+    }
+  }
+
+  @Override
+  protected void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException {
+    // validate dest_file is not absolute
+    if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+      throw new IllegalArgumentException(
+          "Dest_file must not be absolute path: " + configFile.getDestFile());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
new file mode 100644
index 0000000..9d81f66
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public class TarballProviderFactory extends ProviderFactory {
+  private static final ProviderFactory FACTORY = new
+      TarballProviderFactory();
+
+  private TarballProviderFactory() {
+  }
+
+  private static class Client {
+    static final AbstractClientProvider PROVIDER = new TarballClientProvider();
+  }
+
+  private static class Server {
+    static final ProviderService PROVIDER = new TarballProviderService();
+  }
+
+  @Override
+  public AbstractClientProvider createClientProvider() {
+    return Client.PROVIDER;
+  }
+
+  @Override
+  public ProviderService createServerProvider() {
+    return Server.PROVIDER;
+  }
+
+  public static ProviderFactory getInstance() {
+    return FACTORY;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
new file mode 100644
index 0000000..9f29c8b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+
+public class TarballProviderService extends AbstractProviderService {
+
+  @Override
+  public void processArtifact(AbstractLauncher launcher,
+      ComponentInstance instance, SliderFileSystem fileSystem,
+      Service service)
+      throws IOException {
+    Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
+    if (!fileSystem.isFile(artifact)) {
+      throw new IOException(
+          "Package doesn't exist as a resource: " + artifact.toString());
+    }
+    log.info("Adding resource {}", artifact.toString());
+    LocalResourceType type = LocalResourceType.ARCHIVE;
+    LocalResource packageResource = fileSystem.createAmResource(artifact, type);
+    launcher.addLocalResource(APP_LIB_DIR, packageResource);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
new file mode 100644
index 0000000..56634f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+/**
+ * These are constants unique to the Slider AM
+ */
+public class CustomRegistryConstants {
+
+  public static final String MANAGEMENT_REST_API =
+      "classpath:org.apache.slider.management";
+  
+  public static final String REGISTRY_REST_API =
+      "classpath:org.apache.slider.registry";
+  
+  public static final String PUBLISHER_REST_API =
+      "classpath:org.apache.slider.publisher";
+
+  public static final String PUBLISHER_CONFIGURATIONS_API =
+      "classpath:org.apache.slider.publisher.configurations";
+
+  public static final String PUBLISHER_EXPORTS_API =
+      "classpath:org.apache.slider.publisher.exports";
+
+  public static final String PUBLISHER_DOCUMENTS_API =
+      "classpath:org.apache.slider.publisher.documents";
+
+  public static final String AGENT_SECURE_REST_API =
+      "classpath:org.apache.slider.agents.secure";
+
+  public static final String AGENT_ONEWAY_REST_API =
+      "classpath:org.apache.slider.agents.oneway";
+
+  public static final String AM_IPC_PROTOCOL =
+      "classpath:org.apache.slider.appmaster.ipc";
+
+  public static final String AM_REST_BASE =
+      "classpath:org.apache.slider.client.rest";
+
+  public static final String WEB_UI = "http://";
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message