hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject [08/15] hadoop git commit: YARN-6903. Yarn-native-service framework core rewrite. Contributed by Jian He
Date Mon, 14 Aug 2017 21:13:46 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
index 64783b6..235d3da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
@@ -16,7 +16,7 @@
  */
 package org.apache.slider.common.tools;
 
-import org.apache.slider.common.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
 import org.apache.slider.core.exceptions.BadConfigException;
 import org.apache.slider.core.exceptions.SliderException;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 2e1236d..fc57c82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ExitUtil;
@@ -50,10 +49,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.slider.Slider;
 import org.apache.slider.api.RoleKeys;
 import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.params.Arguments;
-import org.apache.slider.common.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
 import org.apache.slider.core.exceptions.BadClusterStateException;
 import org.apache.slider.core.exceptions.BadCommandArgumentsException;
 import org.apache.slider.core.exceptions.BadConfigException;
@@ -111,7 +110,7 @@ import java.util.zip.GZIPOutputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
-import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
+import static org.apache.hadoop.yarn.service.conf.SliderKeys.COMPONENT_SEPARATOR;
 
 /**
  * These are slider-specific Util methods
@@ -373,6 +372,13 @@ public final class SliderUtils {
   private static final PatternValidator clusternamePattern
       = new PatternValidator("[a-z][a-z0-9_-]*");
 
+  private static final PatternValidator compNamePattern
+      = new PatternValidator("[a-z][a-z0-9-]*");
+
+  public static void validateCompName(String compName) {
+    compNamePattern.validate(compName);
+  }
+
   /**
    * Normalize a cluster name then verify that it is valid
    * @param name proposed cluster name

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
index 7f3134a..1430c5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
@@ -18,7 +18,7 @@
 
 package org.apache.slider.core.exceptions;
 
-import org.apache.slider.common.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
 import org.apache.slider.core.main.ServiceLaunchException;
 
 public class SliderException extends ServiceLaunchException implements

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 8f61bf4..55ffbf7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -19,30 +19,17 @@
 package org.apache.slider.core.launch;
 
 import com.google.common.base.Preconditions;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LogAggregationContext;
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.Records;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.common.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
 import org.apache.slider.common.tools.CoreFileSystem;
 import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.MapOperations;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.lang.reflect.Method;
-import java.lang.reflect.InvocationTargetException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -51,12 +38,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import static org.apache.slider.providers.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
+import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
 
 /**
  * Launcher of applications: base class
  */
-public abstract class AbstractLauncher extends Configured {
+public class AbstractLauncher {
   private static final Logger log =
     LoggerFactory.getLogger(AbstractLauncher.class);
   public static final String CLASSPATH = "CLASSPATH";
@@ -68,7 +55,6 @@ public abstract class AbstractLauncher extends Configured {
    * Env vars; set up at final launch stage
    */
   protected final Map<String, String> envVars = new HashMap<>();
-  protected final MapOperations env = new MapOperations("env", envVars);
   protected final ContainerLaunchContext containerLaunchContext =
     Records.newRecord(ContainerLaunchContext.class);
   protected final List<String> commands = new ArrayList<>(20);
@@ -77,36 +63,24 @@ public abstract class AbstractLauncher extends Configured {
   private final Map<String, ByteBuffer> serviceData = new HashMap<>();
   // security
   protected final Credentials credentials;
-  protected LogAggregationContext logAggregationContext;
   protected boolean yarnDockerMode = false;
   protected String dockerImage;
   protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
   protected String dockerHostname;
-  protected String yarnContainerMountPoints;
   protected String runPrivilegedContainer;
 
 
   /**
    * Create instance.
-   * @param conf configuration
    * @param coreFileSystem filesystem
    * @param credentials initial set of credentials -null is permitted
    */
-  protected AbstractLauncher(Configuration conf,
+  public AbstractLauncher(
       CoreFileSystem coreFileSystem,
       Credentials credentials) {
-    super(conf);
     this.coreFileSystem = coreFileSystem;
     this.credentials = credentials != null ? credentials: new Credentials();
   }
-
-  /**
-   * Get the container. Until "completed", this isn't valid to launch.
-   * @return the container to launch
-   */
-  public ContainerLaunchContext getContainerLaunchContext() {
-    return containerLaunchContext;
-  }
   
   public void setYarnDockerMode(boolean yarnDockerMode){
     this.yarnDockerMode = yarnDockerMode;
@@ -116,8 +90,8 @@ public abstract class AbstractLauncher extends Configured {
    * Get the env vars to work on
    * @return env vars
    */
-  public MapOperations getEnv() {
-    return env;
+  public Map<String, String> getEnv() {
+    return envVars;
   }
 
   /**
@@ -128,14 +102,6 @@ public abstract class AbstractLauncher extends Configured {
     return commands;
   }
 
-  /**
-   * Get the map of local resources.
-   * @return the live map of local resources.
-   */
-  public Map<String, LocalResource> getLocalResources() {
-    return localResources;
-  }
-
   public void addLocalResource(String subPath, LocalResource resource) {
     localResources.put(subPath, resource);
   }
@@ -146,18 +112,6 @@ public abstract class AbstractLauncher extends Configured {
   }
 
   /**
-   * Add a set of local resources
-   * @param resourceMap map of name:resource to add
-   */
-  public void addLocalResources(Map<String, LocalResource> resourceMap) {
-    localResources.putAll(resourceMap);
-  }
-
-  public Map<String, ByteBuffer> getServiceData() {
-    return serviceData;
-  }
-
-  /**
    * Accessor to the credentials
    * @return the credentials associated with this launcher
    */
@@ -165,36 +119,12 @@ public abstract class AbstractLauncher extends Configured {
     return credentials;
   }
 
-  /**
-   * Add a command line. It is converted to a single command before being
-   * added.
-   * @param cmd
-   */
-  public void addCommandLine(CommandLineBuilder cmd) {
-    commands.add(cmd.build());
-  }
 
   public void addCommand(String cmd) {
     commands.add(cmd);
   }
 
   /**
-   * Add a list of commands. Each element in the list becomes a single command
-   * @param commandList list of commands
-   */
-  public void addCommands(List<String> commandList) {
-    commands.addAll(commandList);
-  }
-
-  /**
-   * Get all commands as a string, separated by ";". This is for diagnostics
-   * @return a string description of the commands
-   */
-  public String getCommandsAsString() {
-    return SliderUtils.join(getCommands(), "; ");
-  }
-
-  /**
    * Complete the launch context (copy in env vars, etc).
    * @return the container to launch
    */
@@ -211,7 +141,7 @@ public abstract class AbstractLauncher extends Configured {
         log.debug("    \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
       }
     }    
-    containerLaunchContext.setEnvironment(env);
+    containerLaunchContext.setEnvironment(envVars);
 
     //service data
     if (log.isDebugEnabled()) {
@@ -281,120 +211,7 @@ public abstract class AbstractLauncher extends Configured {
   protected void propagateUsernameInInsecureCluster() throws IOException {
     //insecure cluster: propagate user name via env variable
     String userName = UserGroupInformation.getCurrentUser().getUserName();
-    env.put(SliderKeys.HADOOP_USER_NAME, userName);
-  }
-
-  /**
-   * Extract any resource requirements from this component's settings.
-   * All fields that are set will override the existing values -if
-   * unset that resource field will be left unchanged.
-   *
-   * Important: the configuration must already be fully resolved 
-   * in order to pick up global options.
-   * @param resource resource to configure
-   * @param map map of options
-   */
-  public void extractResourceRequirements(Resource resource,
-                                          Map<String, String> map) {
-
-    if (map != null) {
-      MapOperations options = new MapOperations("", map);
-      resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY,
-                                              resource.getMemory()));
-      resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES,
-                                                    resource.getVirtualCores()));
-    }
-  }
-
-  public void extractLogAggregationContext(Map<String, String> map) {
-    if (map != null) {
-      String logPatternSepStr = "\\|";
-      String logPatternJoinStr = "|";
-      MapOperations options = new MapOperations("", map);
-
-      List<String> logIncludePatterns = new ArrayList<>();
-      String includePatternExpression = options.getOption(
-          ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim();
-      if (!includePatternExpression.isEmpty()) {
-        String[] includePatterns = includePatternExpression
-            .split(logPatternSepStr);
-        for (String includePattern : includePatterns) {
-          String trimmedIncludePattern = includePattern.trim();
-          if (!trimmedIncludePattern.isEmpty()) {
-            logIncludePatterns.add(trimmedIncludePattern);
-          }
-        }
-      }
-      String logIncludePattern = StringUtils.join(logIncludePatterns,
-          logPatternJoinStr);
-      log.info("Log include patterns: {}", logIncludePattern);
-
-      List<String> logExcludePatterns = new ArrayList<>();
-      String excludePatternExpression = options.getOption(
-          ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim();
-      if (!excludePatternExpression.isEmpty()) {
-        String[] excludePatterns = excludePatternExpression
-            .split(logPatternSepStr);
-        for (String excludePattern : excludePatterns) {
-          String trimmedExcludePattern = excludePattern.trim();
-          if (!trimmedExcludePattern.isEmpty()) {
-            logExcludePatterns.add(trimmedExcludePattern);
-          }
-        }
-      }
-      String logExcludePattern = StringUtils.join(logExcludePatterns,
-          logPatternJoinStr);
-      log.info("Log exclude patterns: {}", logExcludePattern);
-
-      // SLIDER-810/YARN-3154 - hadoop 2.7.0 onwards a new instance method has
-      // been added for log aggregation for LRS. Existing newInstance method's
-      // behavior has changed and is used for log aggregation only after the
-      // application has finished. This forces Slider users to move to hadoop
-      // 2.7.0+ just for log aggregation, which is not very desirable. So we
-      // decided to use reflection here to find out if the new 2.7.0 newInstance
-      // method is available. If yes, then we use it, so log aggregation will
-      // work in hadoop 2.7.0+ env. If no, then we fallback to the pre-2.7.0
-      // newInstance method, which means log aggregation will work as expected
-      // in hadoop 2.6 as well.
-      // TODO: At some point, say 2-3 Slider releases down, when most users are
-      // running hadoop 2.7.0, we should get rid of the reflection code here.
-      try {
-        Method logAggregationContextMethod = LogAggregationContext.class
-            .getMethod("newInstance", String.class, String.class, String.class,
-                String.class);
-        // Need to set include/exclude patterns appropriately since by default
-        // rolled log aggregation is not done for any files, so defaults are
-        // - include pattern set to ""
-        // - exclude pattern set to "*"
-        // For Slider we want all logs to be uploaded if include/exclude
-        // patterns are left empty by the app owner in resources file
-        if (StringUtils.isEmpty(logIncludePattern)
-            && StringUtils.isEmpty(logExcludePattern)) {
-          logIncludePattern = ".*";
-          logExcludePattern = "";
-        } else if (StringUtils.isEmpty(logIncludePattern)
-            && StringUtils.isNotEmpty(logExcludePattern)) {
-          logIncludePattern = ".*";
-        } else if (StringUtils.isNotEmpty(logIncludePattern)
-            && StringUtils.isEmpty(logExcludePattern)) {
-          logExcludePattern = "";
-        }
-        log.debug("LogAggregationContext newInstance method for rolled logs "
-            + "include/exclude patterns is available");
-        log.info("Modified log include patterns: {}", logIncludePattern);
-        log.info("Modified log exclude patterns: {}", logExcludePattern);
-        logAggregationContext = (LogAggregationContext) logAggregationContextMethod
-            .invoke(null, null, null, logIncludePattern, logExcludePattern);
-      } catch (NoSuchMethodException | SecurityException
-          | IllegalAccessException | IllegalArgumentException
-          | InvocationTargetException e) {
-        log.debug("LogAggregationContext newInstance method for rolled logs "
-            + "include/exclude patterns is not available - fallback to old one");
-        log.debug(e.toString());
-        logAggregationContext = LogAggregationContext.newInstance(
-            logIncludePattern, logExcludePattern);
-      }
-    }
+    envVars.put(SliderKeys.HADOOP_USER_NAME, userName);
   }
 
   /**
@@ -413,57 +230,15 @@ public abstract class AbstractLauncher extends Configured {
   public void setEnv(String var, String value) {
     Preconditions.checkArgument(var != null, "null variable name");
     Preconditions.checkArgument(value != null, "null value");
-    env.put(var, value);
+    envVars.put(var, value);
   }
 
-  /**
-   * Set an environment variable if its value is non-null.
-   * @param var variable name
-   * @param value value (may be null)
-   */
-  public void maybeSetEnv(String var, String value) {
-    if (value != null) {
-      setEnv(var, value);
-    }
-  }
 
   public void putEnv(Map<String, String> map) {
-    env.putAll(map);
+    envVars.putAll(map);
   }
 
 
-  public String[] dumpEnvToString() {
-
-    List<String> nodeEnv = new ArrayList<>();
-
-    for (Map.Entry<String, String> entry : env.entrySet()) {
-      String envElt = String.format("%s=\"%s\"",
-                                    entry.getKey(),
-                                    entry.getValue());
-      log.debug(envElt);
-      nodeEnv.add(envElt);
-    }
-    String[] envDescription = nodeEnv.toArray(new String[nodeEnv.size()]);
-
-    return envDescription;
-  }
-
-  /**
-   * Submit an entire directory
-   * @param srcDir src path in filesystem
-   * @param destRelativeDir relative path under destination local dir
-   * @throws IOException IO problems
-   */
-  public void submitDirectory(Path srcDir, String destRelativeDir)
-      throws IOException {
-    //add the configuration resources
-    Map<String, LocalResource> confResources;
-    confResources = coreFileSystem.submitDirectory(
-      srcDir,
-      destRelativeDir);
-    addLocalResources(confResources);
-  }
-
   public void setDockerImage(String dockerImage) {
     this.dockerImage = dockerImage;
   }
@@ -476,14 +251,6 @@ public abstract class AbstractLauncher extends Configured {
     this.dockerHostname = dockerHostname;
   }
 
-  public void setYarnContainerMountPoints(String yarnContainerMountPoints) {
-    this.yarnContainerMountPoints = yarnContainerMountPoints;
-  }
-
-  public void setRunPrivilegedContainer(String runPrivilegedContainer) {
-    this.runPrivilegedContainer = runPrivilegedContainer;
-  }
-
   public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
     if (runPrivilegedContainer) {
       this.runPrivilegedContainer = Boolean.toString(true);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index d220928..7e164e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -38,35 +38,11 @@ import java.net.InetSocketAddress;
 public class ContainerLauncher extends AbstractLauncher {
   private static final Logger log =
     LoggerFactory.getLogger(ContainerLauncher.class);
-  // Allocated container
-  public final Container container;
 
   public ContainerLauncher(Configuration conf,
       CoreFileSystem coreFileSystem,
       Container container,
       Credentials credentials) {
-    super(conf, coreFileSystem, credentials);
-    this.container = container;
+    super(coreFileSystem, credentials);
   }
-
-  /**
-   * This code is in the dist shell examples -it's been moved here
-   * so that if it is needed, it's still here
-   * @return a remote user with a token to access the container.
-   */
-  public UserGroupInformation setupUGI() {
-    UserGroupInformation user =
-      UserGroupInformation.createRemoteUser(container.getId().toString());
-    String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
-    final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
-
-    org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken();
-    if (containerToken != null) {
-      Token<ContainerTokenIdentifier> token =
-        ConverterUtils.convertFromYarn(containerToken, cmAddress);
-      user.addToken(token);
-    }
-    return user;
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
index 5357cc4..1fd49ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
index 37b36ea..ac8fca5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
@@ -20,7 +20,7 @@ package org.apache.slider.core.registry;
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.slider.common.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
 
 /**
  * Miscellaneous methods to assist slider registry work

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
deleted file mode 100644
index ea92ff7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public abstract class AbstractClientProvider {
-
-  public AbstractClientProvider() {
-  }
-
-  /**
-   * Generates a fixed format of application tags given one or more of
-   * application name, version and description. This allows subsequent query for
-   * an application with a name only, version only or description only or any
-   * combination of those as filters.
-   *
-   * @param appName name of the application
-   * @param appVersion version of the application
-   * @param appDescription brief description of the application
-   * @return
-   */
-  public static final Set<String> createApplicationTags(String appName,
-      String appVersion, String appDescription) {
-    Set<String> tags = new HashSet<>();
-    tags.add(SliderUtils.createNameTag(appName));
-    if (appVersion != null) {
-      tags.add(SliderUtils.createVersionTag(appVersion));
-    }
-    if (appDescription != null) {
-      tags.add(SliderUtils.createDescriptionTag(appDescription));
-    }
-    return tags;
-  }
-
-  /**
-   * Validate the artifact.
-   * @param artifact
-   */
-  public abstract void validateArtifact(Artifact artifact, FileSystem
-      fileSystem) throws IOException;
-
-  protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
-      fileSystem) throws IOException;
-
-  /**
-   * Validate the config files.
-   * @param configFiles config file list
-   * @param fs file system
-   */
-  public void validateConfigFiles(List<ConfigFile> configFiles,
-      FileSystem fs) throws IOException {
-    Set<String> destFileSet = new HashSet<>();
-
-    for (ConfigFile file : configFiles) {
-      if (file.getType() == null) {
-        throw new IllegalArgumentException("File type is empty");
-      }
-
-      if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
-          .isEmpty(file.getSrcFile())) {
-        throw new IllegalArgumentException(
-            "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
-
-      }
-      if (!StringUtils.isEmpty(file.getSrcFile())) {
-        Path p = new Path(file.getSrcFile());
-        if (!fs.exists(p)) {
-          throw new IllegalArgumentException(
-              "Src_file does not exist for config file: " + file
-                  .getSrcFile());
-        }
-      }
-
-      if (StringUtils.isEmpty(file.getDestFile())) {
-        throw new IllegalArgumentException("Dest_file is empty.");
-      }
-
-      if (destFileSet.contains(file.getDestFile())) {
-        throw new IllegalArgumentException(
-            "Duplicated ConfigFile exists: " + file.getDestFile());
-      }
-      destFileSet.add(file.getDestFile());
-
-      java.nio.file.Path destPath = Paths.get(file.getDestFile());
-      if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
-        throw new IllegalArgumentException("Non-absolute dest_file has more " +
-            "than one path element");
-      }
-
-      // provider-specific validation
-      validateConfigFile(file, fs);
-    }
-  }
-
-  /**
-   * Process client operations for applications such as install, configure.
-   * @param fileSystem
-   * @param registryOperations
-   * @param configuration
-   * @param operation
-   * @param clientInstallPath
-   * @param clientPackage
-   * @param clientConfig
-   * @param name
-   * @throws SliderException
-   */
-  public void processClientOperation(SliderFileSystem fileSystem,
-                                     RegistryOperations registryOperations,
-                                     Configuration configuration,
-                                     String operation,
-                                     File clientInstallPath,
-                                     File clientPackage,
-                                     JSONObject clientConfig,
-                                     String name)
-      throws SliderException {
-    throw new SliderException("Provider does not support client operations.");
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index e0d9402..1e1b1b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -23,10 +23,13 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
 import org.apache.slider.api.resource.Application;
 import org.apache.slider.api.resource.Component;
 import org.apache.slider.api.resource.ContainerState;
-import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.core.exceptions.SliderException;
@@ -35,7 +38,6 @@ import org.apache.slider.core.launch.ContainerLauncher;
 import org.apache.slider.core.registry.docstore.PublishedConfiguration;
 import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
 import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -44,14 +46,14 @@ import java.io.IOException;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import static org.apache.slider.util.ServiceApiUtil.$;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
 
 public abstract class AbstractProviderService extends AbstractService
     implements ProviderService, SliderKeys {
 
   protected static final Logger log =
       LoggerFactory.getLogger(AbstractProviderService.class);
-  private static final ProviderUtils providerUtils = new ProviderUtils(log);
+  private static final ProviderUtils providerUtils = new ProviderUtils();
   protected StateAccessForProviders amState;
   protected YarnRegistryViewForProviders yarnRegistry;
   private ServiceTimelinePublisher serviceTimelinePublisher;
@@ -64,15 +66,6 @@ public abstract class AbstractProviderService extends AbstractService
       Application application, RoleInstance roleInstance,
       SliderFileSystem fileSystem) throws IOException;
 
-  @Override
-  public void setAMState(StateAccessForProviders stateAccessor) {
-    this.amState = stateAccessor;
-  }
-
-  @Override
-  public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
-    this.yarnRegistry = yarnRegistry;
-  }
 
   public void buildContainerLaunchContext(ContainerLauncher launcher,
       Application application, Container container, ProviderRole providerRole,
@@ -85,7 +78,7 @@ public abstract class AbstractProviderService extends AbstractService
     // Get pre-defined tokens
     Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
     Map<String, String> tokensForSubstitution = providerUtils
-        .initCompTokensForSubstitute(roleInstance);
+        .initCompTokensForSubstitute(null);
     tokensForSubstitution.putAll(globalTokens);
     // Set the environment variables in launcher
     launcher.putEnv(SliderUtils
@@ -105,8 +98,6 @@ public abstract class AbstractProviderService extends AbstractService
     providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
 
     // create config file on hdfs and add local resource
-    providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
-        component, tokensForSubstitution, roleInstance, amState);
 
     // substitute launch command
     String launchCommand = ProviderUtils
@@ -141,15 +132,6 @@ public abstract class AbstractProviderService extends AbstractService
       return false;
     }
 
-    try {
-      providerUtils.updateServiceRecord(amState, yarnRegistry,
-          containerId.toString(), instance.role, status.getIPs(), status.getHost());
-    } catch (IOException e) {
-      // could not write service record to ZK, log and retry
-      log.warn("Error updating container {} service record in registry, " +
-          "retrying", containerId, e);
-      return true;
-    }
     // TODO publish ip and host
     org.apache.slider.api.resource.Container container =
         instance.providerRole.component.getContainer(containerId.toString());
@@ -163,8 +145,4 @@ public abstract class AbstractProviderService extends AbstractService
     return false;
   }
 
-  @Override
-  public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
-    this.serviceTimelinePublisher = publisher;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
deleted file mode 100644
index 004e81f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-
-public class DefaultClientProvider extends AbstractClientProvider {
-
-  public DefaultClientProvider() {
-  }
-
-  @Override
-  public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
-  }
-
-  @Override
-  protected void validateConfigFile(ConfigFile configFile, FileSystem
-      fileSystem) throws IOException {
-    // validate dest_file is not absolute
-    if (Paths.get(configFile.getDestFile()).isAbsolute()) {
-      throw new IllegalArgumentException(
-          "Dest_file must not be absolute path: " + configFile.getDestFile());
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
deleted file mode 100644
index 09a1423..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-public final class DefaultProviderFactory extends SliderProviderFactory {
-  private static final SliderProviderFactory FACTORY = new
-      DefaultProviderFactory();
-
-  private DefaultProviderFactory() {}
-
-  private static class Client {
-    static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
-  }
-
-  private static class Server {
-    static final ProviderService PROVIDER = new DefaultProviderService();
-  }
-
-  @Override
-  public AbstractClientProvider createClientProvider() {
-    return Client.PROVIDER;
-  }
-
-  @Override
-  public ProviderService createServerProvider() {
-    return Server.PROVIDER;
-  }
-
-  public static SliderProviderFactory getInstance() {
-    return FACTORY;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
deleted file mode 100644
index 8d2725c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-import java.io.IOException;
-
-public class DefaultProviderService extends AbstractProviderService {
-
-  protected DefaultProviderService() {
-    super(DefaultProviderService.class.getSimpleName());
-  }
-
-  @Override
-  public void processArtifact(ContainerLauncher launcher, Application
-      application, RoleInstance roleInstance, SliderFileSystem fileSystem)
-      throws IOException {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
deleted file mode 100644
index edd313b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-
-import java.io.IOException;
-
-public interface ProviderService extends Service {
-
-  /**
-   * Set up the entire container launch context
-   */
-  void buildContainerLaunchContext(ContainerLauncher containerLauncher,
-      Application application, Container container, ProviderRole providerRole,
-      SliderFileSystem sliderFileSystem, RoleInstance roleInstance)
-      throws IOException, SliderException;
-
-
-  void setAMState(StateAccessForProviders stateAccessForProviders);
-
-  /**
-   * Bind to the YARN registry
-   * @param yarnRegistry YARN registry
-   */
-  void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
-
-  /**
-   * Process container status
-   * @return true if status needs to be requested again, false otherwise
-   */
-  boolean processContainerStatus(ContainerId containerId,
-      ContainerStatus status);
-
-  /**
-   * Set service publisher.
-   * @param serviceTimelinePublisher service publisher.
-   */
-  void setServiceTimelinePublisher(
-      ServiceTimelinePublisher serviceTimelinePublisher);
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
deleted file mode 100644
index ecc521f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ /dev/null
@@ -1,525 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.regex.Pattern;
-
-import static org.apache.slider.api.ServiceApiConstants.*;
-import static org.apache.slider.util.ServiceApiUtil.$;
-
-/**
- * This is a factoring out of methods handy for providers. It's bonded to a log
- * at construction time.
- */
-public class ProviderUtils implements RoleKeys, SliderKeys {
-
-  protected final Logger log;
-
-  /**
-   * Create an instance
-   * @param log log directory to use -usually the provider
-   */
-  
-  public ProviderUtils(Logger log) {
-    this.log = log;
-  }
-
-  /**
-   * Add oneself to the classpath. This does not work
-   * on minicluster test runs where the JAR is not built up.
-   * @param providerResources map of provider resources to add these entries to
-   * @param providerClass provider to add
-   * @param jarName name of the jar to use
-   * @param sliderFileSystem target filesystem
-   * @param tempPath path in the cluster FS for temp files
-   * @param libdir relative directory to place resources
-   * @param miniClusterTestRun true if minicluster is being used
-   * @return true if the class was found in a JAR
-   * 
-   * @throws FileNotFoundException if the JAR was not found and this is NOT
-   * a mini cluster test run
-   * @throws IOException IO problems
-   * @throws SliderException any Slider problem
-   */
-  public static boolean addProviderJar(
-      Map<String, LocalResource> providerResources,
-      Class providerClass,
-      String jarName,
-      SliderFileSystem sliderFileSystem,
-      Path tempPath,
-      String libdir,
-      boolean miniClusterTestRun) throws
-      IOException,
-      SliderException {
-    try {
-      SliderUtils.putJar(providerResources,
-          sliderFileSystem,
-          providerClass,
-          tempPath,
-          libdir,
-          jarName);
-      return true;
-    } catch (FileNotFoundException e) {
-      if (miniClusterTestRun) {
-        return false;
-      } else {
-        throw e;
-      }
-    }
-  }
-  
-  /**
-   * Loads all dependency jars from the default path.
-   * @param providerResources map of provider resources to add these entries to
-   * @param sliderFileSystem target filesystem
-   * @param tempPath path in the cluster FS for temp files
-   * @param libDir relative directory to place resources
-   * @param libLocalSrcDir explicitly supplied local libs dir
-   * @throws IOException trouble copying to HDFS
-   * @throws SliderException trouble copying to HDFS
-   */
-  public static void addAllDependencyJars(
-      Map<String, LocalResource> providerResources,
-      SliderFileSystem sliderFileSystem,
-      Path tempPath,
-      String libDir,
-      String libLocalSrcDir)
-      throws IOException, SliderException {
-    if (SliderUtils.isSet(libLocalSrcDir)) {
-      File file = new File(libLocalSrcDir);
-      if (!file.exists() || !file.isDirectory()) {
-        throw new BadCommandArgumentsException(
-            "Supplied lib src dir %s is not valid", libLocalSrcDir);
-      }
-    }
-    SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
-        libDir, libLocalSrcDir);
-  }
-
-  public static String substituteStrWithTokens(String content,
-      Map<String, String> tokensForSubstitution) {
-    for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
-      content =
-          content.replaceAll(Pattern.quote(token.getKey()), token.getValue());
-    }
-    return content;
-  }
-
-  // configs will be substituted by corresponding env in tokenMap
-  public void substituteMapWithTokens(Map<String, String> configs,
-      Map<String, String> tokenMap) {
-    for (Map.Entry<String, String> entry : configs.entrySet()) {
-      String value = entry.getValue();
-      if (tokenMap != null) {
-        for (Map.Entry<String, String> token : tokenMap.entrySet()) {
-          value =
-              value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
-        }
-      }
-      entry.setValue(value);
-    }
-  }
-
-  /**
-   * Get resource requirements from a String value. If value isn't specified,
-   * use the default value. If value is greater than max, use the max value.
-   * @param val string value
-   * @param defVal default value
-   * @param maxVal maximum value
-   * @return int resource requirement
-   */
-  public int getRoleResourceRequirement(String val,
-                                        int defVal,
-                                        int maxVal) {
-    if (val==null) {
-      val = Integer.toString(defVal);
-    }
-    Integer intVal;
-    if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
-      intVal = maxVal;
-    } else {
-      intVal = Integer.decode(val);
-    }
-    return intVal;
-  }
-
-
-  /**
-   * Localize the service keytabs for the application.
-   * @param launcher container launcher
-   * @param fileSystem file system
-   * @throws IOException trouble uploading to HDFS
-   */
-  public void localizeServiceKeytabs(ContainerLauncher launcher,
-      SliderFileSystem fileSystem, Application application) throws IOException {
-
-    Configuration conf = application.getConfiguration();
-    String keytabPathOnHost =
-        conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
-    if (SliderUtils.isUnset(keytabPathOnHost)) {
-      String amKeytabName =
-          conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
-      String keytabDir =
-          conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
-      // we need to localize the keytab files in the directory
-      Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
-          application.getName());
-      boolean serviceKeytabsDeployed = false;
-      if (fileSystem.getFileSystem().exists(keytabDirPath)) {
-        FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
-            keytabDirPath);
-        LocalResource keytabRes;
-        for (FileStatus keytab : keytabs) {
-          if (!amKeytabName.equals(keytab.getPath().getName())
-              && keytab.getPath().getName().endsWith(".keytab")) {
-            serviceKeytabsDeployed = true;
-            log.info("Localizing keytab {}", keytab.getPath().getName());
-            keytabRes = fileSystem.createAmResource(keytab.getPath(),
-                LocalResourceType.FILE);
-            launcher.addLocalResource(KEYTAB_DIR + "/" +
-                    keytab.getPath().getName(),
-                keytabRes);
-          }
-        }
-      }
-      if (!serviceKeytabsDeployed) {
-        log.warn("No service keytabs for the application have been localized.  "
-            + "If the application requires keytabs for secure operation, "
-            + "please ensure that the required keytabs have been uploaded "
-            + "to the folder {}", keytabDirPath);
-      }
-    }
-  }
-
-  // 1. Create all config files for a component on hdfs for localization
-  // 2. Add the config file to localResource
-  public synchronized void createConfigFileAndAddLocalResource(
-      ContainerLauncher launcher, SliderFileSystem fs, Component component,
-      Map<String, String> tokensForSubstitution, RoleInstance roleInstance,
-      StateAccessForProviders appState) throws IOException {
-    Path compDir =
-        new Path(new Path(fs.getAppDir(), "components"), component.getName());
-    Path compInstanceDir =
-        new Path(compDir, roleInstance.getCompInstanceName());
-    if (!fs.getFileSystem().exists(compInstanceDir)) {
-      fs.getFileSystem().mkdirs(compInstanceDir,
-          new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
-      roleInstance.compInstanceDir = compInstanceDir;
-      log.info("Creating component instance dir: " + compInstanceDir);
-    } else {
-      log.info("Component instance conf dir already exists: " + compInstanceDir);
-    }
-
-    log.info("Tokens substitution for component: " + roleInstance
-        .getCompInstanceName() + System.lineSeparator()
-        + tokensForSubstitution);
-
-    for (ConfigFile originalFile : component.getConfiguration().getFiles()) {
-      ConfigFile configFile = originalFile.copy();
-      String fileName = new Path(configFile.getDestFile()).getName();
-
-      // substitute file name
-      for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
-        configFile.setDestFile(configFile.getDestFile()
-            .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
-      }
-
-      Path remoteFile = new Path(compInstanceDir, fileName);
-      if (!fs.getFileSystem().exists(remoteFile)) {
-        log.info("Saving config file on hdfs for component " + roleInstance
-            .getCompInstanceName() + ": " + configFile);
-
-        if (configFile.getSrcFile() != null) {
-          // Load config file template
-          switch (configFile.getType()) {
-          case HADOOP_XML:
-            // Hadoop_xml_template
-            resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
-                tokensForSubstitution, configFile, remoteFile, appState);
-            break;
-          case TEMPLATE:
-            // plain-template
-            resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
-                tokensForSubstitution, configFile, remoteFile, appState);
-            break;
-          default:
-            log.info("Not supporting loading src_file for " + configFile);
-            break;
-          }
-        } else {
-          // non-template
-          resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
-              roleInstance, configFile, fileName, remoteFile);
-        }
-      }
-
-      // Add resource for localization
-      LocalResource configResource =
-          fs.createAmResource(remoteFile, LocalResourceType.FILE);
-      File destFile = new File(configFile.getDestFile());
-      String symlink = APP_CONF_DIR + "/" + fileName;
-      if (destFile.isAbsolute()) {
-        launcher.addLocalResource(symlink, configResource,
-            configFile.getDestFile());
-        log.info("Add config file for localization: " + symlink + " -> "
-            + configResource.getResource().getFile() + ", dest mount path: "
-            + configFile.getDestFile());
-      } else {
-        launcher.addLocalResource(symlink, configResource);
-        log.info("Add config file for localization: " + symlink + " -> "
-            + configResource.getResource().getFile());
-      }
-    }
-  }
-
-  private void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
-      Map<String, String> tokensForSubstitution, RoleInstance roleInstance,
-      ConfigFile configFile, String fileName, Path remoteFile)
-      throws IOException {
-    // substitute non-template configs
-    substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
-
-    // write configs onto hdfs
-    PublishedConfiguration publishedConfiguration =
-        new PublishedConfiguration(fileName,
-            configFile.getProps().entrySet());
-    if (!fs.getFileSystem().exists(remoteFile)) {
-      PublishedConfigurationOutputter configurationOutputter =
-          PublishedConfigurationOutputter.createOutputter(
-              ConfigFormat.resolve(configFile.getType().toString()),
-              publishedConfiguration);
-      try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) {
-        configurationOutputter.save(os);
-        os.flush();
-      }
-    } else {
-      log.info("Component instance = " + roleInstance.getCompInstanceName()
-              + ", config file already exists: " + remoteFile);
-    }
-  }
-
-  // 1. substitute config template - only handle hadoop_xml format
-  // 2. save on hdfs
-  @SuppressWarnings("unchecked")
-  private void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
-      Map<String, String> tokensForSubstitution, ConfigFile configFile,
-      Path remoteFile, StateAccessForProviders appState) throws IOException {
-    Map<String, String> conf;
-    try {
-      conf = (Map<String, String>) appState.getConfigFileCache()
-          .get(configFile);
-    } catch (ExecutionException e) {
-      log.info("Failed to load config file: " + configFile, e);
-      return;
-    }
-    // make a copy for substitution
-    org.apache.hadoop.conf.Configuration confCopy =
-        new org.apache.hadoop.conf.Configuration(false);
-    for (Map.Entry<String, String> entry : conf.entrySet()) {
-      confCopy.set(entry.getKey(), entry.getValue());
-    }
-    // substitute properties
-    for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
-      confCopy.set(entry.getKey(), entry.getValue());
-    }
-    // substitute env variables
-    for (Map.Entry<String, String> entry : confCopy) {
-      String val = entry.getValue();
-      if (val != null) {
-        for (Map.Entry<String, String> token : tokensForSubstitution
-            .entrySet()) {
-          val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue());
-          confCopy.set(entry.getKey(), val);
-        }
-      }
-    }
-    // save on hdfs
-    try (OutputStream output = fs.create(remoteFile)) {
-      confCopy.writeXml(output);
-      log.info("Reading config from: " + configFile.getSrcFile()
-          + ", writing to: " + remoteFile);
-    }
-  }
-
-  // 1) read the template as a string
-  // 2) do token substitution
-  // 3) save on hdfs
-  private void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
-      Map<String, String> tokensForSubstitution, ConfigFile configFile,
-      Path remoteFile, StateAccessForProviders appState) {
-    String content;
-    try {
-      content = (String) appState.getConfigFileCache().get(configFile);
-    } catch (ExecutionException e) {
-      log.info("Failed to load config file: " + configFile, e);
-      return;
-    }
-    // substitute tokens
-    content = substituteStrWithTokens(content, tokensForSubstitution);
-
-    try (OutputStream output = fs.create(remoteFile)) {
-      org.apache.commons.io.IOUtils.write(content, output);
-    } catch (IOException e) {
-      log.info("Failed to create " + remoteFile);
-    }
-  }
-
-  /**
-   * Get initial component token map to be substituted into config values.
-   * @param roleInstance role instance
-   * @return tokens to replace
-   */
-  public Map<String, String> initCompTokensForSubstitute(
-      RoleInstance roleInstance) {
-    Map<String, String> tokens = new HashMap<>();
-    tokens.put(COMPONENT_NAME, roleInstance.role);
-    tokens.put(COMPONENT_NAME_LC, roleInstance.role.toLowerCase());
-    tokens.put(COMPONENT_INSTANCE_NAME, roleInstance.getCompInstanceName());
-    tokens.put(CONTAINER_ID, roleInstance.getContainerId().toString());
-    tokens.put(COMPONENT_ID, String.valueOf(roleInstance.componentId));
-    return tokens;
-  }
-
-  /**
-   * Add ROLE_HOST tokens for substitution into config values.
-   * @param tokens existing tokens
-   * @param amState access to AM state
-   */
-  public void addComponentHostTokens(Map<String, String> tokens,
-      StateAccessForProviders amState) {
-    if (amState == null) {
-      return;
-    }
-    for (Map.Entry<String, Map<String, ClusterNode>> entry :
-        amState.getRoleClusterNodeMapping().entrySet()) {
-      String tokenName = entry.getKey().toUpperCase(Locale.ENGLISH) + "_HOST";
-      String hosts = StringUtils .join(",",
-          getHostsList(entry.getValue().values(), true));
-      tokens.put($(tokenName), hosts);
-    }
-  }
-
-  /**
-   * Return a list of hosts based on current ClusterNodes.
-   * @param values cluster nodes
-   * @param hostOnly whether host or host/server name will be added to list
-   * @return list of hosts
-   */
-  public Iterable<String> getHostsList(Collection<ClusterNode> values,
-      boolean hostOnly) {
-    List<String> hosts = new ArrayList<>();
-    for (ClusterNode cn : values) {
-      hosts.add(hostOnly ? cn.host : cn.host + "/" + cn.name);
-    }
-    return hosts;
-  }
-
-  /**
-   * Update ServiceRecord in Registry with IP and hostname.
-   * @param amState access to AM state
-   * @param yarnRegistry acces to YARN registry
-   * @param containerId container ID
-   * @param roleName component name
-   * @param ip list of IPs
-   * @param hostname hostname
-   */
-  public void updateServiceRecord(StateAccessForProviders amState,
-      YarnRegistryViewForProviders yarnRegistry,
-      String containerId, String roleName, List<String> ip, String hostname)
-      throws IOException {
-    try {
-      RoleInstance role = null;
-      if(ip != null && !ip.isEmpty()){
-        role = amState.getOwnedContainer(containerId);
-        role.ip = ip.get(0);
-      }
-      if(hostname != null && !hostname.isEmpty()){
-        role = amState.getOwnedContainer(containerId);
-        role.hostname = hostname;
-      }
-      if (role != null) {
-        // create and publish updated service record (including hostname & ip)
-        ServiceRecord record = new ServiceRecord();
-        record.set(YarnRegistryAttributes.YARN_ID, containerId);
-        record.description = role.getCompInstanceName();
-        record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
-            PersistencePolicies.CONTAINER);
-        // TODO: use constants from YarnRegistryAttributes
-        if (role.ip != null) {
-          record.set("yarn:ip", role.ip);
-        }
-        if (role.hostname != null) {
-          record.set("yarn:hostname", role.hostname);
-        }
-        yarnRegistry.putComponent(
-            RegistryPathUtils.encodeYarnID(containerId), record);
-      }
-    } catch (NoSuchNodeException e) {
-      // ignore - there is nothing to do if we don't find a container
-      log.warn("Owned container {} not found - {}", containerId, e);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
deleted file mode 100644
index 5ecc374..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.providers.docker.DockerProviderFactory;
-import org.apache.slider.providers.tarball.TarballProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base class for factories.
- */
-public abstract class SliderProviderFactory {
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(SliderProviderFactory.class);
-
-  protected SliderProviderFactory() {}
-
-  public abstract AbstractClientProvider createClientProvider();
-
-  public abstract ProviderService createServerProvider();
-
-  public static synchronized ProviderService getProviderService(Artifact
-      artifact) {
-    return createSliderProviderFactory(artifact).createServerProvider();
-  }
-
-  public static synchronized AbstractClientProvider getClientProvider(Artifact
-      artifact) {
-    return createSliderProviderFactory(artifact).createClientProvider();
-  }
-
-  /**
-   * Create a provider for a specific application
-   * @param artifact artifact
-   * @return provider factory
-   * @throws SliderException on any instantiation problem
-   */
-  public static synchronized SliderProviderFactory createSliderProviderFactory(
-      Artifact artifact) {
-    if (artifact == null || artifact.getType() == null) {
-      LOG.debug("Loading service provider type default");
-      return DefaultProviderFactory.getInstance();
-    }
-    LOG.debug("Loading service provider type {}", artifact.getType());
-    switch (artifact.getType()) {
-      // TODO add handling for custom types?
-      // TODO handle application
-      case DOCKER:
-        return DockerProviderFactory.getInstance();
-      case TARBALL:
-        return TarballProviderFactory.getInstance();
-      default:
-        throw new IllegalArgumentException(String.format("Resolution error, " +
-                "%s should not be passed to createSliderProviderFactory",
-            artifact.getType()));
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
deleted file mode 100644
index 4773327..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.docker;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.util.RestApiErrorMessages;
-
-import java.io.IOException;
-
-public class DockerClientProvider extends AbstractClientProvider
-    implements SliderKeys {
-
-  public DockerClientProvider() {
-    super();
-  }
-
-  @Override
-  public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
-    if (artifact == null) {
-      throw new IllegalArgumentException(
-          RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
-    }
-    if (StringUtils.isEmpty(artifact.getId())) {
-      throw new IllegalArgumentException(
-          RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
-    }
-  }
-
-  @Override
-  protected void validateConfigFile(ConfigFile configFile, FileSystem
-      fileSystem) throws IOException {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
deleted file mode 100644
index 7413ebb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.docker;
-
-public interface DockerKeys {
-  String PROVIDER_DOCKER = "docker";
-  String DOCKER_PREFIX = "docker.";
-  String DOCKER_IMAGE = DOCKER_PREFIX + "image";
-  String DOCKER_NETWORK = DOCKER_PREFIX + "network";
-  String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
-  String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
-
-  String DEFAULT_DOCKER_NETWORK = "bridge";
-  Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
deleted file mode 100644
index 6977e41..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.docker;
-
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.SliderProviderFactory;
-
-public class DockerProviderFactory extends SliderProviderFactory {
-  private static final SliderProviderFactory FACTORY = new
-      DockerProviderFactory();
-
-  private DockerProviderFactory() {
-  }
-
-  private static class Client {
-    static final AbstractClientProvider PROVIDER = new DockerClientProvider();
-  }
-
-  private static class Server {
-    static final ProviderService PROVIDER = new DockerProviderService();
-  }
-
-  @Override
-  public AbstractClientProvider createClientProvider() {
-    return Client.PROVIDER;
-  }
-
-  @Override
-  public ProviderService createServerProvider() {
-    return Server.PROVIDER;
-  }
-
-  public static SliderProviderFactory getInstance() {
-    return FACTORY;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
deleted file mode 100644
index 73783af..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.docker;
-
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-
-public class DockerProviderService extends AbstractProviderService
-    implements DockerKeys {
-
-  protected static final Logger log =
-      LoggerFactory.getLogger(DockerProviderService.class);
-
-  protected DockerProviderService() {
-    super(DockerProviderService.class.getSimpleName());
-  }
-
-  public void processArtifact(ContainerLauncher launcher, Application
-      application, RoleInstance roleInstance, SliderFileSystem fileSystem)
-      throws IOException {
-    Component component = roleInstance.providerRole.component;
-    launcher.setYarnDockerMode(true);
-    launcher.setDockerImage(component.getArtifact().getId());
-    launcher.setDockerNetwork(component.getConfiguration()
-        .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
-    String domain = getConfig().get(RegistryConstants.KEY_DNS_DOMAIN);
-    String hostname;
-    if (domain == null || domain.isEmpty()) {
-      hostname = MessageFormat.format("{0}.{1}.{2}", roleInstance
-          .getCompInstanceName(), application.getName(), RegistryUtils
-          .currentUser());
-    } else {
-      hostname = MessageFormat.format("{0}.{1}.{2}.{3}", roleInstance
-          .getCompInstanceName(), application.getName(), RegistryUtils
-          .currentUser(), domain);
-    }
-    launcher.setDockerHostname(hostname);
-    launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer());
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message