hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [02/50] [abbrv] hadoop git commit: YARN-5505. Create an agent-less docker provider in the native-services framework. Contributed by Billie Rinaldi
Date Tue, 11 Oct 2016 20:37:06 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ba59bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 8203cf0..fdc5be1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -82,6 +82,8 @@ import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
+import static org.apache.slider.common.tools.SliderUtils.getApplicationDefinitionPath;
+
 /** This class implements  the client-side aspects of the agent deployer */
 public class AgentClientProvider extends AbstractClientProvider
     implements AgentKeys, SliderKeys {
@@ -132,13 +134,13 @@ public class AgentClientProvider extends AbstractClientProvider
     sliderFileSystem.verifyFileExists(appDefPath);
 
     String agentConf = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
+        getGlobalOptions().getOption(AGENT_CONF, "");
     if (StringUtils.isNotEmpty(agentConf)) {
       sliderFileSystem.verifyFileExists(new Path(agentConf));
     }
 
     String appHome = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
+        getGlobalOptions().get(PACKAGE_PATH);
     if (SliderUtils.isUnset(appHome)) {
       String agentImage = instanceDefinition.getInternalOperations().
           get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
@@ -173,7 +175,7 @@ public class AgentClientProvider extends AbstractClientProvider
     }
 
     Set<String> names = resources.getComponentNames();
-    names.remove(SliderKeys.COMPONENT_AM);
+    names.remove(COMPONENT_AM);
     Map<Integer, String> priorityMap = new HashMap<Integer, String>();
 
     for (String name : names) {
@@ -271,7 +273,7 @@ public class AgentClientProvider extends AbstractClientProvider
     String agentImage = instanceDefinition.getInternalOperations().
         get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
     if (SliderUtils.isUnset(agentImage)) {
-      Path agentPath = new Path(tempPath.getParent(), AgentKeys.PROVIDER_AGENT);
+      Path agentPath = new Path(tempPath.getParent(), PROVIDER_AGENT);
       log.info("Automatically uploading the agent tarball at {}", agentPath);
       fileSystem.getFileSystem().mkdirs(agentPath);
       if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
@@ -284,6 +286,12 @@ public class AgentClientProvider extends AbstractClientProvider
 
   @Override
   public Set<String> getApplicationTags(SliderFileSystem fileSystem,
+      ConfTreeOperations appConf) throws SliderException {
+    return getApplicationTags(fileSystem,
+        getApplicationDefinitionPath(appConf));
+  }
+
+  public Set<String> getApplicationTags(SliderFileSystem fileSystem,
                                         String appDef) throws SliderException {
     Set<String> tags;
     Metainfo metaInfo = getMetainfo(fileSystem, appDef);
@@ -437,19 +445,19 @@ public class AgentClientProvider extends AbstractClientProvider
         if (config != null) {
           try {
             clientRoot = config.getJSONObject("global")
-                .getString(AgentKeys.APP_CLIENT_ROOT);
+                .getString(APP_CLIENT_ROOT);
           } catch (JSONException e) {
             log.info("Couldn't read {} from provided client config, falling " +
-                "back on default", AgentKeys.APP_CLIENT_ROOT);
+                "back on default", APP_CLIENT_ROOT);
           }
         }
         if (clientRoot == null && defaultConfig != null) {
           try {
             clientRoot = defaultConfig.getJSONObject("global")
-                .getString(AgentKeys.APP_CLIENT_ROOT);
+                .getString(APP_CLIENT_ROOT);
           } catch (JSONException e) {
             log.info("Couldn't read {} from default client config, using {}",
-                AgentKeys.APP_CLIENT_ROOT, clientInstallPath);
+                APP_CLIENT_ROOT, clientInstallPath);
           }
         }
         if (clientRoot == null) {
@@ -500,7 +508,7 @@ public class AgentClientProvider extends AbstractClientProvider
     try {
       String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + "package" +
                                 File.separator + clientScript;
-      List<String> command = Arrays.asList(AgentKeys.PYTHON_EXE,
+      List<String> command = Arrays.asList(PYTHON_EXE,
                "-S",
                clientScriptPath,
                "INSTALL",
@@ -510,12 +518,12 @@ public class AgentClientProvider extends AbstractClientProvider
                "DEBUG");
       ProcessBuilder pb = new ProcessBuilder(command);
       log.info("Command: " + StringUtils.join(pb.command(), " "));
-      pb.environment().put(SliderKeys.PYTHONPATH,
+      pb.environment().put(PYTHONPATH,
                            agentPkgDir.getAbsolutePath()
                            + File.separator + "slider-agent" + File.pathSeparator
                            + agentPkgDir.getAbsolutePath()
                            + File.separator + "slider-agent/jinja2");
-      log.info("{}={}", SliderKeys.PYTHONPATH, pb.environment().get(SliderKeys.PYTHONPATH));
+      log.info("{}={}", PYTHONPATH, pb.environment().get(PYTHONPATH));
 
       Process proc = pb.start();
       InputStream stderr = proc.getErrorStream();
@@ -555,8 +563,8 @@ public class AgentClientProvider extends AbstractClientProvider
 
   private void expandAgentTar(File agentPkgDir) throws IOException {
     String libDirProp =
-        System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
-    File tarFile = new File(libDirProp, SliderKeys.AGENT_TAR);
+        System.getProperty(PROPERTY_LIB_DIR);
+    File tarFile = new File(libDirProp, AGENT_TAR);
     expandTar(tarFile, agentPkgDir);
   }
 
@@ -620,7 +628,7 @@ public class AgentClientProvider extends AbstractClientProvider
                                       String name) throws SliderException {
     try {
       JSONObject pkgList = new JSONObject();
-      pkgList.put(AgentKeys.PACKAGE_LIST,
+      pkgList.put(PACKAGE_LIST,
                   AgentProviderService.getPackageListFromApplication(metainfo.getApplication()));
       JSONObject obj = new JSONObject();
       obj.put("hostLevelParams", pkgList);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ba59bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 9ea984c..c4228e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -23,6 +23,7 @@ package org.apache.slider.providers.agent;
  */
 public interface AgentKeys {
 
+  String AGENT_TAR = "slider-agent.tar.gz";
   String PROVIDER_AGENT = "agent";
   String ROLE_NODE = "echo";
 
@@ -76,23 +77,13 @@ public interface AgentKeys {
   String AGENT_CONF = "agent.conf";
   String ADDON_FOR_ALL_COMPONENTS = "ALL";
 
-  String APP_RESOURCES = "application.resources";
-  String APP_RESOURCES_DIR = "app/resources";
-
-  String APP_CONF_DIR = "app/conf";
-
   String AGENT_INSTALL_DIR = "infra/agent";
   String APP_DEFINITION_DIR = "app/definition";
   String ADDON_DEFINITION_DIR = "addon/definition";
   String AGENT_CONFIG_FILE = "infra/conf/agent.ini";
   String AGENT_VERSION_FILE = "infra/version";
-  String APP_PACKAGES_DIR = "app/packages";
-  String PER_COMPONENT = "per.component";
-  String PER_GROUP = "per.group";
 
-  String JAVA_HOME = "java_home";
   String PACKAGE_LIST = "package_list";
-  String SYSTEM_CONFIGS = "system_configs";
   String WAIT_HEARTBEAT = "wait.heartbeat";
   String PYTHON_EXE = "python";
   String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
@@ -104,7 +95,6 @@ public interface AgentKeys {
   String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
   String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
   String TEST_RELAX_VERIFICATION = "test.relax.validation";
-  String AM_CONFIG_GENERATION = "am.config.generation";
 
   String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ba59bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 2ab5c6f..499812e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -21,20 +21,11 @@ package org.apache.slider.providers.agent;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
 import org.apache.hadoop.registry.client.types.Endpoint;
 import org.apache.hadoop.registry.client.types.ProtocolTypes;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.Container;
@@ -55,7 +46,6 @@ import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.core.conf.AggregateConf;
 import org.apache.slider.core.conf.ConfTreeOperations;
 import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
 import org.apache.slider.core.exceptions.BadConfigException;
 import org.apache.slider.core.exceptions.NoSuchNodeException;
 import org.apache.slider.core.exceptions.SliderException;
@@ -65,7 +55,6 @@ import org.apache.slider.core.registry.docstore.ConfigFormat;
 import org.apache.slider.core.registry.docstore.ConfigUtils;
 import org.apache.slider.core.registry.docstore.ExportEntry;
 import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
 import org.apache.slider.core.registry.docstore.PublishedExports;
 import org.apache.slider.core.registry.info.CustomRegistryConstants;
 import org.apache.slider.providers.AbstractProviderService;
@@ -107,20 +96,15 @@ import org.apache.slider.server.appmaster.web.rest.agent.Register;
 import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
 import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
 import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand;
-import org.apache.slider.server.services.security.CertificateManager;
-import org.apache.slider.server.services.security.SecurityStore;
-import org.apache.slider.server.services.security.StoresGenerator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -157,10 +141,6 @@ public class AgentProviderService extends AbstractProviderService implements
   private static final String LABEL_MAKER = "___";
   private static final String CONTAINER_ID = "container_id";
   private static final String GLOBAL_CONFIG_TAG = "global";
-  private static final String LOG_FOLDERS_TAG = "LogFolders";
-  private static final String HOST_FOLDER_FORMAT = "%s:%s";
-  private static final String CONTAINER_LOGS_TAG = "container_log_dirs";
-  private static final String CONTAINER_PWDS_TAG = "container_work_dirs";
   private static final String COMPONENT_TAG = "component";
   private static final String APPLICATION_TAG = "application";
   private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
@@ -249,13 +229,6 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   @Override
-  public Configuration loadProviderConfigurationInformation(File confDir) throws
-      BadCommandArgumentsException,
-      IOException {
-    return new Configuration(false);
-  }
-
-  @Override
   public void validateInstanceDefinition(AggregateConf instanceDefinition)
       throws
       SliderException {
@@ -265,7 +238,7 @@ public class AgentProviderService extends AbstractProviderService implements
         instanceDefinition.getResourceOperations();
 
     Set<String> names = resources.getComponentNames();
-    names.remove(SliderKeys.COMPONENT_AM);
+    names.remove(COMPONENT_AM);
     for (String name : names) {
       Component componentDef = getApplicationComponent(name);
       if (componentDef == null) {
@@ -350,9 +323,9 @@ public class AgentProviderService extends AbstractProviderService implements
 
           // build a map from component to metainfo
           String addonAppDefString = instanceDefinition.getAppConfOperations()
-              .getGlobalOptions().getOption(AgentKeys.ADDONS, null);
+              .getGlobalOptions().getOption(ADDONS, null);
           if (component != null) {
-            addonAppDefString = component.getOption(AgentKeys.ADDONS, addonAppDefString);
+            addonAppDefString = component.getOption(ADDONS, addonAppDefString);
           }
           log.debug("All addon appdefs: {}", addonAppDefString);
           if (addonAppDefString != null) {
@@ -415,6 +388,7 @@ public class AgentProviderService extends AbstractProviderService implements
     if (isYarnDockerContainer(roleGroup)) {
       launcher.setYarnDockerMode(true);
       launcher.setDockerImage(getConfigFromMetaInfo(roleGroup, "image"));
+      launcher.setDockerNetwork(getConfigFromMetaInfo(roleGroup, "network"));
       launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(roleGroup, "runPriviledgedContainer"));
       launcher
           .setYarnContainerMountPoints(getConfigFromMetaInfoWithAppConfigOverriding(
@@ -423,7 +397,9 @@ public class AgentProviderService extends AbstractProviderService implements
 
     // Set the environment
     launcher.putEnv(SliderUtils.buildEnvMap(appComponent,
-        getStandardTokenMap(getAmState().getAppConfSnapshot(), roleName, roleGroup)));
+        providerUtils.getStandardTokenMap(getAmState().getAppConfSnapshot(),
+            getAmState().getInternalsSnapshot(), roleName, roleGroup,
+            getClusterName())));
 
     String workDir = ApplicationConstants.Environment.PWD.$();
     launcher.setEnv("AGENT_WORK_ROOT", workDir);
@@ -444,17 +420,17 @@ public class AgentProviderService extends AbstractProviderService implements
     //local resources
 
     // TODO: Should agent need to support App Home
-    String scriptPath = new File(AgentKeys.AGENT_MAIN_SCRIPT_ROOT, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
+    String scriptPath = new File(AGENT_MAIN_SCRIPT_ROOT, AGENT_MAIN_SCRIPT).getPath();
     String appHome = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
+        getGlobalOptions().get(PACKAGE_PATH);
     if (SliderUtils.isSet(appHome)) {
-      scriptPath = new File(appHome, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
+      scriptPath = new File(appHome, AGENT_MAIN_SCRIPT).getPath();
     }
 
     // set PYTHONPATH
     List<String> pythonPaths = new ArrayList<String>();
-    pythonPaths.add(AgentKeys.AGENT_MAIN_SCRIPT_ROOT);
-    pythonPaths.add(AgentKeys.AGENT_JINJA2_ROOT);
+    pythonPaths.add(AGENT_MAIN_SCRIPT_ROOT);
+    pythonPaths.add(AGENT_JINJA2_ROOT);
     String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths);
     launcher.setEnv(PYTHONPATH, pythonPath);
     log.info("PYTHONPATH set to {}", pythonPath);
@@ -466,21 +442,21 @@ public class AgentProviderService extends AbstractProviderService implements
       agentImagePath =
           new Path(new Path(new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR),
                                      container.getId().getApplicationAttemptId().getApplicationId().toString()),
-                            AgentKeys.PROVIDER_AGENT),
-                   SliderKeys.AGENT_TAR);
+                            PROVIDER_AGENT),
+                   AGENT_TAR);
     } else {
        agentImagePath = new Path(agentImage);
     }
 
     if (fileSystem.getFileSystem().exists(agentImagePath)) {
       LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
-      launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
+      launcher.addLocalResource(AGENT_INSTALL_DIR, agentImageRes);
     } else {
       String msg =
           String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString());
       MapOperations compOps = appComponent;
       boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps.
-          getOptionBool(AgentKeys.TEST_RELAX_VERIFICATION, false)) : false;
+          getOptionBool(TEST_RELAX_VERIFICATION, false)) : false;
       log.error(msg);
 
       if (!relaxVerificationForTest) {
@@ -492,7 +468,7 @@ public class AgentProviderService extends AbstractProviderService implements
     LocalResource appDefRes = fileSystem.createAmResource(
         fileSystem.getFileSystem().resolvePath(new Path(appDef)),
         LocalResourceType.ARCHIVE);
-    launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes);
+    launcher.addLocalResource(APP_DEFINITION_DIR, appDefRes);
 
     for (Package pkg : getMetaInfo(roleGroup).getApplication().getPackages()) {
       Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
@@ -511,57 +487,61 @@ public class AgentProviderService extends AbstractProviderService implements
       }
       LocalResource packageResource = fileSystem.createAmResource(
           pkgPath, type);
-      launcher.addLocalResource(AgentKeys.APP_PACKAGES_DIR, packageResource);
+      launcher.addLocalResource(APP_PACKAGES_DIR, packageResource);
     }
 
     String agentConf = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
+        getGlobalOptions().getOption(AGENT_CONF, "");
     if (SliderUtils.isSet(agentConf)) {
       LocalResource agentConfRes = fileSystem.createAmResource(fileSystem
                                                                    .getFileSystem().resolvePath(new Path(agentConf)),
                                                                LocalResourceType.FILE);
-      launcher.addLocalResource(AgentKeys.AGENT_CONFIG_FILE, agentConfRes);
+      launcher.addLocalResource(AGENT_CONFIG_FILE, agentConfRes);
     }
 
     String agentVer = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AgentKeys.AGENT_VERSION, null);
+        getGlobalOptions().getOption(AGENT_VERSION, null);
     if (agentVer != null) {
       LocalResource agentVerRes = fileSystem.createAmResource(
           fileSystem.getFileSystem().resolvePath(new Path(agentVer)),
           LocalResourceType.FILE);
-      launcher.addLocalResource(AgentKeys.AGENT_VERSION_FILE, agentVerRes);
+      launcher.addLocalResource(AGENT_VERSION_FILE, agentVerRes);
     }
 
     if (SliderUtils.isHadoopClusterSecure(getConfig())) {
-      localizeServiceKeytabs(launcher, instanceDefinition, fileSystem);
+      providerUtils.localizeServiceKeytabs(launcher, instanceDefinition,
+          fileSystem, getClusterName());
     }
 
     MapOperations amComponent = instanceDefinition.
-        getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
-    boolean twoWayEnabled = amComponent != null ? Boolean.valueOf(amComponent.
-        getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) : false;
-    if (twoWayEnabled) {
-      localizeContainerSSLResources(launcher, container, fileSystem);
+        getAppConfOperations().getComponent(COMPONENT_AM);
+    if (providerUtils.hasTwoWaySSLEnabled(amComponent)) {
+      providerUtils.localizeContainerSSLResources(launcher, container,
+          fileSystem, getClusterName());
     }
 
-    MapOperations compOps = appComponent;
-    if (areStoresRequested(compOps)) {
-      localizeContainerSecurityStores(launcher, container, roleName, fileSystem,
-                                      instanceDefinition, compOps);
+    if (providerUtils.areStoresRequested(appComponent)) {
+      providerUtils.localizeContainerSecurityStores(launcher, container,
+          roleName, fileSystem, instanceDefinition, appComponent,
+          getClusterName());
     }
 
     //add the configuration resources
     launcher.addLocalResources(fileSystem.submitDirectory(
         generatedConfPath,
-        SliderKeys.PROPAGATED_CONF_DIR_NAME));
+        PROPAGATED_CONF_DIR_NAME));
 
-    if (appComponent.getOptionBool(AgentKeys.AM_CONFIG_GENERATION, false)) {
+    if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) {
       // build and localize configuration files
       Map<String, Map<String, String>> configurations =
           buildCommandConfigurations(instanceDefinition.getAppConfOperations(),
+              instanceDefinition.getInternalOperations(),
               container.getId().toString(), roleName, roleGroup);
-      localizeConfigFiles(launcher, roleName, roleGroup, getMetaInfo(roleGroup),
-          configurations, launcher.getEnv(), fileSystem);
+      for (ConfigFile configFile : getMetaInfo(roleGroup)
+          .getComponentConfigFiles(roleGroup)) {
+        localizeConfigFile(launcher, roleName, roleGroup, configFile,
+            configurations, launcher.getEnv(), fileSystem);
+      }
     }
 
     String label = getContainerLabel(container, roleName, roleGroup);
@@ -569,7 +549,7 @@ public class AgentProviderService extends AbstractProviderService implements
 
     String pythonExec = instanceDefinition.getAppConfOperations()
         .getGlobalOptions().getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
-                                      AgentKeys.PYTHON_EXE);
+                                      PYTHON_EXE);
 
     operation.add(pythonExec);
 
@@ -587,13 +567,13 @@ public class AgentProviderService extends AbstractProviderService implements
     }
 
     operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
-        + AgentKeys.AGENT_OUT_FILE + " 2>&1");
+        + AGENT_OUT_FILE + " 2>&1");
 
     launcher.addCommand(operation.build());
 
     // localize addon package
     String addonAppDefString = instanceDefinition.getAppConfOperations()
-        .getGlobalOptions().getOption(AgentKeys.ADDONS, null);
+        .getGlobalOptions().getOption(ADDONS, null);
     log.debug("All addon appdefs: {}", addonAppDefString);
     if (addonAppDefString != null) {
       Scanner scanner = new Scanner(addonAppDefString).useDelimiter(",");
@@ -605,7 +585,7 @@ public class AgentProviderService extends AbstractProviderService implements
         LocalResource addonPkgRes = fileSystem.createAmResource(
             fileSystem.getFileSystem().resolvePath(new Path(addonAppDefPath)),
             LocalResourceType.ARCHIVE);
-        launcher.addLocalResource(AgentKeys.ADDON_DEFINITION_DIR + "/" + addonAppDef, addonPkgRes);
+        launcher.addLocalResource(ADDON_DEFINITION_DIR + "/" + addonAppDef, addonPkgRes);
       }
       log.debug("Metainfo map for master and addon: {}",
           packageMetainfo.toString());
@@ -613,7 +593,7 @@ public class AgentProviderService extends AbstractProviderService implements
 
     // Additional files to localize in addition to the application def
     String appResourcesString = instanceDefinition.getAppConfOperations()
-        .getGlobalOptions().getOption(AgentKeys.APP_RESOURCES, null);
+        .getGlobalOptions().getOption(APP_RESOURCES, null);
     log.info("Configuration value for extra resources to localize: {}", appResourcesString);
     if (null != appResourcesString) {
       try (Scanner scanner = new Scanner(appResourcesString).useDelimiter(",")) {
@@ -623,7 +603,7 @@ public class AgentProviderService extends AbstractProviderService implements
           LocalResource extraResource = fileSystem.createAmResource(
               fileSystem.getFileSystem().resolvePath(resourcePath),
               LocalResourceType.FILE);
-          String destination = AgentKeys.APP_RESOURCES_DIR + "/" + resourcePath.getName();
+          String destination = APP_RESOURCES_DIR + "/" + resourcePath.getName();
           log.info("Localizing {} to {}", resourcePath, destination);
           // TODO Can we try harder to avoid collisions?
           launcher.addLocalResource(destination, extraResource);
@@ -641,7 +621,7 @@ public class AgentProviderService extends AbstractProviderService implements
         log.debug("Current component: {} component in metainfo: {}", roleName,
             comp.getName());
         if (comp.getName().equals(roleGroup)
-            || comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)) {
+            || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) {
           pkgStatuses.put(appPkg.getApplicationPackage().getName(), State.INIT);
         }
       }
@@ -658,253 +638,18 @@ public class AgentProviderService extends AbstractProviderService implements
                                    pkgStatuses));
   }
 
-  private void localizeContainerSecurityStores(ContainerLauncher launcher,
-                                               Container container,
-                                               String role,
-                                               SliderFileSystem fileSystem,
-                                               AggregateConf instanceDefinition,
-                                               MapOperations compOps)
-      throws SliderException, IOException {
-    // substitute CLUSTER_NAME into credentials
-    Map<String,List<String>> newcred = new HashMap<>();
-    for (Entry<String,List<String>> entry : instanceDefinition.getAppConf().credentials.entrySet()) {
-      List<String> resultList = new ArrayList<>();
-      for (String v : entry.getValue()) {
-        resultList.add(v.replaceAll(Pattern.quote("${CLUSTER_NAME}"),
-            clusterName).replaceAll(Pattern.quote("${CLUSTER}"),
-            clusterName));
-      }
-      newcred.put(entry.getKey().replaceAll(Pattern.quote("${CLUSTER_NAME}"),
-          clusterName).replaceAll(Pattern.quote("${CLUSTER}"),
-          clusterName),
-          resultList);
-    }
-    instanceDefinition.getAppConf().credentials = newcred;
-
-    // generate and localize security stores
-    SecurityStore[] stores = generateSecurityStores(container, role,
-                                                    instanceDefinition, compOps);
-    for (SecurityStore store : stores) {
-      LocalResource keystoreResource = fileSystem.createAmResource(
-          uploadSecurityResource(store.getFile(), fileSystem), LocalResourceType.FILE);
-      launcher.addLocalResource(String.format("secstores/%s-%s.p12",
-                                              store.getType(), role),
-                                keystoreResource);
-    }
-  }
-
-  private SecurityStore[] generateSecurityStores(Container container,
-                                                 String role,
-                                                 AggregateConf instanceDefinition,
-                                                 MapOperations compOps)
-      throws SliderException, IOException {
-    return StoresGenerator.generateSecurityStores(container.getNodeId().getHost(),
-                                           container.getId().toString(), role,
-                                           instanceDefinition, compOps);
-  }
-
-  private boolean areStoresRequested(MapOperations compOps) {
-    return compOps != null ? compOps.
-        getOptionBool(SliderKeys.COMP_STORES_REQUIRED_KEY, false) : false;
-  }
-
-  private void localizeContainerSSLResources(ContainerLauncher launcher,
-                                             Container container,
-                                             SliderFileSystem fileSystem)
-      throws SliderException {
-    try {
-      // localize server cert
-      Path certsDir = fileSystem.buildClusterSecurityDirPath(getClusterName());
-      LocalResource certResource = fileSystem.createAmResource(
-          new Path(certsDir, SliderKeys.CRT_FILE_NAME),
-            LocalResourceType.FILE);
-      launcher.addLocalResource(AgentKeys.CERT_FILE_LOCALIZATION_PATH,
-                                certResource);
-
-      // generate and localize agent cert
-      CertificateManager certMgr = new CertificateManager();
-      String hostname = container.getNodeId().getHost();
-      String containerId = container.getId().toString();
-      certMgr.generateContainerCertificate(hostname, containerId);
-      LocalResource agentCertResource = fileSystem.createAmResource(
-          uploadSecurityResource(
-            CertificateManager.getAgentCertficateFilePath(containerId),
-            fileSystem), LocalResourceType.FILE);
-      // still using hostname as file name on the agent side, but the files
-      // do end up under the specific container's file space
-      launcher.addLocalResource(AgentKeys.INFRA_RUN_SECURITY_DIR + hostname +
-                                ".crt", agentCertResource);
-      LocalResource agentKeyResource = fileSystem.createAmResource(
-          uploadSecurityResource(
-              CertificateManager.getAgentKeyFilePath(containerId), fileSystem),
-            LocalResourceType.FILE);
-      launcher.addLocalResource(AgentKeys.INFRA_RUN_SECURITY_DIR + hostname +
-                                ".key", agentKeyResource);
-
-    } catch (Exception e) {
-      throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, e,
-          "Unable to localize certificates.  Two-way SSL cannot be enabled");
-    }
-  }
-
-  private Path uploadSecurityResource(File resource, SliderFileSystem fileSystem)
-      throws IOException {
-    Path certsDir = fileSystem.buildClusterSecurityDirPath(getClusterName());
-    return uploadResource(resource, fileSystem, certsDir);
-  }
-
-  private Path uploadResource(File resource, SliderFileSystem fileSystem,
-      String roleName) throws IOException {
-    Path dir;
-    if (roleName == null) {
-      dir = fileSystem.buildClusterResourcePath(getClusterName());
-    } else {
-      dir = fileSystem.buildClusterResourcePath(getClusterName(), roleName);
-    }
-    return uploadResource(resource, fileSystem, dir);
-  }
-
-  private static synchronized Path uploadResource(File resource,
-      SliderFileSystem fileSystem, Path parentDir) throws IOException {
-    if (!fileSystem.getFileSystem().exists(parentDir)) {
-      fileSystem.getFileSystem().mkdirs(parentDir,
-        new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
-    }
-    Path destPath = new Path(parentDir, resource.getName());
-    if (!fileSystem.getFileSystem().exists(destPath)) {
-      FSDataOutputStream os = null;
-      try {
-        os = fileSystem.getFileSystem().create(destPath);
-        byte[] contents = FileUtils.readFileToByteArray(resource);
-        os.write(contents, 0, contents.length);
-        os.flush();
-      } finally {
-        IOUtils.closeStream(os);
-      }
-      log.info("Uploaded {} to localization path {}", resource, destPath);
-    } else {
-      log.info("Resource {} already existed at localization path {}", resource,
-          destPath);
-    }
-
-    while (!fileSystem.getFileSystem().exists(destPath)) {
-      try {
-        Thread.sleep(500);
-      } catch (InterruptedException e) {
-        // ignore
-      }
-    }
-
-    fileSystem.getFileSystem().setPermission(destPath,
-      new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
-
-    return destPath;
-  }
-
-  private void localizeServiceKeytabs(ContainerLauncher launcher,
-                                      AggregateConf instanceDefinition,
-                                      SliderFileSystem fileSystem)
-      throws IOException {
-    String keytabPathOnHost = instanceDefinition.getAppConfOperations()
-        .getComponent(SliderKeys.COMPONENT_AM).get(
-            SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
-    if (SliderUtils.isUnset(keytabPathOnHost)) {
-      String amKeytabName = instanceDefinition.getAppConfOperations()
-          .getComponent(SliderKeys.COMPONENT_AM).get(
-              SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
-      String keytabDir = instanceDefinition.getAppConfOperations()
-          .getComponent(SliderKeys.COMPONENT_AM).get(
-              SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
-      // we need to localize the keytab files in the directory
-      Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
-                                                      getClusterName());
-      boolean serviceKeytabsDeployed = false;
-      if (fileSystem.getFileSystem().exists(keytabDirPath)) {
-        FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath);
-        LocalResource keytabRes;
-        for (FileStatus keytab : keytabs) {
-          if (!amKeytabName.equals(keytab.getPath().getName())
-              && keytab.getPath().getName().endsWith(".keytab")) {
-            serviceKeytabsDeployed = true;
-            log.info("Localizing keytab {}", keytab.getPath().getName());
-            keytabRes = fileSystem.createAmResource(keytab.getPath(),
-              LocalResourceType.FILE);
-            launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" +
-                                    keytab.getPath().getName(),
-                                    keytabRes);
-          }
-        }
-      }
-      if (!serviceKeytabsDeployed) {
-        log.warn("No service keytabs for the application have been localized.  "
-                 + "If the application requires keytabs for secure operation, "
-                 + "please ensure that the required keytabs have been uploaded "
-                 + "to the folder {}", keytabDirPath);
-      }
-    }
-  }
-
-  private void createConfigFile(SliderFileSystem fileSystem, File file,
-      ConfigFile configFile, Map<String, String> config)
-      throws IOException {
-    ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
-    log.info("Writing {} file {}", configFormat, file);
-
-    ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
-        fileSystem, getClusterName(), file.getName());
-    PublishedConfiguration publishedConfiguration =
-        new PublishedConfiguration(configFile.getDictionaryName(),
-            config.entrySet());
-    PublishedConfigurationOutputter configurationOutputter =
-      PublishedConfigurationOutputter.createOutputter(configFormat,
-          publishedConfiguration);
-    configurationOutputter.save(file);
-  }
-
   @VisibleForTesting
-  protected void localizeConfigFiles(ContainerLauncher launcher,
+  protected void localizeConfigFile(ContainerLauncher launcher,
                                      String roleName, String roleGroup,
-                                     Metainfo metainfo,
+                                     ConfigFile configFile,
                                      Map<String, Map<String, String>> configs,
                                      MapOperations env,
                                      SliderFileSystem fileSystem)
       throws IOException {
-    for (ConfigFile configFile : metainfo.getComponentConfigFiles(roleGroup)) {
-      Map<String, String> config = ConfigUtils.replacePropsInConfig(
-          configs.get(configFile.getDictionaryName()), env.options);
-      String fileName = ConfigUtils.replaceProps(config,
-          configFile.getFileName());
-      File localFile = new File(SliderKeys.RESOURCE_DIR);
-      if (!localFile.exists()) {
-        localFile.mkdir();
-      }
-      localFile = new File(localFile, new File(fileName).getName());
-
-      String folder = null;
-      if ("true".equals(config.get(PER_COMPONENT))) {
-        folder = roleName;
-      } else if ("true".equals(config.get(PER_GROUP))) {
-        folder = roleGroup;
-      }
-
-      log.info("Localizing {} configs to config file {} (destination {}) " +
-          "based on {} configs", config.size(), localFile, fileName,
-          configFile.getDictionaryName());
-      createConfigFile(fileSystem, localFile, configFile, config);
-      Path destPath = uploadResource(localFile, fileSystem, folder);
-      LocalResource configResource = fileSystem.createAmResource(destPath,
-          LocalResourceType.FILE);
-
-      File destFile = new File(fileName);
-      if (destFile.isAbsolute()) {
-        launcher.addLocalResource(
-            SliderKeys.RESOURCE_DIR + "/" + destFile.getName(),
-            configResource, fileName);
-      } else {
-        launcher.addLocalResource(AgentKeys.APP_CONF_DIR + "/" + fileName,
-            configResource);
-      }
-    }
+    ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
+    providerUtils.localizeConfigFile(launcher, roleName, roleGroup,
+        configFile.getDictionaryName(), configFormat, configFile.getFileName(),
+        configs, env, fileSystem, getClusterName());
   }
 
   /**
@@ -1053,7 +798,6 @@ public class AgentProviderService extends AbstractProviderService implements
       doUpgrade = true;
     }
 
-    StateAccessForProviders accessor = getAmState();
     CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup);
     List<ComponentCommand> commands = getApplicationComponent(roleGroup).getCommands();
 
@@ -1089,45 +833,10 @@ public class AgentProviderService extends AbstractProviderService implements
     List<ComponentStatus> statuses = heartBeat.getComponentStatus();
     if (statuses != null && !statuses.isEmpty()) {
       log.info("status from agent: " + statuses.toString());
-      try {
-        for(ComponentStatus status : statuses){
-          RoleInstance role = null;
-          if(status.getIp() != null && !status.getIp().isEmpty()){
-            role = amState.getOwnedContainer(containerId);
-            role.ip = status.getIp();
-          }
-          if(status.getHostname() != null && !status.getHostname().isEmpty()){
-            role = amState.getOwnedContainer(containerId);
-            role.hostname = status.getHostname();
-          }
-          if (role != null) {
-            // create an updated service record (including hostname and ip) and publish...
-            ServiceRecord record = new ServiceRecord();
-            record.set(YarnRegistryAttributes.YARN_ID, containerId);
-            record.description = roleName;
-            record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
-                       PersistencePolicies.CONTAINER);
-            // TODO:  switch record attributes to use constants from YarnRegistryAttributes
-            // when it's been updated.
-            if (role.ip != null) {
-              record.set("yarn:ip", role.ip);
-            }
-            if (role.hostname != null) {
-              record.set("yarn:hostname", role.hostname);
-            }
-            yarnRegistry.putComponent(
-                RegistryPathUtils.encodeYarnID(containerId), record);
-
-          }
-        }
-
-
-      } catch (NoSuchNodeException e) {
-        // ignore - there is nothing to do if we don't find a container
-        log.warn("Owned container {} not found - {}", containerId, e);
-      } catch (IOException e) {
-        log.warn("Error updating container {} service record in registry",
-                 containerId, e);
+      for(ComponentStatus status : statuses){
+        providerUtils.updateServiceRecord(getAmState(), yarnRegistry,
+            containerId, roleName, Collections.singletonList(status.getIp()),
+            status.getHostname());
       }
     }
 
@@ -1179,8 +888,8 @@ public class AgentProviderService extends AbstractProviderService implements
       }
     }
 
-    int waitForCount = accessor.getInstanceDefinitionSnapshot().
-        getAppConfOperations().getComponentOptInt(roleGroup, AgentKeys.WAIT_HEARTBEAT, 0);
+    int waitForCount = getAmState().getInstanceDefinitionSnapshot().
+        getAppConfOperations().getComponentOptInt(roleGroup, WAIT_HEARTBEAT, 0);
 
     if (id < waitForCount) {
       log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id);
@@ -1223,7 +932,7 @@ public class AgentProviderService extends AbstractProviderService implements
             log.debug("Addon component: {} pkg: {} script: {}", comp.getName(),
                 nextPkgToInstall, comp.getCommandScript().getScript());
             if (comp.getName().equals(roleGroup)
-                || comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)) {
+                || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) {
               scriptPath = comp.getCommandScript().getScript();
               if (scriptPath != null) {
                 addInstallCommand(roleName, roleGroup, containerId, response,
@@ -1406,15 +1115,12 @@ public class AgentProviderService extends AbstractProviderService implements
     return details;
   }
 
-  @Override
   public void applyInitialRegistryDefinitions(URL amWebURI,
       URL agentOpsURI,
       URL agentStatusURI,
       ServiceRecord serviceRecord)
     throws IOException {
     super.applyInitialRegistryDefinitions(amWebURI,
-                                          agentOpsURI,
-                                          agentStatusURI,
                                           serviceRecord);
 
     try {
@@ -1450,20 +1156,17 @@ public class AgentProviderService extends AbstractProviderService implements
     ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
     MapOperations clientOperations = appConf.getOrAddComponent(client.getName());
     appConf.resolve();
-    if (!clientOperations.getOptionBool(AgentKeys.AM_CONFIG_GENERATION,
+    if (!clientOperations.getOptionBool(AM_CONFIG_GENERATION,
         false)) {
       log.info("AM config generation is false, not publishing client configs");
       return;
     }
 
     // build and localize configuration files
-    Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
-    Map<String, String> tokens = null;
-    try {
-      tokens = getStandardTokenMap(appConf, client.getName(), client.getName());
-    } catch (SliderException e) {
-      throw new IOException(e);
-    }
+    Map<String, Map<String, String>> configurations = new TreeMap<>();
+    Map<String, String> tokens = providerUtils.getStandardTokenMap(appConf,
+        getAmState().getInternalsSnapshot(), client.getName(),
+        client.getName(), getClusterName());
 
     for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) {
       addNamedConfiguration(configFile.getDictionaryName(),
@@ -1561,7 +1264,7 @@ public class AgentProviderService extends AbstractProviderService implements
    */
   private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
     String hbMonitorInterval = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AgentKeys.HEARTBEAT_MONITOR_INTERVAL,
+        getGlobalOptions().getOption(HEARTBEAT_MONITOR_INTERVAL,
                                      Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
     try {
       setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
@@ -1581,7 +1284,7 @@ public class AgentProviderService extends AbstractProviderService implements
    */
   private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
     String launchParameterStr = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AgentKeys.AGENT_INSTANCE_DEBUG_DATA, "");
+        getGlobalOptions().getOption(AGENT_INSTANCE_DEBUG_DATA, "");
     agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
   }
 
@@ -1699,20 +1402,11 @@ public class AgentProviderService extends AbstractProviderService implements
     return clusterName;
   }
 
-  /**
-   * Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site
-   *
-   * @param name
-   * @param description
-   * @param entries
-   */
+  @VisibleForTesting
   protected void publishApplicationInstanceData(String name, String description,
                                                 Iterable<Map.Entry<String, String>> entries) {
-    PublishedConfiguration pubconf = new PublishedConfiguration();
-    pubconf.description = description;
-    pubconf.putValues(entries);
-    log.info("publishing {}", pubconf);
-    getAmState().getPublishedSliderConfigurations().put(name, pubconf);
+    providerUtils.publishApplicationInstanceData(name, description, entries,
+        getAmState());
   }
 
   /**
@@ -1771,72 +1465,14 @@ public class AgentProviderService extends AbstractProviderService implements
     return stats;
   }
 
-
-  /**
-   * Format the folder locations and publish in the registry service
-   *
-   * @param folders
-   * @param containerId
-   * @param hostFqdn
-   * @param componentName
-   */
+  @VisibleForTesting
   protected void publishFolderPaths(
       Map<String, String> folders, String containerId, String componentName, String hostFqdn) {
-    Date now = new Date();
-    for (Map.Entry<String, String> entry : folders.entrySet()) {
-      ExportEntry exportEntry = new ExportEntry();
-      exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn, entry.getValue()));
-      exportEntry.setContainerId(containerId);
-      exportEntry.setLevel(COMPONENT_TAG);
-      exportEntry.setTag(componentName);
-      exportEntry.setUpdatedTime(now.toString());
-      if (entry.getKey().equals("AGENT_LOG_ROOT")) {
-        synchronized (logFolderExports) {
-          getLogFolderExports().put(containerId, exportEntry);
-        }
-      } else {
-        synchronized (workFolderExports) {
-          getWorkFolderExports().put(containerId, exportEntry);
-        }
-      }
-      log.info("Updating log and pwd folders for container {}", containerId);
-    }
-
-    PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG);
-    exports.setUpdated(now.getTime());
-    synchronized (logFolderExports) {
-      updateExportsFromList(exports, getLogFolderExports());
-    }
-    getAmState().getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports);
-
-    exports = new PublishedExports(CONTAINER_PWDS_TAG);
-    exports.setUpdated(now.getTime());
-    synchronized (workFolderExports) {
-      updateExportsFromList(exports, getWorkFolderExports());
-    }
-    getAmState().getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports);
+    providerUtils.publishFolderPaths(folders, containerId, componentName, hostFqdn,
+        getAmState(), getLogFolderExports(), getWorkFolderExports());
   }
 
   /**
-   * Update the export data from the map
-   * @param exports
-   * @param folderExports
-   */
-  private void updateExportsFromList(PublishedExports exports, Map<String, ExportEntry> folderExports) {
-    Map<String, List<ExportEntry>> perComponentList = new HashMap<String, List<ExportEntry>>();
-    for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet())
-    {
-      String componentName = logEntry.getValue().getTag();
-      if (!perComponentList.containsKey(componentName)) {
-        perComponentList.put(componentName, new ArrayList<ExportEntry>());
-      }
-      perComponentList.get(componentName).add(logEntry.getValue());
-    }
-    exports.putValues(perComponentList.entrySet());
-  }
-
-
-  /**
    * Process return status for component instances
    *
    * @param heartBeat
@@ -1855,7 +1491,7 @@ public class AgentProviderService extends AbstractProviderService implements
 
           if ((!canAnyMasterPublishConfig(componentGroup) || canPublishConfig(componentGroup)) &&
               !getAmState().getAppConfSnapshot().getComponentOptBool(
-                  componentGroup, AgentKeys.AM_CONFIG_GENERATION, false)) {
+                  componentGroup, AM_CONFIG_GENERATION, false)) {
             // If no Master can explicitly publish then publish if its a master
             // Otherwise, wait till the master that can publish is ready
 
@@ -1899,7 +1535,8 @@ public class AgentProviderService extends AbstractProviderService implements
             // publish export groups if any
             Map<String, String> replaceTokens = new HashMap<String, String>();
             for (Map.Entry<String, Map<String, ClusterNode>> entry : getRoleClusterNodeMapping().entrySet()) {
-              String hostName = getHostsList(entry.getValue().values(), true).iterator().next();
+              String hostName = providerUtils.getHostsList(
+                  entry.getValue().values(), true).iterator().next();
               replaceTokens.put(String.format(hostKeyFormat, entry.getKey().toUpperCase(Locale.ENGLISH)), hostName);
             }
 
@@ -1967,28 +1604,24 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   private void publishModifiedExportGroups(Set<String> modifiedGroups) {
-    for (String groupName : modifiedGroups) {
-      Map<String, List<ExportEntry>> entries = this.exportGroups.get(groupName);
-
+    for (String roleGroup : modifiedGroups) {
+      Map<String, List<ExportEntry>> entries = this.exportGroups.get(roleGroup);
       // Publish in old format for the time being
       Map<String, String> simpleEntries = new HashMap<String, String>();
-      for (Map.Entry<String, List<ExportEntry>> entry : entries.entrySet()) {
+      for (Entry<String, List<ExportEntry>> entry : entries.entrySet()) {
         List<ExportEntry> exports = entry.getValue();
         if (SliderUtils.isNotEmpty(exports)) {
           // there is no support for multiple exports per name - so extract only the first one
           simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
         }
       }
-      if (!getAmState().getAppConfSnapshot().getComponentOptBool(
-          groupName, AgentKeys.AM_CONFIG_GENERATION, false)) {
-        publishApplicationInstanceData(groupName, groupName,
-            simpleEntries.entrySet());
-      }
+      publishApplicationInstanceData(roleGroup, roleGroup,
+          simpleEntries.entrySet());
 
-      PublishedExports exports = new PublishedExports(groupName);
+      PublishedExports exports = new PublishedExports(roleGroup);
       exports.setUpdated(new Date().getTime());
       exports.putValues(entries.entrySet());
-      getAmState().getPublishedExportsSet().put(groupName, exports);
+      getAmState().getPublishedExportsSet().put(roleGroup, exports);
     }
   }
 
@@ -2310,7 +1943,8 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setHostLevelParams(hostLevelParams);
 
     Map<String, Map<String, String>> configurations =
-        buildCommandConfigurations(appConf, containerId, roleName, roleGroup);
+        buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(),
+            containerId, roleName, roleGroup);
     cmd.setConfigurations(configurations);
     Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
     cmd.setComponentConfigurations(componentConfigurations);
@@ -2359,7 +1993,8 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setHostLevelParams(hostLevelParams);
 
     Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, containerId, roleName, roleGroup);
+        appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
+        roleGroup);
     cmd.setConfigurations(configurations);
     Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
     cmd.setComponentConfigurations(componentConfigurations);
@@ -2522,7 +2157,9 @@ public class AgentProviderService extends AbstractProviderService implements
 
     cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
 
-    Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, roleName, roleGroup);
+    Map<String, Map<String, String>> configurations =
+        buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(),
+            containerId, roleName, roleGroup);
 
     cmd.setConfigurations(configurations);
 
@@ -2557,7 +2194,8 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
 
     Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, containerId, roleName, roleGroup);
+        appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
+        roleGroup);
     Map<String, String> dockerConfig = new HashMap<String, String>();
     String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand");
     if (statusCommand == null) {
@@ -2598,7 +2236,8 @@ public class AgentProviderService extends AbstractProviderService implements
 
     ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
     Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, containerId, roleName, roleGroup);
+        appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
+        roleGroup);
     Map<String, String> dockerConfig = new HashMap<String, String>();
     String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand");
     if (statusCommand == null) {
@@ -2690,7 +2329,8 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setCommandParams(setCommandParameters(startCommand, timeout, true));
     
     Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, containerId, roleName, roleGroup);
+        appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
+        roleGroup);
     Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
     cmd.setComponentConfigurations(componentConfigurations);
     
@@ -2794,13 +2434,6 @@ public class AgentProviderService extends AbstractProviderService implements
           result = container.getNetwork();
         }
         break;
-      case "useNetworkScript":
-        if (container.getUseNetworkScript() == null || container.getUseNetworkScript().isEmpty()) {
-          result = "yes";
-        } else {
-          result = container.getUseNetworkScript();
-        }
-        break;
       case "statusCommand":
         result = container.getStatusCommand();
         break;
@@ -2903,7 +2536,9 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setRoleParams(roleParams);
     cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
 
-    Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, roleName, roleGroup);
+    Map<String, Map<String, String>> configurations =
+        buildCommandConfigurations(appConf, internalsConf, containerId,
+            roleName, roleGroup);
     cmd.setConfigurations(configurations);
     Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
     cmd.setComponentConfigurations(componentConfigurations);
@@ -2957,7 +2592,7 @@ public class AgentProviderService extends AbstractProviderService implements
 
 
     Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
-        appConf, containerId, roleName, roleGroup);
+        appConf, internalsConf, containerId, roleName, roleGroup);
     cmdStop.setConfigurations(configurationsStop);
     response.addExecutionCommand(cmdStop);
   }
@@ -2989,7 +2624,7 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
 
     Map<String, Map<String, String>> configurations = buildCommandConfigurations(
-        appConf, containerId, roleName, roleGroup);
+        appConf, internalsConf, containerId, roleName, roleGroup);
     cmd.setConfigurations(configurations);
     response.addExecutionCommand(cmd);
   }
@@ -3023,7 +2658,7 @@ public class AgentProviderService extends AbstractProviderService implements
     cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true));
 
     Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
-        appConf, containerId, roleName, roleGroup);
+        appConf, internalsConf, containerId, roleName, roleGroup);
     cmdStop.setConfigurations(configurationsStop);
     response.addExecutionCommand(cmdStop);
   }
@@ -3062,12 +2697,13 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   private Map<String, Map<String, String>> buildCommandConfigurations(
-      ConfTreeOperations appConf, String containerId, String roleName, String roleGroup)
+      ConfTreeOperations appConf, ConfTreeOperations internalsConf,
+      String containerId, String roleName, String roleGroup)
       throws SliderException {
 
-    Map<String, Map<String, String>> configurations =
-        new TreeMap<String, Map<String, String>>();
-    Map<String, String> tokens = getStandardTokenMap(appConf, roleName, roleGroup);
+    Map<String, Map<String, String>> configurations = new TreeMap<>();
+    Map<String, String> tokens = providerUtils.getStandardTokenMap(appConf,
+        internalsConf, roleName, roleGroup, getClusterName());
     tokens.put("${CONTAINER_ID}", containerId);
 
     Set<String> configs = new HashSet<String>();
@@ -3090,111 +2726,16 @@ public class AgentProviderService extends AbstractProviderService implements
     return configurations;
   }
 
+  @VisibleForTesting
   protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
-    Map<String, String> allConfigs = new HashMap<String, String>();
-    String lookupFormat = "${@//site/%s/%s}";
-    for (String configType : configurations.keySet()) {
-      Map<String, String> configBucket = configurations.get(configType);
-      for (String configName : configBucket.keySet()) {
-        allConfigs.put(String.format(lookupFormat, configType, configName), configBucket.get(configName));
-      }
-    }
-
-    boolean finished = false;
-    while (!finished) {
-      finished = true;
-      for (Map.Entry<String, String> entry : allConfigs.entrySet()) {
-        String configValue = entry.getValue();
-        for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) {
-          String lookUpValue = lookUpEntry.getValue();
-          if (lookUpValue.contains("${@//site/")) {
-            continue;
-          }
-          String lookUpKey = lookUpEntry.getKey();
-          if (configValue != null && configValue.contains(lookUpKey)) {
-            configValue = configValue.replace(lookUpKey, lookUpValue);
-          }
-        }
-        if (!configValue.equals(entry.getValue())) {
-          finished = false;
-          allConfigs.put(entry.getKey(), configValue);
-        }
-      }
-    }
-
-    for (String configType : configurations.keySet()) {
-      Map<String, String> configBucket = configurations.get(configType);
-      for (Map.Entry<String, String> entry: configBucket.entrySet()) {
-        String configName = entry.getKey();
-        String configValue = entry.getValue();
-        for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) {
-          String lookUpValue = lookUpEntry.getValue();
-          if (lookUpValue.contains("${@//site/")) {
-            continue;
-          }
-          String lookUpKey = lookUpEntry.getKey();
-          if (configValue != null && configValue.contains(lookUpKey)) {
-            configValue = configValue.replace(lookUpKey, lookUpValue);
-          }
-        }
-        configBucket.put(configName, configValue);
-      }
-    }
-  }
-
-  private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf,
-      String componentName, String componentGroup) throws SliderException {
-    Map<String, String> tokens = new HashMap<String, String>();
-    String nnuri = appConf.get("site.fs.defaultFS");
-    tokens.put("${NN_URI}", nnuri);
-    tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
-    tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
-    tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
-    String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX,
-        null);
-    String dataDirSuffix = "";
-    if (prefix == null) {
-      prefix = "";
-    } else {
-      dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix);
-    }
-    tokens.put("${DEFAULT_DATA_DIR}", getAmState()
-        .getInternalsSnapshot()
-        .getGlobalOptions()
-        .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH) + dataDirSuffix);
-    tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME));
-    tokens.put("${COMPONENT_NAME}", componentName);
-    tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase());
-    tokens.put("${COMPONENT_PREFIX}", prefix);
-    tokens.put("${COMPONENT_PREFIX.lc}", prefix.toLowerCase());
-    if (!componentName.equals(componentGroup) && componentName.startsWith(componentGroup)) {
-      tokens.put("${COMPONENT_ID}", componentName.substring(componentGroup.length()));
-    }
-    tokens.put("${CLUSTER_NAME}", getClusterName());
-    tokens.put("${CLUSTER_NAME.lc}", getClusterName().toLowerCase());
-    tokens.put("${APP_NAME}", getClusterName());
-    tokens.put("${APP_NAME.lc}", getClusterName().toLowerCase());
-    tokens.put("${APP_COMPONENT_NAME}", componentName);
-    tokens.put("${APP_COMPONENT_NAME.lc}", componentName.toLowerCase());
-    return tokens;
+    providerUtils.dereferenceAllConfigs(configurations);
   }
 
   @VisibleForTesting
-  protected List<String> getSystemConfigurationsRequested(ConfTreeOperations appConf) {
-    List<String> configList = new ArrayList<String>();
-
-    String configTypes = appConf.get(AgentKeys.SYSTEM_CONFIGS);
-    if (configTypes != null && configTypes.length() > 0) {
-      String[] configs = configTypes.split(",");
-      for (String config : configs) {
-        configList.add(config.trim());
-      }
-    }
-
-    return new ArrayList<String>(new HashSet<String>(configList));
+  protected Set<String> getSystemConfigurationsRequested(ConfTreeOperations appConf) {
+    return providerUtils.getSystemConfigurationsRequested(appConf);
   }
 
-
   @VisibleForTesting
   protected List<String> getApplicationConfigurationTypes(String roleGroup) {
     List<String> configList = new ArrayList<String>();
@@ -3275,30 +2816,17 @@ public class AgentProviderService extends AbstractProviderService implements
     configurations.put(configName, config);
   }
 
+  @VisibleForTesting
   protected void addRoleRelatedTokens(Map<String, String> tokens) {
-    for (Map.Entry<String, Map<String, ClusterNode>> entry : getRoleClusterNodeMapping().entrySet()) {
-      String tokenName = entry.getKey().toUpperCase(Locale.ENGLISH) + "_HOST";
-      String hosts = StringUtils.join(",", getHostsList(entry.getValue().values(), true));
-      tokens.put("${" + tokenName + "}", hosts);
-    }
-  }
-
-  private Iterable<String> getHostsList(Collection<ClusterNode> values,
-                                        boolean hostOnly) {
-    List<String> hosts = new ArrayList<String>();
-    for (ClusterNode cn : values) {
-      hosts.add(hostOnly ? cn.host : cn.host + "/" + cn.name);
-    }
-
-    return hosts;
+    providerUtils.addRoleRelatedTokens(tokens, getAmState());
   }
 
   private void addDefaultGlobalConfig(Map<String, String> config, String containerId, String roleName) {
     config.put("app_log_dir", "${AGENT_LOG_ROOT}");
     config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
     config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
-    config.put("app_conf_dir", "${AGENT_WORK_ROOT}/" + AgentKeys.APP_CONF_DIR);
-    config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + SliderKeys.PROPAGATED_CONF_DIR_NAME);
+    config.put("app_conf_dir", "${AGENT_WORK_ROOT}/" + APP_CONF_DIR);
+    config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + PROPAGATED_CONF_DIR_NAME);
     config.put("app_container_id", containerId);
     config.put("app_container_tag", tags.getTag(roleName, containerId));
 
@@ -3315,7 +2843,8 @@ public class AgentProviderService extends AbstractProviderService implements
     for (Map.Entry<String, Map<String, ClusterNode>> entry :
         getRoleClusterNodeMapping().entrySet()) {
       details.put(entry.getKey() + " Host(s)/Container(s)",
-                  new MonitorDetail(getHostsList(entry.getValue().values(), false).toString(), false));
+                  new MonitorDetail(providerUtils.getHostsList(
+                      entry.getValue().values(), false).toString(), false));
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ba59bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
new file mode 100644
index 0000000..13473e5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.docker;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE;
+
+public class DockerClientProvider extends AbstractClientProvider
+    implements SliderKeys {
+
+  protected static final Logger log =
+      LoggerFactory.getLogger(DockerClientProvider.class);
+  private static final ProviderUtils providerUtils = new ProviderUtils(log);
+  protected static final String NAME = "docker";
+
+  public DockerClientProvider(Configuration conf) {
+    super(conf);
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+
+  @Override
+  public List<ProviderRole> getRoles() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public void validateInstanceDefinition(AggregateConf instanceDefinition,
+      SliderFileSystem fs) throws SliderException {
+    super.validateInstanceDefinition(instanceDefinition, fs);
+
+    ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
+    ConfTreeOperations resources = instanceDefinition.getResourceOperations();
+
+    for (String roleGroup : resources.getComponentNames()) {
+      if (roleGroup.equals(COMPONENT_AM)) {
+        continue;
+      }
+      if (appConf.getComponentOpt(roleGroup, DOCKER_IMAGE, null) == null &&
+          appConf.getGlobalOptions().get(DOCKER_IMAGE) == null) {
+        throw new BadConfigException("Property " + DOCKER_IMAGE + " not " +
+            "specified for " + roleGroup);
+      }
+
+      providerUtils.getPackages(roleGroup, appConf);
+
+      if (appConf.getComponentOptBool(roleGroup, AM_CONFIG_GENERATION, false)) {
+        // build and localize configuration files
+        Map<String, Map<String, String>> configurations =
+            providerUtils.buildConfigurations(appConf, appConf, null, roleGroup,
+                roleGroup, null);
+        try {
+          providerUtils.localizeConfigFiles(null, roleGroup, roleGroup, appConf,
+              configurations, null, fs, null);
+        } catch (IOException e) {
+          throw new BadConfigException(e.toString());
+        }
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ba59bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
new file mode 100644
index 0000000..40b73a2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.docker;
+
+public interface DockerKeys {
+  String PROVIDER_DOCKER = "docker";
+  String DOCKER_PREFIX = "docker.";
+  String DOCKER_IMAGE = DOCKER_PREFIX + "image";
+  String DOCKER_NETWORK = DOCKER_PREFIX + "network";
+  String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
+  String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
+
+  String DEFAULT_DOCKER_NETWORK = "bridge";
+
+  String OUT_FILE = "stdout.txt";
+  String ERR_FILE = "stderr.txt";
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ba59bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
new file mode 100644
index 0000000..5d2592f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.docker;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.SliderProviderFactory;
+
+public class DockerProviderFactory extends SliderProviderFactory {
+
+  public DockerProviderFactory() {
+  }
+
+  public DockerProviderFactory(Configuration conf) {
+    super(conf);
+  }
+
+  @Override
+  public AbstractClientProvider createClientProvider() {
+    return new DockerClientProvider(getConf());
+  }
+
+  @Override
+  public ProviderService createServerProvider() {
+    return new DockerProviderService();
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message