hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [10/52] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He
Date Fri, 21 Jul 2017 18:39:12 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
deleted file mode 100644
index 9eb7d5c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.Files;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.params.Arguments;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.agent.AgentKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Class to prepare and persist app and add-on definitions.
- *
- * In this case, the app definition and add-on definitions are auto-inferred from the user input rather than explicit
- * inclusion of application package in the config.
- *
- * Processing an app definition involves one or more of the following: - modify appConfig - package definition into a
- * temporary folder - upload to HDFS
- *
- * This class keeps track of all the required operations and allows them to be invoked by build operation
- */
-public class AppDefinitionPersister {
-  private static final Logger log =
-      LoggerFactory.getLogger(AppDefinitionPersister.class);
-
-  private final SliderFileSystem sliderFileSystem;
-  private List<AppDefinition> appDefinitions;
-
-  public AppDefinitionPersister(SliderFileSystem sliderFileSystem) {
-    this.sliderFileSystem = sliderFileSystem;
-    appDefinitions = new ArrayList<>();
-  }
-
-
-  /**
-   * Process the application package or folder by copying it to the cluster path
-   *
-   * @param appDefinition details of application package
-   *
-   * @throws BadConfigException
-   * @throws IOException
-   */
-  private void persistDefinitionPackageOrFolder(AppDefinition appDefinition)
-      throws BadConfigException, IOException {
-    if (!appDefinition.appDefPkgOrFolder.canRead()) {
-      throw new BadConfigException("Pkg/Folder cannot be accessed - "
-                                   + appDefinition.appDefPkgOrFolder.getAbsolutePath());
-    }
-
-    File src = appDefinition.appDefPkgOrFolder;
-    String targetName = appDefinition.pkgName;
-    log.debug("Package name: " + targetName);
-    if (appDefinition.appDefPkgOrFolder.isDirectory()) {
-      log.info("Processing app package/folder {} for {}",
-               appDefinition.appDefPkgOrFolder.getAbsolutePath(),
-               appDefinition.pkgName);
-      File tmpDir = Files.createTempDir();
-      File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + appDefinition.pkgName);
-      SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile);
-      src = zipFile;
-    }
-
-    sliderFileSystem.getFileSystem().copyFromLocalFile(
-        false,
-        false,
-        new Path(src.toURI()),
-        new Path(appDefinition.targetFolderInFs, targetName));
-  }
-
-  public void persistPackages() throws BadConfigException, IOException {
-    for (AppDefinition appDefinition : appDefinitions) {
-      persistDefinitionPackageOrFolder(appDefinition);
-    }
-  }
-
-  public void processSuppliedDefinitions(String clustername,
-                                         AbstractClusterBuildingActionArgs buildInfo,
-                                         ConfTreeOperations appConf)
-      throws BadConfigException, IOException, BadCommandArgumentsException {
-    // if metainfo is provided add to the app instance
-    if (buildInfo.appMetaInfo != null || buildInfo.appMetaInfoJson != null) {
-      if (buildInfo.appMetaInfo != null && buildInfo.appMetaInfoJson != null) {
-        throw new BadConfigException("Both %s and %s cannot be specified",
-            Arguments.ARG_METAINFO, Arguments.ARG_METAINFO_JSON);
-      }
-
-      // Now we know that only one of either file or JSON is used
-      boolean isFileUsed = buildInfo.appMetaInfo != null ? true : false;
-      String argUsed = isFileUsed ? Arguments.ARG_METAINFO
-          : Arguments.ARG_METAINFO_JSON;
-
-      if (buildInfo.appDef != null) {
-        throw new BadConfigException("Both %s and %s cannot be specified",
-            argUsed, Arguments.ARG_APPDEF);
-      }
-      if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
-        throw new BadConfigException(
-            "%s cannot not be set if %s is specified in the cmd line ",
-            AgentKeys.APP_DEF, argUsed);
-      }
-
-      if (isFileUsed) {
-        if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) {
-          throw new BadConfigException(
-              "Path specified with %s either cannot be read or is not a file",
-              Arguments.ARG_METAINFO);
-        }
-      } else {
-        if (StringUtils.isEmpty(buildInfo.appMetaInfoJson.trim())) {
-          throw new BadConfigException("Empty string specified with %s",
-              Arguments.ARG_METAINFO_JSON);
-        }
-      }
-
-      File tempDir = Files.createTempDir();
-      File pkgSrcDir = new File(tempDir, "default");
-      if (!pkgSrcDir.exists() && !pkgSrcDir.mkdirs()) {
-        throw new IOException("Failed to create directory: " + pkgSrcDir);
-      }
-      File destMetaInfo = new File(pkgSrcDir, "metainfo.json");
-      if (isFileUsed) {
-        if (buildInfo.appMetaInfo.getName().endsWith(".xml")) {
-          Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml"));
-        } else {
-          Files.copy(buildInfo.appMetaInfo, destMetaInfo);
-        }
-      } else {
-        Files.write(
-            buildInfo.appMetaInfoJson.getBytes(Charset.forName("UTF-8")),
-            destMetaInfo);
-      }
-
-      Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
-      log.info("Using default app def path {}", appDirPath.toString());
-
-      appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir,
-          SliderKeys.DEFAULT_APP_PKG));
-      Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
-      appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
-      log.info("Setting app package to {}.", appDefPath);
-    }
-
-    if (buildInfo.appDef != null) {
-      if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
-        throw new BadConfigException("application.def must not be set if --appdef is provided.");
-      }
-
-      if (!buildInfo.appDef.exists()) {
-        throw new BadConfigException("--appdef is not a valid path.");
-      }
-
-      Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
-      appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG));
-      Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
-      appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
-      log.info("Setting app package to {}.", appDefPath);
-    }
-
-    if (buildInfo.addonDelegate.getAddonMap().size() > 0) {
-      if (SliderUtils.isUnset(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
-        throw new BadConfigException("addon package can only be specified if main app package is specified.");
-      }
-
-      List<String> addons = new ArrayList<String>();
-      Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap();
-      for (Map.Entry<String, String > entry : addonMap.entrySet()) {
-        String key = entry.getKey();
-        String value = entry.getValue();
-        if (SliderUtils.isUnset(value)) {
-          throw new BadConfigException("Invalid path for addon package " + key);
-        }
-        File defPath = new File(value);
-        if (!defPath.exists()) {
-          throw new BadConfigException("addon folder or package path is not valid.");
-        }
-
-        Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key);
-        String addonPkgName = "addon_" + key + ".zip";
-
-        log.debug(
-            "addonMap.get(key): {} addonPath: {} defPath: {} addonPkgName: {}",
-            addonMap.get(key), addonPath, defPath, addonPkgName);
-
-        appDefinitions.add(new AppDefinition(addonPath, defPath, addonPkgName));
-        String addOnKey = AgentKeys.ADDON_PREFIX + key;
-        Path addonPkgPath = new Path(addonPath, addonPkgName);
-        log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath);
-        appConf.getGlobalOptions().set(addOnKey, addonPkgPath);
-        addons.add(addOnKey);
-      }
-
-      String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS);
-      if (SliderUtils.isUnset(existingList)) {
-        existingList = "";
-      }
-      appConf.getGlobalOptions().set(AgentKeys.ADDONS, existingList + StringUtils.join(addons, ","));
-    }
-  }
-
-
-  @VisibleForTesting
-  public List<AppDefinitionPersister.AppDefinition> getAppDefinitions() {
-    return appDefinitions;
-  }
-
-  // Helper class to hold details for the app and addon packages
-  static class AppDefinition {
-    // The target folder where the package will be stored
-    public Path targetFolderInFs;
-    // The on disk location of the app def package or folder
-    public File appDefPkgOrFolder;
-    // Package name
-    public String pkgName;
-
-    public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) {
-      this.targetFolderInFs = targetFolderInFs;
-      this.appDefPkgOrFolder = appDefPkgOrFolder;
-      this.pkgName = pkgName;
-    }
-
-    @Override
-    public String toString() {
-      return new StringBuilder().append("targetFolderInFs").append(" : ").append(targetFolderInFs.toString())
-          .append(", ")
-          .append("appDefPkgOrFolder").append(" : ").append(appDefPkgOrFolder.toString())
-          .append(", ")
-          .append("pkgName").append(" : ").append(pkgName).toString();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 4f60c06..8fe2549 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -29,6 +29,7 @@ import org.codehaus.jackson.JsonParseException;
 import org.codehaus.jackson.map.DeserializationConfig;
 import org.codehaus.jackson.map.JsonMappingException;
 import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
 import org.codehaus.jackson.map.SerializationConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,6 +66,11 @@ public class JsonSerDeser<T> {
     mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
   }
 
+  public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
+    this(classType);
+    mapper.setPropertyNamingStrategy(namingStrategy);
+  }
+
   /**
    * Convert from JSON
    * @param json input

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index 01444fd..42e103a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -103,19 +103,6 @@ public abstract class AbstractClientProvider extends Configured {
 
 
   /**
-   * Any provider-side alteration of a configuration can take place here.
-   * @param aggregateConf config to patch
-   * @throws IOException IO problems
-   * @throws SliderException Slider-specific issues
-   */
-  public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws
-      SliderException,
-                                                                    IOException {
-    //default: do nothing
-  }
-
-
-  /**
    * Prepare the AM settings for launch
    * @param fileSystem filesystem
    * @param serviceConf configuration of the client
@@ -234,7 +221,7 @@ public abstract class AbstractClientProvider extends Configured {
    * @param appDescription brief description of the application
    * @return
    */
-  public final Set<String> createApplicationTags(String appName,
+  public static final Set<String> createApplicationTags(String appName,
       String appVersion, String appDescription) {
     Set<String> tags = new HashSet<>();
     tags.add(SliderUtils.createNameTag(appName));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
deleted file mode 100644
index 41b26e9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ /dev/null
@@ -1,438 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.types.AddressTypes;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
-import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.services.workflow.ForkedProcessService;
-import org.apache.slider.server.services.workflow.ServiceParent;
-import org.apache.slider.server.services.workflow.WorkflowSequenceService;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * The base class for provider services. It lets the implementations
- * add sequences of operations, and propagates service failures
- * upstream
- */
-public abstract class AbstractProviderService
-    extends WorkflowSequenceService
-    implements
-    ProviderCore,
-    SliderKeys,
-    ProviderService {
-  private static final Logger log =
-    LoggerFactory.getLogger(AbstractProviderService.class);
-  protected StateAccessForProviders amState;
-  protected URL amWebAPI;
-  protected YarnRegistryViewForProviders yarnRegistry;
-  protected QueueAccess queueAccess;
-
-  protected AbstractProviderService(String name) {
-    super(name);
-    setStopIfNoChildServicesAtStartup(false);
-  }
-
-  @Override
-  public Configuration getConf() {
-    return getConfig();
-  }
-
-  public StateAccessForProviders getAmState() {
-    return amState;
-  }
-
-  public QueueAccess getQueueAccess() {
-    return queueAccess;
-  }
-
-  public void setAmState(StateAccessForProviders amState) {
-    this.amState = amState;
-  }
-
-  @Override
-  public String getHumanName() {
-    return getName().toLowerCase(Locale.ENGLISH);
-  }
-  
-  @Override
-  public void bind(StateAccessForProviders stateAccessor,
-      QueueAccess queueAccess,
-      List<Container> liveContainers) {
-    this.amState = stateAccessor;
-    this.queueAccess = queueAccess;
-  }
-
-  @Override
-  public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
-    this.yarnRegistry = yarnRegistry;
-  }
-
-  public YarnRegistryViewForProviders getYarnRegistry() {
-    return yarnRegistry;
-  }
-
-  @Override
-  public void notifyContainerCompleted(ContainerId containerId) {
-  }
-
-  /**
-   * Load default Configuration
-   * @param confDir configuration directory
-   * @return configuration
-   * @throws BadCommandArgumentsException
-   * @throws IOException
-   */
-  @Override
-  public Configuration loadProviderConfigurationInformation(File confDir)
-      throws BadCommandArgumentsException, IOException {
-    return new Configuration(false);
-  }
-
-  /**
-   * Load a specific XML configuration file for the provider config
-   * @param confDir configuration directory
-   * @param siteXMLFilename provider-specific filename
-   * @return a configuration to be included in status
-   * @throws BadCommandArgumentsException argument problems
-   * @throws IOException IO problems
-   */
-  protected Configuration loadProviderConfigurationInformation(File confDir,
-                                                               String siteXMLFilename)
-    throws BadCommandArgumentsException, IOException {
-    Configuration siteConf;
-    File siteXML = new File(confDir, siteXMLFilename);
-    if (!siteXML.exists()) {
-      throw new BadCommandArgumentsException(
-        "Configuration directory %s doesn't contain %s - listing is %s",
-        confDir, siteXMLFilename, SliderUtils.listDir(confDir));
-    }
-
-    //now read it in
-    siteConf = ConfigHelper.loadConfFromFile(siteXML);
-    log.info("{} file is at {}", siteXMLFilename, siteXML);
-    log.info(ConfigHelper.dumpConfigToString(siteConf));
-    return siteConf;
-  }
-
-  /**
-   * No-op implementation of this method.
-   */
-  @Override
-  public void initializeApplicationConfiguration(
-      AggregateConf instanceDefinition, SliderFileSystem fileSystem,
-      String roleGroup)
-      throws IOException, SliderException {
-  }
-
-  /**
-   * No-op implementation of this method.
-   *
-   * {@inheritDoc}
-   */
-  @Override
-  public void validateApplicationConfiguration(AggregateConf instance,
-                                               File confDir,
-                                               boolean secure)
-      throws IOException, SliderException {
-
-  }
-
-  /**
-   * Scan through the roles and see if it is supported.
-   * @param role role to look for
-   * @return true if the role is known about -and therefore
-   * that a launcher thread can be deployed to launch it
-   */
-  @Override
-  public boolean isSupportedRole(String role) {
-    Collection<ProviderRole> roles = getRoles();
-    for (ProviderRole providedRole : roles) {
-      if (providedRole.name.equals(role)) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * override point to allow a process to start executing in this container
-   * @param instanceDefinition cluster description
-   * @param confDir configuration directory
-   * @param env environment
-   * @param execInProgress the callback for the exec events
-   * @return false
-   * @throws IOException
-   * @throws SliderException
-   */
-  @Override
-  public boolean exec(AggregateConf instanceDefinition,
-      File confDir,
-      Map<String, String> env,
-      ProviderCompleted execInProgress) throws IOException, SliderException {
-    return false;
-  }
-
-  @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-  @Override // ExitCodeProvider
-  public int getExitCode() {
-    Throwable cause = getFailureCause();
-    if (cause != null) {
-      //failed for some reason
-      if (cause instanceof ExitCodeProvider) {
-        return ((ExitCodeProvider) cause).getExitCode();
-      }
-    }
-    ForkedProcessService lastProc = latestProcess();
-    if (lastProc == null || !lastProc.isProcessTerminated()) {
-      return 0;
-    } else {
-      return lastProc.getExitCode();
-    }
-  }
-
-  /**
-   * Return the latest forked process service that ran
-   * @return the forkes service
-   */
-  protected ForkedProcessService latestProcess() {
-    Service current = getActiveService();
-    Service prev = getPreviousService();
-
-    Service latest = current != null ? current : prev;
-    if (latest instanceof ForkedProcessService) {
-      return (ForkedProcessService) latest;
-    } else {
-      //its a composite object, so look inside it for a process
-      if (latest instanceof ServiceParent) {
-        return getFPSFromParentService((ServiceParent) latest);
-      } else {
-        //no match
-        return null;
-      }
-    }
-  }
-
-
-  /**
-   * Given a parent service, find the one that is a forked process
-   * @param serviceParent parent
-   * @return the forked process service or null if there is none
-   */
-  protected ForkedProcessService getFPSFromParentService(ServiceParent serviceParent) {
-    List<Service> services = serviceParent.getServices();
-    for (Service s : services) {
-      if (s instanceof ForkedProcessService) {
-        return (ForkedProcessService) s;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * if we are already running, start this service
-   */
-  protected void maybeStartCommandSequence() {
-    if (isInState(STATE.STARTED)) {
-      startNextService();
-    }
-  }
-
-  /**
-   * Create a new forked process service with the given
-   * name, environment and command list -then add it as a child
-   * for execution in the sequence.
-   *
-   * @param name command name
-   * @param env environment
-   * @param commands command line
-   * @throws IOException
-   * @throws SliderException
-   */
-  protected ForkedProcessService queueCommand(String name,
-                              Map<String, String> env,
-                              List<String> commands) throws
-                                                     IOException,
-      SliderException {
-    ForkedProcessService process = buildProcess(name, env, commands);
-    //register the service for lifecycle management; when this service
-    //is terminated, so is the master process
-    addService(process);
-    return process;
-  }
-
-  public ForkedProcessService buildProcess(String name,
-                                           Map<String, String> env,
-                                           List<String> commands) throws
-                                                                  IOException,
-      SliderException {
-    ForkedProcessService process;
-    process = new ForkedProcessService(name);
-    process.init(getConfig());
-    process.build(env, commands);
-    return process;
-  }
-
-  /*
-   * Build the provider status, can be empty
-   * @return the provider status - map of entries to add to the info section
-   */
-  @Override
-  public Map<String, String> buildProviderStatus() {
-    return new HashMap<String, String>();
-  }
-
-  /*
-  Build the monitor details. The base implementation includes all the external URL endpoints
-  in the external view
-   */
-  @Override
-  public Map<String, MonitorDetail> buildMonitorDetails(ClusterDescription clusterDesc) {
-    Map<String, MonitorDetail> details = new LinkedHashMap<String, MonitorDetail>();
-
-    // add in all the endpoints
-    buildEndpointDetails(details);
-
-    return details;
-  }
-
-  @Override
-  public void buildEndpointDetails(Map<String, MonitorDetail> details) {
-    ServiceRecord self = yarnRegistry.getSelfRegistration();
-
-    List<Endpoint> externals = self.external;
-    for (Endpoint endpoint : externals) {
-      String addressType = endpoint.addressType;
-      if (AddressTypes.ADDRESS_URI.equals(addressType)) {
-        try {
-          List<URL> urls = RegistryTypeUtils.retrieveAddressURLs(endpoint);
-          if (!urls.isEmpty()) {
-            details.put(endpoint.api, new MonitorDetail(urls.get(0).toString(), true));
-          }
-        } catch (InvalidRecordException  | MalformedURLException ignored) {
-          // Ignored
-        }
-
-      }
-
-    }
-  }
-
-  @Override
-  public void applyInitialRegistryDefinitions(URL amWebURI,
-      ServiceRecord serviceRecord)
-    throws IOException {
-      this.amWebAPI = amWebURI;
-  }
-
-  /**
-   * {@inheritDoc}
-   * 
-   * 
-   * @return The base implementation returns the most recent containers first.
-   */
-  @Override
-  public ContainerReleaseSelector createContainerReleaseSelector() {
-    return new MostRecentContainerReleaseSelector();
-  }
-
-  @Override
-  public void releaseAssignedContainer(ContainerId containerId) {
-    // no-op
-  }
-
-  @Override
-  public void addContainerRequest(AMRMClient.ContainerRequest req) {
-    // no-op
-  }
-
-  @Override
-  public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
-    // no-op
-  }
-
-  @Override
-  public int cancelContainerRequests(Priority priority1,
-      Priority priority2,
-      int count) {
-    return 0;
-  }
-
-  @Override
-  public void updateBlacklist(List<String> blacklistAdditions,
-      List<String> blacklistRemovals) {
-    // no-op
-  }
-
-  @Override
-  public void execute(List<AbstractRMOperation> operations) {
-    for (AbstractRMOperation operation : operations) {
-      operation.execute(this);
-    }
-  }
-  /**
-   * No-op implementation of this method.
-   */
-  @Override
-  public void rebuildContainerDetails(List<Container> liveContainers,
-      String applicationId, Map<Integer, ProviderRole> providerRoles) {
-  }
-
-  @Override
-  public boolean processContainerStatus(ContainerId containerId,
-      ContainerStatus status) {
-    return false;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
index 9767430..b07fc29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
@@ -19,9 +19,6 @@
 package org.apache.slider.providers;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.exceptions.SliderException;
 
 import java.util.List;
 public interface ProviderCore {
@@ -31,13 +28,4 @@ public interface ProviderCore {
   List<ProviderRole> getRoles();
 
   Configuration getConf();
-
-  /**
-   * Verify that an instance definition is considered valid by the provider
-   * @param instanceDefinition instance definition
-   * @throws SliderException if the configuration is not valid
-   */
-  void validateInstanceDefinition(AggregateConf instanceDefinition) throws
-      SliderException;
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 761ac0f..e0299e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -18,7 +18,7 @@
 
 package org.apache.slider.providers;
 
-import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.resource.Component;
 
 /**
  * Provider role and key for use in app requests.
@@ -34,16 +34,8 @@ public final class ProviderRole {
   public int nodeFailureThreshold;
   public final long placementTimeoutSeconds;
   public final String labelExpression;
+  public final Component component;
 
-  public ProviderRole(String name, int id) {
-    this(name,
-        name,
-        id,
-        PlacementPolicy.DEFAULT,
-        ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
-        ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
-        ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
-  }
 
   /**
    * Create a provider role
@@ -67,7 +59,7 @@ public final class ProviderRole {
         policy,
         nodeFailureThreshold,
         placementTimeoutSeconds,
-        labelExpression);
+        labelExpression, null);
   }
 
   /**
@@ -87,7 +79,8 @@ public final class ProviderRole {
       int policy,
       int nodeFailureThreshold,
       long placementTimeoutSeconds,
-      String labelExpression) {
+      String labelExpression,
+      Component component) {
     this.name = name;
     if (group == null) {
       this.group = name;
@@ -99,6 +92,8 @@ public final class ProviderRole {
     this.nodeFailureThreshold = nodeFailureThreshold;
     this.placementTimeoutSeconds = placementTimeoutSeconds;
     this.labelExpression = labelExpression;
+    this.component = component;
+
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 4ca9326..c80de7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -18,18 +18,15 @@
 
 package org.apache.slider.providers;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.resource.Application;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
 import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.launch.ContainerLauncher;
 import org.apache.slider.core.main.ExitCodeProvider;
@@ -45,128 +42,17 @@ import java.net.URL;
 import java.util.List;
 import java.util.Map;
 
-public interface ProviderService extends ProviderCore,
-    Service,
-    RMOperationHandlerActions,
-    ExitCodeProvider {
+public interface ProviderService extends Service {
 
   /**
    * Set up the entire container launch context
-   * @param containerLauncher
-   * @param instanceDefinition
-   * @param container
-   * @param providerRole
-   * @param sliderFileSystem
-   * @param generatedConfPath
-   * @param appComponent
-   * @param containerTmpDirPath
    */
   void buildContainerLaunchContext(ContainerLauncher containerLauncher,
-      AggregateConf instanceDefinition,
-      Container container,
-      ProviderRole providerRole,
-      SliderFileSystem sliderFileSystem,
-      Path generatedConfPath,
-      MapOperations resourceComponent,
-      MapOperations appComponent,
-      Path containerTmpDirPath) throws
-      IOException,
-      SliderException;
+      Application application, Container container, ProviderRole providerRole,
+      SliderFileSystem sliderFileSystem) throws IOException, SliderException;
 
-  /**
-   * Notify the providers of container completion
-   * @param containerId container that has completed
-   */
-  void notifyContainerCompleted(ContainerId containerId);
-
-  /**
-   * Execute a process in the AM
-   * @param instanceDefinition cluster description
-   * @param confDir configuration directory
-   * @param env environment
-   * @param execInProgress the callback for the exec events
-   * @return true if a process was actually started
-   * @throws IOException
-   * @throws SliderException
-   */
-  boolean exec(AggregateConf instanceDefinition,
-               File confDir,
-               Map<String, String> env,
-               ProviderCompleted execInProgress) throws IOException,
-      SliderException;
-
-  /**
-   * Scan through the roles and see if it is supported.
-   * @param role role to look for
-   * @return true if the role is known about -and therefore
-   * that a launcher thread can be deployed to launch it
-   */
-  boolean isSupportedRole(String role);
-
-  /**
-   * Load a specific XML configuration file for the provider config
-   * @param confDir configuration directory
-   * @return a configuration to be included in status
-   * @throws BadCommandArgumentsException
-   * @throws IOException
-   */
-  Configuration loadProviderConfigurationInformation(File confDir)
-    throws BadCommandArgumentsException, IOException;
 
-  /**
-   * The application configuration should be initialized here
-   * 
-   * @param instanceDefinition
-   * @param fileSystem
-   * @param roleGroup
-   * @throws IOException
-   * @throws SliderException
-   */
-  void initializeApplicationConfiguration(AggregateConf instanceDefinition,
-      SliderFileSystem fileSystem, String roleGroup) throws IOException,
-      SliderException;
-
-  /**
-   * This is a validation of the application configuration on the AM.
-   * Here is where things like the existence of keytabs and other
-   * not-seen-client-side properties can be tested, before
-   * the actual process is spawned. 
-   * @param instanceDefinition clusterSpecification
-   * @param confDir configuration directory
-   * @param secure flag to indicate that secure mode checks must exist
-   * @throws IOException IO problemsn
-   * @throws SliderException any failure
-   */
-  void validateApplicationConfiguration(AggregateConf instanceDefinition,
-                                        File confDir,
-                                        boolean secure
-                                       ) throws IOException, SliderException;
-
-  /*
-     * Build the provider status, can be empty
-     * @return the provider status - map of entries to add to the info section
-     */
-  Map<String, String> buildProviderStatus();
-  
-  /**
-   * Build a map of data intended for the AM webapp that is specific
-   * about this provider. The key is some text to be displayed, and the
-   * value can be a URL that will create an anchor over the key text.
-   * 
-   * If no anchor is needed/desired, insert the key with a null value.
-   * @return the details
-   */
-  Map<String, MonitorDetail> buildMonitorDetails(ClusterDescription clusterSpec);
-
-  /**
-   * Get a human friendly name for web UIs and messages
-   * @return a name string. Default is simply the service instance name.
-   */
-  String getHumanName();
-
-  public void bind(StateAccessForProviders stateAccessor,
-      QueueAccess queueAccess,
-      List<Container> liveContainers);
+  void setAMState(StateAccessForProviders stateAccessForProviders);
 
   /**
    * Bind to the YARN registry
@@ -175,39 +61,6 @@ public interface ProviderService extends ProviderCore,
   void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
 
   /**
-   * Build up the endpoint details for this service
-   * @param details
-   */
-  void buildEndpointDetails(Map<String, MonitorDetail> details);
-
-  /**
-   * Prior to going live -register the initial service registry data
-   * @param amWebURI URL to the AM. This may be proxied, so use relative paths
-   * @param serviceRecord service record to build up
-   */
-  void applyInitialRegistryDefinitions(URL amWebURI,
-      ServiceRecord serviceRecord)
-      throws IOException;
-
-  /**
-   * Create the container release selector for this provider...any policy
-   * can be implemented
-   * @return the selector to use for choosing containers.
-   */
-  ContainerReleaseSelector createContainerReleaseSelector();
-
-  /**
-   * On AM restart (for whatever reason) this API is required to rebuild the AM
-   * internal state with the containers which were already assigned and running
-   * 
-   * @param liveContainers
-   * @param applicationId
-   * @param providerRoles
-   */
-  void rebuildContainerDetails(List<Container> liveContainers,
-      String applicationId, Map<Integer, ProviderRole> providerRoles);
-
-  /**
    * Process container status
    * @return true if status needs to be requested again, false otherwise
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index f33db9b..f8ec976 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -18,7 +18,6 @@
 
 package org.apache.slider.providers;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
@@ -37,24 +36,21 @@ import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.RoleKeys;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.api.resource.Configuration;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
 import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
 import org.apache.slider.core.exceptions.NoSuchNodeException;
 import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.launch.ContainerLauncher;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.ConfigUtils;
-import org.apache.slider.core.registry.docstore.ExportEntry;
 import org.apache.slider.core.registry.docstore.PublishedConfiguration;
 import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.docstore.PublishedExports;
 import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
 import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
@@ -66,16 +62,10 @@ import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Date;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
 import java.util.regex.Pattern;
 
 /**
@@ -114,7 +104,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
    */
   public static boolean addProviderJar(
       Map<String, LocalResource> providerResources,
-      Object provider,
+      Class providerClass,
       String jarName,
       SliderFileSystem sliderFileSystem,
       Path tempPath,
@@ -125,7 +115,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
     try {
       SliderUtils.putJar(providerResources,
           sliderFileSystem,
-          provider.getClass(),
+          providerClass,
           tempPath,
           libdir,
           jarName);
@@ -138,32 +128,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
       }
     }
   }
-
-  /**
-   * Add/overwrite the agent tarball (overwritten every time application is
-   * restarted).
-   * @param provider an instance of a provider class
-   * @param tarName name of the tarball to upload
-   * @param sliderFileSystem the file system
-   * @param agentDir directory to upload to
-   * @return true the location could be determined and the file added
-   * @throws IOException if the upload fails
-   */
-  public static boolean addAgentTar(Object provider,
-                                    String tarName,
-                                    SliderFileSystem sliderFileSystem,
-                                    Path agentDir) throws
-  IOException {
-    File localFile = SliderUtils.findContainingJar(provider.getClass());
-    if(localFile != null) {
-      String parentDir = localFile.getParent();
-      Path agentTarPath = new Path(parentDir, tarName);
-      sliderFileSystem.getFileSystem().copyFromLocalFile(false, true,
-          agentTarPath, agentDir);
-      return true;
-    }
-    return false;
-  }
   
   /**
    * Loads all dependency jars from the default path.
@@ -193,132 +157,24 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
         libDir, libLocalSrcDir);
   }
 
-
-  /**
-   * Validate the requested number of instances of a component.
-   * <p>
-   * If max &lt;= 0:  min &lt;= count
-   * If max &gt; 0:  min &lt;= count &lt;= max
-   * @param instanceDescription configuration
-   * @param name node class name
-   * @param min requested heap size
-   * @param max maximum value.
-   * @throws BadCommandArgumentsException if the values are out of range
-   */
-  public void validateNodeCount(AggregateConf instanceDescription,
-                                String name, int min, int max)
-      throws BadCommandArgumentsException {
-    MapOperations component =
-        instanceDescription.getResourceOperations().getComponent(name);
-    int count;
-    if (component == null) {
-      count = 0;
-    } else {
-      count = component.getOptionInt(ResourceKeys.COMPONENT_INSTANCES, 0);
-    }
-    validateNodeCount(name, count, min, max);
-  }
-  
-  /**
-   * Validate the count is between min and max.
-   * <p>
-   * If max &lt;= 0:  min &lt;= count
-   * If max &gt; 0:  min &lt;= count &lt;= max
-   * @param name node class name
-   * @param count requested node count
-   * @param min requested heap size
-   * @param max maximum value. 
-   * @throws BadCommandArgumentsException if the values are out of range
-   */
-  public void validateNodeCount(String name,
-                                int count,
-                                int min,
-                                int max) throws BadCommandArgumentsException {
-    if (count < min) {
-      throw new BadCommandArgumentsException(
-        "requested no of %s nodes: %d is below the minimum of %d", name, count,
-        min);
-    }
-    if (max > 0 && count > max) {
-      throw new BadCommandArgumentsException(
-        "requested no of %s nodes: %d is above the maximum of %d", name, count,
-        max);
-    }
-  }
-
-  /**
-   * Copy options beginning with "site.configName." prefix from options map
-   * to sitexml map, removing the prefix and substituting the tokens
-   * specified in the tokenMap.
-   * @param options source map
-   * @param sitexml destination map
-   * @param configName optional ".configName" portion of the prefix
-   * @param tokenMap key/value pairs to substitute into the option values
-   */
-  public void propagateSiteOptions(Map<String, String> options,
-      Map<String, String> sitexml,
-      String configName,
-      Map<String,String> tokenMap) {
-    String prefix = OptionKeys.SITE_XML_PREFIX +
-        (!configName.isEmpty() ? configName + "." : "");
-    propagateOptions(options, sitexml, tokenMap, prefix);
-  }
-
-  /**
-   * Copy options beginning with prefix from options map
-   * to sitexml map, removing the prefix and substituting the tokens
-   * specified in the tokenMap.
-   * @param options source map
-   * @param sitexml destination map
-   * @param tokenMap key/value pairs to substitute into the option values
-   * @param prefix which options to copy to destination map
-   */
-  public void propagateOptions(Map<String, String> options,
-                                   Map<String, String> sitexml,
-                                   Map<String,String> tokenMap,
-                                   String prefix) {
-    for (Map.Entry<String, String> entry : options.entrySet()) {
-      String key = entry.getKey();
-      if (key.startsWith(prefix)) {
-        String envName = key.substring(prefix.length());
-        if (!envName.isEmpty()) {
-          String value = entry.getValue();
-          if (tokenMap != null) {
-            for (Map.Entry<String,String> token : tokenMap.entrySet()) {
-              value = value.replaceAll(Pattern.quote(token.getKey()),
-                                       token.getValue());
-            }
-          }
-          sitexml.put(envName, value);
-        }
-      }
-    }
-  }
-
-  /**
-   * Substitute tokens into option map values, returning a new map.
-   * @param options source map
-   * @param tokenMap key/value pairs to substitute into the option values
-   * @return map with substituted values
-   */
-  public Map<String, String> filterSiteOptions(Map<String, String> options,
+  // Build key -> value map
+  // value will be substituted by corresponding data in tokenMap
+  public Map<String, String> substituteConfigs(Map<String, String> configs,
       Map<String, String> tokenMap) {
-    String prefix = OptionKeys.SITE_XML_PREFIX;
     String format = "${%s}";
     Map<String, String> filteredOptions = new HashMap<>();
-    for (Map.Entry<String, String> entry : options.entrySet()) {
+    for (Map.Entry<String, String> entry : configs.entrySet()) {
       String key = entry.getKey();
-      if (key.startsWith(prefix)) {
-        String value = entry.getValue();
-        if (tokenMap != null) {
-          for (Map.Entry<String,String> token : tokenMap.entrySet()) {
-            value = value.replaceAll(Pattern.quote(token.getKey()),
-                token.getValue());
-          }
+      String value = entry.getValue();
+      if (tokenMap != null) {
+        for (Map.Entry<String, String> token : tokenMap.entrySet()) {
+          value =
+              value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
         }
-        filteredOptions.put(String.format(format, key), value);
       }
+      filteredOptions.put(String.format(format, key), value);
     }
+
     return filteredOptions;
   }
 
@@ -345,28 +201,27 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
     return intVal;
   }
 
+
   /**
    * Localize the service keytabs for the application.
    * @param launcher container launcher
-   * @param instanceDefinition app specification
    * @param fileSystem file system
-   * @param clusterName app name
    * @throws IOException trouble uploading to HDFS
    */
   public void localizeServiceKeytabs(ContainerLauncher launcher,
-      AggregateConf instanceDefinition, SliderFileSystem fileSystem,
-      String clusterName) throws IOException {
-    ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
-    String keytabPathOnHost = appConf.getComponent(COMPONENT_AM).get(
-            SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+      SliderFileSystem fileSystem, Application application) throws IOException {
+
+    Configuration conf = application.getConfiguration();
+    String keytabPathOnHost =
+        conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
     if (SliderUtils.isUnset(keytabPathOnHost)) {
-      String amKeytabName = appConf.getComponent(COMPONENT_AM).get(
-              SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
-      String keytabDir = appConf.getComponent(COMPONENT_AM).get(
-              SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+      String amKeytabName =
+          conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+      String keytabDir =
+          conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
       // we need to localize the keytab files in the directory
       Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
-          clusterName);
+          application.getName());
       boolean serviceKeytabsDeployed = false;
       if (fileSystem.getFileSystem().exists(keytabDirPath)) {
         FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
@@ -395,591 +250,119 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
   }
 
 
-  /**
-   * Upload a local file to the cluster security dir in HDFS. If the file
-   * already exists, it is not replaced.
-   * @param resource file to upload
-   * @param fileSystem file system
-   * @param clusterName app name
-   * @return Path of the uploaded file
-   * @throws IOException file cannot be uploaded
-   */
-  private Path uploadSecurityResource(File resource,
-      SliderFileSystem fileSystem, String clusterName) throws IOException {
-    Path certsDir = fileSystem.buildClusterSecurityDirPath(clusterName);
-    return uploadResource(resource, fileSystem, certsDir);
-  }
-
-  /**
-   * Upload a local file to the cluster resources dir in HDFS. If the file
-   * already exists, it is not replaced.
-   * @param resource file to upload
-   * @param fileSystem file system
-   * @param roleName optional subdirectory (for component-specific resources)
-   * @param clusterName app name
-   * @return Path of the uploaded file
-   * @throws IOException file cannot be uploaded
-   */
-  private Path uploadResource(File resource, SliderFileSystem fileSystem,
-      String roleName, String clusterName) throws IOException {
-    Path dir;
-    if (roleName == null) {
-      dir = fileSystem.buildClusterResourcePath(clusterName);
-    } else {
-      dir = fileSystem.buildClusterResourcePath(clusterName, roleName);
-    }
-    return uploadResource(resource, fileSystem, dir);
-  }
-
-  /**
-   * Upload a local file to a specified HDFS directory. If the file already
-   * exists, it is not replaced.
-   * @param resource file to upload
-   * @param fileSystem file system
-   * @param parentDir destination directory in HDFS
-   * @return Path of the uploaded file
-   * @throws IOException file cannot be uploaded
-   */
-  private synchronized Path uploadResource(File resource,
-      SliderFileSystem fileSystem, Path parentDir) throws IOException {
-    if (!fileSystem.getFileSystem().exists(parentDir)) {
-      fileSystem.getFileSystem().mkdirs(parentDir,
+  // 1. Create all config files for a component on hdfs for localization
+  // 2. Add the config file to localResource
+  //TODO handle Template format config file
+  public void createConfigFileAndAddLocalResource(ContainerLauncher launcher,
+      SliderFileSystem fs, Component component,
+      Map<String, String> tokensForSubstitution,
+      StateAccessForProviders amState) throws IOException {
+    Path compDir =
+        new Path(new Path(fs.getAppDir(), "components"), component.getName());
+    if (!fs.getFileSystem().exists(compDir)) {
+      fs.getFileSystem().mkdirs(compDir,
           new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
-    }
-    Path destPath = new Path(parentDir, resource.getName());
-    if (!fileSystem.getFileSystem().exists(destPath)) {
-      FSDataOutputStream os = null;
-      try {
-        os = fileSystem.getFileSystem().create(destPath);
-        byte[] contents = FileUtils.readFileToByteArray(resource);
-        os.write(contents, 0, contents.length);
-        os.flush();
-      } finally {
-        IOUtils.closeStream(os);
-      }
-      log.info("Uploaded {} to localization path {}", resource, destPath);
+      log.info("Creating component dir: " + compDir);
     } else {
-      log.info("Resource {} already existed at localization path {}", resource,
-          destPath);
-    }
-
-    while (!fileSystem.getFileSystem().exists(destPath)) {
-      try {
-        Thread.sleep(500);
-      } catch (InterruptedException e) {
-        // ignore
-      }
-    }
-
-    fileSystem.getFileSystem().setPermission(destPath,
-        new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
-
-    return destPath;
-  }
-
-  /**
-   * Write a configuration property map to a local file in a specified format.
-   * @param fileSystem file system
-   * @param file destination file
-   * @param configFormat file format
-   * @param configFileDN file description
-   * @param config properties to save to the file
-   * @param clusterName app name
-   * @throws IOException file cannot be created
-   */
-  private synchronized void createConfigFile(SliderFileSystem fileSystem,
-      File file, ConfigFormat configFormat, String configFileDN,
-      Map<String, String> config, String clusterName) throws IOException {
-    if (file.exists()) {
-      log.info("Skipping writing {} file {} because it already exists",
-          configFormat, file);
-      return;
-    }
-    log.info("Writing {} file {}", configFormat, file);
-
-    ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
-        fileSystem, clusterName, file.getName());
-    PublishedConfiguration publishedConfiguration =
-        new PublishedConfiguration(configFileDN,
-            config.entrySet());
-    PublishedConfigurationOutputter configurationOutputter =
-        PublishedConfigurationOutputter.createOutputter(configFormat,
-            publishedConfiguration);
-    configurationOutputter.save(file);
-  }
-
-  /**
-   * Determine config files requested in the appConf, generate the files, and
-   * localize them.
-   * @param launcher container launcher
-   * @param roleName component name
-   * @param roleGroup component group
-   * @param appConf app configurations
-   * @param configs configurations grouped by config name
-   * @param env environment variables
-   * @param fileSystem file system
-   * @param clusterName app name
-   * @throws IOException file(s) cannot be uploaded
-   * @throws BadConfigException file name not specified or file format not
-   * supported
-   */
-  public void localizeConfigFiles(ContainerLauncher launcher,
-      String roleName, String roleGroup,
-      ConfTreeOperations appConf,
-      Map<String, Map<String, String>> configs,
-      MapOperations env,
-      SliderFileSystem fileSystem,
-      String clusterName)
-      throws IOException, BadConfigException {
-    for (Entry<String, Map<String, String>> configEntry : configs.entrySet()) {
-      String configFileName = appConf.getComponentOpt(roleGroup,
-          OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
-              .NAME_SUFFIX, null);
-      String configFileType = appConf.getComponentOpt(roleGroup,
-          OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
-              .TYPE_SUFFIX, null);
-      if (configFileName == null && configFileType == null) {
-        // config file not requested, so continue
-        continue;
-      }
-      if (configFileName == null) {
-        throw new BadConfigException("Config file name null for " +
-            configEntry.getKey());
-      }
-      if (configFileType == null) {
-        throw new BadConfigException("Config file type null for " +
-            configEntry.getKey());
-      }
-      ConfigFormat configFormat = ConfigFormat.resolve(configFileType);
-      if (configFormat == null) {
-        throw new BadConfigException("Config format " + configFileType +
-            " doesn't exist");
-      }
-      boolean perComponent = appConf.getComponentOptBool(roleGroup,
-          OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
-              .PER_COMPONENT, false);
-      boolean perGroup = appConf.getComponentOptBool(roleGroup,
-          OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
-              .PER_GROUP, false);
-
-      localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),
-          configFormat, configFileName, configs, env, fileSystem,
-          clusterName, perComponent, perGroup);
-    }
-  }
-
-  /**
-   * Create and localize a config file.
-   * @param launcher container launcher
-   * @param roleName component name
-   * @param roleGroup component group
-   * @param configFileDN config description/name
-   * @param configFormat config format
-   * @param configFileName config file name
-   * @param configs configs grouped by config description/name
-   * @param env environment variables
-   * @param fileSystem file system
-   * @param clusterName app name
-   * @param perComponent true if file should be created per unique component
-   * @param perGroup true if file should be created per component group
-   * @throws IOException file cannot be uploaded
-   */
-  public void localizeConfigFile(ContainerLauncher launcher,
-      String roleName, String roleGroup,
-      String configFileDN, ConfigFormat configFormat, String configFileName,
-      Map<String, Map<String, String>> configs,
-      MapOperations env,
-      SliderFileSystem fileSystem,
-      String clusterName,
-      boolean perComponent,
-      boolean perGroup)
-      throws IOException {
-    if (launcher == null) {
+      log.info("Component conf dir already exists: " + compDir);
       return;
     }
-    Map<String, String> config = ConfigUtils.replacePropsInConfig(
-        configs.get(configFileDN), env.options);
-    String fileName = ConfigUtils.replaceProps(config, configFileName);
-    File localFile = new File(RESOURCE_DIR);
-    if (!localFile.exists()) {
-      if (!localFile.mkdir() && !localFile.exists()) {
-        throw new IOException(RESOURCE_DIR + " could not be created!");
-      }
-    }
-
-    String folder = null;
-    if (perComponent) {
-      folder = roleName;
-    } else if (perGroup) {
-      folder = roleGroup;
-    }
-    if (folder != null) {
-      localFile = new File(localFile, folder);
-      if (!localFile.exists()) {
-        if (!localFile.mkdir() && !localFile.exists()) {
-          throw new IOException(localFile + " could not be created!");
-        }
-      }
-    }
-    localFile = new File(localFile, new File(fileName).getName());
-
-    log.info("Localizing {} configs to config file {} (destination {}) " +
-            "based on {} configs", config.size(), localFile, fileName,
-        configFileDN);
-    if (!localFile.exists()) {
-      createConfigFile(fileSystem, localFile, configFormat, configFileDN,
-          config, clusterName);
-    } else {
-      log.info("Local {} file {} already exists", configFormat, localFile);
-    }
-    Path destPath = uploadResource(localFile, fileSystem, folder, clusterName);
-    LocalResource configResource = fileSystem.createAmResource(destPath,
-        LocalResourceType.FILE);
-
-    File destFile = new File(fileName);
-    if (destFile.isAbsolute()) {
-      launcher.addLocalResource(
-          RESOURCE_DIR + "/" + destFile.getName(),
-          configResource, fileName);
-    } else {
-      launcher.addLocalResource(APP_CONF_DIR + "/" + fileName,
-          configResource);
-    }
-  }
 
-  /**
-   * Localize application tarballs and other resources requested by the app.
-   * @param launcher container launcher
-   * @param fileSystem file system
-   * @param appConf app configurations
-   * @param roleGroup component group
-   * @param clusterName app name
-   * @throws IOException resources cannot be uploaded
-   * @throws BadConfigException package name or type is not specified
-   */
-  public void localizePackages(ContainerLauncher launcher,
-      SliderFileSystem fileSystem, ConfTreeOperations appConf, String roleGroup,
-      String clusterName) throws IOException, BadConfigException {
-    for (Entry<String, Map<String, String>> pkg :
-        getPackages(roleGroup, appConf).entrySet()) {
-      String pkgName = pkg.getValue().get(OptionKeys.NAME_SUFFIX);
-      String pkgType = pkg.getValue().get(OptionKeys.TYPE_SUFFIX);
-      Path pkgPath = fileSystem.buildResourcePath(pkgName);
-      if (!fileSystem.isFile(pkgPath)) {
-        pkgPath = fileSystem.buildResourcePath(clusterName,
-            pkgName);
-      }
-      if (!fileSystem.isFile(pkgPath)) {
-        throw new IOException("Package doesn't exist as a resource: " +
-            pkgName);
+    for (ConfigFile configFile : component.getConfiguration().getFiles()) {
+      String fileName = configFile.getSrcFile();
+      // substitute file name
+      for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+        configFile.setDestFile(configFile.getDestFile()
+            .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
       }
-      log.info("Adding resource {}", pkgName);
-      LocalResourceType type = LocalResourceType.FILE;
-      if ("archive".equals(pkgType)) {
-        type = LocalResourceType.ARCHIVE;
-      }
-      LocalResource packageResource = fileSystem.createAmResource(
-          pkgPath, type);
-      launcher.addLocalResource(APP_PACKAGES_DIR, packageResource);
-    }
-  }
-
-  /**
-   * Build a map of configuration description/name to configuration key/value
-   * properties, with all known tokens substituted into the property values.
-   * @param appConf app configurations
-   * @param internalsConf internal configurations
-   * @param containerId container ID
-   * @param roleName component name
-   * @param roleGroup component group
-   * @param amState access to AM state
-   * @return configuration properties grouped by config description/name
-   */
-  public Map<String, Map<String, String>> buildConfigurations(
-      ConfTreeOperations appConf, ConfTreeOperations internalsConf,
-      String containerId, String clusterName, String roleName, String roleGroup,
-      StateAccessForProviders amState) {
-
-    Map<String, Map<String, String>> configurations = new TreeMap<>();
-    Map<String, String> tokens = getStandardTokenMap(appConf,
-        internalsConf, roleName, roleGroup, containerId, clusterName);
-
-    Set<String> configs = new HashSet<>();
-    configs.addAll(getApplicationConfigurationTypes(roleGroup, appConf));
-    configs.addAll(getSystemConfigurationsRequested(appConf));
-
-    for (String configType : configs) {
-      addNamedConfiguration(configType, appConf.getGlobalOptions().options,
-          configurations, tokens, amState);
-      if (appConf.getComponent(roleGroup) != null) {
-        addNamedConfiguration(configType,
-            appConf.getComponent(roleGroup).options, configurations, tokens,
-            amState);
-      }
-    }
-
-    //do a final replacement of re-used configs
-    dereferenceAllConfigs(configurations);
-
-    return configurations;
-  }
-
-  /**
-   * Substitute "site." prefixed configuration values into other configuration
-   * values where needed. The format for these substitutions is that
-   * {@literal ${@//site/configDN/key}} will be replaced by the value for the
-   * "site.configDN.key" property.
-   * @param configurations configuration properties grouped by config
-   *                       description/name
-   */
-  public void dereferenceAllConfigs(
-      Map<String, Map<String, String>> configurations) {
-    Map<String, String> allConfigs = new HashMap<>();
-    String lookupFormat = "${@//site/%s/%s}";
-    for (Map.Entry<String, Map<String, String>> entry : configurations.entrySet()) {
-      Map<String, String> configBucket = entry.getValue();
-      for(Map.Entry<String, String> config: configBucket.entrySet()) {
-        allConfigs.put(String.format(lookupFormat, entry.getKey(), config.getKey()),
-            config.getValue());
-      }
-    }
-
-    boolean finished = false;
-    while (!finished) {
-      finished = true;
-      for (Map.Entry<String, String> entry : allConfigs.entrySet()) {
-        String configValue = entry.getValue();
-        for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) {
-          String lookUpValue = lookUpEntry.getValue();
-          if (lookUpValue.contains("${@//site/")) {
-            continue;
-          }
-          String lookUpKey = lookUpEntry.getKey();
-          if (configValue != null && configValue.contains(lookUpKey)) {
-            configValue = configValue.replace(lookUpKey, lookUpValue);
-          }
-        }
-        if (configValue != null && !configValue.equals(entry.getValue())) {
-          finished = false;
-          allConfigs.put(entry.getKey(), configValue);
-        }
-      }
-    }
-    for (Map.Entry<String, Map<String, String>> configEntry : configurations
-        .entrySet()) {
-      Map<String, String> configBucket = configEntry.getValue();
-      for (Map.Entry<String, String> entry: configBucket.entrySet()) {
-        String configName = entry.getKey();
-        String configValue = entry.getValue();
-        for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) {
-          String lookUpValue = lookUpEntry.getValue();
-          if (lookUpValue.contains("${@//site/")) {
-            continue;
-          }
-          String lookUpKey = lookUpEntry.getKey();
-          if (configValue != null && configValue.contains(lookUpKey)) {
-            configValue = configValue.replace(lookUpKey, lookUpValue);
+      // substitute configs
+      substituteConfigs(configFile.getProps(), tokensForSubstitution);
+
+      // write configs onto hdfs
+      PublishedConfiguration publishedConfiguration =
+          new PublishedConfiguration(fileName,
+              configFile.getProps().entrySet());
+      Path remoteFile = new Path(compDir, fileName);
+      if (!fs.getFileSystem().exists(remoteFile)) {
+        synchronized (this) {
+          if (!fs.getFileSystem().exists(remoteFile)) {
+            PublishedConfigurationOutputter configurationOutputter =
+                PublishedConfigurationOutputter.createOutputter(
+                    ConfigFormat.resolve(configFile.getType().toString()),
+                    publishedConfiguration);
+            FSDataOutputStream os = null;
+            try {
+              os = fs.getFileSystem().create(remoteFile);
+              configurationOutputter.save(os);
+              os.flush();
+              log.info("Created config file on hdfs: " + remoteFile);
+            } finally {
+              IOUtils.closeStream(os);
+            }
           }
         }
-        configBucket.put(configName, configValue);
-      }
-    }
-  }
-
-  /**
-   * Return a set of configuration description/names represented in the app.
-   * configuration
-   * @param roleGroup component group
-   * @param appConf app configurations
-   * @return set of configuration description/names
-   */
-  public Set<String> getApplicationConfigurationTypes(String roleGroup,
-      ConfTreeOperations appConf) {
-    Set<String> configList = new HashSet<>();
-
-    String prefix = OptionKeys.CONF_FILE_PREFIX;
-    String suffix = OptionKeys.TYPE_SUFFIX;
-    MapOperations component = appConf.getComponent(roleGroup);
-    if (component != null) {
-      addConfsToList(component, configList, prefix, suffix);
-    }
-    addConfsToList(appConf.getGlobalOptions(), configList, prefix, suffix);
-
-    return configList;
-  }
-
-  /**
-   * Finds all configuration description/names of the form
-   * prefixconfigDNsuffix in the configuration (e.g. conf.configDN.type).
-   * @param confMap configuration properties
-   * @param confList set containing configuration description/names
-   * @param prefix configuration key prefix to match
-   * @param suffix configuration key suffix to match
-   */
-  private void addConfsToList(Map<String, String> confMap,
-      Set<String> confList, String prefix, String suffix) {
-    for (Entry<String, String> entry : confMap.entrySet()) {
-      String key = entry.getKey();
-      if (key.startsWith(prefix) && key.endsWith(suffix)) {
-        String confName = key.substring(prefix.length(),
-            key.length() - suffix.length());
-        if (!confName.isEmpty()) {
-          confList.add(confName);
-        }
-      }
-    }
-  }
-
-  /**
-   * Build a map of package description/name to package key/value properties
-   * (there should be two properties, type and name).
-   * @param roleGroup component group
-   * @param appConf app configurations
-   * @return map of package description/name to package key/value properties
-   * @throws BadConfigException package name or type is not specified
-   */
-  public Map<String, Map<String, String>> getPackages(String roleGroup,
-      ConfTreeOperations appConf) throws BadConfigException {
-    Map<String, Map<String, String>> packages = new HashMap<>();
-    String prefix = OptionKeys.PKG_FILE_PREFIX;
-    String typeSuffix = OptionKeys.TYPE_SUFFIX;
-    String nameSuffix = OptionKeys.NAME_SUFFIX;
-    MapOperations component = appConf.getComponent(roleGroup);
-    if (component == null) {
-      component = appConf.getGlobalOptions();
-    }
-    for (Map.Entry<String, String> entry : component.entrySet()) {
-      String key = entry.getKey();
-      if (key.startsWith(prefix)) {
-        String confName;
-        String type;
-        if (key.endsWith(typeSuffix)) {
-          confName = key.substring(prefix.length(), key.length() - typeSuffix.length());
-          type = typeSuffix;
-        } else if (key.endsWith(nameSuffix)) {
-          confName = key.substring(prefix.length(), key.length() - nameSuffix.length());
-          type = nameSuffix;
-        } else {
-          continue;
-        }
-        if (!packages.containsKey(confName)) {
-          packages.put(confName, new HashMap<String, String>());
-        }
-        packages.get(confName).put(type, entry.getValue());
-      }
-    }
-
-    for (Entry<String, Map<String, String>> pkg : packages.entrySet()) {
-      if (!pkg.getValue().containsKey(OptionKeys.TYPE_SUFFIX)) {
-        throw new BadConfigException("Package " + pkg.getKey() + " doesn't " +
-            "have a package type");
       }
-      if (!pkg.getValue().containsKey(OptionKeys.NAME_SUFFIX)) {
-        throw new BadConfigException("Package " + pkg.getKey() + " doesn't " +
-            "have a package name");
-      }
-    }
-
-    return packages;
-  }
-
-  /**
-   * Return system configurations requested by the app.
-   * @param appConf app configurations
-   * @return set of system configurations
-   */
-  public Set<String> getSystemConfigurationsRequested(
-      ConfTreeOperations appConf) {
-    Set<String> configList = new HashSet<>();
 
-    String configTypes = appConf.get(SYSTEM_CONFIGS);
-    if (configTypes != null && configTypes.length() > 0) {
-      String[] configs = configTypes.split(",");
-      for (String config : configs) {
-        configList.add(config.trim());
+      // Publish configs
+      amState.getPublishedSliderConfigurations()
+          .put(configFile.getSrcFile(), publishedConfiguration);
+
+      // Add resource for localization
+      LocalResource configResource =
+          fs.createAmResource(remoteFile, LocalResourceType.FILE);
+      File destFile = new File(configFile.getDestFile());
+      //TODO why to we need to differetiate  RESOURCE_DIR vs APP_CONF_DIR
+      if (destFile.isAbsolute()) {
+        String symlink = RESOURCE_DIR + "/" + fileName;
+        launcher.addLocalResource(symlink, configResource,
+            configFile.getDestFile());
+        log.info("Add config file for localization: " + symlink + " -> "
+            + configResource.getResource().getFile() + ", dest mount path: "
+            + configFile.getDestFile());
+      } else {
+        String symlink = APP_CONF_DIR + "/" + fileName;
+        launcher.addLocalResource(symlink, configResource);
+        log.info("Add config file for localization: " + symlink + " -> "
+            + configResource.getResource().getFile());
       }
     }
-
-    return configList;
-  }
-
-  /**
-   * For a given config description/name, pull out its site configs from the
-   * source config map, remove the site.configDN. prefix from them, and place
-   * them into a new config map using the {@link #propagateSiteOptions} method
-   * (with tokens substituted). This new k/v map is put as the value for the
-   * configDN key in the configurations map.
-   * @param configName config description/name
-   * @param sourceConfig config containing site.* properties
-   * @param configurations configuration map to be populated
-   * @param tokens initial substitution tokens
-   * @param amState access to AM state
-   */
-  private void addNamedConfiguration(String configName,
-      Map<String, String> sourceConfig,
-      Map<String, Map<String, String>> configurations,
-      Map<String, String> tokens, StateAccessForProviders amState) {
-    Map<String, String> config = new HashMap<>();
-    if (configName.equals(GLOBAL_CONFIG_TAG)) {
-      addDefaultGlobalConfig(config);
-    }
-    // add role hosts to tokens
-    addRoleRelatedTokens(tokens, amState);
-    propagateSiteOptions(sourceConfig, config, configName, tokens);
-
-    configurations.put(configName, config);
   }
 
   /**
    * Get initial token map to be substituted into config values.
    * @param appConf app configurations
-   * @param internals internal configurations
-   * @param componentName component name
-   * @param componentGroup component group
-   * @param clusterName app name
-   * @return tokens to replace
-   */
-  public Map<String, String> getStandardTokenMap(ConfTreeOperations appConf,
-      ConfTreeOperations internals, String componentName,
-      String componentGroup, String clusterName) {
-    return getStandardTokenMap(appConf, internals, componentName,
-        componentGroup, null, clusterName);
-  }
-
-  /**
-   * Get initial token map to be substituted into config values.
-   * @param appConf app configurations
-   * @param internals internal configurations
    * @param componentName component name
    * @param componentGroup component group
    * @param containerId container ID
    * @param clusterName app name
    * @return tokens to replace
    */
-  public Map<String, String> getStandardTokenMap(ConfTreeOperations appConf,
-      ConfTreeOperations internals, String componentName,
+  public Map<String, String> getStandardTokenMap(
+      Configuration appConf, Configuration componentConf, String componentName,
       String componentGroup, String containerId, String clusterName) {
 
     Map<String, String> tokens = new HashMap<>();
     if (containerId != null) {
       tokens.put("${CONTAINER_ID}", containerId);
     }
-    String nnuri = appConf.get("site.fs.defaultFS");
-    tokens.put("${NN_URI}", nnuri);
-    tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
-    tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
-    tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
-    String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX,
-        null);
+    String nnuri = appConf.getProperty("fs.defaultFS");
+    if (nnuri != null && !nnuri.isEmpty()) {
+      tokens.put("${NN_URI}", nnuri);
+      tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
+    }
+    tokens.put("${ZK_HOST}", appConf.getProperty(OptionKeys.ZOOKEEPER_HOSTS));
+    tokens.put("${DEFAULT_ZK_PATH}", appConf.getProperty(OptionKeys.ZOOKEEPER_PATH));
+    String prefix = componentConf.getProperty(ROLE_PREFIX);
     String dataDirSuffix = "";
     if (prefix == null) {
       prefix = "";
     } else {
       dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix);
     }
-    tokens.put("${DEFAULT_DATA_DIR}", internals.getGlobalOptions()
-        .getOption(InternalKeys.INTERNAL_DATA_DIR_PATH, null) + dataDirSuffix);
-    tokens.put("${JAVA_HOME}", appConf.get(JAVA_HOME));
+    tokens.put("${DEFAULT_DATA_DIR}",
+        appConf.getProperty(InternalKeys.INTERNAL_DATA_DIR_PATH)
+            + dataDirSuffix);
+    tokens.put("${JAVA_HOME}", appConf.getProperty(JAVA_HOME));
     tokens.put("${COMPONENT_NAME}", componentName);
     tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase());
     tokens.put("${COMPONENT_PREFIX}", prefix);
@@ -1005,7 +388,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
    * @param tokens existing tokens
    * @param amState access to AM state
    */
-  public void addRoleRelatedTokens(Map<String, String> tokens,
+  public void addRoleHostTokens(Map<String, String> tokens,
       StateAccessForProviders amState) {
     if (amState == null) {
       return;
@@ -1020,26 +403,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
   }
 
   /**
-   * Add global configuration properties.
-   * @param config map where default global properties will be added
-   */
-  private void addDefaultGlobalConfig(Map<String, String> config) {
-    config.put("app_log_dir", "${LOG_DIR}");
-    config.put("app_pid_dir", "${WORK_DIR}/app/run");
-    config.put("app_install_dir", "${WORK_DIR}/app/install");
-    config.put("app_conf_dir", "${WORK_DIR}/" + APP_CONF_DIR);
-    config.put("app_input_conf_dir", "${WORK_DIR}/" + PROPAGATED_CONF_DIR_NAME);
-
-    // add optional parameters only if they are not already provided
-    if (!config.containsKey("pid_file")) {
-      config.put("pid_file", "${WORK_DIR}/app/run/component.pid");
-    }
-    if (!config.containsKey("app_root")) {
-      config.put("app_root", "${WORK_DIR}/app/install");
-    }
-  }
-
-  /**
    * Return a list of hosts based on current ClusterNodes.
    * @param values cluster nodes
    * @param hostOnly whether host or host/server name will be added to list
@@ -1101,82 +464,4 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
           containerId, e);
     }
   }
-
-  /**
-   * Publish a named property bag that may contain name-value pairs for app
-   * configurations such as hbase-site.
-   * @param name config file identifying name
-   * @param description config file description
-   * @param entries config file properties
-   * @param amState access to AM state
-   */
-  public void publishApplicationInstanceData(String name, String description,
-      Iterable<Map.Entry<String, String>> entries,
-      StateAccessForProviders amState) {
-    PublishedConfiguration pubconf = new PublishedConfiguration(description,
-        entries);
-    log.info("publishing {}", pubconf);
-    amState.getPublishedSliderConfigurations().put(name, pubconf);
-  }
-
-  /**
-   * Publish an export group.
-   * @param exportGroup export groups
-   * @param amState access to AM state
-   * @param groupName export group name
-   */
-  public void publishExportGroup(
-      Map<String, Set<ExportEntry>> exportGroup,
-      StateAccessForProviders amState, String groupName) {
-    // Publish in old format for the time being
-    Map<String, String> simpleEntries = new HashMap<>();
-    for (Entry<String, Set<ExportEntry>> entry : exportGroup.entrySet()) {
-      Set<ExportEntry> exports = entry.getValue();
-      if (SliderUtils.isNotEmpty(exports)) {
-        Set<String> values = new TreeSet<>();
-        for (ExportEntry export : exports) {
-          values.add(export.getValue());
-        }
-        simpleEntries.put(entry.getKey(), StringUtils.join(",", values));
-      }
-    }
-    publishApplicationInstanceData(groupName, groupName,
-        simpleEntries.entrySet(), amState);
-
-    PublishedExports exports = new PublishedExports(groupName);
-    exports.setUpdated(new Date().getTime());
-    exports.putValues(exportGroup.entrySet());
-    amState.getPublishedExportsSet().put(groupName, exports);
-  }
-
-  public Map<String, String> getExports(ConfTreeOperations appConf,
-      String roleGroup) {
-    Map<String, String> exports = new HashMap<>();
-    propagateOptions(appConf.getComponent(roleGroup).options, exports,
-        null, OptionKeys.EXPORT_PREFIX);
-    return exports;
-  }
-
-  public String getGroupKey(String roleGroup, ConfTreeOperations appConf) {
-    String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
-    return getNameOrGroupKey(rolePrefix, roleGroup);
-  }
-
-  public String getNameKey(String roleName, String roleGroup,
-      ConfTreeOperations appConf) {
-    String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
-    return getNameOrGroupKey(rolePrefix, roleName);
-  }
-
-  public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) {
-    if (!rolePrefix.isEmpty()) {
-      if (!roleNameOrGroup.startsWith(rolePrefix)) {
-        log.warn("Something went wrong, {} doesn't start with {}",
-            roleNameOrGroup, rolePrefix);
-        return null;
-      }
-      roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length());
-    }
-    return roleNameOrGroup.toUpperCase(Locale.ENGLISH);
-  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message