hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [11/17] hadoop git commit: YARN-9075. Add ability to register/remove auxiliary service without restart node manager. Contributed by Billie Rinaldi
Date Wed, 19 Dec 2018 20:41:36 GMT
YARN-9075.  Add ability to register/remove auxiliary service without restart node manager.
            Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7a5a443
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7a5a443
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7a5a443

Branch: refs/heads/HDFS-12943
Commit: c7a5a4435ec3098c2765e8ef53b7771e0532ff61
Parents: a6e9d27
Author: Eric Yang <eyang@apache.org>
Authored: Tue Dec 18 17:05:51 2018 -0500
Committer: Eric Yang <eyang@apache.org>
Committed: Tue Dec 18 17:05:51 2018 -0500

----------------------------------------------------------------------
 .../apache/hadoop/mapred/ShuffleHandler.java    |   3 +
 .../hadoop/yarn/conf/YarnConfiguration.java     |  10 +-
 .../src/main/resources/yarn-default.xml         |  15 +
 .../hadoop/yarn/server/nodemanager/Context.java |   5 +
 .../yarn/server/nodemanager/NodeManager.java    |  13 +
 .../containermanager/AuxServices.java           | 809 ++++++++++++++-----
 .../AuxiliaryServiceWithCustomClassLoader.java  |   8 +-
 .../containermanager/ContainerManagerImpl.java  |   1 +
 .../records/AuxServiceConfiguration.java        | 137 ++++
 .../records/AuxServiceFile.java                 | 137 ++++
 .../records/AuxServiceRecord.java               | 175 ++++
 .../records/AuxServiceRecords.java              |  46 ++
 .../containermanager/records/package-info.java  |  29 +
 .../nodemanager/webapp/NMWebServices.java       |  37 +
 .../webapp/dao/AuxiliaryServiceInfo.java        |  60 ++
 .../webapp/dao/AuxiliaryServicesInfo.java       |  55 ++
 .../amrmproxy/BaseAMRMProxyTest.java            |  10 +
 .../containermanager/TestAuxServices.java       | 330 +++++---
 18 files changed, 1570 insertions(+), 310 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index c222685..e47deb2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -281,6 +281,7 @@ public class ShuffleHandler extends AuxiliaryService {
     }
   }
 
+  private final MetricsSystem ms;
   final ShuffleMetrics metrics;
 
   class ReduceMapFileCount implements ChannelFutureListener {
@@ -397,6 +398,7 @@ public class ShuffleHandler extends AuxiliaryService {
 
   ShuffleHandler(MetricsSystem ms) {
     super(MAPREDUCE_SHUFFLE_SERVICEID);
+    this.ms = ms;
     metrics = ms.register(new ShuffleMetrics());
   }
 
@@ -579,6 +581,7 @@ public class ShuffleHandler extends AuxiliaryService {
     if (stateDb != null) {
       stateDb.close();
     }
+    ms.unregisterSource(ShuffleMetrics.class.getSimpleName());
     super.serviceStop();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2f25284..72cc0d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2222,7 +2222,15 @@ public class YarnConfiguration extends Configuration {
   
   public static final String NM_AUX_SERVICES = 
       NM_PREFIX + "aux-services";
-  
+
+  public static final String NM_AUX_SERVICES_MANIFEST =
+      NM_AUX_SERVICES + ".manifest";
+
+  public static final String NM_AUX_SERVICES_MANIFEST_RELOAD_MS =
+      NM_AUX_SERVICES + ".manifest.reload-ms";
+
+  public static final long DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS = 120000;
+
   public static final String NM_AUX_SERVICE_FMT =
       NM_PREFIX + "aux-services.%s.class";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e7a0e14..ccceaab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1922,6 +1922,21 @@
   </property>
 
   <property>
+    <description>A file containing auxiliary service specifications. If
+      manifest file is specified, yarn.nodemanager.aux-services and other
+      aux services configuration properties will be ignored.</description>
+    <name>yarn.nodemanager.aux-services.manifest</name>
+    <value></value>
+  </property>
+
+  <property>
+    <description>Length of time in ms to wait between reloading aux services
+      manifest. If 0 or less, manifest will not be reloaded.</description>
+    <name>yarn.nodemanager.aux-services.manifest.reload-ms</name>
+    <value></value>
+  </property>
+
+  <property>
     <description>No. of ms to wait between sending a SIGTERM and SIGKILL to a container</description>
     <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
     <value>250</value>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 84b3915..113e6a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
 import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
 import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -137,4 +138,8 @@ public interface Context {
    * @return the NM {@code DeletionService}.
    */
   DeletionService getDeletionService();
+
+  void setAuxServices(AuxServices auxServices);
+
+  AuxServices getAuxServices();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 6eda4a8..c6719d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
 import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
 import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.server.nodemanager.collectormanager.NMCollectorService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
@@ -684,6 +685,8 @@ public class NodeManager extends CompositeService
 
     private NMLogAggregationStatusTracker nmLogAggregationStatusTracker;
 
+    private AuxServices auxServices;
+
     public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
         NMTokenSecretManagerInNM nmTokenSecretManager,
         LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
@@ -934,6 +937,16 @@ public class NodeManager extends CompositeService
     public NMLogAggregationStatusTracker getNMLogAggregationStatusTracker() {
       return nmLogAggregationStatusTracker;
     }
+
+    @Override
+    public void setAuxServices(AuxServices auxServices) {
+      this.auxServices = auxServices;
+    }
+
+    @Override
+    public AuxServices getAuxServices() {
+      return this.auxServices;
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index 77c4dd9..ba936d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -18,15 +18,33 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager;
 
+import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
 import java.util.regex.Pattern;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceFile;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -69,40 +87,69 @@ public class AuxServices extends AbstractService
   public static final FsPermission NM_AUX_SERVICE_DIR_PERM =
       new FsPermission((short) 0700);
 
+  public static final String CLASS_NAME = "class.name";
+  public static final String SYSTEM_CLASSES = "system.classes";
+
   static final String STATE_STORE_ROOT_NAME = "nm-aux-services";
 
   private static final Logger LOG =
        LoggerFactory.getLogger(AuxServices.class);
   private static final String DEL_SUFFIX = "_DEL_";
 
-  protected final Map<String,AuxiliaryService> serviceMap;
-  protected final Map<String,ByteBuffer> serviceMetaData;
+  private final Map<String, AuxiliaryService> serviceMap;
+  private final Map<String, AuxServiceRecord> serviceRecordMap;
+  private final Map<String, ByteBuffer> serviceMetaData;
   private final AuxiliaryLocalPathHandler auxiliaryLocalPathHandler;
   private final LocalDirsHandlerService dirsHandler;
   private final DeletionService delService;
   private final UserGroupInformation userUGI;
 
+  private final FsPermission storeDirPerms = new FsPermission((short)0700);
+  private Path stateStoreRoot = null;
+  private FileSystem stateStoreFs = null;
+
+  private Path manifest;
+  private FileSystem manifestFS;
+  private Timer manifestReloadTimer;
+  private TimerTask manifestReloadTask;
+  private long manifestReloadInterval;
+  private long manifestModifyTS = -1;
+
+  private final ObjectMapper mapper;
+
   private final Pattern p = Pattern.compile("^[A-Za-z_]+[A-Za-z0-9_]*$");
 
-  public AuxServices(AuxiliaryLocalPathHandler auxiliaryLocalPathHandler,
+  AuxServices(AuxiliaryLocalPathHandler auxiliaryLocalPathHandler,
       Context nmContext, DeletionService deletionService) {
     super(AuxServices.class.getName());
     serviceMap =
-      Collections.synchronizedMap(new HashMap<String,AuxiliaryService>());
+      Collections.synchronizedMap(new HashMap<String, AuxiliaryService>());
+    serviceRecordMap =
+        Collections.synchronizedMap(new HashMap<String, AuxServiceRecord>());
     serviceMetaData =
       Collections.synchronizedMap(new HashMap<String,ByteBuffer>());
     this.auxiliaryLocalPathHandler = auxiliaryLocalPathHandler;
     this.dirsHandler = nmContext.getLocalDirsHandler();
     this.delService = deletionService;
     this.userUGI = getRemoteUgi();
+    this.mapper = new ObjectMapper();
+    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
     // Obtain services from configuration in init()
   }
 
+  /**
+   * Adds a service to the service map.
+   *
+   * @param name aux service name
+   * @param service aux service
+   * @param serviceRecord aux service record
+   */
   protected final synchronized void addService(String name,
-      AuxiliaryService service) {
-    LOG.info("Adding auxiliary service " +
-        service.getName() + ", \"" + name + "\"");
+      AuxiliaryService service, AuxServiceRecord serviceRecord) {
+    LOG.info("Adding auxiliary service " + serviceRecord.getName() +
+        " version " + serviceRecord.getVersion());
     serviceMap.put(name, service);
+    serviceRecordMap.put(name, serviceRecord);
   }
 
   Collection<AuxiliaryService> getServices() {
@@ -110,13 +157,21 @@ public class AuxServices extends AbstractService
   }
 
   /**
+   * Gets current aux service records.
+   *
+   * @return a collection of service records
+   */
+  public Collection<AuxServiceRecord> getServiceRecords() {
+    return Collections.unmodifiableCollection(serviceRecordMap.values());
+  }
+
+  /**
    * @return the meta data for all registered services, that have been started.
    * If a service has not been started no metadata will be available. The key
    * is the name of the service as defined in the configuration.
    */
   public Map<String, ByteBuffer> getMetaData() {
-    Map<String, ByteBuffer> metaClone = new HashMap<String, ByteBuffer>(
-        serviceMetaData.size());
+    Map<String, ByteBuffer> metaClone = new HashMap<>(serviceMetaData.size());
     synchronized (serviceMetaData) {
       for (Entry<String, ByteBuffer> entry : serviceMetaData.entrySet()) {
         metaClone.put(entry.getKey(), entry.getValue().duplicate());
@@ -125,11 +180,470 @@ public class AuxServices extends AbstractService
     return metaClone;
   }
 
+  /**
+   * Creates an auxiliary service from a specification using the Configuration
+   * classloader.
+   *
+   * @param service aux service record
+   * @return auxiliary service
+   */
+  private AuxiliaryService createAuxServiceFromConfiguration(AuxServiceRecord
+      service) {
+    Configuration c = new Configuration(false);
+    c.set(CLASS_NAME, getClassName(service));
+    Class<? extends AuxiliaryService> sClass = c.getClass(CLASS_NAME,
+        null, AuxiliaryService.class);
+
+    if (sClass == null) {
+      throw new YarnRuntimeException("No class defined for auxiliary " +
+          "service" + service.getName());
+    }
+    return ReflectionUtils.newInstance(sClass, null);
+  }
+
+  /**
+   * Creates an auxiliary service from a specification using a custom local
+   * classpath.
+   *
+   * @param service aux service record
+   * @param appLocalClassPath local class path
+   * @param conf configuration
+   * @return auxiliary service
+   * @throws IOException
+   * @throws ClassNotFoundException
+   */
+  private AuxiliaryService createAuxServiceFromLocalClasspath(
+      AuxServiceRecord service, String appLocalClassPath, Configuration conf)
+      throws IOException, ClassNotFoundException {
+    Preconditions.checkArgument(appLocalClassPath != null &&
+        !appLocalClassPath.isEmpty(),
+        "local classpath was null in createAuxServiceFromLocalClasspath");
+    final String sName = service.getName();
+    final String className = getClassName(service);
+
+    if (service.getConfiguration() != null && service.getConfiguration()
+        .getFiles().size() > 0) {
+      throw new YarnRuntimeException("The aux service:" + sName
+          + " has configured local classpath:" + appLocalClassPath
+          + " and config files:" + service.getConfiguration().getFiles()
+          + ". Only one of them should be configured.");
+    }
+
+    return AuxiliaryServiceWithCustomClassLoader.getInstance(conf, className,
+        appLocalClassPath, getSystemClasses(service, className));
+  }
+
+  /**
+   * Creates an auxiliary service from a specification.
+   *
+   * @param service aux service record
+   * @param conf configuration
+   * @param fromConfiguration true if from configuration, false if from manifest
+   * @return auxiliary service
+   * @throws IOException
+   * @throws ClassNotFoundException
+   */
+  private AuxiliaryService createAuxService(AuxServiceRecord service,
+      Configuration conf, boolean fromConfiguration) throws IOException,
+      ClassNotFoundException {
+    final String sName = service.getName();
+    final String className = getClassName(service);
+    if (className == null || className.isEmpty()) {
+      throw new YarnRuntimeException("Class name not provided for auxiliary " +
+          "service " + sName);
+    }
+    if (fromConfiguration) {
+      // aux services from the configuration have an additional configuration
+      // option specifying a local classpath that will not be localized
+      final String appLocalClassPath = conf.get(String.format(
+          YarnConfiguration.NM_AUX_SERVICES_CLASSPATH, sName));
+      if (appLocalClassPath != null && !appLocalClassPath.isEmpty()) {
+        return createAuxServiceFromLocalClasspath(service, appLocalClassPath,
+            conf);
+      }
+    }
+    AuxServiceConfiguration serviceConf = service.getConfiguration();
+    List<Path> destFiles = new ArrayList<>();
+    if (serviceConf != null) {
+      List<AuxServiceFile> files = serviceConf.getFiles();
+      if (files != null) {
+        for (AuxServiceFile file : files) {
+          // localize file (if needed) and add it to the list of paths that
+          // will become the classpath
+          destFiles.add(maybeDownloadJars(sName, className, file.getSrcFile(),
+              file.getType(), conf));
+        }
+      }
+    }
+
+    if (destFiles.size() > 0) {
+      // create aux service using a custom localized classpath
+      LOG.info("The aux service:" + sName
+          + " is using the custom classloader with classpath " + destFiles);
+      return AuxiliaryServiceWithCustomClassLoader.getInstance(conf,
+          className, StringUtils.join(File.pathSeparatorChar, destFiles),
+          getSystemClasses(service, className));
+    } else {
+      return createAuxServiceFromConfiguration(service);
+    }
+  }
+
+  /**
+   * Copies the specified remote file to local NM aux service directory. If the
+   * same file already exists (as determined by modification time), the file
+   * will not be copied again.
+   *
+   * @param sName service name
+   * @param className service class name
+   * @param remoteFile location of the file to download
+   * @param type type of file (STATIC for a jar or ARCHIVE for a tarball)
+   * @param conf configuration
+   * @return path of the downloaded file
+   * @throws IOException
+   */
+  private Path maybeDownloadJars(String sName, String className, String
+      remoteFile, AuxServiceFile.TypeEnum type, Configuration conf)
+      throws IOException {
+    // load AuxiliaryService from remote classpath
+    FileContext localLFS = getLocalFileContext(conf);
+    // create NM aux-service dir in NM localdir if it does not exist.
+    Path nmAuxDir = dirsHandler.getLocalPathForWrite("."
+        + Path.SEPARATOR + NM_AUX_SERVICE_DIR);
+    if (!localLFS.util().exists(nmAuxDir)) {
+      try {
+        localLFS.mkdir(nmAuxDir, NM_AUX_SERVICE_DIR_PERM, true);
+      } catch (IOException ex) {
+        throw new YarnRuntimeException("Fail to create dir:"
+            + nmAuxDir.toString(), ex);
+      }
+    }
+    Path src = new Path(remoteFile);
+    FileContext remoteLFS = getRemoteFileContext(src.toUri(), conf);
+    FileStatus scFileStatus = remoteLFS.getFileStatus(src);
+    if (!scFileStatus.getOwner().equals(
+        this.userUGI.getShortUserName())) {
+      throw new YarnRuntimeException("The remote jarfile owner:"
+          + scFileStatus.getOwner() + " is not the same as the NM user:"
+          + this.userUGI.getShortUserName() + ".");
+    }
+    if ((scFileStatus.getPermission().toShort() & 0022) != 0) {
+      throw new YarnRuntimeException("The remote jarfile should not "
+          + "be writable by group or others. "
+          + "The current Permission is "
+          + scFileStatus.getPermission().toShort());
+    }
+    Path downloadDest = new Path(nmAuxDir,
+        className + "_" + scFileStatus.getModificationTime());
+    // check whether we need to re-download the jar
+    // from remote directory
+    Path targetDirPath = new Path(downloadDest,
+        scFileStatus.getPath().getName());
+    FileStatus[] allSubDirs = localLFS.util().listStatus(nmAuxDir);
+    for (FileStatus sub : allSubDirs) {
+      if (sub.getPath().getName().equals(downloadDest.getName())) {
+        return new Path(targetDirPath + Path.SEPARATOR + "*");
+      } else {
+        if (sub.getPath().getName().contains(className) &&
+            !sub.getPath().getName().endsWith(DEL_SUFFIX)) {
+          Path delPath = new Path(sub.getPath().getParent(),
+              sub.getPath().getName() + DEL_SUFFIX);
+          localLFS.rename(sub.getPath(), delPath);
+          LOG.info("delete old aux service jar dir:"
+              + delPath.toString());
+          FileDeletionTask deletionTask = new FileDeletionTask(
+              this.delService, null, delPath, null);
+          this.delService.delete(deletionTask);
+        }
+      }
+    }
+    LocalResourceType srcType;
+    if (type == AuxServiceFile.TypeEnum.STATIC) {
+      srcType = LocalResourceType.FILE;
+    } else if (type == AuxServiceFile.TypeEnum.ARCHIVE) {
+      srcType = LocalResourceType.ARCHIVE;
+    } else {
+      throw new YarnRuntimeException(
+          "Cannot unpack file of type " + type + " from remote-file-path:" +
+              src + "for aux-service:" + ".\n");
+    }
+    LocalResource scRsrc = LocalResource.newInstance(
+        URL.fromURI(src.toUri()),
+        srcType, LocalResourceVisibility.PRIVATE,
+        scFileStatus.getLen(), scFileStatus.getModificationTime());
+    FSDownload download = new FSDownload(localLFS, null, conf,
+        downloadDest, scRsrc, null);
+    try {
+      // don't need to convert downloaded path into a dir
+      // since it's already a jar path.
+      return download.call();
+    } catch (Exception ex) {
+      throw new YarnRuntimeException(
+          "Exception happend while downloading files "
+              + "for aux-service:" + sName + " and remote-file-path:"
+              + src + ".\n" + ex.getMessage());
+    }
+  }
+
+  /**
+   * If recovery is enabled, creates a recovery directory for the named
+   * service and sets it on the service.
+   *
+   * @param sName name of the service
+   * @param s auxiliary service
+   * @throws IOException
+   */
+  private void setStateStoreDir(String sName, AuxiliaryService s) throws
+      IOException {
+    if (stateStoreRoot != null) {
+      Path storePath = new Path(stateStoreRoot, sName);
+      stateStoreFs.mkdirs(storePath, storeDirPerms);
+      s.setRecoveryPath(storePath);
+    }
+  }
+
+  /**
+   * Removes a service from the service map and stops it, if it exists.
+   *
+   * @param sName name of the service
+   */
+  private synchronized void maybeRemoveAuxService(String sName) {
+    AuxiliaryService s;
+    s = serviceMap.remove(sName);
+    serviceRecordMap.remove(sName);
+    serviceMetaData.remove(sName);
+    if (s != null) {
+      stopAuxService(s);
+    }
+  }
+
+  /**
+   * Constructs an AuxiliaryService then configures and initializes it based
+   * on a service specification.
+   *
+   * @param service aux service record
+   * @param conf configuration
+   * @param fromConfiguration true if from configuration, false if from manifest
+   * @return aux service
+   * @throws IOException
+   */
+  private AuxiliaryService initAuxService(AuxServiceRecord service,
+      Configuration conf, boolean fromConfiguration) throws IOException {
+    final String sName = service.getName();
+    AuxiliaryService s;
+    try {
+      Preconditions
+          .checkArgument(
+              validateAuxServiceName(sName),
+              "The auxiliary service name: " + sName + " is invalid. " +
+                  "The valid service name should only contain a-zA-Z0-9_ " +
+                  "and cannot start with numbers.");
+      s = createAuxService(service, conf, fromConfiguration);
+      if (s == null) {
+        throw new YarnRuntimeException("No auxiliary service class loaded for" +
+            " " + sName);
+      }
+      // TODO better use s.getName()?
+      if (!sName.equals(s.getName())) {
+        LOG.warn("The Auxiliary Service named '" + sName + "' in the "
+            + "configuration is for " + s.getClass() + " which has "
+            + "a name of '" + s.getName() + "'. Because these are "
+            + "not the same tools trying to send ServiceData and read "
+            + "Service Meta Data may have issues unless the refer to "
+            + "the name in the config.");
+      }
+      s.setAuxiliaryLocalPathHandler(auxiliaryLocalPathHandler);
+      setStateStoreDir(sName, s);
+      Configuration customConf = new Configuration(conf);
+      if (service.getConfiguration() != null) {
+        for (Entry<String, String> entry : service.getConfiguration()
+            .getProperties().entrySet()) {
+          customConf.set(entry.getKey(), entry.getValue());
+        }
+      }
+      s.init(customConf);
+
+      LOG.info("Initialized auxiliary service " + sName);
+    } catch (RuntimeException e) {
+      LOG.error("Failed to initialize " + sName, e);
+      throw e;
+    } catch (ClassNotFoundException e) {
+      throw new YarnRuntimeException(e);
+    }
+    return s;
+  }
+
+  /**
+   * Reloads auxiliary services manifest. Must be called after service init.
+   *
+   * @throws IOException if manifest can't be loaded
+   */
+  private void reloadManifest() throws IOException {
+    loadManifest(getConfig(), true);
+  }
+
+  /**
+   * Reads the manifest file if it is configured, exists, and has not been
+   * modified since the last read.
+   *
+   * @return aux service records
+   * @throws IOException
+   */
+  private synchronized AuxServiceRecords maybeReadManifestFile() throws
+      IOException {
+    if (manifest == null) {
+      return null;
+    }
+    if (!manifestFS.exists(manifest)) {
+      LOG.info("Manifest file " + manifest + " doesn't exist");
+      return null;
+    }
+    FileStatus status;
+    try {
+      status = manifestFS.getFileStatus(manifest);
+    } catch (FileNotFoundException e) {
+      LOG.info("Manifest file " + manifest + " doesn't exist");
+      return null;
+    }
+    if (status.getModificationTime() == manifestModifyTS) {
+      return null;
+    }
+    manifestModifyTS = status.getModificationTime();
+    LOG.info("Reading auxiliary services manifest " + manifest);
+    try (FSDataInputStream in = manifestFS.open(manifest)) {
+      return mapper.readValue((InputStream) in, AuxServiceRecords.class);
+    }
+  }
+
+  /**
+   * Updates current aux services based on changes found in the manifest.
+   *
+   * @param conf configuration
+   * @param startServices if true starts services, otherwise only inits services
+   * @throws IOException
+   */
+  private synchronized void loadManifest(Configuration conf, boolean
+      startServices) throws IOException {
+    AuxServiceRecords services = maybeReadManifestFile();
+    if (services == null) {
+      // read did not occur or no changes detected
+      return;
+    }
+    Set<String> loadedAuxServices = new HashSet<>();
+    boolean foundChanges = false;
+    if (services.getServices() != null) {
+      for (AuxServiceRecord service : services.getServices()) {
+        AuxServiceRecord existingService = serviceRecordMap.get(service
+            .getName());
+        loadedAuxServices.add(service.getName());
+        if (existingService != null && existingService.equals(service)) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Auxiliary service already loaded: " + service.getName());
+          }
+          continue;
+        }
+        foundChanges = true;
+        try {
+          // stop aux service
+          maybeRemoveAuxService(service.getName());
+          // init aux service
+          AuxiliaryService s = initAuxService(service, conf, false);
+          if (startServices) {
+            // start aux service
+            startAuxService(service.getName(), s, service);
+          }
+          // add aux service to serviceMap
+          addService(service.getName(), s, service);
+        } catch (IOException e) {
+          LOG.error("Failed to load auxiliary service " + service.getName());
+        }
+      }
+    }
+
+    // remove aux services that do not appear in the manifest
+    List<String> servicesToRemove = new ArrayList<>();
+    for (String sName : serviceMap.keySet()) {
+      if (!loadedAuxServices.contains(sName)) {
+        foundChanges = true;
+        servicesToRemove.add(sName);
+      }
+    }
+    for (String sName : servicesToRemove) {
+      LOG.info("Removing aux service " + sName);
+      maybeRemoveAuxService(sName);
+    }
+
+    if (!foundChanges) {
+      LOG.info("No auxiliary services changes detected in manifest");
+    }
+  }
+
+  private static String getClassName(AuxServiceRecord service) {
+    AuxServiceConfiguration serviceConf = service.getConfiguration();
+    if (serviceConf == null) {
+      return null;
+    }
+    return serviceConf.getProperty(CLASS_NAME);
+  }
+
+  private static String[] getSystemClasses(AuxServiceRecord service, String
+      className) {
+    AuxServiceConfiguration serviceConf =
+        service.getConfiguration();
+    if (serviceConf == null) {
+      return new String[]{className};
+    }
+    return StringUtils.split(serviceConf.getProperty(SYSTEM_CLASSES,
+        className));
+  }
+
+  /**
+   * Translates an aux service specified in the Configuration to an aux
+   * service record.
+   *
+   * @param sName aux service name
+   * @param conf configuration
+   * @return
+   */
+  private static AuxServiceRecord createServiceRecordFromConfiguration(String
+      sName, Configuration conf) {
+    String className = conf.get(String.format(
+        YarnConfiguration.NM_AUX_SERVICE_FMT, sName));
+    String remoteClassPath = conf.get(String.format(
+        YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH, sName));
+    String[] systemClasses = conf.getTrimmedStrings(String.format(
+        YarnConfiguration.NM_AUX_SERVICES_SYSTEM_CLASSES, sName));
+
+    AuxServiceConfiguration serviceConf = new AuxServiceConfiguration();
+    if (className != null) {
+      serviceConf.setProperty(CLASS_NAME, className);
+    }
+    if (systemClasses != null) {
+      serviceConf.setProperty(SYSTEM_CLASSES, StringUtils.join(",",
+          systemClasses));
+    }
+    if (remoteClassPath != null) {
+      AuxServiceFile.TypeEnum type;
+      String lcClassPath = StringUtils.toLowerCase(remoteClassPath);
+      if (lcClassPath.endsWith(".jar")) {
+        type = AuxServiceFile.TypeEnum.STATIC;
+      } else if (lcClassPath.endsWith(".zip") ||
+          lcClassPath.endsWith(".tar.gz") || lcClassPath.endsWith(".tgz") ||
+          lcClassPath.endsWith(".tar")) {
+        type = AuxServiceFile.TypeEnum.ARCHIVE;
+      } else {
+        throw new YarnRuntimeException("Cannot unpack file from " +
+            "remote-file-path:" + remoteClassPath + "for aux-service:" +
+            sName + ".\n");
+      }
+      AuxServiceFile file = new AuxServiceFile().srcFile(remoteClassPath)
+          .type(type);
+      serviceConf.getFiles().add(file);
+    }
+    return new AuxServiceRecord().name(sName).configuration(serviceConf);
+  }
+
   @Override
-  public void serviceInit(Configuration conf) throws Exception {
-    final FsPermission storeDirPerms = new FsPermission((short)0700);
-    Path stateStoreRoot = null;
-    FileSystem stateStoreFs = null;
+  public synchronized void serviceInit(Configuration conf) throws Exception {
     boolean recoveryEnabled = conf.getBoolean(
         YarnConfiguration.NM_RECOVERY_ENABLED,
         YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED);
@@ -138,200 +652,77 @@ public class AuxServices extends AbstractService
           STATE_STORE_ROOT_NAME);
       stateStoreFs = FileSystem.getLocal(conf);
     }
-    Collection<String> auxNames = conf.getStringCollection(
-        YarnConfiguration.NM_AUX_SERVICES);
-    for (final String sName : auxNames) {
-      try {
-        Preconditions
-            .checkArgument(
-                validateAuxServiceName(sName),
-                "The ServiceName: " + sName + " set in " +
-                YarnConfiguration.NM_AUX_SERVICES +" is invalid." +
-                "The valid service name should only contain a-zA-Z0-9_ " +
-                "and can not start with numbers");
-        String classKey = String.format(
-            YarnConfiguration.NM_AUX_SERVICE_FMT, sName);
-        String className = conf.get(classKey);
-        final String appLocalClassPath = conf.get(String.format(
-            YarnConfiguration.NM_AUX_SERVICES_CLASSPATH, sName));
-        final String appRemoteClassPath = conf.get(String.format(
-            YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH, sName));
-        AuxiliaryService s = null;
-        boolean useCustomerClassLoader = ((appLocalClassPath != null
-            && !appLocalClassPath.isEmpty()) ||
-            (appRemoteClassPath != null && !appRemoteClassPath.isEmpty()))
-            && className != null && !className.isEmpty();
-        if (useCustomerClassLoader) {
-          // load AuxiliaryService from local class path
-          if (appRemoteClassPath == null || appRemoteClassPath.isEmpty()) {
-            s = AuxiliaryServiceWithCustomClassLoader.getInstance(
-                conf, className, appLocalClassPath);
-          } else {
-            // load AuxiliaryService from remote class path
-            if (appLocalClassPath != null && !appLocalClassPath.isEmpty()) {
-              throw new YarnRuntimeException("The aux serivce:" + sName
-                  + " has configured local classpath:" + appLocalClassPath
-                  + " and remote classpath:" + appRemoteClassPath
-                  + ". Only one of them should be configured.");
-            }
-            FileContext localLFS = getLocalFileContext(conf);
-            // create NM aux-service dir in NM localdir if it does not exist.
-            Path nmAuxDir = dirsHandler.getLocalPathForWrite("."
-                + Path.SEPARATOR + NM_AUX_SERVICE_DIR);
-            if (!localLFS.util().exists(nmAuxDir)) {
-              try {
-                localLFS.mkdir(nmAuxDir, NM_AUX_SERVICE_DIR_PERM, true);
-              } catch (IOException ex) {
-                throw new YarnRuntimeException("Fail to create dir:"
-                    + nmAuxDir.toString(), ex);
-              }
-            }
-            Path src = new Path(appRemoteClassPath);
-            FileContext remoteLFS = getRemoteFileContext(src.toUri(), conf);
-            FileStatus scFileStatus = remoteLFS.getFileStatus(src);
-            if (!scFileStatus.getOwner().equals(
-                this.userUGI.getShortUserName())) {
-              throw new YarnRuntimeException("The remote jarfile owner:"
-                  + scFileStatus.getOwner() + " is not the same as the NM user:"
-                  + this.userUGI.getShortUserName() + ".");
-            }
-            if ((scFileStatus.getPermission().toShort() & 0022) != 0) {
-              throw new YarnRuntimeException("The remote jarfile should not "
-                  + "be writable by group or others. "
-                  + "The current Permission is "
-                  + scFileStatus.getPermission().toShort());
-            }
-            Path dest = null;
-            Path downloadDest = new Path(nmAuxDir,
-                className + "_" + scFileStatus.getModificationTime());
-            // check whether we need to re-download the jar
-            // from remote directory
-            Path targetDirPath = new Path(downloadDest,
-                scFileStatus.getPath().getName());
-            FileStatus[] allSubDirs = localLFS.util().listStatus(nmAuxDir);
-            boolean reDownload = true;
-            for (FileStatus sub : allSubDirs) {
-              if (sub.getPath().getName().equals(downloadDest.getName())) {
-                reDownload = false;
-                dest = new Path(targetDirPath + Path.SEPARATOR + "*");
-                break;
-              } else {
-                if (sub.getPath().getName().contains(className) &&
-                    !sub.getPath().getName().endsWith(DEL_SUFFIX)) {
-                  Path delPath = new Path(sub.getPath().getParent(),
-                      sub.getPath().getName() + DEL_SUFFIX);
-                  localLFS.rename(sub.getPath(), delPath);
-                  LOG.info("delete old aux service jar dir:"
-                      + delPath.toString());
-                  FileDeletionTask deletionTask = new FileDeletionTask(
-                      this.delService, null, delPath, null);
-                  this.delService.delete(deletionTask);
-                }
-              }
-            }
-            if (reDownload) {
-              LocalResourceType srcType = null;
-              String lowerDst = StringUtils.toLowerCase(src.toString());
-              if (lowerDst.endsWith(".jar")) {
-                srcType = LocalResourceType.FILE;
-              } else if (lowerDst.endsWith(".zip") ||
-                  lowerDst.endsWith(".tar.gz") || lowerDst.endsWith(".tgz")
-                  || lowerDst.endsWith(".tar")) {
-                srcType = LocalResourceType.ARCHIVE;
-              } else {
-                throw new YarnRuntimeException(
-                    "Can not unpack file from remote-file-path:" + src
-                        + "for aux-service:" + ".\n");
-              }
-              LocalResource scRsrc = LocalResource.newInstance(
-                  URL.fromURI(src.toUri()),
-                  srcType, LocalResourceVisibility.PRIVATE,
-                  scFileStatus.getLen(), scFileStatus.getModificationTime());
-              FSDownload download = new FSDownload(localLFS, null, conf,
-                  downloadDest, scRsrc, null);
-              try {
-                Path downloaded = download.call();
-                // don't need to convert downloaded path into a dir
-                // since its already a jar path.
-                dest = downloaded;
-              } catch (Exception ex) {
-                throw new YarnRuntimeException(
-                    "Exception happend while downloading files "
-                    + "for aux-service:" + sName + " and remote-file-path:"
-                    + src + ".\n" + ex.getMessage());
-              }
-            }
-            s = AuxiliaryServiceWithCustomClassLoader.getInstance(
-                new Configuration(conf), className, dest.toString());
-          }
-          LOG.info("The aux service:" + sName
-              + " are using the custom classloader");
-        } else {
-          Class<? extends AuxiliaryService> sClass = conf.getClass(
-              classKey, null, AuxiliaryService.class);
-
-          if (sClass == null) {
-            throw new RuntimeException("No class defined for " + sName);
-          }
-          s = ReflectionUtils.newInstance(sClass, new Configuration(conf));
-        }
-        if (s == null) {
-          throw new RuntimeException("No object created for " + sName);
-        }
-        // TODO better use s.getName()?
-        if(!sName.equals(s.getName())) {
-          LOG.warn("The Auxiliary Service named '"+sName+"' in the "
-              +"configuration is for "+s.getClass()+" which has "
-              +"a name of '"+s.getName()+"'. Because these are "
-              +"not the same tools trying to send ServiceData and read "
-              +"Service Meta Data may have issues unless the refer to "
-              +"the name in the config.");
-        }
-        s.setAuxiliaryLocalPathHandler(auxiliaryLocalPathHandler);
-        addService(sName, s);
-        if (recoveryEnabled) {
-          Path storePath = new Path(stateStoreRoot, sName);
-          stateStoreFs.mkdirs(storePath, storeDirPerms);
-          s.setRecoveryPath(storePath);
-        }
-        s.init(new Configuration(conf));
-      } catch (RuntimeException e) {
-        LOG.error("Failed to initialize " + sName, e);
-        throw e;
+    String manifestStr = conf.get(YarnConfiguration.NM_AUX_SERVICES_MANIFEST);
+    if (manifestStr == null) {
+      Collection<String> auxNames = conf.getStringCollection(
+          YarnConfiguration.NM_AUX_SERVICES);
+      for (final String sName : auxNames) {
+        AuxServiceRecord service = createServiceRecordFromConfiguration(sName,
+            conf);
+        maybeRemoveAuxService(sName);
+        AuxiliaryService s = initAuxService(service, conf, true);
+        addService(sName, s, service);
       }
+    } else {
+      manifest = new Path(manifestStr);
+      manifestFS = FileSystem.get(new URI(manifestStr), conf);
+      loadManifest(conf, false);
     }
+    manifestReloadInterval = conf.getLong(
+        YarnConfiguration.NM_AUX_SERVICES_MANIFEST_RELOAD_MS,
+        YarnConfiguration.DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS);
+    manifestReloadTask = new ManifestReloadTask();
+
     super.serviceInit(conf);
   }
 
+  private void startAuxService(String name, AuxiliaryService service,
+      AuxServiceRecord serviceRecord) {
+    service.start();
+    service.registerServiceListener(this);
+    ByteBuffer meta = service.getMetaData();
+    if (meta != null) {
+      serviceMetaData.put(name, meta);
+    }
+    serviceRecord.setLaunchTime(new Date());
+  }
+
+  private void stopAuxService(Service service) {
+    if (service.getServiceState() == Service.STATE.STARTED) {
+      service.unregisterServiceListener(this);
+      service.stop();
+    }
+  }
+
   @Override
-  public void serviceStart() throws Exception {
+  public synchronized void serviceStart() throws Exception {
     // TODO fork(?) services running as configured user
     //      monitor for health, shutdown/restart(?) if any should die
     for (Map.Entry<String, AuxiliaryService> entry : serviceMap.entrySet()) {
       AuxiliaryService service = entry.getValue();
       String name = entry.getKey();
-      service.start();
-      service.registerServiceListener(this);
-      ByteBuffer meta = service.getMetaData();
-      if(meta != null) {
-        serviceMetaData.put(name, meta);
-      }
+      startAuxService(name, service, serviceRecordMap.get(name));
+    }
+    if (manifest != null && manifestReloadInterval > 0) {
+      manifestReloadTimer = new Timer("AuxServicesManifestRelaod-Timer",
+          true);
+      manifestReloadTimer.schedule(manifestReloadTask,
+          manifestReloadInterval, manifestReloadInterval);
     }
     super.serviceStart();
   }
 
   @Override
-  public void serviceStop() throws Exception {
+  public synchronized void serviceStop() throws Exception {
     try {
-      synchronized (serviceMap) {
-        for (Service service : serviceMap.values()) {
-          if (service.getServiceState() == Service.STATE.STARTED) {
-            service.unregisterServiceListener(this);
-            service.stop();
-          }
-        }
-        serviceMap.clear();
-        serviceMetaData.clear();
+      for (Service service : serviceMap.values()) {
+        stopAuxService(service);
+      }
+      serviceMap.clear();
+      serviceRecordMap.clear();
+      serviceMetaData.clear();
+      if (manifestReloadTimer != null) {
+        manifestReloadTimer.cancel();
       }
     } finally {
       super.serviceStop();
@@ -340,9 +731,9 @@ public class AuxServices extends AbstractService
 
   @Override
   public void stateChanged(Service service) {
-    LOG.error("Service " + service.getName() + " changed state: " +
+    // services changing state is expected on reload
+    LOG.info("Service " + service.getName() + " changed state: " +
         service.getServiceState());
-    stop();
   }
 
   @Override
@@ -448,4 +839,38 @@ public class AuxServices extends AbstractService
     }
     return remoteUgi;
   }
+
+  protected static AuxServiceRecord newAuxService(String name, String
+      className) {
+    AuxServiceConfiguration serviceConf = new AuxServiceConfiguration();
+    serviceConf.setProperty(CLASS_NAME, className);
+    return new AuxServiceRecord().name(name).configuration(serviceConf);
+  }
+
+  protected static void setClasspath(AuxServiceRecord service, String
+      classpath) {
+    service.getConfiguration().getFiles().add(new AuxServiceFile()
+        .srcFile(classpath).type(AuxServiceFile.TypeEnum.STATIC));
+  }
+
+  protected static void setSystemClasses(AuxServiceRecord service, String
+      systemClasses) {
+    service.getConfiguration().setProperty(SYSTEM_CLASSES, systemClasses);
+  }
+
+  /**
+   * Class which is used by the {@link Timer} class to periodically execute the
+   * manifest reload.
+   */
+  private final class ManifestReloadTask extends TimerTask {
+    @Override
+    public void run() {
+      try {
+        reloadManifest();
+      } catch (Throwable t) {
+        // Prevent uncaught exceptions from killing this thread
+        LOG.warn("Error while reloading manifest: ", t);
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java
index c764fbd..d0b039e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxiliaryServiceWithCustomClassLoader.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ApplicationClassLoader;
 import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
 import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
 import org.apache.hadoop.yarn.server.api.AuxiliaryService;
@@ -159,11 +158,8 @@ final class AuxiliaryServiceWithCustomClassLoader extends AuxiliaryService {
   }
 
   public static AuxiliaryServiceWithCustomClassLoader getInstance(
-      Configuration conf, String className, String appClassPath)
-      throws IOException, ClassNotFoundException {
-    String[] systemClasses = conf.getTrimmedStrings(String.format(
-        YarnConfiguration.NM_AUX_SERVICES_SYSTEM_CLASSES,
-        className));
+      Configuration conf, String className, String appClassPath, String[]
+      systemClasses) throws IOException, ClassNotFoundException {
     ClassLoader customClassLoader = createAuxServiceClassLoader(
         appClassPath, systemClasses);
     Class<?> clazz = Class.forName(className, true,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8a12c3c..42ea59c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -258,6 +258,7 @@ public class ContainerManagerImpl extends CompositeService implements
     auxiliaryServices = new AuxServices(auxiliaryLocalPathHandler,
         this.context, this.deletionService);
     auxiliaryServices.registerServiceListener(this);
+    context.setAuxServices(auxiliaryServices);
     addService(auxiliaryServices);
 
     // initialize the metrics publisher if the timeline service v.2 is enabled

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceConfiguration.java
new file mode 100644
index 0000000..24383ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceConfiguration.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Set of configuration properties that can be injected into the service
+ * components via envs, files and custom pluggable helper docker containers.
+ * Files of several standard formats like xml, properties, json, yaml and
+ * templates will be supported.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class AuxServiceConfiguration {
+
+  private Map<String, String> properties = new HashMap<>();
+  private List<AuxServiceFile> files = new ArrayList<>();
+
+  /**
+   * A blob of key-value pairs of common service properties.
+   **/
+  public AuxServiceConfiguration properties(Map<String, String> props) {
+    this.properties = props;
+    return this;
+  }
+
+  @JsonProperty("properties")
+  public Map<String, String> getProperties() {
+    return properties;
+  }
+
+  public void setProperties(Map<String, String> properties) {
+    this.properties = properties;
+  }
+
+  /**
+   * Array of list of files that needs to be created and made available as
+   * volumes in the service component containers.
+   **/
+  public AuxServiceConfiguration files(List<AuxServiceFile> fileList) {
+    this.files = fileList;
+    return this;
+  }
+
+  @JsonProperty("files")
+  public List<AuxServiceFile> getFiles() {
+    return files;
+  }
+
+  public void setFiles(List<AuxServiceFile> files) {
+    this.files = files;
+  }
+
+  public String getProperty(String name, String defaultValue) {
+    String value = getProperty(name);
+    if (StringUtils.isEmpty(value)) {
+      return defaultValue;
+    }
+    return value;
+  }
+
+  public void setProperty(String name, String value) {
+    properties.put(name, value);
+  }
+
+  public String getProperty(String name) {
+    return properties.get(name.trim());
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    AuxServiceConfiguration configuration = (AuxServiceConfiguration) o;
+    return Objects.equals(this.properties, configuration.properties)
+        && Objects.equals(this.files, configuration.files);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(properties, files);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("class Configuration {\n");
+
+    sb.append("    properties: ").append(toIndentedString(properties))
+        .append("\n");
+    sb.append("    files: ").append(toIndentedString(files)).append("\n");
+    sb.append("}");
+    return sb.toString();
+  }
+
+  /**
+   * Convert the given object to string with each line indented by 4 spaces
+   * (except the first line).
+   */
+  private String toIndentedString(java.lang.Object o) {
+    if (o == null) {
+      return "null";
+    }
+    return o.toString().replace("\n", "\n    ");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceFile.java
new file mode 100644
index 0000000..6b79b9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceFile.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.util.Objects;
+
+/**
+ * A config file that needs to be created and made available as a volume in an
+ * service component container.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class AuxServiceFile {
+
+  /**
+   * Config Type.
+   **/
+  public enum TypeEnum {
+    STATIC("STATIC"), ARCHIVE("ARCHIVE");
+
+    private String value;
+
+    TypeEnum(String type) {
+      this.value = type;
+    }
+
+    @Override
+    @JsonValue
+    public String toString() {
+      return value;
+    }
+  }
+
+  private TypeEnum type = null;
+  private String srcFile = null;
+
+  /**
+   * Config file in the standard format like xml, properties, json, yaml,
+   * template.
+   **/
+  public AuxServiceFile type(TypeEnum t) {
+    this.type = t;
+    return this;
+  }
+
+  @JsonProperty("type")
+  public TypeEnum getType() {
+    return type;
+  }
+
+  public void setType(TypeEnum type) {
+    this.type = type;
+  }
+
+  /**
+   * This provides the source location of the configuration file, the content
+   * of which is dumped to dest_file post property substitutions, in the format
+   * as specified in type. Typically the src_file would point to a source
+   * controlled network accessible file maintained by tools like puppet, chef,
+   * or hdfs etc. Currently, only hdfs is supported.
+   **/
+  public AuxServiceFile srcFile(String file) {
+    this.srcFile = file;
+    return this;
+  }
+
+  @JsonProperty("src_file")
+  public String getSrcFile() {
+    return srcFile;
+  }
+
+  public void setSrcFile(String srcFile) {
+    this.srcFile = srcFile;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    AuxServiceFile auxServiceFile = (AuxServiceFile) o;
+    return Objects.equals(this.type, auxServiceFile.type)
+        && Objects.equals(this.srcFile, auxServiceFile.srcFile);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(type, srcFile);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("class AuxServiceFile {\n");
+
+    sb.append("    type: ").append(toIndentedString(type)).append("\n");
+    sb.append("    srcFile: ").append(toIndentedString(srcFile)).append("\n");
+    sb.append("}");
+    return sb.toString();
+  }
+
+  /**
+   * Convert the given object to string with each line indented by 4 spaces
+   * (except the first line).
+   */
+  private String toIndentedString(java.lang.Object o) {
+    if (o == null) {
+      return "null";
+    }
+    return o.toString().replace("\n", "\n    ");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceRecord.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceRecord.java
new file mode 100644
index 0000000..40882be
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceRecord.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.util.Date;
+import java.util.Objects;
+
+/**
+ * An Service resource has the following attributes.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ "name", "version", "description", "launch_time",
+    "configuration" })
+public class AuxServiceRecord {
+
+  private String name = null;
+  private String version = null;
+  private String description = null;
+  private Date launchTime = null;
+  private AuxServiceConfiguration configuration = new AuxServiceConfiguration();
+
+  /**
+   * A unique service name.
+   **/
+  public AuxServiceRecord name(String n) {
+    this.name = n;
+    return this;
+  }
+
+  @JsonProperty("name")
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  @JsonProperty("version")
+  public String getVersion() {
+    return version;
+  }
+
+  public void setVersion(String version) {
+    this.version = version;
+  }
+
+  /**
+   * Version of the service.
+   */
+  public AuxServiceRecord version(String v) {
+    this.version = v;
+    return this;
+  }
+
+  @JsonProperty("description")
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  /**
+   * Description of the service.
+   */
+  public AuxServiceRecord description(String d) {
+    this.description = d;
+    return this;
+  }
+
+  /**
+   * The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
+   **/
+  public AuxServiceRecord launchTime(Date time) {
+    this.launchTime = time == null ? null : (Date) time.clone();
+    return this;
+  }
+
+  @JsonProperty("launch_time")
+  public Date getLaunchTime() {
+    return launchTime == null ? null : (Date) launchTime.clone();
+  }
+
+  public void setLaunchTime(Date time) {
+    this.launchTime = time == null ? null : (Date) time.clone();
+  }
+  /**
+   * Config properties of an service. Configurations provided at the
+   * service/global level are available to all the components. Specific
+   * properties can be overridden at the component level.
+   **/
+  public AuxServiceRecord configuration(AuxServiceConfiguration conf) {
+    this.configuration = conf;
+    return this;
+  }
+
+  @JsonProperty("configuration")
+  public AuxServiceConfiguration getConfiguration() {
+    return configuration;
+  }
+
+  public void setConfiguration(AuxServiceConfiguration conf) {
+    this.configuration = conf;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    AuxServiceRecord service = (AuxServiceRecord) o;
+    return Objects.equals(this.name, service.name) && Objects.equals(this
+        .version, service.version);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(name, version);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("class Service {\n");
+
+    sb.append("    name: ").append(toIndentedString(name)).append("\n");
+    sb.append("    version: ").append(toIndentedString(version)).append("\n");
+    sb.append("    description: ").append(toIndentedString(description))
+        .append("\n");
+    sb.append("    configuration: ").append(toIndentedString(configuration))
+        .append("\n");
+    sb.append("}");
+    return sb.toString();
+  }
+
+  /**
+   * Convert the given object to string with each line indented by 4 spaces
+   * (except the first line).
+   */
+  private String toIndentedString(java.lang.Object o) {
+    if (o == null) {
+      return "null";
+    }
+    return o.toString().replace("\n", "\n    ");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceRecords.java
new file mode 100644
index 0000000..c2454e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/AuxServiceRecords.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A list of Services.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class AuxServiceRecords {
+  private List<AuxServiceRecord> services = new ArrayList<>();
+
+  public AuxServiceRecords serviceList(AuxServiceRecord... serviceList) {
+    for (AuxServiceRecord service : serviceList) {
+      this.services.add(service);
+    }
+    return this;
+  }
+
+  public List<AuxServiceRecord> getServices() {
+    return services;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/package-info.java
new file mode 100644
index 0000000..bfdc33f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/records/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Package used for auxiliary services manifest records. To the extent
+ * possible, the format matches the YARN service framework service
+ * specification.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index ca08897..4b82f37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -24,13 +24,16 @@ import java.io.OutputStream;
 import java.nio.charset.Charset;
 import java.security.Principal;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AuxiliaryServicesInfo;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -556,6 +559,25 @@ public class NMWebServices {
     return new NMResourceInfo();
   }
 
+  @GET
+  @Path("/auxiliaryservices")
+  @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+      MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+  public AuxiliaryServicesInfo getAuxiliaryServices(@javax.ws.rs.core.Context
+      HttpServletRequest hsr) {
+    init();
+    AuxiliaryServicesInfo auxiliaryServices = new AuxiliaryServicesInfo();
+    if (!hasAdminAccess(hsr)) {
+      return auxiliaryServices;
+    }
+    Collection<AuxServiceRecord> loadedServices = nmContext.getAuxServices()
+        .getServiceRecords();
+    if (loadedServices != null) {
+      auxiliaryServices.addAll(loadedServices);
+    }
+    return auxiliaryServices;
+  }
+
   @PUT
   @Path("/yarn/sysfs/{user}/{appId}")
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@@ -625,6 +647,21 @@ public class NMWebServices {
     return true;
   }
 
+  protected Boolean hasAdminAccess(HttpServletRequest hsr) {
+    // Check for the authorization.
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+
+    if (callerUGI == null) {
+      return false;
+    }
+
+    if (!this.nmContext.getApplicationACLsManager().isAdmin(callerUGI)) {
+      return false;
+    }
+
+    return true;
+  }
+
   private UserGroupInformation getCallerUserGroupInformation(
       HttpServletRequest hsr, boolean usePrincipal) {
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AuxiliaryServiceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AuxiliaryServiceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AuxiliaryServiceInfo.java
new file mode 100644
index 0000000..873314d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AuxiliaryServiceInfo.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+/**
+ * Information about a loaded auxiliary service.
+ */
+@XmlRootElement(name = "service")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AuxiliaryServiceInfo {
+  private String name;
+  private String version;
+  private String startTime;
+
+  public AuxiliaryServiceInfo() {
+    // JAXB needs this
+  }
+
+  public AuxiliaryServiceInfo(String name, String version, Date startTime) {
+    DateFormat dateFormat =
+        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    this.name = name;
+    this.version = version;
+    this.startTime = dateFormat.format(startTime);
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getVersion() {
+    return version;
+  }
+
+  public String getStartTime() {
+    return startTime;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AuxiliaryServicesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AuxiliaryServicesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AuxiliaryServicesInfo.java
new file mode 100644
index 0000000..e11b9c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AuxiliaryServicesInfo.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * A list of loaded auxiliary services.
+ */
+@XmlRootElement(name = "services")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AuxiliaryServicesInfo {
+  private ArrayList<AuxiliaryServiceInfo> services = new
+      ArrayList<>();
+
+  public AuxiliaryServicesInfo() {
+    // JAXB needs this
+  }
+
+  public void add(AuxServiceRecord s) {
+    services.add(new AuxiliaryServiceInfo(s.getName(), s.getVersion(), s
+        .getLaunchTime()));
+  }
+
+  public void addAll(Collection<AuxServiceRecord> serviceList) {
+    for (AuxServiceRecord service : serviceList) {
+      add(service);
+    }
+  }
+
+  public ArrayList<AuxiliaryServiceInfo> getServices() {
+    return services;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 2794857..2ecbf31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
 import org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitor;
 import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -827,5 +828,14 @@ public abstract class BaseAMRMProxyTest {
     public NMLogAggregationStatusTracker getNMLogAggregationStatusTracker() {
       return null;
     }
+
+    @Override
+    public void setAuxServices(AuxServices auxServices) {
+    }
+
+    @Override
+    public AuxServices getAuxServices() {
+      return null;
+    }
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message