hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [57/58] [abbrv] hadoop git commit: YARN-7423. Improve service client loggings. Contributed by Jian He
Date Mon, 06 Nov 2017 22:02:31 GMT
YARN-7423. Improve service client loggings. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a127f7b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a127f7b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a127f7b0

Branch: refs/heads/yarn-native-services
Commit: a127f7b0fbde3942ca4329d5463bb8fe421daf7b
Parents: a6c4bd7
Author: Billie Rinaldi <billie@apache.org>
Authored: Sat Nov 4 12:27:42 2017 -0700
Committer: Jian He <jianhe@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/service/ServiceMaster.java      |   4 +-
 .../yarn/service/api/records/Configuration.java |   8 +-
 .../yarn/service/client/ServiceClient.java      |  50 +-
 .../yarn/service/component/Component.java       |  10 +-
 .../component/instance/ComponentInstance.java   |   4 +-
 .../containerlaunch/AbstractLauncher.java       |   6 +-
 .../containerlaunch/ClasspathConstructor.java   |   4 +-
 .../containerlaunch/CommandLineBuilder.java     |   4 +-
 .../containerlaunch/JavaCommandLineBuilder.java |   4 +-
 .../yarn/service/monitor/probe/HttpProbe.java   |   4 +-
 .../yarn/service/monitor/probe/PortProbe.java   |   4 +-
 .../provider/AbstractClientProvider.java        |   8 +-
 .../provider/AbstractProviderService.java       |   4 +-
 .../yarn/service/provider/ProviderUtils.java    |  10 +-
 .../registry/YarnRegistryViewForProviders.java  |   6 +-
 .../yarn/service/utils/CoreFileSystem.java      |   2 +-
 .../hadoop/yarn/service/utils/PortScanner.java  |   4 +-
 .../yarn/service/utils/ServiceApiUtil.java      |   2 +-
 .../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++++++++++++++
 .../hadoop/yarn/service/utils/SliderUtils.java  | 564 -------------------
 .../yarn/service/utils/ZookeeperUtils.java      |   4 +-
 .../service/conf/TestValidateServiceNames.java  |   1 -
 22 files changed, 614 insertions(+), 638 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 2d86a5a..b0b4f06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
 import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -112,7 +112,7 @@ public class ServiceMaster extends CompositeService {
   }
 
   protected ContainerId getAMContainerId() throws BadClusterStateException {
-    return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
+    return ContainerId.fromString(ServiceUtils.mandatoryEnvVariable(
         ApplicationConstants.Environment.CONTAINER_ID.name()));
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
index dc6dff8..859c1ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -24,7 +24,7 @@ import io.swagger.annotations.ApiModelProperty;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -201,9 +201,9 @@ public class Configuration implements Serializable {
    * this ConfigFile.
    */
   public synchronized void mergeFrom(Configuration that) {
-    SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+    ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
         .getProperties());
-    SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+    ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
 
     Map<String, ConfigFile> thatMap = new HashMap<>();
     for (ConfigFile file : that.getFiles()) {
@@ -212,7 +212,7 @@ public class Configuration implements Serializable {
     for (ConfigFile thisFile : files) {
       if(thatMap.containsKey(thisFile.getDestFile())) {
         ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
-        SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
+        ServiceUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
             thatFile.getProperties());
         thatMap.remove(thisFile.getDestFile());
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 775a642..11cd30d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -69,7 +69,7 @@ import org.apache.hadoop.yarn.service.provider.ProviderUtils;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.Times;
@@ -80,19 +80,13 @@ import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.text.MessageFormat;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
 import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
 import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
 import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
-import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
+import static org.apache.hadoop.yarn.service.utils.ServiceUtils.*;
 
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
@@ -169,7 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
           fileName);
     }
     Path filePath = new Path(file.getAbsolutePath());
-    LOG.info("Loading service definition from: " + filePath);
+    LOG.info("Loading service definition from local FS: " + filePath);
     Service service = jsonSerDeser
         .load(FileSystem.getLocal(getConfig()), filePath);
     if (!StringUtils.isEmpty(serviceName)) {
@@ -482,7 +476,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
         getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
 
     // though if neither is set: trouble
-    if (SliderUtils.isUnset(registryQuorum)) {
+    if (ServiceUtils.isUnset(registryQuorum)) {
       throw new BadConfigException(
           "No Zookeeper quorum provided in the" + " configuration property "
               + RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
@@ -505,7 +499,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
     types.add(YarnServiceConstants.APP_TYPE);
     Set<String> tags = null;
     if (serviceName != null) {
-      tags = Collections.singleton(SliderUtils.createNameTag(serviceName));
+      tags = Collections.singleton(ServiceUtils.createNameTag(serviceName));
     }
     GetApplicationsRequest request = GetApplicationsRequest.newInstance();
     request.setApplicationTypes(types);
@@ -652,7 +646,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
     // write out the path output
     CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
     String cmdStr = CLI.build();
-    LOG.info("AM launch command: {}", cmdStr);
+    LOG.debug("AM launch command: {}", cmdStr);
     return cmdStr;
   }
 
@@ -671,12 +665,12 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
     }
     if (!UserGroupInformation.isSecurityEnabled()) {
       String userName = UserGroupInformation.getCurrentUser().getUserName();
-      LOG.info("Run as user " + userName);
+      LOG.debug("Run as user " + userName);
       // HADOOP_USER_NAME env is used by UserGroupInformation when log in
       // This env makes AM run as this user
       env.put("HADOOP_USER_NAME", userName);
     }
-    LOG.info("AM env: \n{}", stringifyMap(env));
+    LOG.debug("AM env: \n{}", stringifyMap(env));
     return env;
   }
 
@@ -689,11 +683,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
             libPath, "lib", false);
     Path dependencyLibTarGzip = fs.getDependencyTarGzip();
     if (fs.isFile(dependencyLibTarGzip)) {
-      LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+      LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
           + dependencyLibTarGzip);
-      SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
+      fs.submitTarGzipAndUpdate(localResources);
     } else {
-      String[] libs = SliderUtils.getLibDirs();
+      String[] libs = ServiceUtils.getLibDirs();
+      LOG.info("Uploading all dependency jars to HDFS. For faster submission of" +
+          " apps, pre-upload dependency jars to HDFS "
+          + "using command: yarn app -enableFastLaunch");
       for (String libDirProp : libs) {
         ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
             libDirProp);
@@ -740,7 +737,8 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
     ApplicationId appId = submitApp(service);
     service.setId(appId.toString());
     // write app definition on to hdfs
-    createDirAndPersistApp(appDir, service);
+    Path appJson = persistAppDef(appDir, service);
+    LOG.info("Persisted service " + service.getName() + " at " + appJson);
     return 0;
   }
 
@@ -763,16 +761,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
       throws IOException, SliderException {
     FsPermission appDirPermission = new FsPermission("750");
     fs.createWithPermissions(appDir, appDirPermission);
-    persistAppDef(appDir, service);
+    Path appJson = persistAppDef(appDir, service);
+    LOG.info("Persisted service " + service.getName() + " at " + appJson);
   }
 
-  private void persistAppDef(Path appDir, Service service)
-      throws IOException {
+  private Path persistAppDef(Path appDir, Service service) throws IOException {
     Path appJson = new Path(appDir, service.getName() + ".json");
-    jsonSerDeser
-        .save(fs.getFileSystem(), appJson, service, true);
-    LOG.info(
-        "Persisted service " + service.getName() + " at " + appJson);
+    jsonSerDeser.save(fs.getFileSystem(), appJson, service, true);
+    return appJson;
   }
 
   private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
@@ -922,7 +918,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
       return EXIT_SUCCESS;
     }
 
-    String[] libDirs = SliderUtils.getLibDirs();
+    String[] libDirs = ServiceUtils.getLibDirs();
     if (libDirs.length > 0) {
       File tempLibTarGzipFile = File.createTempFile(
           YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 7667a53..7208f39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
 import org.apache.hadoop.yarn.service.monitor.probe.Probe;
 import org.slf4j.Logger;
@@ -419,7 +419,7 @@ public class Component implements EventHandler<ComponentEvent> {
 
   public boolean areDependenciesReady() {
     List<String> dependencies = componentSpec.getDependencies();
-    if (SliderUtils.isEmpty(dependencies)) {
+    if (ServiceUtils.isEmpty(dependencies)) {
       return true;
     }
     for (String dependency : dependencies) {
@@ -445,7 +445,7 @@ public class Component implements EventHandler<ComponentEvent> {
   public Map<String, String> getDependencyHostIpTokens() {
     Map<String, String> tokens = new HashMap<>();
     List<String> dependencies = componentSpec.getDependencies();
-    if (SliderUtils.isEmpty(dependencies)) {
+    if (ServiceUtils.isEmpty(dependencies)) {
       return tokens;
     }
     for (String dependency : dependencies) {
@@ -455,8 +455,8 @@ public class Component implements EventHandler<ComponentEvent> {
         if (instance.getContainerStatus() == null) {
           continue;
         }
-        if (SliderUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
-            SliderUtils.isUnset(instance.getContainerStatus().getHost())) {
+        if (ServiceUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
+            ServiceUtils.isUnset(instance.getContainerStatus().getHost())) {
           continue;
         }
         String ip = instance.getContainerStatus().getIPs().get(0);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 2492454..9e5f98c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
 import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
 import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
@@ -503,7 +503,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
             + nodeId + ", will try again", e);
         return;
       }
-      if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils
+      if (ServiceUtils.isEmpty(status.getIPs()) || ServiceUtils
           .isUnset(status.getHost())) {
         return;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
index e4eae20..2d7c3bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -132,7 +132,7 @@ public class AbstractLauncher {
    */
   public ContainerLaunchContext completeContainerLaunch() throws IOException {
     
-    String cmdStr = SliderUtils.join(commands, " ", false);
+    String cmdStr = ServiceUtils.join(commands, " ", false);
     log.debug("Completed setting up container command {}", cmdStr);
     containerLaunchContext.setCommands(commands);
 
@@ -205,7 +205,7 @@ public class AbstractLauncher {
 
         String key = entry.getKey();
         LocalResource val = entry.getValue();
-        log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
+        log.debug(key + "=" + ServiceUtils.stringify(val.getResource()));
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
index 22b3877..711abb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -63,7 +63,7 @@ public class ClasspathConstructor {
   }
 
   public String buildClasspath() {
-    return SliderUtils.join(pathElements,
+    return ServiceUtils.join(pathElements,
         CLASS_PATH_SEPARATOR,
         false);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
index 7baa284..5ed56e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.containerlaunch;
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -81,6 +81,6 @@ public class CommandLineBuilder {
    * @return the command line
    */
   public String build() {
-    return SliderUtils.join(argumentList, " ");
+    return ServiceUtils.join(argumentList, " ");
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
index cbcb0d6..7f6cc04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -22,7 +22,7 @@ package org.apache.hadoop.yarn.service.containerlaunch;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
 
 import java.util.Map;
@@ -53,7 +53,7 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
    * trimmed.
    */
   public void setJVMHeap(String heap) {
-    if (SliderUtils.isSet(heap)) {
+    if (ServiceUtils.isSet(heap)) {
       add("-Xmx" + heap.trim());
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
index 1923086..1ed13a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
@@ -21,7 +21,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -75,7 +75,7 @@ public class HttpProbe extends Probe {
   public ProbeStatus ping(ComponentInstance instance) {
     ProbeStatus status = new ProbeStatus();
     ContainerStatus containerStatus = instance.getContainerStatus();
-    if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
+    if (containerStatus == null || ServiceUtils.isEmpty(containerStatus.getIPs())
         || StringUtils.isEmpty(containerStatus.getHost())) {
       status.fail(this, new IOException("IP is not available yet"));
       return status;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
index aba5859..85569f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.service.monitor.probe;
 
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -67,7 +67,7 @@ public class PortProbe extends Probe {
   public ProbeStatus ping(ComponentInstance instance) {
     ProbeStatus status = new ProbeStatus();
 
-    if (instance.getContainerStatus() == null || SliderUtils
+    if (instance.getContainerStatus() == null || ServiceUtils
         .isEmpty(instance.getContainerStatus().getIPs())) {
       status.fail(this, new IOException(
           instance.getCompInstanceName() + ": IP is not available yet"));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
index 75c647b..fc8953c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.service.api.records.Artifact;
 import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 
 import java.io.IOException;
 import java.nio.file.Paths;
@@ -53,12 +53,12 @@ public abstract class AbstractClientProvider {
   public static final Set<String> createApplicationTags(String appName,
       String appVersion, String appDescription) {
     Set<String> tags = new HashSet<>();
-    tags.add(SliderUtils.createNameTag(appName));
+    tags.add(ServiceUtils.createNameTag(appName));
     if (appVersion != null) {
-      tags.add(SliderUtils.createVersionTag(appVersion));
+      tags.add(ServiceUtils.createVersionTag(appVersion));
     }
     if (appDescription != null) {
-      tags.add(SliderUtils.createDescriptionTag(appDescription));
+      tags.add(ServiceUtils.createDescriptionTag(appDescription));
     }
     return tags;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 6ffb84d..6d74061 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.apache.hadoop.yarn.service.exceptions.SliderException;
 import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
 import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
@@ -70,7 +70,7 @@ public abstract class AbstractProviderService implements ProviderService,
         .initCompTokensForSubstitute(instance);
     tokensForSubstitution.putAll(globalTokens);
     // Set the environment variables in launcher
-    launcher.putEnv(SliderUtils
+    launcher.putEnv(ServiceUtils
         .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
     launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
     launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 582c5ac..e074dd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.service.exceptions.SliderException;
 import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
 import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -94,7 +94,7 @@ public class ProviderUtils implements YarnServiceConstants {
       IOException,
       SliderException {
     try {
-      SliderUtils.putJar(providerResources,
+      ServiceUtils.putJar(providerResources,
           sliderFileSystem,
           providerClass,
           tempPath,
@@ -127,14 +127,14 @@ public class ProviderUtils implements YarnServiceConstants {
       String libDir,
       String libLocalSrcDir)
       throws IOException, SliderException {
-    if (SliderUtils.isSet(libLocalSrcDir)) {
+    if (ServiceUtils.isSet(libLocalSrcDir)) {
       File file = new File(libLocalSrcDir);
       if (!file.exists() || !file.isDirectory()) {
         throw new BadCommandArgumentsException(
             "Supplied lib src dir %s is not valid", libLocalSrcDir);
       }
     }
-    SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
+    ServiceUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
         libDir, libLocalSrcDir);
   }
 
@@ -174,7 +174,7 @@ public class ProviderUtils implements YarnServiceConstants {
     Configuration conf = service.getConfiguration();
     String keytabPathOnHost =
         conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
-    if (SliderUtils.isUnset(keytabPathOnHost)) {
+    if (ServiceUtils.isUnset(keytabPathOnHost)) {
       String amKeytabName =
           conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
       String keytabDir =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
index d418b59..cecca5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
 
 import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -70,9 +70,9 @@ public class YarnRegistryViewForProviders {
     Preconditions.checkArgument(registryOperations != null,
         "null registry operations");
     Preconditions.checkArgument(user != null, "null user");
-    Preconditions.checkArgument(SliderUtils.isSet(serviceClass),
+    Preconditions.checkArgument(ServiceUtils.isSet(serviceClass),
         "unset service class");
-    Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+    Preconditions.checkArgument(ServiceUtils.isSet(instanceName),
         "instanceName");
     Preconditions.checkArgument(applicationAttemptId != null,
         "null applicationAttemptId");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 281e1dfe..6b9d4d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -409,7 +409,7 @@ public class CoreFileSystem {
    */
   public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
           throws IOException, SliderException {
-    File localFile = SliderUtils.findContainingJarOrFail(clazz);
+    File localFile = ServiceUtils.findContainingJarOrFail(clazz);
     return submitFile(localFile, tempPath, subdir, jarName);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
index 2dbf37f..1d64ed6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
@@ -85,7 +85,7 @@ public class PortScanner {
     if (remainingPortsToCheck != null) {
       return getAvailablePortViaPortArray();
     } else {
-      return SliderUtils.getOpenPort();
+      return ServiceUtils.getOpenPort();
     }
   }
 
@@ -95,7 +95,7 @@ public class PortScanner {
     Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
     while (portsToCheck.hasNext() && !found) {
       int portToCheck = portsToCheck.next();
-      found = SliderUtils.isPortAvailable(portToCheck);
+      found = ServiceUtils.isPortAvailable(portToCheck);
       if (found) {
         availablePort = portToCheck;
         portsToCheck.remove();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 68db0bb..b58cea8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -369,7 +369,7 @@ public class ServiceApiUtil {
         continue;
       }
       boolean dependenciesAlreadySorted = true;
-      if (!SliderUtils.isEmpty(component.getDependencies())) {
+      if (!ServiceUtils.isEmpty(component.getDependencies())) {
         for (String dependency : component.getDependencies()) {
           if (!sortedComponents.containsKey(dependency)) {
             dependenciesAlreadySorted = false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
new file mode 100644
index 0000000..e18bcae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class ServiceUtils {
+
+  private static final Logger log = LoggerFactory.getLogger(ServiceUtils.class);
+
+  private ServiceUtils() {
+  }
+
+  /**
+   * Implementation of set-ness, groovy definition of true/false for a string
+   * @param s string
+   * @return true iff the string is neither null nor empty
+   */
+  public static boolean isUnset(String s) {
+    return s == null || s.isEmpty();
+  }
+
+  public static boolean isSet(String s) {
+    return !isUnset(s);
+  }
+
+  public static boolean isEmpty(Collection l) {
+    return l == null || l.isEmpty();
+  }
+
+  /**
+   * Find a containing JAR
+   * @param clazz class to find
+   * @return the file
+   * @throws IOException any IO problem, including the class not having a
+   * classloader
+   * @throws FileNotFoundException if the class did not resolve to a file
+   */
+  public static File findContainingJarOrFail(Class clazz) throws IOException {
+    File localFile = ServiceUtils.findContainingJar(clazz);
+    if (null == localFile) {
+      throw new FileNotFoundException("Could not find JAR containing " + clazz);
+    }
+    return localFile;
+  }
+
+
+  /**
+   * Find a containing JAR
+   * @param my_class class to find
+   * @return the file or null if it is not found
+   * @throws IOException any IO problem, including the class not having a
+   * classloader
+   */
+  public static File findContainingJar(Class my_class) throws IOException {
+    ClassLoader loader = my_class.getClassLoader();
+    if (loader == null) {
+      throw new IOException(
+          "Class " + my_class + " does not have a classloader!");
+    }
+    String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+    Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+    for (; urlEnumeration.hasMoreElements(); ) {
+      URL url = urlEnumeration.nextElement();
+      if ("jar".equals(url.getProtocol())) {
+        String toReturn = url.getPath();
+        if (toReturn.startsWith("file:")) {
+          toReturn = toReturn.substring("file:".length());
+        }
+        // URLDecoder is a misnamed class, since it actually decodes
+        // x-www-form-urlencoded MIME type rather than actual
+        // URL encoding (which the file path has). Therefore it would
+        // decode +s to ' 's which is incorrect (spaces are actually
+        // either unencoded or encoded as "%20"). Replace +s first, so
+        // that they are kept sacred during the decoding process.
+        toReturn = toReturn.replaceAll("\\+", "%2B");
+        toReturn = URLDecoder.decode(toReturn, "UTF-8");
+        String jarFilePath = toReturn.replaceAll("!.*$", "");
+        return new File(jarFilePath);
+      } else {
+        log.info("could not locate JAR containing {} URL={}", my_class, url);
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Copy a file to a new FS -both paths must be qualified.
+   * @param conf conf file
+   * @param srcFile src file
+   * @param destFile dest file
+   */
+  @SuppressWarnings("deprecation")
+  public static void copy(Configuration conf,
+      Path srcFile,
+      Path destFile) throws
+      IOException,
+      BadClusterStateException {
+    FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+    //list all paths in the src.
+    if (!srcFS.exists(srcFile)) {
+      throw new FileNotFoundException("Source file not found " + srcFile);
+    }
+    if (!srcFS.isFile(srcFile)) {
+      throw new FileNotFoundException(
+          "Source file not a file " + srcFile);
+    }
+    FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+    FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+  }
+
+  /**
+   * Take a collection, return a list containing the string value of every
+   * element in the collection.
+   * @param c collection
+   * @return a stringified list
+   */
+  public static List<String> collectionToStringList(Collection c) {
+    List<String> l = new ArrayList<>(c.size());
+    for (Object o : c) {
+      l.add(o.toString());
+    }
+    return l;
+  }
+
+  /**
+   * Join an collection of objects with a separator that appears after every
+   * instance in the list -including at the end
+   * @param collection collection to call toString() on each element
+   * @param separator separator string
+   * @return the joined entries
+   */
+  public static String join(Collection collection, String separator) {
+    return join(collection, separator, true);
+  }
+
+  /**
+   * Join an collection of objects with a separator that appears after every
+   * instance in the list -optionally at the end
+   * @param collection collection to call toString() on each element
+   * @param separator separator string
+   * @param trailing add a trailing entry or not
+   * @return the joined entries
+   */
+  public static String join(Collection collection,
+      String separator,
+      boolean trailing) {
+    StringBuilder b = new StringBuilder();
+    // fast return on empty collection
+    if (collection.isEmpty()) {
+      return trailing ? separator : "";
+    }
+    for (Object o : collection) {
+      b.append(o);
+      b.append(separator);
+    }
+    int length = separator.length();
+    String s = b.toString();
+    return (trailing || s.isEmpty()) ?
+           s : (b.substring(0, b.length() - length));
+  }
+
+  /**
+   * Join an array of strings with a separator that appears after every
+   * instance in the list -optionally at the end
+   * @param collection strings
+   * @param separator separator string
+   * @param trailing add a trailing entry or not
+   * @return the joined entries
+   */
+  public static String join(String[] collection, String separator,
+      boolean trailing) {
+    return join(Arrays.asList(collection), separator, trailing);
+  }
+
+  /**
+   * Resolve a mandatory environment variable
+   * @param key env var
+   * @return the resolved value
+   * @throws BadClusterStateException
+   */
+  public static String mandatoryEnvVariable(String key) throws
+      BadClusterStateException {
+    String v = System.getenv(key);
+    if (v == null) {
+      throw new BadClusterStateException("Missing Environment variable " + key);
+    }
+    return v;
+  }
+
+  /**
+   * Generic map merge logic
+   * @param first first map
+   * @param second second map
+   * @param <T1> key type
+   * @param <T2> value type
+   * @return 'first' merged with the second
+   */
+  public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+      Map<T1, T2> second) {
+    Preconditions.checkArgument(first != null, "Null 'first' value");
+    Preconditions.checkArgument(second != null, "Null 'second' value");
+    for (Map.Entry<T1, T2> entry : second.entrySet()) {
+      T1 key = entry.getKey();
+      if (!first.containsKey(key)) {
+        first.put(key, entry.getValue());
+      }
+    }
+    return first;
+  }
+
+  /**
+   * Convert a map to a multi-line string for printing
+   * @param map map to stringify
+   * @return a string representation of the map
+   */
+  public static String stringifyMap(Map<String, String> map) {
+    StringBuilder builder = new StringBuilder();
+    for (Map.Entry<String, String> entry : map.entrySet()) {
+      builder.append(entry.getKey())
+             .append("=\"")
+             .append(entry.getValue())
+             .append("\"\n");
+
+    }
+    return builder.toString();
+  }
+
+  /**
+   * Convert a YARN URL into a string value of a normal URL
+   * @param url URL
+   * @return string representatin
+   */
+  public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+    StringBuilder builder = new StringBuilder();
+    builder.append(url.getScheme()).append("://");
+    if (url.getHost() != null) {
+      builder.append(url.getHost()).append(":").append(url.getPort());
+    }
+    builder.append(url.getFile());
+    return builder.toString();
+  }
+
+  /**
+   * Get a random open port
+   * @return true if the port was available for listening on
+   */
+  public static int getOpenPort() throws IOException {
+    ServerSocket socket = null;
+    try {
+      socket = new ServerSocket(0);
+      return socket.getLocalPort();
+    } finally {
+      if (socket != null) {
+        socket.close();
+      }
+    }
+  }
+
+  /**
+   * See if a port is available for listening on by trying to listen
+   * on it and seeing if that works or fails.
+   * @param port port to listen to
+   * @return true if the port was available for listening on
+   */
+  public static boolean isPortAvailable(int port) {
+    try {
+      ServerSocket socket = new ServerSocket(port);
+      socket.close();
+      return true;
+    } catch (IOException e) {
+      return false;
+    }
+  }
+
+  // Build env map: key -> value;
+  // value will be replaced by the corresponding value in tokenMap, if any.
+  public static Map<String, String> buildEnvMap(
+      org.apache.hadoop.yarn.service.api.records.Configuration conf,
+      Map<String,String> tokenMap) {
+    if (tokenMap == null) {
+      return conf.getEnv();
+    }
+    Map<String, String> env = new HashMap<>();
+    for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
+      String key = entry.getKey();
+      String val = entry.getValue();
+      for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+        val = val.replaceAll(Pattern.quote(token.getKey()),
+            token.getValue());
+      }
+      env.put(key,val);
+    }
+    return env;
+  }
+
+  public static String[] getLibDirs() {
+    String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
+    if (isUnset(libDirStr)) {
+      return ArrayUtils.EMPTY_STRING_ARRAY;
+    }
+    return StringUtils.split(libDirStr, ',');
+  }
+
+  /**
+   * Submit a JAR containing a specific class and map it
+   * @param providerResources provider map to build up
+   * @param sliderFileSystem remote fs
+   * @param clazz class to look for
+   * @param libdir lib directory
+   * @param jarName <i>At the destination</i>
+   * @return the local resource ref
+   * @throws IOException trouble copying to HDFS
+   */
+  public static LocalResource putJar(Map<String, LocalResource> providerResources,
+      SliderFileSystem sliderFileSystem,
+      Class clazz,
+      Path tempPath,
+      String libdir,
+      String jarName
+  )
+      throws IOException, SliderException {
+    LocalResource res = sliderFileSystem.submitJarWithClass(
+        clazz,
+        tempPath,
+        libdir,
+        jarName);
+    providerResources.put(libdir + "/" + jarName, res);
+    return res;
+  }
+
+  /**
+   * Submit a JAR containing and map it
+   * @param providerResources provider map to build up
+   * @param sliderFileSystem remote fs
+   * @param libDir lib directory
+   * @param srcPath copy jars from
+   */
+  public static void putAllJars(Map<String, LocalResource> providerResources,
+                                SliderFileSystem sliderFileSystem,
+                                Path tempPath,
+                                String libDir,
+                                String srcPath) throws IOException, SliderException {
+    log.debug("Loading all dependencies from {}", srcPath);
+    if (ServiceUtils.isSet(srcPath)) {
+      File srcFolder = new File(srcPath);
+      FilenameFilter jarFilter = createJarFilter();
+      File[] listOfJars = srcFolder.listFiles(jarFilter);
+      if (listOfJars == null || listOfJars.length == 0) {
+        return;
+      }
+      for (File jarFile : listOfJars) {
+        LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+        providerResources.put(libDir + "/" + jarFile.getName(), res);
+      }
+    }
+  }
+
+  /**
+   * Accept all filenames ending with {@code .jar}
+   * @return a filename filter
+   */
+  public static FilenameFilter createJarFilter() {
+    return new FilenameFilter() {
+      public boolean accept(File dir, String name) {
+        return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+      }
+    };
+  }
+
+  /**
+   * Create a file:// path from a local file
+   * @param file file to point the path
+   * @return a new Path
+   */
+  public static Path createLocalPath(File file) {
+    return new Path(file.toURI());
+  }
+
+  /**
+   * Build up the classpath for execution
+   * -behaves very differently on a mini test cluster vs a production
+   * production one.
+   *
+   * @param sliderConfDir relative path to the dir containing slider config
+   *                      options to put on the classpath -or null
+   * @param libdir directory containing the JAR files
+   * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+   * (and hence the current classpath should be used, not anything built up)
+   * @return a classpath
+   */
+  public static ClasspathConstructor buildClasspath(String sliderConfDir,
+      String libdir,
+      SliderFileSystem sliderFileSystem,
+      boolean usingMiniMRCluster) {
+
+    ClasspathConstructor classpath = new ClasspathConstructor();
+    classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+
+    // add the runtime classpath needed for tests to work
+    if (usingMiniMRCluster) {
+      // for mini cluster we pass down the java CP properties
+      // and nothing else
+      classpath.appendAll(classpath.localJVMClasspath());
+    } else {
+      if (sliderConfDir != null) {
+        classpath.addClassDirectory(sliderConfDir);
+      }
+      classpath.addLibDir(libdir);
+      if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+        classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
+      }
+      classpath.addRemoteClasspathEnvVar();
+      classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
+    }
+    return classpath;
+  }
+
+  /**
+   * Given a source folder create a tar.gz file
+   * 
+   * @param libDirs
+   * @param tarGzipFile
+   * 
+   * @throws IOException
+   */
+  public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
+      FilenameFilter filter) throws IOException {
+    log.info("Tar-gzipping folders {} to {}", libDirs,
+        tarGzipFile.getAbsolutePath());
+
+    try(TarArchiveOutputStream taos =
+            new TarArchiveOutputStream(new GZIPOutputStream(
+        new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
+      for (String libDir : libDirs) {
+        File srcFolder = new File(libDir);
+        List<String> files = new ArrayList<>();
+        generateFileList(files, srcFolder, srcFolder, true, filter);
+        for (String file : files) {
+          File srcFile = new File(srcFolder, file);
+          TarArchiveEntry tarEntry = new TarArchiveEntry(
+              srcFile, file);
+          taos.putArchiveEntry(tarEntry);
+          try(FileInputStream in = new FileInputStream(srcFile)) {
+            org.apache.commons.io.IOUtils.copy(in, taos);
+          }
+          taos.flush();
+          taos.closeArchiveEntry();
+        }
+      }
+    }
+  }
+
+  private static void generateFileList(List<String> fileList, File node,
+      File rootFolder, Boolean relative, FilenameFilter filter) {
+    if (node.isFile()) {
+      String fileFullPath = node.toString();
+      if (relative) {
+        fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
+            fileFullPath.length()));
+      } else {
+        fileList.add(fileFullPath);
+      }
+    }
+
+    if (node.isDirectory()) {
+      String[] subNode = node.list(filter);
+      if (subNode == null || subNode.length == 0) {
+          return;
+      }
+      for (String filename : subNode) {
+        generateFileList(fileList, new File(node, filename), rootFolder,
+            relative, filter);
+      }
+    }
+  }
+
+  public static String createNameTag(String name) {
+    return "Name: " + name;
+  }
+
+  public static String createVersionTag(String version) {
+    return "Version: " + version;
+  }
+
+  public static String createDescriptionTag(String description) {
+    return "Description: " + description;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
deleted file mode 100644
index 2809dfb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ /dev/null
@@ -1,564 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPOutputStream;
-
-/**
- * These are slider-specific Util methods
- */
-public final class SliderUtils {
-
-  private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
-
-  private SliderUtils() {
-  }
-
-  /**
-   * Implementation of set-ness, groovy definition of true/false for a string
-   * @param s string
-   * @return true iff the string is neither null nor empty
-   */
-  public static boolean isUnset(String s) {
-    return s == null || s.isEmpty();
-  }
-
-  public static boolean isSet(String s) {
-    return !isUnset(s);
-  }
-
-  public static boolean isEmpty(Collection l) {
-    return l == null || l.isEmpty();
-  }
-
-  /**
-   * Find a containing JAR
-   * @param clazz class to find
-   * @return the file
-   * @throws IOException any IO problem, including the class not having a
-   * classloader
-   * @throws FileNotFoundException if the class did not resolve to a file
-   */
-  public static File findContainingJarOrFail(Class clazz) throws IOException {
-    File localFile = SliderUtils.findContainingJar(clazz);
-    if (null == localFile) {
-      throw new FileNotFoundException("Could not find JAR containing " + clazz);
-    }
-    return localFile;
-  }
-
-
-  /**
-   * Find a containing JAR
-   * @param my_class class to find
-   * @return the file or null if it is not found
-   * @throws IOException any IO problem, including the class not having a
-   * classloader
-   */
-  public static File findContainingJar(Class my_class) throws IOException {
-    ClassLoader loader = my_class.getClassLoader();
-    if (loader == null) {
-      throw new IOException(
-          "Class " + my_class + " does not have a classloader!");
-    }
-    String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
-    Enumeration<URL> urlEnumeration = loader.getResources(class_file);
-    for (; urlEnumeration.hasMoreElements(); ) {
-      URL url = urlEnumeration.nextElement();
-      if ("jar".equals(url.getProtocol())) {
-        String toReturn = url.getPath();
-        if (toReturn.startsWith("file:")) {
-          toReturn = toReturn.substring("file:".length());
-        }
-        // URLDecoder is a misnamed class, since it actually decodes
-        // x-www-form-urlencoded MIME type rather than actual
-        // URL encoding (which the file path has). Therefore it would
-        // decode +s to ' 's which is incorrect (spaces are actually
-        // either unencoded or encoded as "%20"). Replace +s first, so
-        // that they are kept sacred during the decoding process.
-        toReturn = toReturn.replaceAll("\\+", "%2B");
-        toReturn = URLDecoder.decode(toReturn, "UTF-8");
-        String jarFilePath = toReturn.replaceAll("!.*$", "");
-        return new File(jarFilePath);
-      } else {
-        log.info("could not locate JAR containing {} URL={}", my_class, url);
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Copy a file to a new FS -both paths must be qualified.
-   * @param conf conf file
-   * @param srcFile src file
-   * @param destFile dest file
-   */
-  @SuppressWarnings("deprecation")
-  public static void copy(Configuration conf,
-      Path srcFile,
-      Path destFile) throws
-      IOException,
-      BadClusterStateException {
-    FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
-    //list all paths in the src.
-    if (!srcFS.exists(srcFile)) {
-      throw new FileNotFoundException("Source file not found " + srcFile);
-    }
-    if (!srcFS.isFile(srcFile)) {
-      throw new FileNotFoundException(
-          "Source file not a file " + srcFile);
-    }
-    FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
-    FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
-  }
-
-  /**
-   * Take a collection, return a list containing the string value of every
-   * element in the collection.
-   * @param c collection
-   * @return a stringified list
-   */
-  public static List<String> collectionToStringList(Collection c) {
-    List<String> l = new ArrayList<>(c.size());
-    for (Object o : c) {
-      l.add(o.toString());
-    }
-    return l;
-  }
-
-  /**
-   * Join an collection of objects with a separator that appears after every
-   * instance in the list -including at the end
-   * @param collection collection to call toString() on each element
-   * @param separator separator string
-   * @return the joined entries
-   */
-  public static String join(Collection collection, String separator) {
-    return join(collection, separator, true);
-  }
-
-  /**
-   * Join an collection of objects with a separator that appears after every
-   * instance in the list -optionally at the end
-   * @param collection collection to call toString() on each element
-   * @param separator separator string
-   * @param trailing add a trailing entry or not
-   * @return the joined entries
-   */
-  public static String join(Collection collection,
-      String separator,
-      boolean trailing) {
-    StringBuilder b = new StringBuilder();
-    // fast return on empty collection
-    if (collection.isEmpty()) {
-      return trailing ? separator : "";
-    }
-    for (Object o : collection) {
-      b.append(o);
-      b.append(separator);
-    }
-    int length = separator.length();
-    String s = b.toString();
-    return (trailing || s.isEmpty()) ?
-           s : (b.substring(0, b.length() - length));
-  }
-
-  /**
-   * Join an array of strings with a separator that appears after every
-   * instance in the list -optionally at the end
-   * @param collection strings
-   * @param separator separator string
-   * @param trailing add a trailing entry or not
-   * @return the joined entries
-   */
-  public static String join(String[] collection, String separator,
-      boolean trailing) {
-    return join(Arrays.asList(collection), separator, trailing);
-  }
-
-  /**
-   * Resolve a mandatory environment variable
-   * @param key env var
-   * @return the resolved value
-   * @throws BadClusterStateException
-   */
-  public static String mandatoryEnvVariable(String key) throws
-      BadClusterStateException {
-    String v = System.getenv(key);
-    if (v == null) {
-      throw new BadClusterStateException("Missing Environment variable " + key);
-    }
-    return v;
-  }
-
-  /**
-   * Generic map merge logic
-   * @param first first map
-   * @param second second map
-   * @param <T1> key type
-   * @param <T2> value type
-   * @return 'first' merged with the second
-   */
-  public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
-      Map<T1, T2> second) {
-    Preconditions.checkArgument(first != null, "Null 'first' value");
-    Preconditions.checkArgument(second != null, "Null 'second' value");
-    for (Map.Entry<T1, T2> entry : second.entrySet()) {
-      T1 key = entry.getKey();
-      if (!first.containsKey(key)) {
-        first.put(key, entry.getValue());
-      }
-    }
-    return first;
-  }
-
-  /**
-   * Convert a map to a multi-line string for printing
-   * @param map map to stringify
-   * @return a string representation of the map
-   */
-  public static String stringifyMap(Map<String, String> map) {
-    StringBuilder builder = new StringBuilder();
-    for (Map.Entry<String, String> entry : map.entrySet()) {
-      builder.append(entry.getKey())
-             .append("=\"")
-             .append(entry.getValue())
-             .append("\"\n");
-
-    }
-    return builder.toString();
-  }
-
-  /**
-   * Convert a YARN URL into a string value of a normal URL
-   * @param url URL
-   * @return string representatin
-   */
-  public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
-    StringBuilder builder = new StringBuilder();
-    builder.append(url.getScheme()).append("://");
-    if (url.getHost() != null) {
-      builder.append(url.getHost()).append(":").append(url.getPort());
-    }
-    builder.append(url.getFile());
-    return builder.toString();
-  }
-
-  /**
-   * Get a random open port
-   * @return true if the port was available for listening on
-   */
-  public static int getOpenPort() throws IOException {
-    ServerSocket socket = null;
-    try {
-      socket = new ServerSocket(0);
-      return socket.getLocalPort();
-    } finally {
-      if (socket != null) {
-        socket.close();
-      }
-    }
-  }
-
-  /**
-   * See if a port is available for listening on by trying to listen
-   * on it and seeing if that works or fails.
-   * @param port port to listen to
-   * @return true if the port was available for listening on
-   */
-  public static boolean isPortAvailable(int port) {
-    try {
-      ServerSocket socket = new ServerSocket(port);
-      socket.close();
-      return true;
-    } catch (IOException e) {
-      return false;
-    }
-  }
-
-  // Build env map: key -> value;
-  // value will be replaced by the corresponding value in tokenMap, if any.
-  public static Map<String, String> buildEnvMap(
-      org.apache.hadoop.yarn.service.api.records.Configuration conf,
-      Map<String,String> tokenMap) {
-    if (tokenMap == null) {
-      return conf.getEnv();
-    }
-    Map<String, String> env = new HashMap<>();
-    for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
-      String key = entry.getKey();
-      String val = entry.getValue();
-      for (Map.Entry<String,String> token : tokenMap.entrySet()) {
-        val = val.replaceAll(Pattern.quote(token.getKey()),
-            token.getValue());
-      }
-      env.put(key,val);
-    }
-    return env;
-  }
-
-  public static String[] getLibDirs() {
-    String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
-    if (isUnset(libDirStr)) {
-      return ArrayUtils.EMPTY_STRING_ARRAY;
-    }
-    return StringUtils.split(libDirStr, ',');
-  }
-
-  /**
-   * Submit a JAR containing a specific class and map it
-   * @param providerResources provider map to build up
-   * @param sliderFileSystem remote fs
-   * @param clazz class to look for
-   * @param libdir lib directory
-   * @param jarName <i>At the destination</i>
-   * @return the local resource ref
-   * @throws IOException trouble copying to HDFS
-   */
-  public static LocalResource putJar(Map<String, LocalResource> providerResources,
-      SliderFileSystem sliderFileSystem,
-      Class clazz,
-      Path tempPath,
-      String libdir,
-      String jarName
-  )
-      throws IOException, SliderException {
-    LocalResource res = sliderFileSystem.submitJarWithClass(
-        clazz,
-        tempPath,
-        libdir,
-        jarName);
-    providerResources.put(libdir + "/" + jarName, res);
-    return res;
-  }
-
-  /**
-   * Submit a JAR containing and map it
-   * @param providerResources provider map to build up
-   * @param sliderFileSystem remote fs
-   * @param libDir lib directory
-   * @param srcPath copy jars from
-   */
-  public static void putAllJars(Map<String, LocalResource> providerResources,
-                                SliderFileSystem sliderFileSystem,
-                                Path tempPath,
-                                String libDir,
-                                String srcPath) throws IOException, SliderException {
-    log.info("Loading all dependencies from {}", srcPath);
-    if (SliderUtils.isSet(srcPath)) {
-      File srcFolder = new File(srcPath);
-      FilenameFilter jarFilter = createJarFilter();
-      File[] listOfJars = srcFolder.listFiles(jarFilter);
-      if (listOfJars == null || listOfJars.length == 0) {
-        return;
-      }
-      for (File jarFile : listOfJars) {
-        LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
-        providerResources.put(libDir + "/" + jarFile.getName(), res);
-      }
-    }
-  }
-
-  /**
-   * Accept all filenames ending with {@code .jar}
-   * @return a filename filter
-   */
-  public static FilenameFilter createJarFilter() {
-    return new FilenameFilter() {
-      public boolean accept(File dir, String name) {
-        return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
-      }
-    };
-  }
-
-  /**
-   * Submit the AM tar.gz containing all dependencies and map it
-   * @param providerResources provider map to build up
-   * @param sliderFileSystem remote fs
-   */
-  public static void putAmTarGzipAndUpdate(
-      Map<String, LocalResource> providerResources,
-      SliderFileSystem sliderFileSystem
-  ) throws IOException, SliderException {
-    log.info("Loading all dependencies from {}{}",
-        YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
-        YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
-    sliderFileSystem.submitTarGzipAndUpdate(providerResources);
-  }
-
-  /**
-   * Create a file:// path from a local file
-   * @param file file to point the path
-   * @return a new Path
-   */
-  public static Path createLocalPath(File file) {
-    return new Path(file.toURI());
-  }
-
-  /**
-   * Build up the classpath for execution
-   * -behaves very differently on a mini test cluster vs a production
-   * production one.
-   *
-   * @param sliderConfDir relative path to the dir containing slider config
-   *                      options to put on the classpath -or null
-   * @param libdir directory containing the JAR files
-   * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
-   * (and hence the current classpath should be used, not anything built up)
-   * @return a classpath
-   */
-  public static ClasspathConstructor buildClasspath(String sliderConfDir,
-      String libdir,
-      SliderFileSystem sliderFileSystem,
-      boolean usingMiniMRCluster) {
-
-    ClasspathConstructor classpath = new ClasspathConstructor();
-    classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
-
-    // add the runtime classpath needed for tests to work
-    if (usingMiniMRCluster) {
-      // for mini cluster we pass down the java CP properties
-      // and nothing else
-      classpath.appendAll(classpath.localJVMClasspath());
-    } else {
-      if (sliderConfDir != null) {
-        classpath.addClassDirectory(sliderConfDir);
-      }
-      classpath.addLibDir(libdir);
-      if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
-        classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
-      } else {
-        log.info(
-            "For faster submission of apps, upload dependencies using cmd " +
-                "enableFastLaunch");
-      }
-      classpath.addRemoteClasspathEnvVar();
-      classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
-    }
-    return classpath;
-  }
-
-  /**
-   * Given a source folder create a tar.gz file
-   * 
-   * @param libDirs
-   * @param tarGzipFile
-   * 
-   * @throws IOException
-   */
-  public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
-      FilenameFilter filter) throws IOException {
-    log.info("Tar-gzipping folders {} to {}", libDirs,
-        tarGzipFile.getAbsolutePath());
-
-    try(TarArchiveOutputStream taos =
-            new TarArchiveOutputStream(new GZIPOutputStream(
-        new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
-      for (String libDir : libDirs) {
-        File srcFolder = new File(libDir);
-        List<String> files = new ArrayList<>();
-        generateFileList(files, srcFolder, srcFolder, true, filter);
-        for (String file : files) {
-          File srcFile = new File(srcFolder, file);
-          TarArchiveEntry tarEntry = new TarArchiveEntry(
-              srcFile, file);
-          taos.putArchiveEntry(tarEntry);
-          try(FileInputStream in = new FileInputStream(srcFile)) {
-            org.apache.commons.io.IOUtils.copy(in, taos);
-          }
-          taos.flush();
-          taos.closeArchiveEntry();
-        }
-      }
-    }
-  }
-
-  private static void generateFileList(List<String> fileList, File node,
-      File rootFolder, Boolean relative, FilenameFilter filter) {
-    if (node.isFile()) {
-      String fileFullPath = node.toString();
-      if (relative) {
-        fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
-            fileFullPath.length()));
-      } else {
-        fileList.add(fileFullPath);
-      }
-    }
-
-    if (node.isDirectory()) {
-      String[] subNode = node.list(filter);
-      if (subNode == null || subNode.length == 0) {
-          return;
-      }
-      for (String filename : subNode) {
-        generateFileList(fileList, new File(node, filename), rootFolder,
-            relative, filter);
-      }
-    }
-  }
-
-  public static String createNameTag(String name) {
-    return "Name: " + name;
-  }
-
-  public static String createVersionTag(String version) {
-    return "Version: " + version;
-  }
-
-  public static String createDescriptionTag(String description) {
-    return "Description: " + description;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
index 1fa07ce..c6e8525 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
@@ -32,7 +32,7 @@ public class ZookeeperUtils {
     String zkPort = Integer.toString(port);
     //parse the hosts
     String[] hostlist = zkHosts.split(",", 0);
-    String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
+    String quorum = ServiceUtils.join(hostlist, ":" + zkPort + ",", false);
     return quorum;
   }
 
@@ -117,7 +117,7 @@ public class ZookeeperUtils {
     for (HostAndPort hostAndPort : hostAndPorts) {
       entries.add(buildQuorumEntry(hostAndPort, defaultPort));
     }
-    return SliderUtils.join(entries, ",", false);
+    return ServiceUtils.join(entries, ",", false);
   }
   
   public static String convertToHostsOnlyList(String quorum) throws

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
index 6159215..d7fa9a04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.service.conf;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.junit.Assert;
 import org.junit.Test;
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message