hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [73/93] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi
Date Fri, 13 Oct 2017 18:22:19 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
index 6e6f4dd..7e53d18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -24,35 +24,17 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.service.client.params.Arguments;
 import org.apache.hadoop.yarn.service.client.params.SliderActions;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
 import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
 import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.zookeeper.server.util.KerberosUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,32 +45,19 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.io.Serializable;
-import java.net.InetSocketAddress;
 import java.net.ServerSocket;
-import java.net.Socket;
 import java.net.URL;
 import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 
 /**
  * These are slider-specific Util methods
@@ -97,41 +66,6 @@ public final class SliderUtils {
 
   private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
 
-  /**
-   * Atomic bool to track whether or not process security has already been
-   * turned on (prevents re-entrancy)
-   */
-  private static final AtomicBoolean processSecurityAlreadyInitialized =
-      new AtomicBoolean(false);
-  public static final String JAVA_SECURITY_KRB5_REALM =
-      "java.security.krb5.realm";
-  public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
-  /**
-   * Winutils
-   */
-  public static final String WINUTILS = "WINUTILS.EXE";
-  /**
-   * name of openssl program
-   */
-  public static final String OPENSSL = "openssl";
-
-  /**
-   * name of python program
-   */
-  public static final String PYTHON = "python";
-
-  /**
-   * type of docker standalone service
-   */
-  public static final String DOCKER = "docker";
-  /**
-   * type of docker on yarn service
-   */
-  public static final String DOCKER_YARN = "yarn_docker";
-
-  public static final int NODE_LIST_LIMIT = 10;
-
   private SliderUtils() {
   }
 
@@ -153,96 +87,6 @@ public final class SliderUtils {
   }
 
   /**
-   * Probe for a collection existing and not being empty
-   * @param l collection
-   * @return true if the reference is valid and it contains entries
-   */
-
-  public static boolean isNotEmpty(Collection l) {
-    return l != null && !l.isEmpty();
-  }
-
-  /**
-   * Probe for a map existing and not being empty
-   * @param m map
-   * @return true if the reference is valid and it contains map entries
-   */
-  public static boolean isNotEmpty(Map m) {
-    return m != null && !m.isEmpty();
-  }
-  
-  /*
-   * Validates whether num is an integer
-   * @param num
-   * @param msg the message to be shown in exception
-   */
-  @SuppressWarnings("ResultOfMethodCallIgnored")
-  private static void validateNumber(String num, String msg) throws
-      BadConfigException {
-    try {
-      Integer.parseInt(num);
-    } catch (NumberFormatException nfe) {
-      throw new BadConfigException(msg + num);
-    }
-  }
-
-  /*
-   * Translates the trailing JVM heapsize unit: g, G, m, M
-   * This assumes designated unit of 'm'
-   * @param heapsize
-   * @return heapsize in MB
-   */
-  public static String translateTrailingHeapUnit(String heapsize) throws
-      BadConfigException {
-    String errMsg = "Bad heapsize: ";
-    if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
-      String num = heapsize.substring(0, heapsize.length() - 1);
-      validateNumber(num, errMsg);
-      return num;
-    }
-    if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
-      String num = heapsize.substring(0, heapsize.length() - 1) + "000";
-      validateNumber(num, errMsg);
-      return num;
-    }
-    // check if specified heap size is a number
-    validateNumber(heapsize, errMsg);
-    return heapsize;
-  }
-
-  /**
-   * recursive directory delete
-   * @param dir dir to delete
-   * @throws IOException on any problem
-   */
-  public static void deleteDirectoryTree(File dir) throws IOException {
-    if (dir.exists()) {
-      if (dir.isDirectory()) {
-        log.info("Cleaning up {}", dir);
-        //delete the children
-        File[] files = dir.listFiles();
-        if (files == null) {
-          throw new IOException("listfiles() failed for " + dir);
-        }
-        for (File file : files) {
-          log.info("deleting {}", file);
-          if (!file.delete()) {
-            log.warn("Unable to delete " + file);
-          }
-        }
-        if (!dir.delete()) {
-          log.warn("Unable to delete " + dir);
-        }
-      } else {
-        throw new IOException("Not a directory " + dir);
-      }
-    } else {
-      //not found, do nothing
-      log.debug("No output dir yet");
-    }
-  }
-
-  /**
    * Find a containing JAR
    * @param clazz class to find
    * @return the file
@@ -298,127 +142,6 @@ public final class SliderUtils {
     return null;
   }
 
-  public static void checkPort(String hostname, int port, int connectTimeout)
-      throws IOException {
-    InetSocketAddress addr = new InetSocketAddress(hostname, port);
-    checkPort(hostname, addr, connectTimeout);
-  }
-
-  @SuppressWarnings("SocketOpenedButNotSafelyClosed")
-  public static void checkPort(String name,
-      InetSocketAddress address,
-      int connectTimeout)
-      throws IOException {
-    try(Socket socket = new Socket()) {
-      socket.connect(address, connectTimeout);
-    } catch (Exception e) {
-      throw new IOException("Failed to connect to " + name
-                            + " at " + address
-                            + " after " + connectTimeout + "milliseconds"
-                            + ": " + e,
-          e);
-    }
-  }
-
-  public static void checkURL(String name, String url, int timeout) throws
-      IOException {
-    InetSocketAddress address = NetUtils.createSocketAddr(url);
-    checkPort(name, address, timeout);
-  }
-
-  /**
-   * A required file
-   * @param role role of the file (for errors)
-   * @param filename the filename
-   * @throws ExitUtil.ExitException if the file is missing
-   * @return the file
-   */
-  public static File requiredFile(String filename, String role) throws
-      IOException {
-    if (filename.isEmpty()) {
-      throw new ExitUtil.ExitException(-1, role + " file not defined");
-    }
-    File file = new File(filename);
-    if (!file.exists()) {
-      throw new ExitUtil.ExitException(-1,
-          role + " file not found: " +
-          file.getCanonicalPath());
-    }
-    return file;
-  }
-
-  private static final PatternValidator clusternamePattern
-      = new PatternValidator("[a-z][a-z0-9_-]*");
-
-  /**
-   * Normalize a cluster name then verify that it is valid
-   * @param name proposed cluster name
-   * @return true iff it is valid
-   */
-  public static boolean isClusternameValid(String name) {
-    return name != null && clusternamePattern.matches(name);
-  }
-
-  /**
-   * Copy a directory to a new FS -both paths must be qualified. If
-   * a directory needs to be created, supplied permissions can override
-   * the default values. Existing directories are not touched
-   * @param conf conf file
-   * @param srcDirPath src dir
-   * @param destDirPath dest dir
-   * @param permission permission for the dest directory; null means "default"
-   * @return # of files copies
-   */
-  @SuppressWarnings("deprecation")
-  public static int copyDirectory(Configuration conf,
-      Path srcDirPath,
-      Path destDirPath,
-      FsPermission permission) throws
-      IOException,
-      BadClusterStateException {
-    FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
-    FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
-    //list all paths in the src.
-    if (!srcFS.exists(srcDirPath)) {
-      throw new FileNotFoundException("Source dir not found " + srcDirPath);
-    }
-    if (!srcFS.isDirectory(srcDirPath)) {
-      throw new FileNotFoundException(
-          "Source dir not a directory " + srcDirPath);
-    }
-    GlobFilter dotFilter = new GlobFilter("[!.]*");
-    FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
-    int srcFileCount = entries.length;
-    if (srcFileCount == 0) {
-      return 0;
-    }
-    if (permission == null) {
-      permission = FsPermission.getDirDefault();
-    }
-    if (!destFS.exists(destDirPath)) {
-      new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
-          permission);
-    }
-    Path[] sourcePaths = new Path[srcFileCount];
-    for (int i = 0; i < srcFileCount; i++) {
-      FileStatus e = entries[i];
-      Path srcFile = e.getPath();
-      if (srcFS.isDirectory(srcFile)) {
-        String msg = "Configuration dir " + srcDirPath
-                     + " contains a directory " + srcFile;
-        log.warn(msg);
-        throw new IOException(msg);
-      }
-      log.debug("copying src conf file {}", srcFile);
-      sourcePaths[i] = srcFile;
-    }
-    log.debug("Copying {} files from {} to dest {}", srcFileCount,
-        srcDirPath,
-        destDirPath);
-    FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
-    return srcFileCount;
-  }
-
   /**
    * Copy a file to a new FS -both paths must be qualified.
    * @param conf conf file
@@ -497,19 +220,6 @@ public final class SliderUtils {
 
   /**
    * Join an array of strings with a separator that appears after every
-   * instance in the list -including at the end
-   * @param collection strings
-   * @param separator separator string
-   * @return the joined entries
-   */
-  public static String join(String[] collection, String separator) {
-    return join(collection, separator, true);
-
-
-  }
-
-  /**
-   * Join an array of strings with a separator that appears after every
    * instance in the list -optionally at the end
    * @param collection strings
    * @param separator separator string
@@ -522,30 +232,6 @@ public final class SliderUtils {
   }
 
   /**
-   * Join an array of strings with a separator that appears after every
-   * instance in the list -except at the end
-   * @param collection strings
-   * @param separator separator string
-   * @return the list
-   */
-  public static String joinWithInnerSeparator(String separator,
-      Object... collection) {
-    StringBuilder b = new StringBuilder();
-    boolean first = true;
-
-    for (Object o : collection) {
-      if (first) {
-        first = false;
-      } else {
-        b.append(separator);
-      }
-      b.append(o.toString());
-      b.append(separator);
-    }
-    return b.toString();
-  }
-
-  /**
    * Resolve a mandatory environment variable
    * @param key env var
    * @return the resolved value
@@ -560,170 +246,6 @@ public final class SliderUtils {
     return v;
   }
 
-  public static String appReportToString(ApplicationReport r,
-      String separator) {
-    StringBuilder builder = new StringBuilder(512);
-    builder.append("service ")
-           .append(
-               r.getName())
-           .append("/")
-           .append(r.getApplicationType())
-           .append(separator);
-    Set<String> tags = r.getApplicationTags();
-    if (!tags.isEmpty()) {
-      for (String tag : tags) {
-        builder.append(tag).append(separator);
-      }
-    }
-    DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
-    dateFormat.setTimeZone(TimeZone.getDefault());
-    builder.append("state: ").append(r.getYarnApplicationState());
-    String trackingUrl = r.getTrackingUrl();
-    if (isSet(trackingUrl)) {
-      builder.append(separator).append("URL: ").append(trackingUrl);
-    }
-    builder.append(separator)
-           .append("Started: ")
-           .append(dateFormat.format(new Date(r.getStartTime())));
-    long finishTime = r.getFinishTime();
-    if (finishTime > 0) {
-      builder.append(separator)
-             .append("Finished: ")
-             .append(dateFormat.format(new Date(finishTime)));
-    }
-    String rpcHost = r.getHost();
-    if (!isSet(rpcHost)) {
-      builder.append(separator)
-             .append("RPC :")
-             .append(rpcHost)
-             .append(':')
-             .append(r.getRpcPort());
-    }
-    String diagnostics = r.getDiagnostics();
-    if (!isSet(diagnostics)) {
-      builder.append(separator).append("Diagnostics :").append(diagnostics);
-    }
-    return builder.toString();
-  }
-
-  /**
-   * Filter a string value given a single filter
-   * 
-   * @param value
-   *          the string value to check
-   * @param filter
-   *          a single string filter
-   * @return return true if value should be trapped, false if it should be let
-   *         through
-   */
-  public static boolean filter(String value, String filter) {
-    return !(StringUtils.isEmpty(filter) || filter.equals(value));
-  }
-
-  /**
-   * Filter a string value given a set of filters
-   * 
-   * @param value
-   *          the string value to check
-   * @param filters
-   *          a set of string filters
-   * @return return true if value should be trapped, false if it should be let
-   *         through
-   */
-  public static boolean filter(String value, Set<String> filters) {
-    return !(filters.isEmpty() || filters.contains(value));
-  }
-
-  /**
-   * Sorts the given list of service reports, most recently started
-   * or finished instance first.
-   *
-   * @param instances list of instances
-   */
-  public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
-    Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
-  }
-
-  /**
-   * Sorts the given list of service reports
-   * Finished instances are ordered by finished time and running/accepted instances are
-   * ordered by start time
-   * Finally Instance are order by finished instances coming after running instances
-   *
-   * @param instances list of instances
-   */
-  public static void sortApplicationReport(List<ApplicationReport> instances) {
-    if (instances.size() <= 1) {
-      return;
-    }
-    List<ApplicationReport> nonLiveInstance =
-        new ArrayList<>(instances.size());
-    List<ApplicationReport> liveInstance =
-        new ArrayList<>(instances.size());
-
-    for (ApplicationReport report : instances) {
-      if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
-          ||
-          report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
-        liveInstance.add(report);
-      } else {
-        nonLiveInstance.add(report);
-      }
-    }
-
-    if (liveInstance.size() > 1) {
-      Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
-    }
-    if (nonLiveInstance.size() > 1) {
-      Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
-    }
-    instances.clear();
-    instances.addAll(liveInstance);
-    instances.addAll(nonLiveInstance);
-  }
-
-  /**
-   * Merge in one map to another -all entries in the second map are
-   * merged into the first -overwriting any duplicate keys.
-   * @param first first map -the updated one.
-   * @param second the map that is merged in
-   * @return the first map
-   */
-  public static Map<String, String> mergeMap(Map<String, String> first,
-      Map<String, String> second) {
-    first.putAll(second);
-    return first;
-  }
-
-  /**
-   * Merge a set of entries into a map. This will take the entryset of
-   * a map, or a Hadoop collection itself
-   * @param dest destination
-   * @param entries entries
-   * @return dest -with the entries merged in
-   */
-  public static Map<String, String> mergeEntries(Map<String, String> dest,
-      Iterable<Map.Entry<String, String>> entries) {
-    for (Map.Entry<String, String> entry : entries) {
-      dest.put(entry.getKey(), entry.getValue());
-    }
-    return dest;
-  }
-
-  /**
-   * Generic map merge logic
-   * @param first first map
-   * @param second second map
-   * @param <T1> key type
-   * @param <T2> value type
-   * @return 'first' merged with the second
-   */
-  public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
-      Map<T1, T2> second) {
-    first.putAll(second);
-    return first;
-  }
-
   /**
    * Generic map merge logic
    * @param first first map
@@ -763,125 +285,6 @@ public final class SliderUtils {
   }
 
   /**
-   * Parse an int value, replacing it with defval if undefined;
-   * @param errorKey key to use in exceptions
-   * @param defVal default value to use if the key is not in the map
-   * @param min min value or -1 for do not check
-   * @param max max value or -1 for do not check
-   * @return the int value the integer value
-   * @throws BadConfigException if the value could not be parsed
-   */
-  public static int parseAndValidate(String errorKey,
-      String valS,
-      int defVal,
-      int min, int max) throws
-      BadConfigException {
-    if (valS == null) {
-      valS = Integer.toString(defVal);
-    }
-    String trim = valS.trim();
-    int val;
-    try {
-      val = Integer.decode(trim);
-    } catch (NumberFormatException e) {
-      throw new BadConfigException("Failed to parse value of "
-                                   + errorKey + ": \"" + trim + "\"");
-    }
-    if (min >= 0 && val < min) {
-      throw new BadConfigException("Value of "
-                                   + errorKey + ": " + val + ""
-                                   + "is less than the minimum of " + min);
-    }
-    if (max >= 0 && val > max) {
-      throw new BadConfigException("Value of "
-                                   + errorKey + ": " + val + ""
-                                   + "is more than the maximum of " + max);
-    }
-    return val;
-  }
-
-  public static InetSocketAddress getRmAddress(Configuration conf) {
-    return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
-        YarnConfiguration.DEFAULT_RM_ADDRESS,
-        YarnConfiguration.DEFAULT_RM_PORT);
-  }
-
-  public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
-    return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
-  }
-
-  /**
-   * probe to see if the RM scheduler is defined
-   * @param conf config
-   * @return true if the RM scheduler address is set to
-   * something other than 0.0.0.0
-   */
-  public static boolean isRmSchedulerAddressDefined(Configuration conf) {
-    InetSocketAddress address = getRmSchedulerAddress(conf);
-    return isAddressDefined(address);
-  }
-
-  /**
-   * probe to see if the address
-   * @param address network address
-   * @return true if the scheduler address is set to
-   * something other than 0.0.0.0
-   */
-  public static boolean isAddressDefined(InetSocketAddress address) {
-    if (address == null || address.getHostString() == null) {
-      return false;
-    }
-    return !(address.getHostString().equals("0.0.0.0"));
-  }
-
-  public static void setRmAddress(Configuration conf, String rmAddr) {
-    conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
-  }
-
-  public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
-    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
-  }
-
-  public static boolean hasAppFinished(ApplicationReport report) {
-    return report == null ||
-           report.getYarnApplicationState().ordinal() >=
-           YarnApplicationState.FINISHED.ordinal();
-  }
-
-  public static String containerToString(Container container) {
-    if (container == null) {
-      return "null container";
-    }
-    return String.format(Locale.ENGLISH,
-        "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
-        container.getId(),
-        container.getNodeId(),
-        container.getNodeHttpAddress(),
-        container.getPriority(),
-        container.getResource());
-  }
-
-  /**
-   * convert an AM report to a string for diagnostics
-   * @param report the report
-   * @return the string value
-   */
-  public static String reportToString(ApplicationReport report) {
-    if (report == null) {
-      return "Null service report";
-    }
-
-    return "App " + report.getName() + "/" + report.getApplicationType() +
-           "# " +
-           report.getApplicationId() + " user " + report.getUser() +
-           " is in state " + report.getYarnApplicationState() +
-           " RPC: " + report.getHost() + ":" + report.getRpcPort() +
-           " URL: " + report.getOriginalTrackingUrl();
-  }
-
-  /**
    * Convert a YARN URL into a string value of a normal URL
    * @param url URL
    * @return string representatin
@@ -949,151 +352,6 @@ public final class SliderUtils {
     return env;
   }
 
-  /**
-   * Apply a set of command line options to a cluster role map
-   * @param clusterRoleMap cluster role map to merge onto
-   * @param commandOptions command opts
-   */
-  public static void applyCommandLineRoleOptsToRoleMap(
-      Map<String, Map<String, String>> clusterRoleMap,
-      Map<String, Map<String, String>> commandOptions) {
-    for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
-      String key = entry.getKey();
-      Map<String, String> optionMap = entry.getValue();
-      Map<String, String> existingMap = clusterRoleMap.get(key);
-      if (existingMap == null) {
-        existingMap = new HashMap<String, String>();
-      }
-      log.debug("Overwriting role options with command line values {}",
-          stringifyMap(optionMap));
-      mergeMap(existingMap, optionMap);
-      //set or overwrite the role
-      clusterRoleMap.put(key, existingMap);
-    }
-  }
-
-  /**
-   * Verify that a Kerberos principal has been set -if not fail
-   * with an error message that actually tells you what is missing
-   * @param conf configuration to look at
-   * @param principal key of principal
-   * @throws BadConfigException if the key is not set
-   */
-  public static void verifyPrincipalSet(Configuration conf,
-      String principal) throws
-      BadConfigException {
-    String principalName = conf.get(principal);
-    if (principalName == null) {
-      throw new BadConfigException("Unset Kerberos principal : %s",
-          principal);
-    }
-    log.debug("Kerberos princial {}={}", principal, principalName);
-  }
-
-  /**
-   * Flag to indicate whether the cluster is in secure mode
-   * @param conf configuration to look at
-   * @return true if the slider client/service should be in secure mode
-   */
-  public static boolean isHadoopClusterSecure(Configuration conf) {
-    return SecurityUtil.getAuthenticationMethod(conf) !=
-           UserGroupInformation.AuthenticationMethod.SIMPLE;
-  }
-
-  /**
-   * Init security if the cluster configuration declares the cluster is secure
-   * @param conf configuration to look at
-   * @return true if the cluster is secure
-   * @throws IOException cluster is secure
-   * @throws SliderException the configuration/process is invalid
-   */
-  public static boolean maybeInitSecurity(Configuration conf) throws
-      IOException,
-      SliderException {
-    boolean clusterSecure = isHadoopClusterSecure(conf);
-    if (clusterSecure) {
-      log.debug("Enabling security");
-      initProcessSecurity(conf);
-    }
-    return clusterSecure;
-  }
-
-  /**
-   * Turn on security. This is setup to only run once.
-   * @param conf configuration to build up security
-   * @return true if security was initialized in this call
-   * @throws IOException IO/Net problems
-   * @throws BadConfigException the configuration and system state are inconsistent
-   */
-  public static boolean initProcessSecurity(Configuration conf) throws
-      IOException,
-      SliderException {
-
-    if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
-      //security is already inited
-      return false;
-    }
-
-    log.info("JVM initialized into secure mode with kerberos realm {}",
-        SliderUtils.getKerberosRealm());
-    //this gets UGI to reset its previous world view (i.e simple auth)
-    //security
-    log.debug("java.security.krb5.realm={}",
-        System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
-    log.debug("java.security.krb5.kdc={}",
-        System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
-    log.debug("hadoop.security.authentication={}",
-        conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
-    log.debug("hadoop.security.authorization={}",
-        conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
-    UserGroupInformation.setConfiguration(conf);
-    UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
-    log.debug("Authenticating as {}", authUser);
-    log.debug("Login user is {}", UserGroupInformation.getLoginUser());
-    if (!UserGroupInformation.isSecurityEnabled()) {
-      throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
-          "Although secure mode is enabled," +
-         "the service has already set up its user as an insecure entity %s",
-          authUser);
-    }
-    if (authUser.getAuthenticationMethod() ==
-        UserGroupInformation.AuthenticationMethod.SIMPLE) {
-      throw new BadConfigException("Auth User is not Kerberized %s" +
-         " -security has already been set up with the wrong authentication method. "
-         + "This can occur if a file system has already been created prior to the loading of "
-         + "the security configuration.",
-          authUser);
-
-    }
-
-    SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
-    SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal");
-    return true;
-  }
-
-  /**
-   * Force an early login: This catches any auth problems early rather than
-   * in RPC operations
-   * @throws IOException if the login fails
-   */
-  public static void forceLogin() throws IOException {
-    if (UserGroupInformation.isSecurityEnabled()) {
-      if (UserGroupInformation.isLoginKeytabBased()) {
-        UserGroupInformation.getLoginUser().reloginFromKeytab();
-      } else {
-        UserGroupInformation.getLoginUser().reloginFromTicketCache();
-      }
-    }
-  }
-
-  public static String getLibDir() {
-    String[] libDirs = getLibDirs();
-    if (libDirs == null || libDirs.length == 0) {
-      return null;
-    }
-    return libDirs[0];
-  }
-
   public static String[] getLibDirs() {
     String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
     if (isUnset(libDirStr)) {
@@ -1183,39 +441,6 @@ public final class SliderUtils {
     sliderFileSystem.submitTarGzipAndUpdate(providerResources);
   }
 
-  public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
-    Map<String, Map<String, String>> dest = new HashMap<>();
-    for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
-      dest.put(entry.getKey(), stringMapClone(entry.getValue()));
-    }
-    return dest;
-  }
-
-  public static Map<String, String> stringMapClone(Map<String, String> src) {
-    Map<String, String> dest = new HashMap<>();
-    return mergeEntries(dest, src.entrySet());
-  }
-
-  /**
-   * List a directory in the local filesystem
-   * @param dir directory
-   * @return a listing, one to a line
-   */
-  public static String listDir(File dir) {
-    if (dir == null) {
-      return "";
-    }
-    String[] confDirEntries = dir.list();
-    if (confDirEntries == null) {
-      return "";
-    }
-    StringBuilder builder = new StringBuilder();
-    for (String entry : confDirEntries) {
-      builder.append(entry).append("\n");
-    }
-    return builder.toString();
-  }
-
   /**
    * Create a file:// path from a local file
    * @param file file to point the path
@@ -1225,16 +450,6 @@ public final class SliderUtils {
     return new Path(file.toURI());
   }
 
-  public static String getKerberosRealm() {
-    try {
-      return KerberosUtil.getDefaultRealm();
-    } catch (Exception e) {
-      log.debug("introspection into JVM internals failed", e);
-      return "(unknown)";
-
-    }
-  }
-
   /**
    * Build up the classpath for execution
    * -behaves very differently on a mini test cluster vs a production
@@ -1279,171 +494,6 @@ public final class SliderUtils {
   }
 
   /**
-   * Verify that a path refers to a directory. If not
-   * logs the parent dir then throws an exception
-   * @param dir the directory
-   * @param errorlog log for output on an error
-   * @throws FileNotFoundException if it is not a directory
-   */
-  public static void verifyIsDir(File dir, Logger errorlog) throws
-      FileNotFoundException {
-    if (!dir.exists()) {
-      errorlog.warn("contents of {}: {}", dir,
-          listDir(dir.getParentFile()));
-      throw new FileNotFoundException(dir.toString());
-    }
-    if (!dir.isDirectory()) {
-      errorlog.info("contents of {}: {}", dir,
-          listDir(dir.getParentFile()));
-      throw new FileNotFoundException(
-          "Not a directory: " + dir);
-    }
-  }
-
-  /**
-   * Verify that a file exists
-   * @param file file
-   * @param errorlog log for output on an error
-   * @throws FileNotFoundException
-   */
-  public static void verifyFileExists(File file, Logger errorlog) throws
-      FileNotFoundException {
-    if (!file.exists()) {
-      errorlog.warn("contents of {}: {}", file,
-          listDir(file.getParentFile()));
-      throw new FileNotFoundException(file.toString());
-    }
-    if (!file.isFile()) {
-      throw new FileNotFoundException("Not a file: " + file.toString());
-    }
-  }
-
-  /**
-   * verify that a config option is set
-   * @param configuration config
-   * @param key key
-   * @return the value, in case it needs to be verified too
-   * @throws BadConfigException if the key is missing
-   */
-  public static String verifyOptionSet(Configuration configuration, String key,
-      boolean allowEmpty) throws BadConfigException {
-    String val = configuration.get(key);
-    if (val == null) {
-      throw new BadConfigException(
-          "Required configuration option \"%s\" not defined ", key);
-    }
-    if (!allowEmpty && val.isEmpty()) {
-      throw new BadConfigException(
-          "Configuration option \"%s\" must not be empty", key);
-    }
-    return val;
-  }
-
-  /**
-   * Verify that a keytab property is defined and refers to a non-empty file
-   *
-   * @param siteConf configuration
-   * @param prop property to look for
-   * @return the file referenced
-   * @throws BadConfigException on a failure
-   */
-  public static File verifyKeytabExists(Configuration siteConf,
-      String prop) throws
-      BadConfigException {
-    String keytab = siteConf.get(prop);
-    if (keytab == null) {
-      throw new BadConfigException("Missing keytab property %s",
-          prop);
-
-    }
-    File keytabFile = new File(keytab);
-    if (!keytabFile.exists()) {
-      throw new BadConfigException("Missing keytab file %s defined in %s",
-          keytabFile,
-          prop);
-    }
-    if (keytabFile.length() == 0 || !keytabFile.isFile()) {
-      throw new BadConfigException("Invalid keytab file %s defined in %s",
-          keytabFile,
-          prop);
-    }
-    return keytabFile;
-  }
-
-  /**
-   * Add a subpath to an existing URL. This extends
-   * the path, inserting a / between all entries
-   * if needed.
-   * @param base base path/URL
-   * @param path subpath
-   * @return base+"/"+subpath
-   */
-  public static String appendToURL(String base, String path) {
-    StringBuilder fullpath = new StringBuilder(base);
-    if (!base.endsWith("/")) {
-      fullpath.append("/");
-    }
-    if (path.startsWith("/")) {
-      fullpath.append(path.substring(1));
-    } else {
-      fullpath.append(path);
-    }
-    return fullpath.toString();
-  }
-
-  /**
-   * Truncate the given string to a maximum length provided
-   * with a pad (...) added to the end if expected size if more than 10.
-   * @param toTruncate string to truncate; may be null
-   * @param maxSize maximum size
-   * @return the truncated/padded string. 
-   */
-  public static String truncate(String toTruncate, int maxSize) {
-    if (toTruncate == null || maxSize < 1
-        || toTruncate.length() <= maxSize) {
-      return toTruncate;
-    }
-
-    String pad = "...";
-    if (maxSize < 10) {
-      pad = "";
-    }
-    return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
-  }
-
-  /**
-   * Given a source folder create zipped file
-   *
-   * @param srcFolder
-   * @param zipFile
-   *
-   * @throws IOException
-   */
-  public static void zipFolder(File srcFolder, File zipFile) throws IOException {
-    log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
-    List<String> files = new ArrayList<>();
-    generateFileList(files, srcFolder, srcFolder, true);
-
-    byte[] buffer = new byte[1024];
-
-    try (FileOutputStream fos = new FileOutputStream(zipFile)) {
-      try (ZipOutputStream zos = new ZipOutputStream(fos)) {
-
-        for (String file : files) {
-          ZipEntry ze = new ZipEntry(file);
-          zos.putNextEntry(ze);
-          try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
-            int len;
-            while ((len = in.read(buffer)) > 0) {
-              zos.write(buffer, 0, len);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  /**
    * Given a source folder create a tar.gz file
    * 
    * @param libDirs
@@ -1479,11 +529,6 @@ public final class SliderUtils {
   }
 
   private static void generateFileList(List<String> fileList, File node,
-      File rootFolder, Boolean relative) {
-    generateFileList(fileList, node, rootFolder, relative, null);
-  }
-
-  private static void generateFileList(List<String> fileList, File node,
       File rootFolder, Boolean relative, FilenameFilter filter) {
     if (node.isFile()) {
       String fileFullPath = node.toString();
@@ -1507,134 +552,6 @@ public final class SliderUtils {
     }
   }
 
-  /**
-   * Check for any needed libraries being present. On Unix none are needed;
-   * on windows they must be present
-   * @return true if all is well
-   */
-  public static String checkForRequiredNativeLibraries() {
-
-    if (!Shell.WINDOWS) {
-      return "";
-    }
-    StringBuilder errorText = new StringBuilder("");
-    if (!NativeIO.isAvailable()) {
-      errorText.append("No native IO library. ");
-    }
-    try {
-      String path = Shell.getQualifiedBinPath(WINUTILS);
-      log.debug("winutils is at {}", path);
-    } catch (IOException e) {
-      errorText.append("No " + WINUTILS);
-      log.warn("No winutils: {}", e, e);
-    }
-    try {
-      File target = new File("target");
-      FileUtil.canRead(target);
-    } catch (UnsatisfiedLinkError e) {
-      log.warn("Failing to link to native IO methods: {}", e, e);
-      errorText.append("No native IO methods");
-    }
-    return errorText.toString();
-  }
-
-  /**
-   * Strictly verify that windows utils is present.
-   * Checks go as far as opening the file and looking for
-   * the headers. 
-   * @throws IOException on any problem reading the file
-   * @throws FileNotFoundException if the file is not considered valid
-   */
-  public static void maybeVerifyWinUtilsValid() throws
-      IOException,
-      SliderException {
-    String errorText = SliderUtils.checkForRequiredNativeLibraries();
-    if (!errorText.isEmpty()) {
-      throw new BadClusterStateException(errorText);
-    }
-  }
-
-  /**
-   * Write bytes to a file
-   * @param outfile output file
-   * @param data data to write
-   * @throws IOException on any IO problem
-   */
-  public static void write(File outfile, byte[] data)
-      throws IOException {
-    File parentDir = outfile.getCanonicalFile().getParentFile();
-    if (parentDir == null) {
-      throw new IOException(outfile.getPath() + " has no parent dir");
-    }
-    if (!parentDir.exists()) {
-      if(!parentDir.mkdirs()) {
-        throw new IOException("Failed to create parent directory " + parentDir);
-      }
-    }
-    SliderUtils.verifyIsDir(parentDir, log);
-    try(FileOutputStream out = new FileOutputStream(outfile)) {
-      out.write(data);
-    }
-  }
-
-  /**
-   * Compare the times of two applications: most recent app comes first
-   * Specifically: the one whose start time value is greater.
-   */
-  private static class MostRecentlyStartedAppFirst
-      implements Comparator<ApplicationReport>, Serializable {
-    @Override
-    public int compare(ApplicationReport r1, ApplicationReport r2) {
-      long x = r1.getStartTime();
-      long y = r2.getStartTime();
-      return compareTwoLongsReverse(x, y);
-    }
-  }
-  
-  /**
-   * Compare the times of two applications: most recent app comes first.
-   * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
-   */
-  private static class MostRecentlyStartedOrFinishedFirst
-      implements Comparator<ApplicationReport>, Serializable {
-    @Override
-    public int compare(ApplicationReport r1, ApplicationReport r2) {
-      long started1 = r1.getStartTime();
-      long started2 = r2.getStartTime();
-      long finished1 = r1.getFinishTime();
-      long finished2 = r2.getFinishTime();
-      long lastEvent1 = Math.max(started1, finished1);
-      long lastEvent2 = Math.max(started2, finished2);
-      return compareTwoLongsReverse(lastEvent1, lastEvent2);
-    }
-  }
-
-  /**
-   * Compare the times of two applications: most recently finished app comes first
-   * Specifically: the one whose finish time value is greater.
-   */
-  private static class MostRecentAppFinishFirst
-      implements Comparator<ApplicationReport>, Serializable {
-    @Override
-    public int compare(ApplicationReport r1, ApplicationReport r2) {
-      long x = r1.getFinishTime();
-      long y = r2.getFinishTime();
-      return compareTwoLongsReverse(x, y);
-    }
-  }
-
-  /**
-   * Compare two long values for sorting. As the return value for 
-   * comparators must be int, the simple value of <code>x-y</code>
-   * is inapplicable
-   * @param x x value
-   * @param y y value
-   * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
-   */
-  public static int compareTwoLongsReverse(long x, long y) {
-    return (x < y) ? 1 : ((x == y) ? 0 : -1);
-  }
-
   public static String createNameTag(String name) {
     return "Name: " + name;
   }
@@ -1646,9 +563,4 @@ public final class SliderUtils {
   public static String createDescriptionTag(String description) {
     return "Description: " + description;
   }
-
-  public static final String DAYS = ".days";
-  public static final String HOURS = ".hours";
-  public static final String MINUTES = ".minutes";
-  public static final String SECONDS = ".seconds";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 0f4f598..a2edbc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -18,12 +18,22 @@
 
 package org.apache.hadoop.yarn.service;
 
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Resource;
 import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
 import org.codehaus.jackson.map.PropertyNamingStrategy;
 
+import java.io.IOException;
+
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class ServiceTestUtils {
 
   public static final JsonSerDeser<Service> JSON_SER_DESER =
@@ -56,4 +66,22 @@ public class ServiceTestUtils {
     resource.setCpus(1);
     return comp1;
   }
+
+  public static SliderFileSystem initMockFs() throws IOException {
+    return initMockFs(null);
+  }
+
+  public static SliderFileSystem initMockFs(Service ext) throws IOException {
+    SliderFileSystem sfs = mock(SliderFileSystem.class);
+    FileSystem mockFs = mock(FileSystem.class);
+    JsonSerDeser<Service> jsonSerDeser = mock(JsonSerDeser.class);
+    when(sfs.getFileSystem()).thenReturn(mockFs);
+    when(sfs.buildClusterDirPath(anyObject())).thenReturn(
+        new Path("cluster_dir_path"));
+    if (ext != null) {
+      when(jsonSerDeser.load(anyObject(), anyObject())).thenReturn(ext);
+    }
+    ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+    return sfs;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
index be36335..959e4d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -18,8 +18,6 @@
 package org.apache.hadoop.yarn.service;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.service.api.records.Service;
@@ -27,7 +25,6 @@ import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
 import org.apache.hadoop.yarn.service.api.records.Artifact;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
 import org.junit.Assert;
@@ -44,10 +41,8 @@ import java.util.List;
 import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
 import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
 import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
-import static org.easymock.EasyMock.*;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 
 /**
  * Test for ServiceApiUtil helper methods.
@@ -78,7 +73,7 @@ public class TestServiceApiUtil {
     assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
         .length());
 
-    SliderFileSystem sfs = initMock(null);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs();
 
     Service app = new Service();
 
@@ -230,7 +225,7 @@ public class TestServiceApiUtil {
 
   @Test
   public void testArtifacts() throws IOException {
-    SliderFileSystem sfs = initMock(null);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs();
 
     Service app = new Service();
     app.setName("name");
@@ -309,27 +304,10 @@ public class TestServiceApiUtil {
     return app;
   }
 
-  private static SliderFileSystem initMock(Service ext) throws IOException {
-    SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-    FileSystem mockFs = createNiceMock(FileSystem.class);
-    JsonSerDeser<Service> jsonSerDeser = createNiceMock(JsonSerDeser
-        .class);
-    expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-    expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-        new Path("cluster_dir_path")).anyTimes();
-    if (ext != null) {
-      expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
-          .anyTimes();
-    }
-    replay(sfs, mockFs, jsonSerDeser);
-    ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
-    return sfs;
-  }
-
   @Test
   public void testExternalApplication() throws IOException {
     Service ext = createValidApplication("comp1");
-    SliderFileSystem sfs = initMock(ext);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext);
 
     Service app = createValidApplication(null);
 
@@ -350,7 +328,7 @@ public class TestServiceApiUtil {
 
   @Test
   public void testDuplicateComponents() throws IOException {
-    SliderFileSystem sfs = initMock(null);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs();
 
     String compName = "comp1";
     Service app = createValidApplication(compName);
@@ -368,7 +346,7 @@ public class TestServiceApiUtil {
   @Test
   public void testExternalDuplicateComponent() throws IOException {
     Service ext = createValidApplication("comp1");
-    SliderFileSystem sfs = initMock(ext);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext);
 
     Service app = createValidApplication("comp1");
     Artifact artifact = new Artifact();
@@ -387,7 +365,7 @@ public class TestServiceApiUtil {
   @Test
   public void testExternalComponent() throws IOException {
     Service ext = createValidApplication("comp1");
-    SliderFileSystem sfs = initMock(ext);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext);
 
     Service app = createValidApplication("comp2");
     Artifact artifact = new Artifact();
@@ -454,7 +432,7 @@ public class TestServiceApiUtil {
               e)), ex.getMessage());
     }
 
-    SliderFileSystem sfs = initMock(null);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs();
     Service service = createValidApplication(null);
     service.setComponents(Arrays.asList(c, d, e));
     try {
@@ -470,7 +448,7 @@ public class TestServiceApiUtil {
 
   @Test
   public void testInvalidComponent() throws IOException {
-    SliderFileSystem sfs = initMock(null);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs();
     testComponent(sfs);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 63aa9c6..30f2aeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -19,8 +19,6 @@
 package org.apache.hadoop.yarn.service;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.curator.test.TestingCluster;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -51,6 +49,8 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -78,8 +78,8 @@ import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_B
  */
 public class TestYarnNativeServices extends ServiceTestUtils{
 
-  private static final Log LOG =
-      LogFactory.getLog(TestYarnNativeServices.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestYarnNativeServices.class);
 
   private MiniYARNCluster yarnCluster = null;
   private MiniDFSCluster hdfsCluster = null;
@@ -416,7 +416,7 @@ public class TestYarnNativeServices extends ServiceTestUtils{
         LOG.info("Num Components " + retrievedApp.getComponents().size());
         for (Component component : retrievedApp.getComponents()) {
           LOG.info("looking for  " + component.getName());
-          LOG.info(component);
+          LOG.info(component.toString());
           if (component.getContainers() != null) {
             if (component.getContainers().size() == exampleApp
                 .getComponent(component.getName()).getNumberOfContainers()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index ecc529d..c53ee2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -45,7 +45,6 @@ import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONN
 import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
 import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
 import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.*;
 
 public class TestServiceCLI {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
index 04ec526..8739382 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
@@ -18,13 +18,11 @@
 
 package org.apache.hadoop.yarn.service.conf;
 
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.api.records.ConfigFile;
 import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
 import org.junit.Assert;
@@ -40,7 +38,6 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
-import static org.easymock.EasyMock.*;
 
 /**
  * Test global configuration resolution.
@@ -78,12 +75,7 @@ public class TestAppJsonResolve extends Assert {
     assertEquals("1000", worker.getProperty("timeout"));
 
     // here is the resolution
-    SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-    FileSystem mockFs = createNiceMock(FileSystem.class);
-    expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-    expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-        new Path("cluster_dir_path")).anyTimes();
-    replay(sfs, mockFs);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs();
     ServiceApiUtil.validateAndResolveService(orig, sfs, new
         YarnConfiguration());
 
@@ -162,27 +154,13 @@ public class TestAppJsonResolve extends Assert {
     assertEquals(0, other.getProperties().size());
 
     // load the external service
-    SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-    FileSystem mockFs = createNiceMock(FileSystem.class);
-    expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-    expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-        new Path("cluster_dir_path")).anyTimes();
-    replay(sfs, mockFs);
+    SliderFileSystem sfs = ServiceTestUtils.initMockFs();
     Service ext = ExampleAppJson.loadResource(APP_JSON);
     ServiceApiUtil.validateAndResolveService(ext, sfs, new
         YarnConfiguration());
-    reset(sfs, mockFs);
 
     // perform the resolution on original service
-    JsonSerDeser<Service> jsonSerDeser = createNiceMock(JsonSerDeser
-        .class);
-    expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-    expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-        new Path("cluster_dir_path")).anyTimes();
-    expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
-        .anyTimes();
-    replay(sfs, mockFs, jsonSerDeser);
-    ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+    sfs = ServiceTestUtils.initMockFs(ext);
     ServiceApiUtil.validateAndResolveService(orig, sfs, new
         YarnConfiguration());
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
index 83e9502..a813da3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
@@ -18,9 +18,8 @@
 
 package org.apache.hadoop.yarn.service.conf;
 
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
@@ -33,7 +32,6 @@ import java.util.Arrays;
 import java.util.Collection;
 
 import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.*;
 
 /**
  * Test loading example resources.
@@ -62,12 +60,7 @@ public class TestLoadExampleAppJson extends Assert {
     try {
       Service service = JSON_SER_DESER.fromResource(resource);
 
-      SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-      FileSystem mockFs = createNiceMock(FileSystem.class);
-      expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-      expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-          new Path("cluster_dir_path")).anyTimes();
-      replay(sfs, mockFs);
+      SliderFileSystem sfs = ServiceTestUtils.initMockFs();
 
       ServiceApiUtil.validateAndResolveService(service, sfs,
           new YarnConfiguration());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
index 5b24a1d..79406e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.yarn.service.providers;
 
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.service.api.records.Artifact;
 import org.apache.hadoop.yarn.service.api.records.ConfigFile;
 import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
@@ -29,7 +28,9 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.easymock.EasyMock.*;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Test the AbstractClientProvider shared methods.
@@ -55,9 +56,8 @@ public class TestAbstractClientProvider {
   @Test
   public void testConfigFiles() throws IOException {
     ClientProvider clientProvider = new ClientProvider();
-    FileSystem mockFs = createNiceMock(FileSystem.class);
-    expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
-    replay(mockFs);
+    FileSystem mockFs = mock(FileSystem.class);
+    when(mockFs.exists(anyObject())).thenReturn(true);
 
     ConfigFile configFile = new ConfigFile();
     List<ConfigFile> configFiles = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a37378a/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 3cbbaa7..1b3c5f0 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -82,6 +82,10 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-router</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-services-core</artifactId>
+    </dependency>
   </dependencies>
 
   <build>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message