hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [26/52] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy unit tests to yarn native services. Contributed by Billie Rinaldi
Date Fri, 21 Jul 2017 18:39:28 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
new file mode 100644
index 0000000..fc51e31
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
@@ -0,0 +1,901 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.internal.AssumptionViolatedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.UUID;
+
+/**
+ * Utilities used across test cases to make assertions about filesystems
+ * -assertions which fail with useful information.
+ * This is lifted from Hadoop common Test; that JAR isn't published, so
+ * we have to make do.
+ */
+public class ContractTestUtils extends Assert {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ContractTestUtils.class);
+
+  public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+  // For scale testing, we can repeatedly write small chunk data to generate
+  // a large file.
+  public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size";
+  public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128;
+  public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size";
+  public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128;
+
+  /**
+   * Assert that a property in the property set matches the expected value
+   * @param props property set
+   * @param key property name
+   * @param expected expected value. If null, the property must not be in the set
+   */
+  public static void assertPropertyEquals(Properties props,
+                                          String key,
+                                          String expected) {
+    String val = props.getProperty(key);
+    if (expected == null) {
+      assertNull("Non null property " + key + " = " + val, val);
+    } else {
+      assertEquals("property " + key + " = " + val,
+                          expected,
+                          val);
+    }
+  }
+
+  /**
+   *
+   * Write a file and read it in, validating the result. Optional flags control
+   * whether file overwrite operations should be enabled, and whether the
+   * file should be deleted afterwards.
+   *
+   * If there is a mismatch between what was written and what was expected,
+   * a small range of bytes either side of the first error are logged to aid
+   * diagnosing what problem occurred -whether it was a previous file
+   * or a corrupting of the current file. This assumes that two
+   * sequential runs to the same path use datasets with different character
+   * moduli.
+   *
+   * @param fs filesystem
+   * @param path path to write to
+   * @param len length of data
+   * @param overwrite should the create option allow overwrites?
+   * @param delete should the file be deleted afterwards? -with a verification
+   * that it worked. Deletion is not attempted if an assertion has failed
+   * earlier -it is not in a <code>finally{}</code> block.
+   * @throws IOException IO problems
+   */
+  public static void writeAndRead(FileSystem fs,
+                                  Path path,
+                                  byte[] src,
+                                  int len,
+                                  int blocksize,
+                                  boolean overwrite,
+                                  boolean delete) throws IOException {
+    fs.mkdirs(path.getParent());
+
+    writeDataset(fs, path, src, len, blocksize, overwrite);
+
+    byte[] dest = readDataset(fs, path, len);
+
+    compareByteArrays(src, dest, len);
+
+    if (delete) {
+      rejectRootOperation(path);
+      boolean deleted = fs.delete(path, false);
+      assertTrue("Deleted", deleted);
+      assertPathDoesNotExist(fs, "Cleanup failed", path);
+    }
+  }
+
+  /**
+   * Write a file.
+   * Optional flags control
+   * whether file overwrite operations should be enabled
+   * @param fs filesystem
+   * @param path path to write to
+   * @param len length of data
+   * @param overwrite should the create option allow overwrites?
+   * @throws IOException IO problems
+   */
+  public static void writeDataset(FileSystem fs,
+                                   Path path,
+                                   byte[] src,
+                                   int len,
+                                   int buffersize,
+                                   boolean overwrite) throws IOException {
+    assertTrue(
+      "Not enough data in source array to write " + len + " bytes",
+      src.length >= len);
+    FSDataOutputStream out = fs.create(path,
+                                       overwrite,
+                                       fs.getConf()
+                                         .getInt(IO_FILE_BUFFER_SIZE,
+                                                 4096),
+                                       (short) 1,
+                                       buffersize);
+    out.write(src, 0, len);
+    out.close();
+    assertFileHasLength(fs, path, len);
+  }
+
+  /**
+   * Read the file and convert to a byte dataset.
+   * This implements readfully internally, so that it will read
+   * in the file without ever having to seek()
+   * @param fs filesystem
+   * @param path path to read from
+   * @param len length of data to read
+   * @return the bytes
+   * @throws IOException IO problems
+   */
+  public static byte[] readDataset(FileSystem fs, Path path, int len)
+      throws IOException {
+    FSDataInputStream in = fs.open(path);
+    byte[] dest = new byte[len];
+    int offset =0;
+    int nread = 0;
+    try {
+      while (nread < len) {
+        int nbytes = in.read(dest, offset + nread, len - nread);
+        if (nbytes < 0) {
+          throw new EOFException("End of file reached before reading fully.");
+        }
+        nread += nbytes;
+      }
+    } finally {
+      in.close();
+    }
+    return dest;
+  }
+
+  /**
+   * Read a file, verify its length and contents match the expected array
+   * @param fs filesystem
+   * @param path path to file
+   * @param original original dataset
+   * @throws IOException IO Problems
+   */
+  public static void verifyFileContents(FileSystem fs,
+                                        Path path,
+                                        byte[] original) throws IOException {
+    FileStatus stat = fs.getFileStatus(path);
+    String statText = stat.toString();
+    assertTrue("not a file " + statText, stat.isFile());
+    assertEquals("wrong length " + statText, original.length, stat.getLen());
+    byte[] bytes = readDataset(fs, path, original.length);
+    compareByteArrays(original,bytes,original.length);
+  }
+
+  /**
+   * Verify that the read at a specific offset in a stream
+   * matches that expected
+   * @param stm stream
+   * @param fileContents original file contents
+   * @param seekOff seek offset
+   * @param toRead number of bytes to read
+   * @throws IOException IO problems
+   */
+  public static void verifyRead(FSDataInputStream stm, byte[] fileContents,
+                                int seekOff, int toRead) throws IOException {
+    byte[] out = new byte[toRead];
+    stm.seek(seekOff);
+    stm.readFully(out);
+    byte[] expected = Arrays.copyOfRange(fileContents, seekOff,
+                                         seekOff + toRead);
+    compareByteArrays(expected, out,toRead);
+  }
+
+  /**
+   * Assert that tthe array original[0..len] and received[] are equal.
+   * A failure triggers the logging of the bytes near where the first
+   * difference surfaces.
+   * @param original source data
+   * @param received actual
+   * @param len length of bytes to compare
+   */
+  public static void compareByteArrays(byte[] original,
+                                       byte[] received,
+                                       int len) {
+    assertEquals("Number of bytes read != number written",
+                        len, received.length);
+    int errors = 0;
+    int first_error_byte = -1;
+    for (int i = 0; i < len; i++) {
+      if (original[i] != received[i]) {
+        if (errors == 0) {
+          first_error_byte = i;
+        }
+        errors++;
+      }
+    }
+
+    if (errors > 0) {
+      String message = String.format(" %d errors in file of length %d",
+                                     errors, len);
+      LOG.warn(message);
+      // the range either side of the first error to print
+      // this is a purely arbitrary number, to aid user debugging
+      final int overlap = 10;
+      for (int i = Math.max(0, first_error_byte - overlap);
+           i < Math.min(first_error_byte + overlap, len);
+           i++) {
+        byte actual = received[i];
+        byte expected = original[i];
+        String letter = toChar(actual);
+        String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+        if (expected != actual) {
+          line = String.format("[%04d] %2x %s -expected %2x %s\n",
+                               i,
+                               actual,
+                               letter,
+                               expected,
+                               toChar(expected));
+        }
+        LOG.warn(line);
+      }
+      fail(message);
+    }
+  }
+
+  /**
+   * Convert a byte to a character for printing. If the
+   * byte value is < 32 -and hence unprintable- the byte is
+   * returned as a two digit hex value
+   * @param b byte
+   * @return the printable character string
+   */
+  public static String toChar(byte b) {
+    if (b >= 0x20) {
+      return Character.toString((char) b);
+    } else {
+      return String.format("%02x", b);
+    }
+  }
+
+  /**
+   * Convert a buffer to a string, character by character
+   * @param buffer input bytes
+   * @return a string conversion
+   */
+  public static String toChar(byte[] buffer) {
+    StringBuilder builder = new StringBuilder(buffer.length);
+    for (byte b : buffer) {
+      builder.append(toChar(b));
+    }
+    return builder.toString();
+  }
+
+  public static byte[] toAsciiByteArray(String s) {
+    char[] chars = s.toCharArray();
+    int len = chars.length;
+    byte[] buffer = new byte[len];
+    for (int i = 0; i < len; i++) {
+      buffer[i] = (byte) (chars[i] & 0xff);
+    }
+    return buffer;
+  }
+
+  /**
+   * Cleanup at the end of a test run
+   * @param action action triggering the operation (for use in logging)
+   * @param fileSystem filesystem to work with. May be null
+   * @param cleanupPath path to delete as a string
+   */
+  public static void cleanup(String action,
+                             FileSystem fileSystem,
+                             String cleanupPath) {
+    if (fileSystem == null) {
+      return;
+    }
+    Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(),
+        fileSystem.getWorkingDirectory());
+    cleanup(action, fileSystem, path);
+  }
+
+  /**
+   * Cleanup at the end of a test run
+   * @param action action triggering the operation (for use in logging)
+   * @param fileSystem filesystem to work with. May be null
+   * @param path path to delete
+   */
+  public static void cleanup(String action, FileSystem fileSystem, Path path) {
+    noteAction(action);
+    try {
+      rm(fileSystem, path, true, false);
+    } catch (Exception e) {
+      LOG.error("Error deleting in "+ action + " - "  + path + ": " + e, e);
+    }
+  }
+
+  /**
+   * Delete a directory. There's a safety check for operations against the
+   * root directory -these are intercepted and rejected with an IOException
+   * unless the allowRootDelete flag is true
+   * @param fileSystem filesystem to work with. May be null
+   * @param path path to delete
+   * @param recursive flag to enable recursive delete
+   * @param allowRootDelete can the root directory be deleted?
+   * @throws IOException on any problem.
+   */
+  public static boolean rm(FileSystem fileSystem,
+      Path path,
+      boolean recursive,
+      boolean allowRootDelete) throws
+      IOException {
+    if (fileSystem != null) {
+      rejectRootOperation(path, allowRootDelete);
+      if (fileSystem.exists(path)) {
+        return fileSystem.delete(path, recursive);
+      }
+    }
+    return false;
+
+  }
+
+  /**
+   * Block any operation on the root path. This is a safety check
+   * @param path path in the filesystem
+   * @param allowRootOperation can the root directory be manipulated?
+   * @throws IOException if the operation was rejected
+   */
+  public static void rejectRootOperation(Path path,
+      boolean allowRootOperation) throws IOException {
+    if (path.isRoot() && !allowRootOperation) {
+      throw new IOException("Root directory operation rejected: " + path);
+    }
+  }
+
+  /**
+   * Block any operation on the root path. This is a safety check
+   * @param path path in the filesystem
+   * @throws IOException if the operation was rejected
+   */
+  public static void rejectRootOperation(Path path) throws IOException {
+    rejectRootOperation(path, false);
+  }
+
+
+  public static void noteAction(String action) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("==============  "+ action +" =============");
+    }
+  }
+
+  /**
+   * downgrade a failure to a message and a warning, then an
+   * exception for the Junit test runner to mark as failed
+   * @param message text message
+   * @param failure what failed
+   * @throws AssumptionViolatedException always
+   */
+  public static void downgrade(String message, Throwable failure) {
+    LOG.warn("Downgrading test " + message, failure);
+    AssumptionViolatedException ave =
+      new AssumptionViolatedException(failure, null);
+    throw ave;
+  }
+
+  /**
+   * report an overridden test as unsupported
+   * @param message message to use in the text
+   * @throws AssumptionViolatedException always
+   */
+  public static void unsupported(String message) {
+    skip(message);
+  }
+
+  /**
+   * report a test has been skipped for some reason
+   * @param message message to use in the text
+   * @throws AssumptionViolatedException always
+   */
+  public static void skip(String message) {
+    LOG.info("Skipping: {}", message);
+    throw new AssumptionViolatedException(message);
+  }
+
+  /**
+   * Fail with an exception that was received
+   * @param text text to use in the exception
+   * @param thrown a (possibly null) throwable to init the cause with
+   * @throws AssertionError with the text and throwable -always
+   */
+  public static void fail(String text, Throwable thrown) {
+    AssertionError e = new AssertionError(text);
+    e.initCause(thrown);
+    throw e;
+  }
+
+  /**
+   * Make an assertion about the length of a file
+   * @param fs filesystem
+   * @param path path of the file
+   * @param expected expected length
+   * @throws IOException on File IO problems
+   */
+  public static void assertFileHasLength(FileSystem fs, Path path,
+                                         int expected) throws IOException {
+    FileStatus status = fs.getFileStatus(path);
+    assertEquals(
+      "Wrong file length of file " + path + " status: " + status,
+      expected,
+      status.getLen());
+  }
+
+  /**
+   * Assert that a path refers to a directory
+   * @param fs filesystem
+   * @param path path of the directory
+   * @throws IOException on File IO problems
+   */
+  public static void assertIsDirectory(FileSystem fs,
+                                       Path path) throws IOException {
+    FileStatus fileStatus = fs.getFileStatus(path);
+    assertIsDirectory(fileStatus);
+  }
+
+  /**
+   * Assert that a path refers to a directory
+   * @param fileStatus stats to check
+   */
+  public static void assertIsDirectory(FileStatus fileStatus) {
+    assertTrue("Should be a directory -but isn't: " + fileStatus,
+               fileStatus.isDirectory());
+  }
+
+  /**
+   * Write the text to a file, returning the converted byte array
+   * for use in validating the round trip
+   * @param fs filesystem
+   * @param path path of file
+   * @param text text to write
+   * @param overwrite should the operation overwrite any existing file?
+   * @return the read bytes
+   * @throws IOException on IO problems
+   */
+  public static byte[] writeTextFile(FileSystem fs,
+                                   Path path,
+                                   String text,
+                                   boolean overwrite) throws IOException {
+    byte[] bytes = new byte[0];
+    if (text != null) {
+      bytes = toAsciiByteArray(text);
+    }
+    createFile(fs, path, overwrite, bytes);
+    return bytes;
+  }
+
+  /**
+   * Create a file
+   * @param fs filesystem
+   * @param path       path to write
+   * @param overwrite overwrite flag
+   * @param data source dataset. Can be null
+   * @throws IOException on any problem
+   */
+  public static void createFile(FileSystem fs,
+                                 Path path,
+                                 boolean overwrite,
+                                 byte[] data) throws IOException {
+    FSDataOutputStream stream = fs.create(path, overwrite);
+    if (data != null && data.length > 0) {
+      stream.write(data);
+    }
+    stream.close();
+  }
+
+  /**
+   * Touch a file
+   * @param fs filesystem
+   * @param path path
+   * @throws IOException IO problems
+   */
+  public static void touch(FileSystem fs,
+                           Path path) throws IOException {
+    createFile(fs, path, true, null);
+  }
+
+  /**
+   * Delete a file/dir and assert that delete() returned true
+   * <i>and</i> that the path no longer exists. This variant rejects
+   * all operations on root directories
+   * @param fs filesystem
+   * @param file path to delete
+   * @param recursive flag to enable recursive delete
+   * @throws IOException IO problems
+   */
+  public static void assertDeleted(FileSystem fs,
+                                   Path file,
+                                   boolean recursive) throws IOException {
+    assertDeleted(fs, file, recursive, false);
+  }
+
+  /**
+   * Delete a file/dir and assert that delete() returned true
+   * <i>and</i> that the path no longer exists. This variant rejects
+   * all operations on root directories
+   * @param fs filesystem
+   * @param file path to delete
+   * @param recursive flag to enable recursive delete
+   * @param allowRootOperations can the root dir be deleted?
+   * @throws IOException IO problems
+   */
+  public static void assertDeleted(FileSystem fs,
+      Path file,
+      boolean recursive,
+      boolean allowRootOperations) throws IOException {
+    rejectRootOperation(file, allowRootOperations);
+    assertPathExists(fs, "about to be deleted file", file);
+    boolean deleted = fs.delete(file, recursive);
+    String dir = ls(fs, file.getParent());
+    assertTrue("Delete failed on " + file + ": " + dir, deleted);
+    assertPathDoesNotExist(fs, "Deleted file", file);
+  }
+
+  /**
+   * Read in "length" bytes, convert to an ascii string
+   * @param fs filesystem
+   * @param path path to read
+   * @param length #of bytes to read.
+   * @return the bytes read and converted to a string
+   * @throws IOException IO problems
+   */
+  public static String readBytesToString(FileSystem fs,
+                                  Path path,
+                                  int length) throws IOException {
+    FSDataInputStream in = fs.open(path);
+    try {
+      byte[] buf = new byte[length];
+      in.readFully(0, buf);
+      return toChar(buf);
+    } finally {
+      in.close();
+    }
+  }
+
+  /**
+   * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
+   * @param stats array of stats
+   * @param separator separator after every entry
+   * @return a stringified set
+   */
+  public static String fileStatsToString(FileStatus[] stats, String separator) {
+    StringBuilder buf = new StringBuilder(stats.length * 128);
+    for (int i = 0; i < stats.length; i++) {
+      buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+    }
+    return buf.toString();
+  }
+
+  /**
+   * List a directory
+   * @param fileSystem FS
+   * @param path path
+   * @return a directory listing or failure message
+   * @throws IOException
+   */
+  public static String ls(FileSystem fileSystem, Path path) throws IOException {
+    if (path == null) {
+      //surfaces when someone calls getParent() on something at the top of the path
+      return "/";
+    }
+    FileStatus[] stats;
+    String pathtext = "ls " + path;
+    try {
+      stats = fileSystem.listStatus(path);
+    } catch (FileNotFoundException e) {
+      return pathtext + " -file not found";
+    } catch (IOException e) {
+      return pathtext + " -failed: " + e;
+    }
+    return dumpStats(pathtext, stats);
+  }
+
+  public static String dumpStats(String pathname, FileStatus[] stats) {
+    return pathname + fileStatsToString(stats, "\n");
+  }
+
+   /**
+   * Assert that a file exists and whose {@link FileStatus} entry
+   * declares that this is a file and not a symlink or directory.
+   * @param fileSystem filesystem to resolve path against
+   * @param filename name of the file
+   * @throws IOException IO problems during file operations
+   */
+  public static void assertIsFile(FileSystem fileSystem, Path filename) throws
+                                                                 IOException {
+    assertPathExists(fileSystem, "Expected file", filename);
+    FileStatus status = fileSystem.getFileStatus(filename);
+    assertIsFile(filename, status);
+  }
+
+  /**
+   * Assert that a file exists and whose {@link FileStatus} entry
+   * declares that this is a file and not a symlink or directory.
+   * @param filename name of the file
+   * @param status file status
+   */
+  public static void assertIsFile(Path filename, FileStatus status) {
+    String fileInfo = filename + "  " + status;
+    assertFalse("File claims to be a directory " + fileInfo,
+                status.isDirectory());
+    assertFalse("File claims to be a symlink " + fileInfo,
+                       status.isSymlink());
+  }
+
+  /**
+   * Create a dataset for use in the tests; all data is in the range
+   * base to (base+modulo-1) inclusive
+   * @param len length of data
+   * @param base base of the data
+   * @param modulo the modulo
+   * @return the newly generated dataset
+   */
+  public static byte[] dataset(int len, int base, int modulo) {
+    byte[] dataset = new byte[len];
+    for (int i = 0; i < len; i++) {
+      dataset[i] = (byte) (base + (i % modulo));
+    }
+    return dataset;
+  }
+
+  /**
+   * Assert that a path exists -but make no assertions as to the
+   * type of that entry
+   *
+   * @param fileSystem filesystem to examine
+   * @param message message to include in the assertion failure message
+   * @param path path in the filesystem
+   * @throws FileNotFoundException raised if the path is missing
+   * @throws IOException IO problems
+   */
+  public static void assertPathExists(FileSystem fileSystem, String message,
+                               Path path) throws IOException {
+    if (!fileSystem.exists(path)) {
+      //failure, report it
+      String listing = ls(fileSystem, path.getParent());
+      throw new FileNotFoundException(message + ": not found " + path
+        + " in \"" + path.getParent() + "\" :\n" + listing);
+    }
+  }
+
+  /**
+   * Assert that a path does not exist
+   *
+   * @param fileSystem filesystem to examine
+   * @param message message to include in the assertion failure message
+   * @param path path in the filesystem
+   * @throws IOException IO problems
+   */
+  public static void assertPathDoesNotExist(FileSystem fileSystem,
+                                            String message,
+                                            Path path) throws IOException {
+    try {
+      FileStatus status = fileSystem.getFileStatus(path);
+      fail(message + ": unexpectedly found " + path + " as  " + status);
+    } catch (FileNotFoundException expected) {
+      //this is expected
+
+    }
+  }
+
+  /**
+   * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
+   * @param fs filesystem
+   * @param dir directory to scan
+   * @param subdir full path to look for
+   * @throws IOException IO probles
+   */
+  public static void assertListStatusFinds(FileSystem fs,
+                                           Path dir,
+                                           Path subdir) throws IOException {
+    FileStatus[] stats = fs.listStatus(dir);
+    boolean found = false;
+    StringBuilder builder = new StringBuilder();
+    for (FileStatus stat : stats) {
+      builder.append(stat.toString()).append('\n');
+      if (stat.getPath().equals(subdir)) {
+        found = true;
+      }
+    }
+    assertTrue("Path " + subdir
+                      + " not found in directory " + dir + ":" + builder,
+                      found);
+  }
+
+  /**
+   * Test for the host being an OSX machine
+   * @return true if the JVM thinks that is running on OSX
+   */
+  public static boolean isOSX() {
+    return System.getProperty("os.name").contains("OS X");
+  }
+
+  /**
+   * compare content of file operations using a double byte array
+   * @param concat concatenated files
+   * @param bytes bytes
+   */
+  public static void validateFileContent(byte[] concat, byte[][] bytes) {
+    int idx = 0;
+    boolean mismatch = false;
+
+    for (byte[] bb : bytes) {
+      for (byte b : bb) {
+        if (b != concat[idx++]) {
+          mismatch = true;
+          break;
+        }
+      }
+      if (mismatch)
+        break;
+    }
+    assertFalse("File content of file is not as expected at offset " + idx,
+                mismatch);
+  }
+
+  /**
+   * Receives test data from the given input file and checks the size of the
+   * data as well as the pattern inside the received data.
+   *
+   * @param fs FileSystem
+   * @param path Input file to be checked
+   * @param expectedSize the expected size of the data to be read from the
+   *        input file in bytes
+   * @param bufferLen Pattern length
+   * @param modulus   Pattern modulus
+   * @throws IOException
+   *         thrown if an error occurs while reading the data
+   */
+  public static void verifyReceivedData(FileSystem fs, Path path,
+                                      final long expectedSize,
+                                      final int bufferLen,
+                                      final int modulus) throws IOException {
+    final byte[] testBuffer = new byte[bufferLen];
+
+    long totalBytesRead = 0;
+    int nextExpectedNumber = 0;
+    final InputStream inputStream = fs.open(path);
+    try {
+      while (true) {
+        final int bytesRead = inputStream.read(testBuffer);
+        if (bytesRead < 0) {
+          break;
+        }
+
+        totalBytesRead += bytesRead;
+
+        for (int i = 0; i < bytesRead; ++i) {
+          if (testBuffer[i] != nextExpectedNumber) {
+            throw new IOException("Read number " + testBuffer[i]
+                + " but expected " + nextExpectedNumber);
+          }
+
+          ++nextExpectedNumber;
+
+          if (nextExpectedNumber == modulus) {
+            nextExpectedNumber = 0;
+          }
+        }
+      }
+
+      if (totalBytesRead != expectedSize) {
+        throw new IOException("Expected to read " + expectedSize +
+            " bytes but only received " + totalBytesRead);
+      }
+    } finally {
+      inputStream.close();
+    }
+  }
+
+  /**
+   * Generates test data of the given size according to some specific pattern
+   * and writes it to the provided output file.
+   *
+   * @param fs FileSystem
+   * @param path Test file to be generated
+   * @param size The size of the test data to be generated in bytes
+   * @param bufferLen Pattern length
+   * @param modulus   Pattern modulus
+   * @throws IOException
+   *         thrown if an error occurs while writing the data
+   */
+  public static long generateTestFile(FileSystem fs, Path path,
+                                      final long size,
+                                      final int bufferLen,
+                                      final int modulus) throws IOException {
+    final byte[] testBuffer = new byte[bufferLen];
+    for (int i = 0; i < testBuffer.length; ++i) {
+      testBuffer[i] = (byte) (i % modulus);
+    }
+
+    final OutputStream outputStream = fs.create(path, false);
+    long bytesWritten = 0;
+    try {
+      while (bytesWritten < size) {
+        final long diff = size - bytesWritten;
+        if (diff < testBuffer.length) {
+          outputStream.write(testBuffer, 0, (int) diff);
+          bytesWritten += diff;
+        } else {
+          outputStream.write(testBuffer);
+          bytesWritten += testBuffer.length;
+        }
+      }
+
+      return bytesWritten;
+    } finally {
+      outputStream.close();
+    }
+  }
+
+  /**
+   * Creates and reads a file with the given size. The test file is generated
+   * according to a specific pattern so it can be easily verified even if it's
+   * a multi-GB one.
+   * During the read phase the incoming data stream is also checked against
+   * this pattern.
+   *
+   * @param fs FileSystem
+   * @param parent Test file parent dir path
+   * @throws IOException
+   *    thrown if an I/O error occurs while writing or reading the test file
+   */
+  public static void createAndVerifyFile(FileSystem fs, Path parent, final long fileSize)
+      throws IOException {
+    int testBufferSize = fs.getConf()
+        .getInt(IO_CHUNK_BUFFER_SIZE, DEFAULT_IO_CHUNK_BUFFER_SIZE);
+    int modulus = fs.getConf()
+        .getInt(IO_CHUNK_MODULUS_SIZE, DEFAULT_IO_CHUNK_MODULUS_SIZE);
+
+    final String objectName = UUID.randomUUID().toString();
+    final Path objectPath = new Path(parent, objectName);
+
+    // Write test file in a specific pattern
+    assertEquals(fileSize,
+        generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus));
+    assertPathExists(fs, "not created successful", objectPath);
+
+    // Now read the same file back and verify its content
+    try {
+      verifyReceivedData(fs, objectPath, fileSize, testBufferSize, modulus);
+    } finally {
+      // Delete test file
+      fs.delete(objectPath, false);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
new file mode 100644
index 0000000..cf96407
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXMLConfKeysForTesting;
+
+/**
+ * Keys shared across tests.
+ */
+public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting {
+  /**
+   * Username for all clusters, ZK, etc.
+   */
+  String USERNAME = "bigdataborat";
+
+  int WAIT_TIME = 120;
+  String WAIT_TIME_ARG =  Integer.toString(WAIT_TIME);
+
+  String SLIDER_TEST_XML = "slider-test.xml";
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
new file mode 100644
index 0000000..be452f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService;
+import org.apache.hadoop.registry.server.services.MicroZookeeperService;
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Test ZK cluster.
+ */
+public class MicroZKCluster implements Closeable {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MicroZKCluster.class);
+
+  public static final String HOSTS = "127.0.0.1";
+  private MicroZookeeperService zkService;
+  private String zkBindingString;
+  private final Configuration conf;
+  private RegistryOperations registryOperations;
+
+  MicroZKCluster() {
+    this(SliderUtils.createConfiguration());
+  }
+
+  MicroZKCluster(Configuration conf) {
+    this.conf = conf;
+  }
+
+  String getZkBindingString() {
+    return zkBindingString;
+  }
+
+  void createCluster(String name) {
+    zkService = new MicroZookeeperService(name);
+
+    zkService.init(conf);
+    zkService.start();
+    zkBindingString = zkService.getConnectionString();
+    LOG.info("Created {}", this);
+    registryOperations = new RegistryOperationsService(
+        "registry",
+        zkService);
+    registryOperations.init(conf);
+    registryOperations.start();
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (registryOperations != null) {
+      registryOperations.stop();
+    }
+    if (zkService != null) {
+      zkService.stop();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "Micro ZK cluster as " + zkBindingString;
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
new file mode 100644
index 0000000..52875d3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+/**
+ * Outcome for probes.
+ */
+public final class Outcome {
+
+  private final String name;
+
+  private Outcome(String name) {
+    this.name = name;
+  }
+
+  public static final Outcome SUCCESS = new Outcome(
+      "Success");
+  public static final Outcome RETRY = new Outcome("Retry");
+  public static final Outcome FAIL = new Outcome("Fail");
+
+  /**
+   * Build from a bool, where false is mapped to retry.
+   * @param b boolean
+   * @return an outcome
+   */
+  static Outcome fromBool(boolean b) {
+    return b ? SUCCESS : RETRY;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
new file mode 100644
index 0000000..f7da585
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.slider.common.SliderXMLConfKeysForTesting;
+import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import java.io.File;
+
+
+/**
+ * Base class for unit tests as well as ones starting mini clusters
+ * -the foundational code and methods.
+ *
+ */
+public abstract class SliderTestBase extends SliderTestUtils {
+
+  /**
+   * Singleton metric registry.
+   */
+  public static final MetricsAndMonitoring METRICS = new MetricsAndMonitoring();
+  public static final int WEB_STARTUP_TIME = 30000;
+
+  @Rule
+  public TestName methodName = new TestName();
+
+  @BeforeClass
+  public static void nameThread() {
+    Thread.currentThread().setName("JUnit");
+  }
+
+  @Before
+  public void setup() throws Exception {
+    setSliderClientClassName(DEFAULT_SLIDER_CLIENT);
+    FileUtil.fullyDelete(new File(SliderXMLConfKeysForTesting
+        .TEST_SECURITY_DIR));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
new file mode 100644
index 0000000..fc29b5e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
@@ -0,0 +1,1065 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParser;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Container;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.core.main.ServiceLaunchException;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.server.services.workflow.ForkedProcessService;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.slider.common.params.Arguments.ARG_OPTION;
+
+/**
+ * Static utils for tests in this package and in other test projects.
+ *
+ * It is designed to work with mini clusters as well as remote ones
+ *
+ * This class is not final and may be extended for test cases.
+ *
+ * Some of these methods are derived from the SwiftUtils and SwiftTestUtils
+ * classes -replicated here so that they are available in Hadoop-2.0 code
+ */
+public class SliderTestUtils extends Assert {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SliderTestUtils.class);
+  public static final String DEFAULT_SLIDER_CLIENT = SliderClient.class
+      .getName();
+  private static String sliderClientClassName = DEFAULT_SLIDER_CLIENT;
+
+  public static final Map<String, String> EMPTY_MAP = Collections.emptyMap();
+  public static final Map<String, Integer> EMPTY_INT_MAP = Collections
+      .emptyMap();
+  public static final List<String> EMPTY_LIST = Collections.emptyList();
+
+  public static final ObjectReader OBJECT_READER;
+  public static final ObjectWriter OBJECT_WRITER;
+
+  public static final JsonSerDeser<Application> JSON_SER_DESER =
+      new JsonSerDeser<>(Application.class,
+          PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+
+  static {
+    ObjectMapper mapper = new ObjectMapper();
+    OBJECT_READER = mapper.readerFor(Object.class);
+    OBJECT_WRITER = mapper.writer();
+  }
+
+  /**
+   * Action that returns an object.
+   */
+  public interface Action {
+    Object invoke() throws Exception;
+  }
+
+  /**
+   * Probe that returns an Outcome.
+   */
+  public interface Probe {
+    Outcome invoke(Map args) throws Exception;
+  }
+
+  public static void setSliderClientClassName(String sliderClientClassName) {
+    sliderClientClassName = sliderClientClassName;
+  }
+
+  public static void describe(String s) {
+    LOG.info("");
+    LOG.info("===============================");
+    LOG.info(s);
+    LOG.info("===============================");
+    LOG.info("");
+  }
+
+  /**
+   * Convert a JSON string to something readable.
+   * @param json
+   * @return a string for printing
+   */
+  public static String prettyPrintJson(String json) {
+    Gson gson = new GsonBuilder().setPrettyPrinting().create();
+    return gson.toJson(new JsonParser().parse(json));
+  }
+
+  /**
+   * Convert an object to something readable.
+   * @param src
+   * @return a string for printing
+   */
+  public static String prettyPrintAsJson(Object src)
+      throws JsonProcessingException, UnsupportedEncodingException {
+    return new String(OBJECT_WRITER.writeValueAsBytes(src), "UTF8");
+  }
+
+  /**
+   * Skip the test with a message.
+   * @param message message logged and thrown
+   */
+  public static void skip(String message) {
+    LOG.warn("Skipping test: {}", message);
+    Assume.assumeTrue(message, false);
+  }
+
+  /**
+   * Skip the test with a message if condition holds.
+   * @param condition predicate
+   * @param message message logged and thrown
+   */
+  public static void assume(boolean condition, String message) {
+    if (!condition) {
+      skip(message);
+    }
+  }
+
+  /**
+   * Skip a test if not running on Windows.
+   */
+  public static void assumeWindows() {
+    assume(Shell.WINDOWS, "not windows");
+  }
+
+  /**
+   * Skip a test if running on Windows.
+   */
+  public static void assumeNotWindows() {
+    assume(!Shell.WINDOWS, "windows");
+  }
+
+  /**
+   * Skip a test on windows.
+   */
+  public static void skipOnWindows() {
+    assumeNotWindows();
+  }
+
+  /**
+   * Equality size for a list.
+   * @param left
+   * @param right
+   */
+  public static void assertListEquals(List left, List right) {
+    String lval = collectionToString(left);
+    String rval = collectionToString(right);
+    String text = "comparing " + lval + " to " + rval;
+    assertEquals(text, left.size(), right.size());
+    for (int i = 0; i < left.size(); i++) {
+      assertEquals(text, left.get(i), right.get(i));
+    }
+  }
+
+  /**
+   * Assert a list has a given length.
+   * @param list list
+   * @param size size to have
+   */
+  public static void assertListLength(List list, int size) {
+    String lval = collectionToString(list);
+    assertEquals(lval, size, list.size());
+  }
+
+  /**
+   * Stringify a collection with [ ] at either end.
+   * @param collection collection
+   * @return string value
+   */
+  public static String collectionToString(List collection) {
+    return "[" + SliderUtils.join(collection, ", ", false) + "]";
+  }
+
+  /**
+   * Assume that a string option is set and not equal to "".
+   * @param conf configuration file
+   * @param key key to look for
+   */
+  public static void assumeStringOptionSet(Configuration conf, String key) {
+    if (SliderUtils.isUnset(conf.getTrimmed(key))) {
+      skip("Configuration key " + key + " not set");
+    }
+  }
+
+  /**
+   * assert that a string option is set and not equal to "".
+   * @param conf configuration file
+   * @param key key to look for
+   */
+  public static void assertStringOptionSet(Configuration conf, String key) {
+    getRequiredConfOption(conf, key);
+  }
+
+  /**
+   * Assume that a boolean option is set and true.
+   * Unset or false triggers a test skip
+   * @param conf configuration file
+   * @param key key to look for
+   */
+  public static void assumeBoolOptionTrue(Configuration conf, String key) {
+    assumeBoolOption(conf, key, false);
+  }
+
+  /**
+   * Assume that a boolean option is true.
+   * False triggers a test skip
+   * @param conf configuration file
+   * @param key key to look for
+   * @param defval default value if the property is not defined
+   */
+  public static void assumeBoolOption(
+      Configuration conf, String key, boolean defval) {
+    assume(conf.getBoolean(key, defval),
+        "Configuration key " + key + " is false");
+  }
+
+  /**
+   * Get a required config option (trimmed, incidentally).
+   * Test will fail if not set
+   * @param conf configuration
+   * @param key key
+   * @return the string
+   */
+  public static String getRequiredConfOption(Configuration conf, String key) {
+    String val = conf.getTrimmed(key);
+    if (SliderUtils.isUnset(val)) {
+      fail("Missing configuration option " + key);
+    }
+    return val;
+  }
+
+  /**
+   * Fails a test because required behavior has not been implemented.
+   */
+  public static void failNotImplemented() {
+    fail("Not implemented");
+  }
+
+  /**
+   * Assert that any needed libraries being present. On Unix none are needed;
+   * on windows they must be present
+   */
+  public static void assertNativeLibrariesPresent() {
+    String errorText = SliderUtils.checkForRequiredNativeLibraries();
+    if (SliderUtils.isSet(errorText)) {
+      fail(errorText);
+    }
+  }
+
+  protected static String[] toArray(List<Object> args) {
+    String[] converted = new String[args.size()];
+    for (int i = 0; i < args.size(); i++) {
+      Object elt = args.get(i);
+      assertNotNull(args.get(i));
+      converted[i] = elt.toString();
+    }
+    return converted;
+  }
+
+  public static void waitWhileClusterLive(SliderClient client, int timeout)
+      throws IOException, YarnException {
+    Duration duration = new Duration(timeout);
+    duration.start();
+    while (client.actionExists(client.getDeployedClusterName(), true) ==
+        LauncherExitCodes.EXIT_SUCCESS && !duration.getLimitExceeded()) {
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+      }
+    }
+    if (duration.getLimitExceeded()) {
+      fail("Cluster " + client.getDeployedClusterName() + " still live after " +
+          timeout + " ms");
+    }
+  }
+
+  public static void waitUntilClusterLive(SliderClient client, int timeout)
+      throws IOException, YarnException {
+    Duration duration = new Duration(timeout);
+    duration.start();
+    while (LauncherExitCodes.EXIT_SUCCESS != client.actionExists(
+        client.getDeployedClusterName(), true) &&
+           !duration.getLimitExceeded()) {
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+      }
+    }
+    if (duration.getLimitExceeded()) {
+      fail("Cluster " + client.getDeployedClusterName() + " not live after " +
+          timeout + " ms");
+    }
+  }
+
+  public static void dumpClusterDescription(
+      String text,
+      Application status) throws IOException {
+    describe(text);
+    LOG.info(JSON_SER_DESER.toJson(status));
+  }
+
+  /**
+   * Assert that a service operation succeeded.
+   * @param service service
+   */
+  public static void assertSucceeded(ServiceLauncher service) {
+    assertEquals(0, service.getServiceExitCode());
+  }
+
+  public static void assertContainersLive(Application application,
+      String component, int expected) {
+    LOG.info("Asserting component {} expected count {}", component, expected);
+    int actual = extractLiveContainerCount(application, component);
+    if (expected != actual) {
+      LOG.warn("{} actual={}, expected {} in \n{}\n", component, actual,
+          expected, application);
+    }
+    assertEquals(expected, actual);
+  }
+
+  /**
+   * Robust extraction of live container count.
+   * @param application status
+   * @param component component to resolve
+   * @return the number of containers live.
+   */
+  public static int extractLiveContainerCount(
+      Application application,
+      String component) {
+    int actual = 0;
+    if (application.getContainers() != null) {
+      for (Container container : application.getContainers()) {
+        if (container.getComponentName().equals(component)) {
+          actual++;
+        }
+      }
+    }
+    return actual;
+  }
+
+  /**
+   * Exec a set of commands, wait a few seconds for it to finish.
+   * @param status code
+   * @param commands
+   * @return the process
+   */
+  public static ForkedProcessService exec(int status, List<String> commands)
+      throws IOException, TimeoutException {
+    ForkedProcessService process = exec(commands);
+
+    Integer exitCode = process.getExitCode();
+    assertNotNull(exitCode);
+    assertEquals(status, exitCode.intValue());
+    return process;
+  }
+
+  /**
+   * Exec a set of commands, wait a few seconds for it to finish.
+   * @param commands
+   * @return
+   */
+  public static ForkedProcessService exec(List<String> commands)
+      throws IOException, TimeoutException {
+    ForkedProcessService process;
+    process = new ForkedProcessService(
+        commands.get(0),
+        EMPTY_MAP,
+        commands);
+    process.init(new Configuration());
+    process.start();
+    int timeoutMillis = 5000;
+    if (!process.waitForServiceToStop(timeoutMillis)) {
+      throw new TimeoutException(
+          "Process did not stop in " + timeoutMillis + "mS");
+    }
+    return process;
+  }
+
+  /**
+   * Determine whether an application exists. Run the commands and if the
+   * operation fails with a FileNotFoundException, then
+   * this method returns false.
+   * <p>
+   *   Run something harmless like a -version command, something
+   *   which must return 0
+   *
+   * @param commands
+   * @return true if the command sequence succeeded
+   * false if they failed with no file
+   * @throws Exception on any other failure cause
+   */
+  public static boolean doesAppExist(List<String> commands)
+      throws IOException, TimeoutException {
+    try {
+      exec(0, commands);
+      return true;
+    } catch (ServiceStateException e) {
+      if (!(e.getCause() instanceof FileNotFoundException)) {
+        throw e;
+      }
+      return false;
+    }
+  }
+
+  /**
+   * Locate an executable on the path.
+   * @param exe executable name. If it is an absolute path which
+   * exists then it will returned direct
+   * @return the path to an exe or null for no match
+   */
+  public static File locateExecutable(String exe) {
+    File exeNameAsPath = new File(exe).getAbsoluteFile();
+    if (exeNameAsPath.exists()) {
+      return exeNameAsPath;
+    }
+
+    File exepath = null;
+    String path = extractPath();
+    String[] dirs = path.split(System.getProperty("path.separator"));
+    for (String dirname : dirs) {
+      File dir = new File(dirname);
+
+      File possible = new File(dir, exe);
+      if (possible.exists()) {
+        exepath = possible;
+      }
+    }
+    return exepath;
+  }
+
+  /**
+   * Lookup the PATH env var.
+   * @return the path or null
+   */
+  public static String extractPath() {
+    return extractEnvVar("PATH");
+  }
+
+  /**
+   * Find an environment variable. Uses case independent checking for
+   * the benefit of windows.
+   * Will fail if the var is not found.
+   * @param var path variable <i>in upper case</i>
+   * @return the env var
+   */
+  public static String extractEnvVar(String var) {
+    String realkey = "";
+
+    for (String it : System.getenv().keySet()) {
+      if (it.toUpperCase(Locale.ENGLISH).equals(var)) {
+        realkey = it;
+      }
+    }
+
+    if (SliderUtils.isUnset(realkey)) {
+      fail("No environment variable " + var + " found");
+    }
+    String val = System.getenv(realkey);
+
+    LOG.info("{} = {}", realkey, val);
+    return val;
+  }
+
+  /**
+   * Create a temp JSON file. After coming up with the name, the file
+   * is deleted
+   * @return the filename
+   */
+  public static  File createTempJsonFile() throws IOException {
+    return tmpFile(".json");
+  }
+
+  /**
+   * Create a temp file with the specific name. It's deleted after creation,
+   * to avoid  "file exists exceptions"
+   * @param suffix suffix, e.g. ".txt"
+   * @return a path to a file which may be created
+   */
+  public static File tmpFile(String suffix) throws IOException {
+    File reportFile = File.createTempFile(
+        "temp",
+        suffix,
+        new File("target"));
+    reportFile.delete();
+    return reportFile;
+  }
+
+  /**
+   * Execute a closure, assert it fails with a given exit code and text.
+   * @param exitCode exit code
+   * @param text text (can be "")
+   * @param action action
+   * @return
+   */
+  public void  assertFailsWithException(int exitCode,
+      String text,
+      Action action) throws Exception {
+    try {
+      action.invoke();
+      fail("Operation was expected to fail —but it succeeded");
+    } catch (ServiceLaunchException e) {
+      assertExceptionDetails(e, exitCode, text);
+    }
+  }
+
+  /**
+   * Execute a closure, assert it fails with a given exit code and text.
+   * @param text text (can be "")
+   * @param action action
+   * @return
+   */
+  public void assertFailsWithExceptionClass(Class clazz,
+      String text,
+      Action action) throws Exception {
+    try {
+      action.invoke();
+      fail("Operation was expected to fail —but it succeeded");
+    } catch (Exception e) {
+      assertExceptionDetails(e, clazz, text);
+    }
+  }
+
+  public static void assertExceptionDetails(
+      ServiceLaunchException ex,
+      int exitCode) {
+    assertExceptionDetails(ex, exitCode, null);
+  }
+
+  /**
+   * Make an assertion about the exit code of an exception.
+   * @param ex exception
+   * @param exitCode exit code
+   * @param text error text to look for in the exception
+   */
+  public static void assertExceptionDetails(
+      ServiceLaunchException ex,
+      int exitCode,
+      String text) {
+    if (exitCode != ex.getExitCode()) {
+      String message = String.format("Wrong exit code, expected %d but" +
+              " got %d in %s", exitCode, ex.getExitCode(), ex);
+      LOG.warn(message, ex);
+      throw new AssertionError(message, ex);
+    }
+    if (SliderUtils.isSet(text)) {
+      if (!(ex.toString().contains(text))) {
+        String message = String.format("String match for \"%s\"failed in %s",
+            text, ex);
+        LOG.warn(message, ex);
+        throw new AssertionError(message, ex);
+      }
+    }
+  }
+
+  /**
+   * Make an assertion about the class of an exception.
+   * @param ex exception
+   * @param clazz exit code
+   * @param text error text to look for in the exception
+   */
+  static void assertExceptionDetails(
+      Exception ex,
+      Class clazz,
+      String text) throws Exception {
+    if (ex.getClass() != clazz) {
+      throw ex;
+    }
+    if (SliderUtils.isSet(text) && !(ex.toString().contains(text))) {
+      throw ex;
+    }
+  }
+
+  /**
+   * Launch the slider client with the specific args; no validation
+   * of return code takes place.
+   * @param conf configuration
+   * @param args arg list
+   * @return the launcher
+   */
+  protected static ServiceLauncher<SliderClient> execSliderCommand(
+      Configuration conf,
+      List args) throws Throwable {
+    ServiceLauncher<SliderClient> serviceLauncher =
+        new ServiceLauncher<>(sliderClientClassName);
+
+    LOG.debug("slider {}", SliderUtils.join(args, " ", false));
+    serviceLauncher.launchService(conf,
+        toArray(args),
+        false);
+    return serviceLauncher;
+  }
+
+  /**
+   * Launch a slider command to a given exit code.
+   * Most failures will trigger exceptions; this is for the exit code of the
+   * runService() call.
+   * @param exitCode desired exit code
+   * @param conf configuration
+   * @param args arg list
+   * @return the launcher
+   */
+  protected static ServiceLauncher<SliderClient> execSliderCommand(
+      int exitCode,
+      Configuration conf,
+      List args) throws Throwable {
+    ServiceLauncher<SliderClient> serviceLauncher = execSliderCommand(conf,
+        args);
+    assertEquals(exitCode, serviceLauncher.getServiceExitCode());
+    return serviceLauncher;
+  }
+
+  public static ServiceLauncher launch(Class serviceClass,
+      Configuration conf,
+      List<Object> args) throws
+      Throwable {
+    ServiceLauncher serviceLauncher =
+        new ServiceLauncher(serviceClass.getName());
+
+    String joinedArgs = SliderUtils.join(args, " ", false);
+    LOG.debug("slider {}", joinedArgs);
+
+    serviceLauncher.launchService(conf,
+        toArray(args),
+        false);
+    return serviceLauncher;
+  }
+
+  public static Throwable launchExpectingException(Class serviceClass,
+      Configuration conf,
+      String expectedText,
+      List args)
+      throws Throwable {
+    try {
+      ServiceLauncher launch = launch(serviceClass, conf, args);
+      throw new AssertionError("Expected an exception with text containing " +
+          expectedText + " -but the service completed with exit code " +
+          launch.getServiceExitCode());
+    } catch (AssertionError error) {
+      throw error;
+    } catch (Throwable thrown) {
+      if (SliderUtils.isSet(expectedText) && !thrown.toString().contains(
+          expectedText)) {
+        //not the right exception -rethrow
+        LOG.warn("Caught Exception did not contain expected text" +
+                 "\"" + expectedText + "\"");
+        throw thrown;
+      }
+      return thrown;
+    }
+  }
+
+
+  public static ServiceLauncher<SliderClient> launchClientAgainstRM(
+      String address,
+      List<String> args,
+      Configuration conf) throws Throwable {
+    assertNotNull(address);
+    LOG.info("Connecting to rm at {}", address);
+    if (!args.contains(Arguments.ARG_MANAGER)) {
+      args.add(Arguments.ARG_MANAGER);
+      args.add(address);
+    }
+    ServiceLauncher<SliderClient> launcher = execSliderCommand(conf, args);
+    return launcher;
+  }
+
+  /**
+   * Add a configuration parameter as a cluster configuration option.
+   * @param extraArgs extra arguments
+   * @param conf config
+   * @param option option
+   */
+  public static void addClusterConfigOption(
+      List<String> extraArgs,
+      YarnConfiguration conf,
+      String option) {
+
+    conf.getTrimmed(option);
+    extraArgs.add(ARG_OPTION);
+    extraArgs.add(option);
+    extraArgs.add(getRequiredConfOption(conf, option));
+  }
+
+  /**
+   * Assert that a path refers to a directory.
+   * @param fs filesystem
+   * @param path path of the directory
+   * @throws IOException on File IO problems
+   */
+  public static void assertIsDirectory(FileSystem fs,
+      Path path) throws IOException {
+    FileStatus fileStatus = fs.getFileStatus(path);
+    assertIsDirectory(fileStatus);
+  }
+
+  /**
+   * Assert that a path refers to a directory.
+   * @param fileStatus stats to check
+   */
+  public static void assertIsDirectory(FileStatus fileStatus) {
+    assertTrue("Should be a dir -but isn't: " + fileStatus,
+        fileStatus.isDirectory());
+  }
+
+  /**
+   * Assert that a path exists -but make no assertions as to the
+   * type of that entry.
+   *
+   * @param fileSystem filesystem to examine
+   * @param message message to include in the assertion failure message
+   * @param path path in the filesystem
+   * @throws IOException IO problems
+   */
+  public static void assertPathExists(
+      FileSystem fileSystem,
+      String message,
+      Path path) throws IOException {
+    if (!fileSystem.exists(path)) {
+      //failure, report it
+      fail(
+          message + ": not found \"" + path + "\" in " + path.getParent() +
+          "-" +
+          ls(fileSystem, path.getParent()));
+    }
+  }
+
+  /**
+   * Assert that a path does not exist.
+   *
+   * @param fileSystem filesystem to examine
+   * @param message message to include in the assertion failure message
+   * @param path path in the filesystem
+   * @throws IOException IO problems
+   */
+  public static void assertPathDoesNotExist(
+      FileSystem fileSystem,
+      String message,
+      Path path) throws IOException {
+    try {
+      FileStatus status = fileSystem.getFileStatus(path);
+      // a status back implies there is a file here
+      fail(message + ": unexpectedly found " + path + " as  " + status);
+    } catch (FileNotFoundException expected) {
+      //this is expected
+
+    }
+  }
+
+  /**
+   * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry.
+   * @param fs filesystem
+   * @param dir directory to scan
+   * @param subdir full path to look for
+   * @throws IOException IO probles
+   */
+  public static void assertListStatusFinds(FileSystem fs,
+      Path dir,
+      Path subdir) throws IOException {
+    FileStatus[] stats = fs.listStatus(dir);
+    boolean found = false;
+    StringBuilder builder = new StringBuilder();
+    for (FileStatus stat : stats) {
+      builder.append(stat.toString()).append('\n');
+      if (stat.getPath().equals(subdir)) {
+        found = true;
+      }
+    }
+    assertTrue("Path " + subdir
+        + " not found in directory " + dir + ":" + builder,
+        found);
+  }
+
+  /**
+   * List a a path to string.
+   * @param fileSystem filesystem
+   * @param path directory
+   * @return a listing of the filestatuses of elements in the directory, one
+   * to a line, precedeed by the full path of the directory
+   * @throws IOException connectivity problems
+   */
+  public static String ls(FileSystem fileSystem, Path path)
+      throws IOException {
+    if (path == null) {
+      //surfaces when someone calls getParent() on something at the top of
+      // the path
+      return "/";
+    }
+    FileStatus[] stats;
+    String pathtext = "ls " + path;
+    try {
+      stats = fileSystem.listStatus(path);
+    } catch (FileNotFoundException e) {
+      return pathtext + " -file not found";
+    } catch (IOException e) {
+      return pathtext + " -failed: " + e;
+    }
+    return pathtext + fileStatsToString(stats, "\n");
+  }
+
+  /**
+   * Take an array of filestats and convert to a string (prefixed w/ a [01]
+   * counter).
+   * @param stats array of stats
+   * @param separator separator after every entry
+   * @return a stringified set
+   */
+  public static String fileStatsToString(FileStatus[] stats, String separator) {
+    StringBuilder buf = new StringBuilder(stats.length * 128);
+    for (int i = 0; i < stats.length; i++) {
+      buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+    }
+    return buf.toString();
+  }
+
+  public static void waitWhileClusterLive(SliderClient sliderClient)
+      throws IOException, YarnException {
+    waitWhileClusterLive(sliderClient, 30000);
+  }
+
+  public static void dumpRegistryInstances(
+      Map<String, ServiceRecord> instances) {
+    describe("service registry slider instances");
+    for (Entry<String, ServiceRecord> it : instances.entrySet()) {
+      LOG.info(" {} : {}", it.getKey(), it.getValue());
+    }
+    describe("end list service registry slider instances");
+  }
+
+
+  public static void dumpRegistryInstanceIDs(List<String> instanceIds) {
+    describe("service registry instance IDs");
+    dumpCollection(instanceIds);
+  }
+
+  public static void dumpRegistryServiceTypes(Collection<String> entries) {
+    describe("service registry types");
+    dumpCollection(entries);
+  }
+
+  public static <V> void dumpCollection(Collection<V> entries) {
+    LOG.info("number of entries: {}", entries.size());
+    for (V it : entries) {
+      LOG.info(it.toString());
+    }
+  }
+
+  public static void dumpArray(Object[] entries) {
+    LOG.info("number of entries: {}", entries.length);
+    for (Object it : entries) {
+      LOG.info(it.toString());
+    }
+  }
+
+  public static <K, V> void dumpMap(Map<K, V> map) {
+    for (Entry<K, V> it : map.entrySet()) {
+      LOG.info("\"{}\": \"{}\"", it.getKey().toString(), it.getValue()
+          .toString());
+    }
+  }
+
+  /**
+   * Get a time option in seconds if set, otherwise the default value (also
+   * in seconds).
+   * This operation picks up the time value as a system property if set -that
+   * value overrides anything in the test file
+   * @param conf
+   * @param key
+   * @param defValMillis
+   * @return
+   */
+  public static int getTimeOptionMillis(
+      Configuration conf,
+      String key,
+      int defValMillis) {
+    int val = conf.getInt(key, 0);
+    val = Integer.getInteger(key, val);
+    int time = 1000 * val;
+    if (time == 0) {
+      time = defValMillis;
+    }
+    return time;
+  }
+
+  public void dumpConfigurationSet(PublishedConfigSet confSet) {
+    for (String key : confSet.keys()) {
+      PublishedConfiguration config = confSet.get(key);
+      LOG.info("{} -- {}", key, config.description);
+    }
+  }
+
+  /**
+   * Convert a file to a URI suitable for use in an argument.
+   * @param file file
+   * @return a URI string valid on all platforms
+   */
+  public String toURIArg(File file) {
+    return file.getAbsoluteFile().toURI().toString();
+  }
+
+  /**
+   * Assert a file exists; fails with a listing of the parent dir.
+   * @param text text for front of message
+   * @param file file to look for
+   * @throws FileNotFoundException
+   */
+  public void assertFileExists(String text, File file)
+      throws FileNotFoundException {
+    if (!file.exists()) {
+      File parent = file.getParentFile();
+      String[] files = parent.list();
+      StringBuilder builder = new StringBuilder();
+      builder.append(parent.getAbsolutePath());
+      builder.append(":\n");
+      for (String name : files) {
+        builder.append("  ");
+        builder.append(name);
+        builder.append("\n");
+      }
+      throw new FileNotFoundException(text + ": " + file + " not found in " +
+          builder);
+    }
+  }
+
+  /**
+   * Repeat a probe until it succeeds, if it does not execute a failure
+   * closure then raise an exception with the supplied message.
+   * @param probe probe
+   * @param timeout time in millis before giving up
+   * @param sleepDur sleep between failing attempts
+   * @param args map of arguments to the probe
+   * @param failIfUnsuccessful if the probe fails after all the attempts
+   * —should it raise an exception
+   * @param failureMessage message to include in exception raised
+   * @param failureHandler closure to invoke prior to the failure being raised
+   */
+  protected void repeatUntilSuccess(
+      String action,
+      Probe probe,
+      int timeout,
+      int sleepDur,
+      Map args,
+      boolean failIfUnsuccessful,
+      String failureMessage,
+      Action failureHandler) throws Exception {
+    LOG.debug("Probe {} timelimit {}", action, timeout);
+    if (timeout < 1000) {
+      fail("Timeout " + timeout + " too low: milliseconds are expected, not " +
+          "seconds");
+    }
+    int attemptCount = 1;
+    boolean succeeded = false;
+    boolean completed = false;
+    Duration duration = new Duration(timeout);
+    duration.start();
+    while (!completed) {
+      Outcome outcome = probe.invoke(args);
+      if (outcome.equals(Outcome.SUCCESS)) {
+        // success
+        LOG.debug("Success after {} attempt(s)", attemptCount);
+        succeeded = true;
+        completed = true;
+      } else if (outcome.equals(Outcome.RETRY)) {
+        // failed but retry possible
+        attemptCount++;
+        completed = duration.getLimitExceeded();
+        if (!completed) {
+          LOG.debug("Attempt {} failed", attemptCount);
+          try {
+            Thread.sleep(sleepDur);
+          } catch (InterruptedException e) {
+          }
+        }
+      } else if (outcome.equals(Outcome.FAIL)) {
+        // fast fail
+        LOG.debug("Fast fail of probe");
+        completed = true;
+      }
+    }
+    if (!succeeded) {
+      if (duration.getLimitExceeded()) {
+        LOG.info("probe timed out after {} and {} attempts", timeout,
+            attemptCount);
+      }
+      if (failureHandler != null) {
+        failureHandler.invoke();
+      }
+      if (failIfUnsuccessful) {
+        fail(failureMessage);
+      }
+    }
+  }
+
+  /**
+   * Get a value from a map; raise an assertion if it is not there.
+   * @param map map to look up
+   * @param key key
+   * @return the string value
+   */
+  public <K, V> String requiredMapValue(Map<K, V> map, String key) {
+    assertNotNull(map.get(key));
+    return map.get(key).toString();
+  }
+
+  public static void assertStringContains(String expected, String text) {
+    assertNotNull("null text", text);
+    if (!text.contains(expected)) {
+      String message = String.format("did not find %s in \"%s\"", expected,
+          text);
+      LOG.error(message);
+      fail(message);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
new file mode 100644
index 0000000..9806ac3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.slider.api.resource.Application;
+import org.junit.Test;
+
+import java.util.Collections;
+
+/**
+ * Test for some of the command test base operations.
+ */
+public class TestAssertions {
+
+  public static final String CLUSTER_JSON = "json/cluster.json";
+
+  @Test
+  public void testNoInstances() throws Throwable {
+    Application application = new Application();
+    application.setContainers(null);
+    SliderTestUtils.assertContainersLive(application, "example", 0);
+  }
+
+  @Test
+  public void testEmptyInstances() throws Throwable {
+    Application application = new Application();
+    application.setContainers(Collections.emptyList());
+    SliderTestUtils.assertContainersLive(application, "example", 0);
+  }
+
+// TODO test metrics retrieval
+//  @Test
+//  public void testLiveInstances() throws Throwable {
+//    InputStream stream = getClass().getClassLoader().getResourceAsStream(
+//        CLUSTER_JSON);
+//    assertNotNull("could not load " + CLUSTER_JSON, stream);
+//    ClusterDescription liveCD = ClusterDescription.fromStream(stream);
+//    assertNotNull(liveCD);
+//    SliderTestUtils.assertContainersLive(liveCD, "SLEEP_LONG", 4);
+//    assertEquals((Integer) 1, liveCD.statistics.get("SLEEP_LONG").get(
+//        StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING));
+//  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
new file mode 100644
index 0000000..5493198
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.utils;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.Assert;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+/**
+ *  Various utility methods
+ *  Byte comparison methods are from
+ *  <code>org.apache.hadoop.fs.contract.ContractTestUtils</code>
+ */
+public class TestUtility {
+  protected static final Logger log =
+      LoggerFactory.getLogger(TestUtility.class);
+
+  public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, String prefix) throws IOException {
+    for (File file : dirObj.listFiles()) {
+      if (file.isDirectory()) {
+        addDir(file, zipFile, prefix + file.getName() + File.separator);
+      } else {
+        log.info("Adding to zip - " + prefix + file.getName());
+        zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName()));
+        IOUtils.copy(new FileInputStream(file), zipFile);
+        zipFile.closeArchiveEntry();
+      }
+    }
+  }
+
+  public static void zipDir(String zipFile, String dir) throws IOException {
+    File dirObj = new File(dir);
+    ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile));
+    log.info("Creating : {}", zipFile);
+    try {
+      addDir(dirObj, out, "");
+    } finally {
+      out.close();
+    }
+  }
+
+  public static String createAppPackage(
+      TemporaryFolder folder, String subDir, String pkgName, String srcPath) throws IOException {
+    String zipFileName;
+    File pkgPath = folder.newFolder(subDir);
+    File zipFile = new File(pkgPath, pkgName).getAbsoluteFile();
+    zipFileName = zipFile.getAbsolutePath();
+    TestUtility.zipDir(zipFileName, srcPath);
+    log.info("Created temporary zip file at {}", zipFileName);
+    return zipFileName;
+  }
+
+
+  /**
+   * Assert that tthe array original[0..len] and received[] are equal.
+   * A failure triggers the logging of the bytes near where the first
+   * difference surfaces.
+   * @param original source data
+   * @param received actual
+   * @param len length of bytes to compare
+   */
+  public static void compareByteArrays(byte[] original,
+      byte[] received,
+      int len) {
+    Assert.assertEquals("Number of bytes read != number written",
+        len, received.length);
+    int errors = 0;
+    int first_error_byte = -1;
+    for (int i = 0; i < len; i++) {
+      if (original[i] != received[i]) {
+        if (errors == 0) {
+          first_error_byte = i;
+        }
+        errors++;
+      }
+    }
+
+    if (errors > 0) {
+      String message = String.format(" %d errors in file of length %d",
+          errors, len);
+      log.warn(message);
+      // the range either side of the first error to print
+      // this is a purely arbitrary number, to aid user debugging
+      final int overlap = 10;
+      for (int i = Math.max(0, first_error_byte - overlap);
+           i < Math.min(first_error_byte + overlap, len);
+           i++) {
+        byte actual = received[i];
+        byte expected = original[i];
+        String letter = toChar(actual);
+        String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+        if (expected != actual) {
+          line = String.format("[%04d] %2x %s -expected %2x %s\n",
+              i,
+              actual,
+              letter,
+              expected,
+              toChar(expected));
+        }
+        log.warn(line);
+      }
+      Assert.fail(message);
+    }
+  }
+  /**
+   * Convert a byte to a character for printing. If the
+   * byte value is < 32 -and hence unprintable- the byte is
+   * returned as a two digit hex value
+   * @param b byte
+   * @return the printable character string
+   */
+  public static String toChar(byte b) {
+    if (b >= 0x20) {
+      return Character.toString((char) b);
+    } else {
+      return String.format("%02x", b);
+    }
+  }
+
+  /**
+   * Convert a buffer to a string, character by character
+   * @param buffer input bytes
+   * @return a string conversion
+   */
+  public static String toChar(byte[] buffer) {
+    StringBuilder builder = new StringBuilder(buffer.length);
+    for (byte b : buffer) {
+      builder.append(toChar(b));
+    }
+    return builder.toString();
+  }
+
+  public static byte[] toAsciiByteArray(String s) {
+    char[] chars = s.toCharArray();
+    int len = chars.length;
+    byte[] buffer = new byte[len];
+    for (int i = 0; i < len; i++) {
+      buffer[i] = (byte) (chars[i] & 0xff);
+    }
+    return buffer;
+  }
+
+  /**
+   * Create a dataset for use in the tests; all data is in the range
+   * base to (base+modulo-1) inclusive
+   * @param len length of data
+   * @param base base of the data
+   * @param modulo the modulo
+   * @return the newly generated dataset
+   */
+  public static byte[] dataset(int len, int base, int modulo) {
+    byte[] dataset = new byte[len];
+    for (int i = 0; i < len; i++) {
+      dataset[i] = (byte) (base + (i % modulo));
+    }
+    return dataset;
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message