hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [2/2] hadoop git commit: HADOOP-10775. Shell operations to fail with meaningful errors on windows if winutils.exe not found. (stevel)
Date Tue, 13 Oct 2015 20:49:39 GMT
HADOOP-10775. Shell operations to fail with meaningful errors on windows if winutils.exe not found. (stevel)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c59af2fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c59af2fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c59af2fd

Branch: refs/heads/trunk
Commit: c59af2fdf8f7f53117c626a895dab7fd78cf08ec
Parents: eb50c4f
Author: Steve Loughran <stevel@apache.org>
Authored: Tue Oct 13 21:29:50 2015 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Tue Oct 13 21:49:28 2015 +0100

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../java/org/apache/hadoop/fs/FileUtil.java     |  29 -
 .../java/org/apache/hadoop/fs/HardLink.java     |   8 +
 .../org/apache/hadoop/util/DataChecksum.java    |   2 +-
 .../hadoop/util/NativeLibraryChecker.java       |  25 +-
 .../main/java/org/apache/hadoop/util/Shell.java | 730 ++++++++++++++-----
 .../org/apache/hadoop/util/SysInfoWindows.java  |   5 +-
 .../org/apache/hadoop/fs/SymlinkBaseTest.java   |  30 +-
 .../java/org/apache/hadoop/fs/TestFileUtil.java |   8 +-
 .../apache/hadoop/fs/TestSymlinkLocalFS.java    |  11 -
 .../security/TestUserGroupInformation.java      |   3 +-
 .../java/org/apache/hadoop/util/TestShell.java  | 265 ++++++-
 .../org/apache/hadoop/util/TestWinUtils.java    | 124 ++--
 .../yarn/util/WindowsBasedProcessTree.java      |  10 +-
 .../server/nodemanager/ContainerExecutor.java   |   2 +-
 .../WindowsSecureContainerExecutor.java         |   3 +-
 .../launcher/ContainerLaunch.java               |  14 +-
 .../nodemanager/TestContainerExecutor.java      |   1 +
 .../launcher/TestContainerLaunch.java           |  31 +-
 19 files changed, 918 insertions(+), 386 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f3c341c..4b66830 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -885,6 +885,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12344. Improve validateSocketPathSecurity0 error message (Casey
     Brotherton via Colin P. McCabe)
 
+    HADOOP-10775. Shell operations to fail with meaningful errors on windows if
+    winutils.exe not found. (stevel)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 3c0e90d..e74c41c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -784,35 +784,6 @@ public class FileUtil {
     File linkFile = new File(
         Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
 
-    // If not on Java7+, copy a file instead of creating a symlink since
-    // Java6 has close to no support for symlinks on Windows. Specifically
-    // File#length and File#renameTo do not work as expected.
-    // (see HADOOP-9061 for additional details)
-    // We still create symlinks for directories, since the scenario in this
-    // case is different. The directory content could change in which
-    // case the symlink loses its purpose (for example task attempt log folder
-    // is symlinked under userlogs and userlogs are generated afterwards).
-    if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
-      try {
-        LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " +
-            "of creating a symlink. Copying " + target + " -> " + linkname);
-
-        if (!linkFile.getParentFile().exists()) {
-          LOG.warn("Parent directory " + linkFile.getParent() +
-              " does not exist.");
-          return 1;
-        } else {
-          org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
-        }
-      } catch (IOException ex) {
-        LOG.warn("FileUtil#symlink failed to copy the file with error: "
-            + ex.getMessage());
-        // Exit with non-zero exit code
-        return 1;
-      }
-      return 0;
-    }
-
     String[] cmd = Shell.getSymlinkCommand(
         targetFile.toString(),
         linkFile.toString());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
index 209ba69..0de019d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
@@ -122,6 +122,12 @@ public class HardLink {
   @VisibleForTesting
   static class HardLinkCGWin extends HardLinkCommandGetter {
 
+    /**
+     * Build the windows link command. This must not
+     * use an exception-raising reference to WINUTILS, as
+     * some tests examine the command.
+     */
+    @SuppressWarnings("deprecation")
     static String[] getLinkCountCommand = {
         Shell.WINUTILS, "hardlink", "stat", null};
 
@@ -130,6 +136,8 @@ public class HardLink {
      */
     @Override
     String[] linkCount(File file) throws IOException {
+      // trigger the check for winutils
+      Shell.getWinutilsFile();
       String[] buf = new String[getLinkCountCommand.length];
       System.arraycopy(getLinkCountCommand, 0, buf, 0, 
                        getLinkCountCommand.length);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index a38ec32..d9dc7af 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -75,7 +75,7 @@ public class DataChecksum implements Checksum {
    * is chosen depending on the platform.
    */
   public static Checksum newCrc32() {
-    return Shell.isJava7OrAbove()? new CRC32(): new PureJavaCrc32();
+    return new CRC32();
   }
 
   public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
index 81448ab..9d84ced 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.util;
 
-import org.apache.hadoop.util.NativeCodeLoader;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.OpensslCipher;
 import org.apache.hadoop.io.compress.Lz4Codec;
@@ -27,10 +26,17 @@ import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
 import org.apache.hadoop.io.compress.zlib.ZlibFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class NativeLibraryChecker {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(NativeLibraryChecker.class);
+
   /**
    * A tool to test native library availability, 
    */
@@ -99,12 +105,17 @@ public class NativeLibraryChecker {
       }
     }
 
-    // winutils.exe is required on Windows
-    winutilsPath = Shell.getWinUtilsPath();
-    if (winutilsPath != null) {
-      winutilsExists = true;
-    } else {
-      winutilsPath = "";
+    if (Shell.WINDOWS) {
+      // winutils.exe is required on Windows
+      try {
+        winutilsPath = Shell.getWinutilsFile().getCanonicalPath();
+        winutilsExists = true;
+      } catch (IOException e) {
+        LOG.debug("No Winutils: ", e);
+        winutilsPath = e.getMessage();
+        winutilsExists = false;
+      }
+      System.out.printf("winutils: %b %s%n", winutilsExists, winutilsPath);
     }
 
     System.out.println("Native library checking:");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index ca70ef3..4370d89 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.util;
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.InputStream;
@@ -30,41 +31,74 @@ import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-/** 
- * A base class for running a Unix command.
- * 
- * <code>Shell</code> can be used to run unix commands like <code>du</code> or
+/**
+ * A base class for running a Shell command.
+ *
+ * <code>Shell</code> can be used to run shell commands like <code>du</code> or
  * <code>df</code>. It also offers facilities to gate commands by 
  * time-intervals.
  */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Unstable
-abstract public class Shell {
-  
-  public static final Log LOG = LogFactory.getLog(Shell.class);
-  
-  private static boolean IS_JAVA7_OR_ABOVE =
-      System.getProperty("java.version").substring(0, 3).compareTo("1.7") >= 0;
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class Shell {
+  public static final Logger LOG = LoggerFactory.getLogger(Shell.class);
+
+  /**
+   * Text to include when there are windows-specific problems.
+   * {@value}
+   */
+  private static final String WINDOWS_PROBLEMS =
+      "https://wiki.apache.org/hadoop/WindowsProblems";
+
+  /**
+   * Name of the windows utils binary: {@value}.
+   */
+  static final String WINUTILS_EXE = "winutils.exe";
+
+  /**
+   * System property for the Hadoop home directory: {@value}.
+   */
+  public static final String SYSPROP_HADOOP_HOME_DIR = "hadoop.home.dir";
 
+  /**
+   * Environment variable for Hadoop's home dir: {@value}.
+   */
+  public static final String ENV_HADOOP_HOME = "HADOOP_HOME";
+
+  /**
+   * query to see if system is Java 7 or later.
+   * Now that Hadoop requires Java 7 or later, this always returns true.
+   * @deprecated This call isn't needed any more: please remove uses of it.
+   * @return true, always.
+   */
+  @Deprecated
   public static boolean isJava7OrAbove() {
-    return IS_JAVA7_OR_ABOVE;
+    return true;
   }
 
   /**
    * Maximum command line length in Windows
    * KB830473 documents this as 8191
    */
-  public static final int WINDOWS_MAX_SHELL_LENGHT = 8191;
+  public static final int WINDOWS_MAX_SHELL_LENGTH = 8191;
 
   /**
-   * Checks if a given command (String[]) fits in the Windows maximum command line length
-   * Note that the input is expected to already include space delimiters, no extra count
-   * will be added for delimiters.
+   * mis-spelling of {@link #WINDOWS_MAX_SHELL_LENGTH}.
+   * @deprecated use the correctly spelled constant.
+   */
+  @Deprecated
+  public static final int WINDOWS_MAX_SHELL_LENGHT = WINDOWS_MAX_SHELL_LENGTH;
+
+  /**
+   * Checks if a given command (String[]) fits in the Windows maximum command
+   * line length Note that the input is expected to already include space
+   * delimiters, no extra count will be added for delimiters.
    *
    * @param commands command parts, including any space delimiters
    */
@@ -74,19 +108,19 @@ abstract public class Shell {
     for (String s: commands) {
       len += s.length();
     }
-    if (len > WINDOWS_MAX_SHELL_LENGHT) {
+    if (len > WINDOWS_MAX_SHELL_LENGTH) {
       throw new IOException(String.format(
-          "The command line has a length of %d exceeds maximum allowed length of %d. " +
-          "Command starts with: %s",
-          len, WINDOWS_MAX_SHELL_LENGHT,
-          StringUtils.join("", commands).substring(0, 100)));
+        "The command line has a length of %d exceeds maximum allowed length" +
+            " of %d. Command starts with: %s",
+        len, WINDOWS_MAX_SHELL_LENGTH,
+        StringUtils.join("", commands).substring(0, 100)));
     }
   }
 
-  /** a Unix command to get the current user's name */
-  public final static String USER_NAME_COMMAND = "whoami";
+  /** a Unix command to get the current user's name: {@value}. */
+  public static final String USER_NAME_COMMAND = "whoami";
 
-  /** Windows CreateProcess synchronization object */
+  /** Windows <code>CreateProcess</code> synchronization object. */
   public static final Object WindowsProcessLaunchLock = new Object();
 
   // OSType detection
@@ -100,9 +134,13 @@ abstract public class Shell {
     OS_TYPE_OTHER
   }
 
+  /**
+   * Get the type of the operating system, as determined from parsing
+   * the <code>os.name</code> property.
+   */
   public static final OSType osType = getOSType();
 
-  static private OSType getOSType() {
+  private static OSType getOSType() {
     String osName = System.getProperty("os.name");
     if (osName.startsWith("Windows")) {
       return OSType.OS_TYPE_WIN;
@@ -131,46 +169,49 @@ abstract public class Shell {
   public static final boolean PPC_64
                 = System.getProperties().getProperty("os.arch").contains("ppc64");
 
-  /** a Unix command to get the current user's groups list */
+  /** a Unix command to get the current user's groups list. */
   public static String[] getGroupsCommand() {
     return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
                     : new String[]{"bash", "-c", "groups"};
   }
 
   /**
-   * a Unix command to get a given user's groups list.
+   * A command to get a given user's groups list.
    * If the OS is not WINDOWS, the command will get the user's primary group
    * first and finally get the groups list which includes the primary group.
    * i.e. the user's primary group will be included twice.
    */
   public static String[] getGroupsForUserCommand(final String user) {
-    //'groups username' command return is non-consistent across different unixes
-    return (WINDOWS)? new String[] { WINUTILS, "groups", "-F", "\"" + user + "\""}
-                    : new String [] {"bash", "-c", "id -gn " + user
-                                     + "&& id -Gn " + user};
+    //'groups username' command return is inconsistent across different unixes
+    return WINDOWS ?
+      new String[]
+          { getWinutilsPath(), "groups", "-F", "\"" + user + "\"" }
+      : new String [] {"bash", "-c", "id -gn " + user + "&& id -Gn " + user};
   }
 
-  /** a Unix command to get a given netgroup's user list */
+  /** A command to get a given netgroup's user list. */
   public static String[] getUsersForNetgroupCommand(final String netgroup) {
     //'groups username' command return is non-consistent across different unixes
-    return (WINDOWS)? new String [] {"cmd", "/c", "getent netgroup " + netgroup}
+    return WINDOWS ? new String [] {"cmd", "/c", "getent netgroup " + netgroup}
                     : new String [] {"bash", "-c", "getent netgroup " + netgroup};
   }
 
   /** Return a command to get permission information. */
   public static String[] getGetPermissionCommand() {
-    return (WINDOWS) ? new String[] { WINUTILS, "ls", "-F" }
+    return (WINDOWS) ? new String[] { getWinutilsPath(), "ls", "-F" }
                      : new String[] { "/bin/ls", "-ld" };
   }
 
-  /** Return a command to set permission */
+  /** Return a command to set permission. */
   public static String[] getSetPermissionCommand(String perm, boolean recursive) {
     if (recursive) {
-      return (WINDOWS) ? new String[] { WINUTILS, "chmod", "-R", perm }
-                         : new String[] { "chmod", "-R", perm };
+      return (WINDOWS) ?
+          new String[] { getWinutilsPath(), "chmod", "-R", perm }
+          : new String[] { "chmod", "-R", perm };
     } else {
-      return (WINDOWS) ? new String[] { WINUTILS, "chmod", perm }
-                       : new String[] { "chmod", perm };
+      return (WINDOWS) ?
+          new String[] { getWinutilsPath(), "chmod", perm }
+          : new String[] { "chmod", perm };
     }
   }
 
@@ -182,45 +223,52 @@ abstract public class Shell {
    * @param file String file to set
    * @return String[] containing command and arguments
    */
-  public static String[] getSetPermissionCommand(String perm, boolean recursive,
-                                                 String file) {
+  public static String[] getSetPermissionCommand(String perm,
+      boolean recursive, String file) {
     String[] baseCmd = getSetPermissionCommand(perm, recursive);
     String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1);
     cmdWithFile[cmdWithFile.length - 1] = file;
     return cmdWithFile;
   }
 
-  /** Return a command to set owner */
+  /** Return a command to set owner. */
   public static String[] getSetOwnerCommand(String owner) {
-    return (WINDOWS) ? new String[] { WINUTILS, "chown", "\"" + owner + "\"" }
-                     : new String[] { "chown", owner };
+    return (WINDOWS) ?
+        new String[] { getWinutilsPath(), "chown", "\"" + owner + "\"" }
+        : new String[] { "chown", owner };
   }
-  
-  /** Return a command to create symbolic links */
+
+  /** Return a command to create symbolic links. */
   public static String[] getSymlinkCommand(String target, String link) {
-    return WINDOWS ? new String[] { WINUTILS, "symlink", link, target }
-                   : new String[] { "ln", "-s", target, link };
+    return WINDOWS ?
+       new String[] { getWinutilsPath(), "symlink", link, target }
+       : new String[] { "ln", "-s", target, link };
   }
 
-  /** Return a command to read the target of the a symbolic link*/
+  /** Return a command to read the target of the a symbolic link. */
   public static String[] getReadlinkCommand(String link) {
-    return WINDOWS ? new String[] { WINUTILS, "readlink", link }
+    return WINDOWS ?
+        new String[] { getWinutilsPath(), "readlink", link }
         : new String[] { "readlink", link };
   }
 
-  /** Return a command for determining if process with specified pid is alive. */
+  /**
+   * Return a command for determining if process with specified pid is alive.
+   * @param pid process ID
+   * @return a <code>kill -0</code> command or equivalent
+   */
   public static String[] getCheckProcessIsAliveCommand(String pid) {
     return getSignalKillCommand(0, pid);
   }
 
-  /** Return a command to send a signal to a given pid */
+  /** Return a command to send a signal to a given pid. */
   public static String[] getSignalKillCommand(int code, String pid) {
     // Code == 0 means check alive
     if (Shell.WINDOWS) {
       if (0 == code) {
-        return new String[] { Shell.WINUTILS, "task", "isAlive", pid };
+        return new String[] {Shell.getWinutilsPath(), "task", "isAlive", pid };
       } else {
-        return new String[] { Shell.WINUTILS, "task", "kill", pid };
+        return new String[] {Shell.getWinutilsPath(), "task", "kill", pid };
       }
     }
 
@@ -232,18 +280,20 @@ abstract public class Shell {
     }
   }
 
+  /** Regular expression for environment variables: {@value}. */
   public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
-  /** Return a regular expression string that match environment variables */
+
+  /** Return a regular expression string that match environment variables. */
   public static String getEnvironmentVariableRegex() {
     return (WINDOWS)
         ? "%(" + ENV_NAME_REGEX + "?)%"
         : "\\$(" + ENV_NAME_REGEX + ")";
   }
-  
+
   /**
    * Returns a File referencing a script with the given basename, inside the
-   * given parent directory.  The file extension is inferred by platform: ".cmd"
-   * on Windows, or ".sh" otherwise.
+   * given parent directory.  The file extension is inferred by platform:
+   * <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
    * 
    * @param parent File parent directory
    * @param basename String script file basename
@@ -254,9 +304,11 @@ abstract public class Shell {
   }
 
   /**
-   * Returns a script file name with the given basename.  The file extension is
-   * inferred by platform: ".cmd" on Windows, or ".sh" otherwise.
-   * 
+   * Returns a script file name with the given basename.
+   *
+   * The file extension is inferred by platform:
+   * <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+   *
    * @param basename String script file basename
    * @return String script file name
    */
@@ -267,129 +319,372 @@ abstract public class Shell {
   /**
    * Returns a command to run the given script.  The script interpreter is
    * inferred by platform: cmd on Windows or bash otherwise.
-   * 
+   *
    * @param script File script to run
    * @return String[] command to run the script
    */
   public static String[] getRunScriptCommand(File script) {
     String absolutePath = script.getAbsolutePath();
-    return WINDOWS ? new String[] { "cmd", "/c", absolutePath } :
-      new String[] { "/bin/bash", absolutePath };
+    return WINDOWS ?
+      new String[] { "cmd", "/c", absolutePath }
+      : new String[] { "/bin/bash", absolutePath };
   }
 
-  /** a Unix command to set permission */
+  /** a Unix command to set permission: {@value}. */
   public static final String SET_PERMISSION_COMMAND = "chmod";
-  /** a Unix command to set owner */
+  /** a Unix command to set owner: {@value}. */
   public static final String SET_OWNER_COMMAND = "chown";
 
-  /** a Unix command to set the change user's groups list */
+  /** a Unix command to set the change user's groups list: {@value}. */
   public static final String SET_GROUP_COMMAND = "chgrp";
-  /** a Unix command to create a link */
+  /** a Unix command to create a link: {@value}. */
   public static final String LINK_COMMAND = "ln";
-  /** a Unix command to get a link target */
+  /** a Unix command to get a link target: {@value}. */
   public static final String READ_LINK_COMMAND = "readlink";
 
-  /**Time after which the executing script would be timedout*/
+  /**Time after which the executing script would be timedout. */
   protected long timeOutInterval = 0L;
   /** If or not script timed out*/
-  private AtomicBoolean timedOut;
+  private final AtomicBoolean timedOut = new AtomicBoolean(false);
 
-
-  /** Centralized logic to discover and validate the sanity of the Hadoop 
-   *  home directory. Returns either NULL or a directory that exists and 
-   *  was specified via either -Dhadoop.home.dir or the HADOOP_HOME ENV 
-   *  variable.  This does a lot of work so it should only be called 
+  /**
+   *  Centralized logic to discover and validate the sanity of the Hadoop
+   *  home directory.
+   *
+   *  This does a lot of work so it should only be called
    *  privately for initialization once per process.
-   **/
-  private static String checkHadoopHome() {
+   *
+   * @return A directory that exists and via was specified on the command line
+   * via <code>-Dhadoop.home.dir</code> or the <code>HADOOP_HOME</code>
+   * environment variable.
+   * @throws FileNotFoundException if the properties are absent or the specified
+   * path is not a reference to a valid directory.
+   */
+  private static File checkHadoopHome() throws FileNotFoundException {
 
     // first check the Dflag hadoop.home.dir with JVM scope
-    String home = System.getProperty("hadoop.home.dir");
+    String home = System.getProperty(SYSPROP_HADOOP_HOME_DIR);
 
     // fall back to the system/user-global env variable
     if (home == null) {
-      home = System.getenv("HADOOP_HOME");
+      home = System.getenv(ENV_HADOOP_HOME);
     }
+    return checkHadoopHomeInner(home);
+  }
 
-    try {
-       // couldn't find either setting for hadoop's home directory
-       if (home == null) {
-         throw new IOException("HADOOP_HOME or hadoop.home.dir are not set.");
-       }
+  /*
+  A set of exception strings used to construct error messages;
+  these are referred to in tests
+  */
+  static final String E_DOES_NOT_EXIST = "does not exist";
+  static final String E_IS_RELATIVE = "is not an absolute path.";
+  static final String E_NOT_DIRECTORY = "is not a directory.";
+  static final String E_NO_EXECUTABLE = "Could not locate Hadoop executable";
+  static final String E_NOT_EXECUTABLE_FILE = "Not an executable file";
+  static final String E_HADOOP_PROPS_UNSET = ENV_HADOOP_HOME + " and "
+      + SYSPROP_HADOOP_HOME_DIR + " are unset.";
+  static final String E_HADOOP_PROPS_EMPTY = ENV_HADOOP_HOME + " or "
+      + SYSPROP_HADOOP_HOME_DIR + " set to an empty string";
+  static final String E_NOT_A_WINDOWS_SYSTEM = "Not a Windows system";
 
-       if (home.startsWith("\"") && home.endsWith("\"")) {
-         home = home.substring(1, home.length()-1);
-       }
+  /**
+   *  Validate the accessibility of the Hadoop home directory.
+   *
+   * @return A directory that is expected to be the hadoop home directory
+   * @throws FileNotFoundException if the specified
+   * path is not a reference to a valid directory.
+   */
+  @VisibleForTesting
+  static File checkHadoopHomeInner(String home) throws FileNotFoundException {
+    // couldn't find either setting for hadoop's home directory
+    if (home == null) {
+      throw new FileNotFoundException(E_HADOOP_PROPS_UNSET);
+    }
+    // strip off leading and trailing double quotes
+    while (home.startsWith("\"")) {
+      home = home.substring(1);
+    }
+    while (home.endsWith("\"")) {
+      home = home.substring(0, home.length() - 1);
+    }
 
-       // check that the home setting is actually a directory that exists
-       File homedir = new File(home);
-       if (!homedir.isAbsolute() || !homedir.exists() || !homedir.isDirectory()) {
-         throw new IOException("Hadoop home directory " + homedir
-           + " does not exist, is not a directory, or is not an absolute path.");
-       }
+    // after stripping any quotes, check for home dir being non-empty
+    if (home.isEmpty()) {
+      throw new FileNotFoundException(E_HADOOP_PROPS_EMPTY);
+    }
+
+    // check that the hadoop home dir value
+    // is an absolute reference to a directory
+    File homedir = new File(home);
+    if (!homedir.isAbsolute()) {
+      throw new FileNotFoundException("Hadoop home directory " + homedir
+          + " " + E_IS_RELATIVE);
+    }
+    if (!homedir.exists()) {
+      throw new FileNotFoundException("Hadoop home directory " + homedir
+          + " " + E_DOES_NOT_EXIST);
+    }
+    if (!homedir.isDirectory()) {
+      throw new FileNotFoundException("Hadoop home directory " + homedir
+          + " "+ E_NOT_DIRECTORY);
+    }
+    return homedir;
+  }
 
-       home = homedir.getCanonicalPath();
+  /**
+   * The Hadoop home directory.
+   */
+  private static final File HADOOP_HOME_FILE;
+
+  /**
+   * Rethrowable cause for the failure to determine the hadoop
+   * home directory
+   */
+  private static final IOException HADOOP_HOME_DIR_FAILURE_CAUSE;
 
+  static {
+    File home;
+    IOException ex;
+    try {
+      home = checkHadoopHome();
+      ex = null;
     } catch (IOException ioe) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Failed to detect a valid hadoop home directory", ioe);
       }
+      ex = ioe;
       home = null;
     }
-    
-    return home;
+    HADOOP_HOME_FILE = home;
+    HADOOP_HOME_DIR_FAILURE_CAUSE = ex;
   }
-  private static String HADOOP_HOME_DIR = checkHadoopHome();
 
-  // Public getter, throws an exception if HADOOP_HOME failed validation
-  // checks and is being referenced downstream.
-  public static final String getHadoopHome() throws IOException {
-    if (HADOOP_HOME_DIR == null) {
-      throw new IOException("Misconfigured HADOOP_HOME cannot be referenced.");
-    }
+  /**
+   * Optionally extend an error message with some OS-specific text.
+   * @param message core error message
+   * @return error message, possibly with some extra text
+   */
+  private static String addOsText(String message) {
+    return WINDOWS ? (message + " -see " + WINDOWS_PROBLEMS) : message;
+  }
 
-    return HADOOP_HOME_DIR;
+  /**
+   * Create a {@code FileNotFoundException} with the inner nested cause set
+   * to the given exception. Compensates for the fact that FNFE doesn't
+   * have an initializer that takes an exception.
+   * @param text error text
+   * @param ex inner exception
+   * @return a new exception to throw.
+   */
+  private static FileNotFoundException fileNotFoundException(String text,
+      Exception ex) {
+    return (FileNotFoundException) new FileNotFoundException(text)
+        .initCause(ex);
+  }
+
+  /**
+   * Get the Hadoop home directory. Raises an exception if not found
+   * @return the home dir
+   * @throws IOException if the home directory cannot be located.
+   */
+  public static String getHadoopHome() throws IOException {
+    return getHadoopHomeDir().getCanonicalPath();
   }
 
-  /** fully qualify the path to a binary that should be in a known hadoop 
+  /**
+   * Get the Hadoop home directory. If it is invalid,
+   * throw an exception.
+   * @return a path referring to hadoop home.
+   * @throws FileNotFoundException if the directory doesn't exist.
+   */
+  private static File getHadoopHomeDir() throws FileNotFoundException {
+    if (HADOOP_HOME_DIR_FAILURE_CAUSE != null) {
+      throw fileNotFoundException(
+          addOsText(HADOOP_HOME_DIR_FAILURE_CAUSE.toString()),
+          HADOOP_HOME_DIR_FAILURE_CAUSE);
+    }
+    return HADOOP_HOME_FILE;
+  }
+
+  /**
+   *  Fully qualify the path to a binary that should be in a known hadoop
    *  bin location. This is primarily useful for disambiguating call-outs 
    *  to executable sub-components of Hadoop to avoid clashes with other 
    *  executables that may be in the path.  Caveat:  this call doesn't 
    *  just format the path to the bin directory.  It also checks for file 
    *  existence of the composed path. The output of this call should be 
    *  cached by callers.
-   * */
-  public static final String getQualifiedBinPath(String executable) 
-  throws IOException {
+   *
+   * @param executable executable
+   * @return executable file reference
+   * @throws FileNotFoundException if the path does not exist
+   */
+  public static File getQualifiedBin(String executable)
+      throws FileNotFoundException {
     // construct hadoop bin path to the specified executable
-    String fullExeName = HADOOP_HOME_DIR + File.separator + "bin" 
-      + File.separator + executable;
+    return getQualifiedBinInner(getHadoopHomeDir(), executable);
+  }
+
+  /**
+   * Inner logic of {@link #getQualifiedBin(String)}, accessible
+   * for tests.
+   * @param hadoopHomeDir home directory (assumed to be valid)
+   * @param executable executable
+   * @return path to the binary
+   * @throws FileNotFoundException if the executable was not found/valid
+   */
+  static File getQualifiedBinInner(File hadoopHomeDir, String executable)
+      throws FileNotFoundException {
+    String binDirText = "Hadoop bin directory ";
+    File bin = new File(hadoopHomeDir, "bin");
+    if (!bin.exists()) {
+      throw new FileNotFoundException(addOsText(binDirText + E_DOES_NOT_EXIST
+          + ": " + bin));
+    }
+    if (!bin.isDirectory()) {
+      throw new FileNotFoundException(addOsText(binDirText + E_NOT_DIRECTORY
+          + ": " + bin));
+    }
 
-    File exeFile = new File(fullExeName);
+    File exeFile = new File(bin, executable);
     if (!exeFile.exists()) {
-      throw new IOException("Could not locate executable " + fullExeName
-        + " in the Hadoop binaries.");
+      throw new FileNotFoundException(
+          addOsText(E_NO_EXECUTABLE + ": " + exeFile));
+    }
+    if (!exeFile.isFile()) {
+      throw new FileNotFoundException(
+          addOsText(E_NOT_EXECUTABLE_FILE + ": " + exeFile));
     }
+    try {
+      return exeFile.getCanonicalFile();
+    } catch (IOException e) {
+      // this isn't going to happen, because of all the upfront checks.
+      // so if it does, it gets converted to a FNFE and rethrown
+      throw fileNotFoundException(e.toString(), e);
+    }
+  }
 
-    return exeFile.getCanonicalPath();
+  /**
+   *  Fully qualify the path to a binary that should be in a known hadoop
+   *  bin location. This is primarily useful for disambiguating call-outs
+   *  to executable sub-components of Hadoop to avoid clashes with other
+   *  executables that may be in the path.  Caveat:  this call doesn't
+   *  just format the path to the bin directory.  It also checks for file
+   *  existence of the composed path. The output of this call should be
+   *  cached by callers.
+   *
+   * @param executable executable
+   * @return executable file reference
+   * @throws FileNotFoundException if the path does not exist
+   * @throws IOException on path canonicalization failures
+   */
+  public static String getQualifiedBinPath(String executable)
+      throws IOException {
+    return getQualifiedBin(executable).getCanonicalPath();
   }
 
-  /** a Windows utility to emulate Unix commands */
-  public static final String WINUTILS = getWinUtilsPath();
+  /**
+   * Location of winutils as a string; null if not found.
+   * <p>
+   * <i>Important: caller must check for this value being null</i>.
+   * The lack of such checks has led to many support issues being raised.
+   * <p>
+   * @deprecated use one of the exception-raising getter methods,
+   * specifically {@link #getWinutilsPath()} or {@link #getWinutilsFile()}
+   */
+  @Deprecated
+  public static final String WINUTILS;
+
+  /** Canonical path to winutils, private to Shell. */
+  private static final String WINUTILS_PATH;
 
-  public static final String getWinUtilsPath() {
-    String winUtilsPath = null;
+  /** file reference to winutils. */
+  private static final File WINUTILS_FILE;
 
-    try {
-      if (WINDOWS) {
-        winUtilsPath = getQualifiedBinPath("winutils.exe");
+  /** the exception raised on a failure to init the WINUTILS fields. */
+  private static final IOException WINUTILS_FAILURE;
+
+  /*
+   * Static WINUTILS_* field initializer.
+   * On non-Windows systems sets the paths to null, and
+   * adds a specific exception to the failure cause, so
+   * that on any attempt to resolve the paths will raise
+   * a meaningful exception.
+   */
+  static {
+    IOException ioe = null;
+    String path = null;
+    File file = null;
+    // invariant: either there's a valid file and path,
+    // or there is a cached IO exception.
+    if (WINDOWS) {
+      try {
+        file = getQualifiedBin(WINUTILS_EXE);
+        path = file.getCanonicalPath();
+        ioe = null;
+      } catch (IOException e) {
+        LOG.warn("Did not find {}: {}", WINUTILS_EXE, e);
+        // stack trace comes at debug level
+        LOG.debug("Failed to find " + WINUTILS_EXE, e);
+        file = null;
+        path = null;
+        ioe = e;
       }
-    } catch (IOException ioe) {
-       LOG.error("Failed to locate the winutils binary in the hadoop binary path",
-         ioe);
+    } else {
+      // on a non-windows system, the invariant is kept
+      // by adding an explicit exception.
+      ioe = new FileNotFoundException(E_NOT_A_WINDOWS_SYSTEM);
+    }
+    WINUTILS_PATH = path;
+    WINUTILS_FILE = file;
+
+    WINUTILS = path;
+    WINUTILS_FAILURE = ioe;
+  }
+
+  /**
+   * Predicate to indicate whether or not the path to winutils is known.
+   *
+   * If true, then {@link #WINUTILS} is non-null, and both
+   * {@link #getWinutilsPath()} and {@link #getWinutilsFile()}
+   * will successfully return this value. Always false on non-windows systems.
+   * @return true if there is a valid path to the binary
+   */
+  public static boolean hasWinutilsPath() {
+    return WINUTILS_PATH != null;
+  }
+
+  /**
+   * Locate the winutils binary, or fail with a meaningful
+   * exception and stack trace as an RTE.
+   * This method is for use in methods which don't explicitly throw
+   * an <code>IOException</code>.
+   * @return the path to {@link #WINUTILS_EXE}
+   * @throws RuntimeException if the path is not resolvable
+   */
+  public static String getWinutilsPath() {
+    if (WINUTILS_FAILURE == null) {
+      return WINUTILS_PATH;
+    } else {
+      throw new RuntimeException(WINUTILS_FAILURE.toString(),
+          WINUTILS_FAILURE);
     }
+  }
 
-    return winUtilsPath;
+  /**
+   * Get a file reference to winutils.
+   * Always raises an exception if there isn't one
+   * @return the file instance referring to the winutils bin.
+   * @throws FileNotFoundException on any failure to locate that file.
+   */
+  public static File getWinutilsFile() throws FileNotFoundException {
+    if (WINUTILS_FAILURE == null) {
+      return WINUTILS_FILE;
+    } else {
+      // raise a new exception to generate a new stack trace
+      throw fileNotFoundException(WINUTILS_FAILURE.toString(),
+          WINUTILS_FAILURE);
+    }
   }
 
   public static final boolean isBashSupported = checkIsBashSupported();
@@ -412,7 +707,15 @@ abstract public class Shell {
     return supported;
   }
 
+  /**
+   * Flag which is true if setsid exists.
+   */
   public static final boolean isSetsidAvailable = isSetsidSupported();
+
+  /**
+   * Look for <code>setsid</code>.
+   * @return true if <code>setsid</code> was present
+   */
   private static boolean isSetsidSupported() {
     if (Shell.WINDOWS) {
       return false;
@@ -427,7 +730,8 @@ abstract public class Shell {
       LOG.debug("setsid is not available on this machine. So not using it.");
       setsidSupported = false;
     }  catch (Error err) {
-      if (err.getMessage().contains("posix_spawn is not " +
+      if (err.getMessage() != null
+          && err.getMessage().contains("posix_spawn is not " +
           "a supported process launch mechanism")
           && (Shell.FREEBSD || Shell.MAC)) {
         // HADOOP-11924: This is a workaround to avoid failure of class init
@@ -444,69 +748,86 @@ abstract public class Shell {
     return setsidSupported;
   }
 
-  /** Token separator regex used to parse Shell tool outputs */
+  /** Token separator regex used to parse Shell tool outputs. */
   public static final String TOKEN_SEPARATOR_REGEX
                 = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
 
-  private long    interval;   // refresh interval in msec
-  private long    lastTime;   // last time the command was performed
-  final private boolean redirectErrorStream; // merge stdout and stderr
+  private long interval;   // refresh interval in msec
+  private long lastTime;   // last time the command was performed
+  private final boolean redirectErrorStream; // merge stdout and stderr
   private Map<String, String> environment; // env for the command execution
   private File dir;
   private Process process; // sub process used to execute the command
   private int exitCode;
 
-  /**If or not script finished executing*/
-  private volatile AtomicBoolean completed;
-  
-  public Shell() {
+  /** Flag to indicate whether or not the script has finished executing. */
+  private final AtomicBoolean completed = new AtomicBoolean(false);
+
+  /**
+   * Create an instance with no minimum interval between runs; stderr is
+   * not merged with stdout.
+   */
+  protected Shell() {
     this(0L);
   }
-  
-  public Shell(long interval) {
+
+  /**
+   * Create an instance with a minimum interval between executions; stderr is
+   * not merged with stdout.
+   * @param interval interval in milliseconds between command executions.
+   */
+  protected Shell(long interval) {
     this(interval, false);
   }
 
   /**
-   * @param interval the minimum duration to wait before re-executing the 
-   *        command.
+   * Create a shell instance which can be re-executed when the {@link #run()}
+   * method is invoked with a given elapsed time between calls.
+   *
+   * @param interval the minimum duration in milliseconds to wait before
+   *        re-executing the command. If set to 0, there is no minimum.
+   * @param redirectErrorStream should the error stream be merged with
+   *        the normal output stream?
    */
-  public Shell(long interval, boolean redirectErrorStream) {
+  protected Shell(long interval, boolean redirectErrorStream) {
     this.interval = interval;
-    this.lastTime = (interval<0) ? 0 : -interval;
+    this.lastTime = (interval < 0) ? 0 : -interval;
     this.redirectErrorStream = redirectErrorStream;
   }
-  
-  /** set the environment for the command 
+
+  /**
+   * Set the environment for the command.
    * @param env Mapping of environment variables
    */
   protected void setEnvironment(Map<String, String> env) {
     this.environment = env;
   }
 
-  /** set the working directory 
-   * @param dir The directory where the command would be executed
+  /**
+   * Set the working directory.
+   * @param dir The directory where the command will be executed
    */
   protected void setWorkingDirectory(File dir) {
     this.dir = dir;
   }
 
-  /** check to see if a command needs to be executed and execute if needed */
+  /** Check to see if a command needs to be executed and execute if needed. */
   protected void run() throws IOException {
-    if (lastTime + interval > Time.monotonicNow())
+    if (lastTime + interval > Time.monotonicNow()) {
       return;
+    }
     exitCode = 0; // reset for next run
     runCommand();
   }
 
-  /** Run a command */
+  /** Run the command. */
   private void runCommand() throws IOException { 
     ProcessBuilder builder = new ProcessBuilder(getExecString());
     Timer timeOutTimer = null;
     ShellTimeoutTimerTask timeoutTimerTask = null;
-    timedOut = new AtomicBoolean(false);
-    completed = new AtomicBoolean(false);
-    
+    timedOut.set(false);
+    completed.set(false);
+
     if (environment != null) {
       builder.environment().putAll(this.environment);
     }
@@ -539,11 +860,11 @@ abstract public class Shell {
     final BufferedReader errReader = 
             new BufferedReader(new InputStreamReader(
                 process.getErrorStream(), Charset.defaultCharset()));
-    BufferedReader inReader = 
+    BufferedReader inReader =
             new BufferedReader(new InputStreamReader(
                 process.getInputStream(), Charset.defaultCharset()));
     final StringBuffer errMsg = new StringBuffer();
-    
+
     // read error and input streams as this would free up the buffers
     // free the error stream buffer
     Thread errThread = new Thread() {
@@ -641,28 +962,30 @@ abstract public class Shell {
     }
   }
 
-  /** return an array containing the command name & its parameters */ 
+  /** return an array containing the command name and its parameters. */
   protected abstract String[] getExecString();
-  
+
   /** Parse the execution result */
   protected abstract void parseExecResult(BufferedReader lines)
   throws IOException;
 
-  /** 
-   * Get the environment variable
+  /**
+   * Get an environment variable.
+   * @param env the environment var
+   * @return the value or null if it was unset.
    */
   public String getEnvironment(String env) {
     return environment.get(env);
   }
-  
-  /** get the current sub-process executing the given command 
+
+  /** get the current sub-process executing the given command.
    * @return process executing the command
    */
   public Process getProcess() {
     return process;
   }
 
-  /** get the exit code 
+  /** get the exit code.
    * @return the exit code of the process
    */
   public int getExitCode() {
@@ -674,12 +997,12 @@ abstract public class Shell {
    */
   public static class ExitCodeException extends IOException {
     private final int exitCode;
-    
+
     public ExitCodeException(int exitCode, String message) {
       super(message);
       this.exitCode = exitCode;
     }
-    
+
     public int getExitCode() {
       return exitCode;
     }
@@ -694,7 +1017,7 @@ abstract public class Shell {
       return sb.toString();
     }
   }
-  
+
   public interface CommandExecutor {
 
     void execute() throws IOException;
@@ -704,33 +1027,33 @@ abstract public class Shell {
     String getOutput() throws IOException;
 
     void close();
-    
+
   }
-  
+
   /**
    * A simple shell command executor.
    * 
-   * <code>ShellCommandExecutor</code>should be used in cases where the output 
-   * of the command needs no explicit parsing and where the command, working 
-   * directory and the environment remains unchanged. The output of the command 
+   * <code>ShellCommandExecutor</code>should be used in cases where the output
+   * of the command needs no explicit parsing and where the command, working
+   * directory and the environment remains unchanged. The output of the command
    * is stored as-is and is expected to be small.
    */
   public static class ShellCommandExecutor extends Shell 
       implements CommandExecutor {
-    
+
     private String[] command;
     private StringBuffer output;
-    
-    
+
+
     public ShellCommandExecutor(String[] execString) {
       this(execString, null);
     }
-    
+
     public ShellCommandExecutor(String[] execString, File dir) {
       this(execString, dir, null);
     }
-   
-    public ShellCommandExecutor(String[] execString, File dir, 
+
+    public ShellCommandExecutor(String[] execString, File dir,
                                  Map<String, String> env) {
       this(execString, dir, env , 0L);
     }
@@ -746,7 +1069,7 @@ abstract public class Shell {
      *            key-value pairs specified in the map. If null, the current
      *            environment is not modified.
      * @param timeout Specifies the time in milliseconds, after which the
-     *                command will be killed and the status marked as timedout.
+     *                command will be killed and the status marked as timed-out.
      *                If 0, the command will not be timed out. 
      */
     public ShellCommandExecutor(String[] execString, File dir, 
@@ -760,10 +1083,19 @@ abstract public class Shell {
       }
       timeOutInterval = timeout;
     }
-        
 
-    /** Execute the shell command. */
+    /**
+     * Execute the shell command.
+     * @throws IOException if the command fails, or if the command is
+     * not well constructed.
+     */
     public void execute() throws IOException {
+      for (String s : command) {
+        if (s == null) {
+          throw new IOException("(null) entry in command string: "
+              + StringUtils.join(" ", command));
+        }
+      }
       this.run();    
     }
 
@@ -781,8 +1113,8 @@ abstract public class Shell {
         output.append(buf, 0, nRead);
       }
     }
-    
-    /** Get the output of the shell command.*/
+
+    /** Get the output of the shell command. */
     public String getOutput() {
       return (output == null) ? "" : output.toString();
     }
@@ -813,7 +1145,7 @@ abstract public class Shell {
     public void close() {
     }
   }
-  
+
   /**
    * To check if the passed script to shell command executor timed out or
    * not.
@@ -823,15 +1155,15 @@ abstract public class Shell {
   public boolean isTimedOut() {
     return timedOut.get();
   }
-  
+
   /**
-   * Set if the command has timed out.
+   * Declare that the command has timed out.
    * 
    */
   private void setTimedOut() {
     this.timedOut.set(true);
   }
-  
+
   /** 
    * Static method to execute a shell command. 
    * Covers most of the simple cases without requiring the user to implement  
@@ -842,17 +1174,18 @@ abstract public class Shell {
   public static String execCommand(String ... cmd) throws IOException {
     return execCommand(null, cmd, 0L);
   }
-  
-  /** 
-   * Static method to execute a shell command. 
-   * Covers most of the simple cases without requiring the user to implement  
+
+  /**
+   * Static method to execute a shell command.
+   * Covers most of the simple cases without requiring the user to implement
    * the <code>Shell</code> interface.
    * @param env the map of environment key=value
    * @param cmd shell command to execute.
    * @param timeout time in milliseconds after which script should be marked timeout
-   * @return the output of the executed command.o
+   * @return the output of the executed command.
+   * @throws IOException on any problem.
    */
-  
+
   public static String execCommand(Map<String, String> env, String[] cmd,
       long timeout) throws IOException {
     ShellCommandExecutor exec = new ShellCommandExecutor(cmd, null, env, 
@@ -861,25 +1194,26 @@ abstract public class Shell {
     return exec.getOutput();
   }
 
-  /** 
-   * Static method to execute a shell command. 
-   * Covers most of the simple cases without requiring the user to implement  
+  /**
+   * Static method to execute a shell command.
+   * Covers most of the simple cases without requiring the user to implement
    * the <code>Shell</code> interface.
    * @param env the map of environment key=value
    * @param cmd shell command to execute.
    * @return the output of the executed command.
+   * @throws IOException on any problem.
    */
-  public static String execCommand(Map<String,String> env, String ... cmd) 
+  public static String execCommand(Map<String,String> env, String ... cmd)
   throws IOException {
     return execCommand(env, cmd, 0L);
   }
-  
+
   /**
    * Timer which is used to timeout scripts spawned off by shell.
    */
   private static class ShellTimeoutTimerTask extends TimerTask {
 
-    private Shell shell;
+    private final Shell shell;
 
     public ShellTimeoutTimerTask(Shell shell) {
       this.shell = shell;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
index f3fb364..1fd036e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
@@ -70,9 +70,10 @@ public class SysInfoWindows extends SysInfo {
   }
 
   String getSystemInfoInfoFromShell() {
-    ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
-        new String[] {Shell.WINUTILS, "systeminfo" });
     try {
+      ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
+          new String[] {Shell.getWinutilsFile().getCanonicalPath(),
+              "systeminfo" });
       shellExecutor.execute();
       return shellExecutor.getOutput();
     } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
index 8018946..9d66732 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
@@ -52,13 +52,6 @@ public abstract class SymlinkBaseTest {
   abstract protected String testBaseDir2() throws IOException;
   abstract protected URI testURI();
 
-  // Returns true if the filesystem is emulating symlink support. Certain
-  // checks will be bypassed if that is the case.
-  //
-  protected boolean emulatingSymlinksOnWindows() {
-    return false;
-  }
-
   protected IOException unwrapException(IOException e) {
     return e;
   }
@@ -235,7 +228,6 @@ public abstract class SymlinkBaseTest {
   @Test(timeout=10000)
   /** Stat a link to a file */
   public void testStatLinkToFile() throws IOException {
-    assumeTrue(!emulatingSymlinksOnWindows());
     Path file = new Path(testBaseDir1()+"/file");
     Path linkToFile = new Path(testBaseDir1()+"/linkToFile");
     createAndWriteFile(file);
@@ -362,11 +354,6 @@ public abstract class SymlinkBaseTest {
   private void checkLink(Path linkAbs, Path expectedTarget, Path targetQual)
       throws IOException {
 
-    // If we are emulating symlinks then many of these checks will fail
-    // so we skip them.
-    //
-    assumeTrue(!emulatingSymlinksOnWindows());
-
     Path dir = new Path(testBaseDir1());
     // isFile/Directory
     assertTrue(wrapper.isFile(linkAbs));
@@ -663,7 +650,6 @@ public abstract class SymlinkBaseTest {
   @Test(timeout=10000)
   /** Create symlink through a symlink */
   public void testCreateLinkViaLink() throws IOException {
-    assumeTrue(!emulatingSymlinksOnWindows());
     Path dir1        = new Path(testBaseDir1());
     Path file        = new Path(testBaseDir1(), "file");
     Path linkToDir   = new Path(testBaseDir2(), "linkToDir");
@@ -706,7 +692,6 @@ public abstract class SymlinkBaseTest {
   @Test(timeout=10000)
   /** Test create symlink using the same path */
   public void testCreateLinkTwice() throws IOException {
-    assumeTrue(!emulatingSymlinksOnWindows());
     Path file = new Path(testBaseDir1(), "file");
     Path link = new Path(testBaseDir1(), "linkToFile");
     createAndWriteFile(file);
@@ -895,8 +880,7 @@ public abstract class SymlinkBaseTest {
     assertFalse(wrapper.exists(linkViaLink));
     // Check that we didn't rename the link target
     assertTrue(wrapper.exists(file));
-    assertTrue(wrapper.getFileLinkStatus(linkNewViaLink).isSymlink() ||
-        emulatingSymlinksOnWindows());
+    assertTrue(wrapper.getFileLinkStatus(linkNewViaLink).isSymlink());
     readFile(linkNewViaLink);
   }
 
@@ -1034,8 +1018,7 @@ public abstract class SymlinkBaseTest {
     createAndWriteFile(file);
     wrapper.createSymlink(file, link1, false);
     wrapper.rename(link1, link2);
-    assertTrue(wrapper.getFileLinkStatus(link2).isSymlink() ||
-        emulatingSymlinksOnWindows());
+    assertTrue(wrapper.getFileLinkStatus(link2).isSymlink());
     readFile(link2);
     readFile(file);
     assertFalse(wrapper.exists(link1));
@@ -1059,11 +1042,8 @@ public abstract class SymlinkBaseTest {
     }
     wrapper.rename(link, file1, Rename.OVERWRITE);
     assertFalse(wrapper.exists(link));
-
-    if (!emulatingSymlinksOnWindows()) {
-      assertTrue(wrapper.getFileLinkStatus(file1).isSymlink());
-      assertEquals(file2, wrapper.getLinkTarget(file1));
-    }
+    assertTrue(wrapper.getFileLinkStatus(file1).isSymlink());
+    assertEquals(file2, wrapper.getLinkTarget(file1));
   }
 
   @Test(timeout=10000)
@@ -1125,7 +1105,6 @@ public abstract class SymlinkBaseTest {
   @Test(timeout=10000)
   /** Rename a symlink */
   public void testRenameSymlink() throws IOException {
-    assumeTrue(!emulatingSymlinksOnWindows());
     Path file  = new Path(testBaseDir1(), "file");
     Path link1 = new Path(testBaseDir1(), "linkToFile1");
     Path link2 = new Path(testBaseDir1(), "linkToFile2");
@@ -1223,7 +1202,6 @@ public abstract class SymlinkBaseTest {
   @Test(timeout=10000)
   /** Test rename the symlink's target */
   public void testRenameLinkTarget() throws IOException {
-    assumeTrue(!emulatingSymlinksOnWindows());
     Path file    = new Path(testBaseDir1(), "file");
     Path fileNew = new Path(testBaseDir1(), "fileNew");
     Path link    = new Path(testBaseDir1(), "linkToFile");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 3418ade..5fc0b2d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -983,13 +983,7 @@ public class TestFileUtil {
     file.delete();
     Assert.assertFalse(file.exists());
 
-    if (Shell.WINDOWS && !Shell.isJava7OrAbove()) {
-      // On Java6 on Windows, we copied the file
-      Assert.assertEquals(data.length, link.length());
-    } else {
-      // Otherwise, the target file size is zero
-      Assert.assertEquals(0, link.length());
-    }
+    Assert.assertEquals(0, link.length());
 
     link.delete();
     Assert.assertFalse(link.exists());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java
index 602af97..ab55f00 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java
@@ -69,16 +69,6 @@ abstract public class TestSymlinkLocalFS extends SymlinkBaseTest {
   }
 
   @Override
-  protected boolean emulatingSymlinksOnWindows() {
-    // Java 6 on Windows has very poor symlink support. Specifically
-    // Specifically File#length and File#renameTo do not work as expected.
-    // (see HADOOP-9061 for additional details)
-    // Hence some symlink tests will be skipped.
-    //
-    return (Shell.WINDOWS && !Shell.isJava7OrAbove());
-  }
-
-  @Override
   public void testCreateDanglingLink() throws IOException {
     // Dangling symlinks are not supported on Windows local file system.
     assumeTrue(!Path.WINDOWS);
@@ -186,7 +176,6 @@ abstract public class TestSymlinkLocalFS extends SymlinkBaseTest {
    * file scheme (eg file://host/tmp/test).
    */  
   public void testGetLinkStatusPartQualTarget() throws IOException {
-    assumeTrue(!emulatingSymlinksOnWindows());
     Path fileAbs  = new Path(testBaseDir1()+"/file");
     Path fileQual = new Path(testURI().toString(), fileAbs);
     Path dir      = new Path(testBaseDir1());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 5b8eac6..6e27964 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -218,7 +218,8 @@ public class TestUserGroupInformation {
     }
     // get the groups
     pp = Runtime.getRuntime().exec(Shell.WINDOWS ?
-      Shell.WINUTILS + " groups -F" : "id -Gn");
+      Shell.getWinutilsPath() + " groups -F"
+      : "id -Gn");
     br = new BufferedReader(new InputStreamReader(pp.getInputStream()));
     String line = br.readLine();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c59af2fd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index fc202da..138d025 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -17,11 +17,12 @@
  */
 package org.apache.hadoop.util;
 
-import junit.framework.TestCase;
+import org.apache.commons.io.FileUtils;
 import org.junit.Assert;
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -30,8 +31,31 @@ import java.lang.management.ThreadInfo;
 import java.lang.management.ThreadMXBean;
 
 import org.apache.hadoop.fs.FileUtil;
+import static org.apache.hadoop.util.Shell.*;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
 
-public class TestShell extends TestCase {
+public class TestShell extends Assert {
+  /**
+   * Set the timeout for every test
+   */
+  @Rule
+  public Timeout testTimeout = new Timeout(30000);
+
+  @Rule
+  public TestName methodName = new TestName();
+
+  private File rootTestDir = new File(System.getProperty("test.build.data", "target/"));
+
+  /**
+   * A filename generated uniquely for each test method. The file
+   * itself is neither created nor deleted during test setup/teardown.
+   */
+  private File methodDir;
 
   private static class Command extends Shell {
     private int runCount = 0;
@@ -45,7 +69,7 @@ public class TestShell extends TestCase {
       // There is no /bin/echo equivalent on Windows so just launch it as a
       // shell built-in.
       //
-      return Shell.WINDOWS ?
+      return WINDOWS ?
           (new String[] {"cmd.exe", "/c", "echo", "hello"}) :
           (new String[] {"echo", "hello"});
     }
@@ -60,6 +84,14 @@ public class TestShell extends TestCase {
     }
   }
 
+  @Before
+  public void setup() {
+    rootTestDir.mkdirs();
+    assertTrue("Not a directory " + rootTestDir, rootTestDir.isDirectory());
+    methodDir = new File(rootTestDir, methodName.getMethodName());
+  }
+
+  @Test
   public void testInterval() throws IOException {
     testInterval(Long.MIN_VALUE / 60000);  // test a negative interval
     testInterval(0L);  // test a zero interval
@@ -79,6 +111,7 @@ public class TestShell extends TestCase {
     }
   }
 
+  @Test
   public void testShellCommandExecutorToString() throws Throwable {
     Shell.ShellCommandExecutor sce=new Shell.ShellCommandExecutor(
             new String[] { "ls", "..","arg 2"});
@@ -87,30 +120,28 @@ public class TestShell extends TestCase {
     assertInString(command, " .. ");
     assertInString(command, "\"arg 2\"");
   }
-  
+
+  @Test
   public void testShellCommandTimeout() throws Throwable {
-    if(Shell.WINDOWS) {
-      // setExecutable does not work on Windows
-      return;
-    }
-    String rootDir = new File(System.getProperty(
-        "test.build.data", "/tmp")).getAbsolutePath();
+    Assume.assumeFalse(WINDOWS);
+    String rootDir = rootTestDir.getAbsolutePath();
     File shellFile = new File(rootDir, "timeout.sh");
     String timeoutCommand = "sleep 4; echo \"hello\"";
-    PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
-    writer.println(timeoutCommand);
-    writer.close();
+    Shell.ShellCommandExecutor shexc;
+    try (PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile))) {
+      writer.println(timeoutCommand);
+      writer.close();
+    }
     FileUtil.setExecutable(shellFile, true);
-    Shell.ShellCommandExecutor shexc 
-    = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
-                                      null, null, 100);
+    shexc = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
+        null, null, 100);
     try {
       shexc.execute();
     } catch (Exception e) {
       //When timing out exception is thrown.
     }
     shellFile.delete();
-    assertTrue("Script didnt not timeout" , shexc.isTimedOut());
+    assertTrue("Script did not timeout" , shexc.isTimedOut());
   }
   
   private static int countTimerThreads() {
@@ -129,7 +160,8 @@ public class TestShell extends TestCase {
     }
     return count;
   }
-  
+
+  @Test
   public void testShellCommandTimerLeak() throws Exception {
     String quickCommand[] = new String[] {"/bin/sleep", "100"};
     
@@ -152,16 +184,17 @@ public class TestShell extends TestCase {
     assertEquals(timersBefore, timersAfter);
   }
 
+  @Test
   public void testGetCheckProcessIsAliveCommand() throws Exception {
     String anyPid = "9999";
-    String[] checkProcessAliveCommand = Shell.getCheckProcessIsAliveCommand(
+    String[] checkProcessAliveCommand = getCheckProcessIsAliveCommand(
         anyPid);
 
     String[] expectedCommand;
 
     if (Shell.WINDOWS) {
       expectedCommand =
-          new String[]{ Shell.WINUTILS, "task", "isAlive", anyPid };
+          new String[]{getWinutilsPath(), "task", "isAlive", anyPid };
     } else if (Shell.isSetsidAvailable) {
       expectedCommand = new String[] { "bash", "-c", "kill -0 -- -" + anyPid };
     } else {
@@ -170,17 +203,18 @@ public class TestShell extends TestCase {
     Assert.assertArrayEquals(expectedCommand, checkProcessAliveCommand);
   }
 
+  @Test
   public void testGetSignalKillCommand() throws Exception {
     String anyPid = "9999";
     int anySignal = 9;
-    String[] checkProcessAliveCommand = Shell.getSignalKillCommand(anySignal,
+    String[] checkProcessAliveCommand = getSignalKillCommand(anySignal,
         anyPid);
 
     String[] expectedCommand;
 
     if (Shell.WINDOWS) {
       expectedCommand =
-          new String[]{ Shell.WINUTILS, "task", "isAlive", anyPid };
+          new String[]{getWinutilsPath(), "task", "kill", anyPid };
     } else if (Shell.isSetsidAvailable) {
       expectedCommand = new String[] { "bash", "-c", "kill -9 -- -" + anyPid };
     } else {
@@ -188,7 +222,6 @@ public class TestShell extends TestCase {
     }
     Assert.assertArrayEquals(expectedCommand, checkProcessAliveCommand);
   }
-  
 
   private void testInterval(long interval) throws IOException {
     Command command = new Command(interval);
@@ -203,4 +236,190 @@ public class TestShell extends TestCase {
       assertEquals(2, command.getRunCount());
     }
   }
+
+  @Test
+  public void testHadoopHomeUnset() throws Throwable {
+    assertHomeResolveFailed(null, "unset");
+  }
+
+  @Test
+  public void testHadoopHomeEmpty() throws Throwable {
+    assertHomeResolveFailed("", E_HADOOP_PROPS_EMPTY);
+  }
+
+  @Test
+  public void testHadoopHomeEmptyDoubleQuotes() throws Throwable {
+    assertHomeResolveFailed("\"\"", E_HADOOP_PROPS_EMPTY);
+  }
+
+  @Test
+  public void testHadoopHomeEmptySingleQuote() throws Throwable {
+    assertHomeResolveFailed("\"", E_HADOOP_PROPS_EMPTY);
+  }
+
+  @Test
+  public void testHadoopHomeValid() throws Throwable {
+    File f = checkHadoopHomeInner(rootTestDir.getCanonicalPath());
+    assertEquals(rootTestDir, f);
+  }
+
+  @Test
+  public void testHadoopHomeValidQuoted() throws Throwable {
+    File f = checkHadoopHomeInner('"'+ rootTestDir.getCanonicalPath() + '"');
+    assertEquals(rootTestDir, f);
+  }
+
+  @Test
+  public void testHadoopHomeNoDir() throws Throwable {
+    assertHomeResolveFailed(methodDir.getCanonicalPath(), E_DOES_NOT_EXIST);
+  }
+
+  @Test
+  public void testHadoopHomeNotADir() throws Throwable {
+    File touched = touch(methodDir);
+    try {
+      assertHomeResolveFailed(touched.getCanonicalPath(), E_NOT_DIRECTORY);
+    } finally {
+      FileUtils.deleteQuietly(touched);
+    }
+  }
+
+  @Test
+  public void testHadoopHomeRelative() throws Throwable {
+    assertHomeResolveFailed("./target", E_IS_RELATIVE);
+  }
+
+  @Test
+  public void testBinDirMissing() throws Throwable {
+    FileNotFoundException ex = assertWinutilsResolveFailed(methodDir,
+        E_DOES_NOT_EXIST);
+    assertInString(ex.toString(), "Hadoop bin directory");
+  }
+
+  @Test
+  public void testHadoopBinNotADir() throws Throwable {
+    File bin = new File(methodDir, "bin");
+    touch(bin);
+    try {
+      assertWinutilsResolveFailed(methodDir, E_NOT_DIRECTORY);
+    } finally {
+      FileUtils.deleteQuietly(methodDir);
+    }
+  }
+
+  @Test
+  public void testBinWinUtilsFound() throws Throwable {
+    try {
+      File bin = new File(methodDir, "bin");
+      File winutils = new File(bin, WINUTILS_EXE);
+      touch(winutils);
+      assertEquals(winutils.getCanonicalPath(),
+          getQualifiedBinInner(methodDir, WINUTILS_EXE).getCanonicalPath());
+    } finally {
+      FileUtils.deleteQuietly(methodDir);
+    }
+  }
+
+  @Test
+  public void testBinWinUtilsNotAFile() throws Throwable {
+    try {
+      File bin = new File(methodDir, "bin");
+      File winutils = new File(bin, WINUTILS_EXE);
+      winutils.mkdirs();
+      assertWinutilsResolveFailed(methodDir, E_NOT_EXECUTABLE_FILE);
+    } finally {
+      FileUtils.deleteDirectory(methodDir);
+    }
+  }
+
+  /**
+   * This test takes advantage of the invariant winutils path is valid
+   * or access to it will raise an exception holds on Linux, and without
+   * any winutils binary even if HADOOP_HOME points to a real hadoop
+   * directory, the exception reporting can be validated
+   */
+  @Test
+  public void testNoWinutilsOnUnix() throws Throwable {
+    Assume.assumeFalse(WINDOWS);
+    try {
+      getWinutilsFile();
+    } catch (FileNotFoundException ex) {
+      assertExContains(ex, E_NOT_A_WINDOWS_SYSTEM);
+    }
+    try {
+      getWinutilsPath();
+    } catch (RuntimeException ex) {
+      assertExContains(ex, E_NOT_A_WINDOWS_SYSTEM);
+      if ( ex.getCause() == null
+          || !(ex.getCause() instanceof FileNotFoundException)) {
+        throw ex;
+      }
+    }
+  }
+
+  /**
+   * Touch a file; creating parent dirs on demand.
+   * @param path path of file
+   * @return the file created
+   * @throws IOException on any failure to write
+   */
+  private File touch(File path) throws IOException {
+    path.getParentFile().mkdirs();
+    FileUtils.writeByteArrayToFile(path, new byte[]{});
+    return path;
+  }
+
+  /**
+   * Assert that an attept to resolve the hadoop home dir failed with
+   * an expected text in the exception string value.
+   * @param path input
+   * @param expectedText expected exception text
+   * @return the caught exception
+   * @throws FileNotFoundException any FileNotFoundException that was thrown
+   * but which did not contain the expected text
+   */
+  private FileNotFoundException assertHomeResolveFailed(String path,
+      String expectedText) throws Exception {
+    try {
+      File f = checkHadoopHomeInner(path);
+      fail("Expected an exception with the text `" + expectedText + "`"
+          + " -but got the path " + f);
+      // unreachable
+      return null;
+    } catch (FileNotFoundException ex) {
+      assertExContains(ex, expectedText);
+      return ex;
+    }
+  }
+
+  /**
+   * Assert that an attept to resolve the {@code bin/winutils.exe} failed with
+   * an expected text in the exception string value.
+   * @param hadoopHome hadoop home directory
+   * @param expectedText expected exception text
+   * @return the caught exception
+   * @throws Exception any Exception that was thrown
+   * but which did not contain the expected text
+   */
+  private FileNotFoundException assertWinutilsResolveFailed(File hadoopHome,
+      String expectedText) throws Exception {
+    try {
+      File f = getQualifiedBinInner(hadoopHome, WINUTILS_EXE);
+      fail("Expected an exception with the text `" + expectedText + "`"
+          + " -but got the path " + f);
+      // unreachable
+      return null;
+    } catch (FileNotFoundException ex) {
+      assertExContains(ex, expectedText);
+      return ex;
+    }
+  }
+
+  private void assertExContains(Exception ex, String expectedText)
+      throws Exception {
+    if (!ex.toString().contains(expectedText)) {
+      throw ex;
+    }
+  }
+
 }


Mime
View raw message