hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1389653 - in /hadoop/common/branches/branch-1-win: ./ src/contrib/streaming/src/test/org/apache/hadoop/streaming/ src/test/org/apache/hadoop/mapred/
Date Mon, 24 Sep 2012 23:27:51 GMT
Author: vinodkv
Date: Mon Sep 24 23:27:50 2012
New Revision: 1389653

URL: http://svn.apache.org/viewvc?rev=1389653&view=rev
Log:
HADOOP-8733. Fix the tests TestStreamingTaskLog, TestJvmManager and TestLinuxTaskControllerLaunchArgs
to not fail on Windows. Contributed by Ivan Mitic.

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
    hadoop/common/branches/branch-1-win/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestLinuxTaskControllerLaunchArgs.java

Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1389653&r1=1389652&r2=1389653&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Mon Sep 24 23:27:50 2012
@@ -133,3 +133,7 @@ BUG FIXES
 
     HADOOP-8457. Address file ownership issue for users in Admin group on
     windows. (Ivan Mitic via suresh)
+
+    HADOOP-8733. Fix the tests TestStreamingTaskLog, TestJvmManager
+    and TestLinuxTaskControllerLaunchArgs to not fail on Windows. (Ivan Mitic via
+    vinodkv)

Modified: hadoop/common/branches/branch-1-win/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java?rev=1389653&r1=1389652&r2=1389653&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java
(original)
+++ hadoop/common/branches/branch-1-win/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java
Mon Sep 24 23:27:50 2012
@@ -22,6 +22,7 @@ import java.io.*;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
@@ -77,7 +78,7 @@ public class TestStreamingTaskLog extend
       }
       fs.mkdirs(testDir);
       File scriptFile = createScript(
-          testDir.toString() + "/testTaskLog.sh");
+          testDir.toString());
       mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1);
       
       writeInputFile(fs, inputPath);
@@ -97,16 +98,19 @@ public class TestStreamingTaskLog extend
     }
   }
 
-  private File createScript(String script) throws IOException {
-    File scriptFile = new File(script);
+  private File createScript(String testDir) throws IOException {
+    File scriptFile = new File(testDir,
+      "testTaskLog" + ((Shell.WINDOWS) ? ".cmd" : ".sh"));
     UtilTest.recursiveDelete(scriptFile);
     FileOutputStream in = new FileOutputStream(scriptFile);
-    in.write(("cat > /dev/null 2>&1\n" +
-              "echo $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS").getBytes());
+    in.write((Shell.WINDOWS ?
+      "cat > NUL 2>&1\necho %HADOOP_ROOT_LOGGER% %HADOOP_CLIENT_OPTS%" :
+      "cat > /dev/null 2>&1\necho $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS")
+      .getBytes());
     in.close();
-    
-    Shell.execCommand(new String[]{"chmod", "+x",
-                                   scriptFile.getAbsolutePath()});
+
+    FileUtil.chmod(scriptFile.getAbsolutePath(), "+x", false);
+
     return scriptFile;
   }
   

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java?rev=1389653&r1=1389652&r2=1389653&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
Mon Sep 24 23:27:50 2012
@@ -37,6 +37,8 @@ import org.apache.hadoop.mapred.UtilsFor
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.server.tasktracker.userlogs.UserLogManager;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
 import org.junit.After;
 import static org.junit.Assert.*;
 import org.junit.Before;
@@ -52,6 +54,11 @@ public class TestJvmManager {
   private JobConf ttConf;
   private boolean threadCaughtException = false;
   private String user;
+  private final String sleepScriptName = Shell.WINDOWS ? "SLEEP.cmd" : "SLEEP";
+  private final String sleepCommand = "sleep 60\n";
+  private final String lsScriptName = Shell.WINDOWS ? "LS.cmd" : "LS";
+  private final String lsCommand =
+      StringUtils.join(" ", Shell.getGetPermissionCommand());
 
   @Before
   public void setUp() {
@@ -89,10 +96,13 @@ public class TestJvmManager {
   private File writeScript(String fileName, String cmd, File pidFile) throws IOException
{
     File script = new File(TEST_DIR, fileName);
     FileOutputStream out = new FileOutputStream(script);
-    // write pid into a file
-    out.write(("echo $$ >" + pidFile.toString() + ";").getBytes());
-    // ignore SIGTERM
-    out.write(("trap '' 15\n").getBytes());
+    // write pid into a file and ignore SIGTERM
+    String command = Shell.WINDOWS
+        // On Windows we pass back the attempt id that was passed to the task
+        // through the environment.
+      ? "echo %ATTEMPT_ID% > " + pidFile.toString() + "\r\n"
+      : "echo $$ >" + pidFile.toString() + ";\n trap '' 15\n";
+    out.write((command).getBytes());
     // write the actual command it self.
     out.write(cmd.getBytes());
     out.close();
@@ -130,8 +140,11 @@ public class TestJvmManager {
       newTaskDistributedCacheManager(attemptID.getJobID(), taskConf);
     final TaskRunner taskRunner = task.createRunner(tt, tip, rjob);
     // launch a jvm which sleeps for 60 seconds
+    final Vector<String> setup = new Vector<String>(1);
+    setup.add((Shell.WINDOWS ? "set " : "export ")
+      + "ATTEMPT_ID=" + attemptID.toString());
     final Vector<String> vargs = new Vector<String>(2);
-    vargs.add(writeScript("SLEEP", "sleep 60\n", pidFile).getAbsolutePath());
+    vargs.add(writeScript(sleepScriptName, sleepCommand, pidFile).getAbsolutePath());
     final File workDir = new File(TEST_DIR, "work");
     final File stdout = new File(TEST_DIR, "stdout");
     final File stderr = new File(TEST_DIR, "stderr");
@@ -140,7 +153,7 @@ public class TestJvmManager {
     Thread launcher = new Thread() {
       public void run() {
         try {
-          taskRunner.launchJvmAndWait(null, vargs, stdout, stderr, 100,
+          taskRunner.launchJvmAndWait(setup, vargs, stdout, stderr, 100,
               workDir);
         } catch (InterruptedException e) {
           e.printStackTrace();
@@ -204,7 +217,7 @@ public class TestJvmManager {
     TaskRunner taskRunner2 = task.createRunner(tt, tip, rjob);
     // build dummy vargs to call ls
     Vector<String> vargs2 = new Vector<String>(1);
-    vargs2.add(writeScript("LS", "ls", pidFile).getAbsolutePath());
+    vargs2.add(writeScript(lsScriptName, lsCommand, pidFile).getAbsolutePath());
     File workDir2 = new File(TEST_DIR, "work2");
     File stdout2 = new File(TEST_DIR, "stdout2");
     File stderr2 = new File(TEST_DIR, "stderr2");

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestLinuxTaskControllerLaunchArgs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestLinuxTaskControllerLaunchArgs.java?rev=1389653&r1=1389652&r2=1389653&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestLinuxTaskControllerLaunchArgs.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestLinuxTaskControllerLaunchArgs.java
Mon Sep 24 23:27:50 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.mapred.TaskTracker.LocalStorage;
 import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.util.Shell;
 
 import junit.framework.TestCase;
 
@@ -120,6 +121,11 @@ public class TestLinuxTaskControllerLaun
    * necessary java properties to launch JobLocalizer successfully.
    */
   public void testLTCCallInitializeJob() throws Exception {
+    if (!Shell.LINUX) {
+      // LinuxTaskContoller is not supported on non-Linux platforms
+      return;
+    }
+
     if (!initialized) {
       initMyTest();
     }
@@ -139,6 +145,11 @@ public class TestLinuxTaskControllerLaun
    * necessary java properties to launch JobLocalizer successfully.
    */
   public void testLTCCallTruncateLogsAsUser() throws Exception {
+    if (!Shell.LINUX) {
+      // LinuxTaskContoller is not supported on non-Linux platforms
+      return;
+    }
+
     if (!initialized) {
       initMyTest();
     }



Mime
View raw message