hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r758584 - in /hadoop/core/trunk: ./ src/mapred/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/
Date Thu, 26 Mar 2009 10:24:41 GMT
Author: ddas
Date: Thu Mar 26 10:24:22 2009
New Revision: 758584

URL: http://svn.apache.org/viewvc?rev=758584&view=rev
Log:
HADOOP-5194. Disables setsid for tasks run on cygwin. Contributed by Ravi Gummadi.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/mapred/mapred-default.xml
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Mar 26 10:24:22 2009
@@ -333,6 +333,9 @@
     HADOOP-5477. Fix rare failure in TestCLI for hosts returning variations of
     'localhost'. (Jacob Homan via cdouglas)
 
+    HADOOP-5194. Disables setsid for tasks run on cygwin. 
+    (Ravi Gummadi via ddas)
+
 Release 0.20.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/mapred/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/mapred-default.xml?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/mapred-default.xml (original)
+++ hadoop/core/trunk/src/mapred/mapred-default.xml Thu Mar 26 10:24:22 2009
@@ -315,7 +315,8 @@
   <name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
   <value>5000</value>
   <description>The time, in milliseconds, the tasktracker waits for sending a
-  SIGKILL to a process, after it has been sent a SIGTERM.
+  SIGKILL to a task, after it has been sent a SIGTERM. This is currently
+  not used on WINDOWS where tasks are just sent a SIGTERM.
   </description>
 </property>
 

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java Thu Mar 26 10:24:22 2009
@@ -35,6 +35,7 @@
 import org.apache.hadoop.metrics.MetricsContext;
 import org.apache.hadoop.metrics.MetricsUtil;
 import org.apache.hadoop.metrics.jvm.JvmMetrics;
+import org.apache.hadoop.util.Shell;
 import org.apache.log4j.LogManager;
 
 /** 
@@ -123,13 +124,13 @@
         //are viewable immediately
         TaskLog.syncLogs(firstTaskid, taskid, isCleanup);
         JobConf job = new JobConf(task.getJobFile());
-        if (srcPidPath == null) {
+        if (srcPidPath == null && !Shell.WINDOWS) {
           // get the first task's path for the first time
           srcPidPath = new Path(task.getPidFile());
         }
         //since the JVM is running multiple tasks potentially, we need
         //to do symlink stuff only for the subsequent tasks
-        if (!taskid.equals(firstTaskid)) {
+        if (!taskid.equals(firstTaskid) && !Shell.WINDOWS) {
           dstPidPath = new Path(task.getPidFile());
           FileUtil.symLink(srcPidPath.toUri().getPath(), 
               dstPidPath.toUri().getPath());
@@ -155,7 +156,7 @@
           task.run(job, umbilical);             // run the task
         } finally {
           TaskLog.syncLogs(firstTaskid, taskid, isCleanup);
-          if (!taskid.equals(firstTaskid)) {
+          if (!taskid.equals(firstTaskid) && !Shell.WINDOWS) {
             // delete the pid-file's symlink
             new File(dstPidPath.toUri().getPath()).delete();
           }

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JvmManager.java Thu Mar 26 10:24:22
2009
@@ -33,6 +33,7 @@
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 import org.apache.hadoop.util.ProcessTree;
 
@@ -366,19 +367,24 @@
         if (shexec != null) {
           Process process = shexec.getProcess();
           if (process != null) {
-            Path pidFilePath = new Path(env.pidFile);
-            String pid = ProcessTree.getPidFromPidFile(
+            if (Shell.WINDOWS) {
+              process.destroy();
+            }
+            else {
+              Path pidFilePath = new Path(env.pidFile);
+              String pid = ProcessTree.getPidFromPidFile(
                                                     pidFilePath.toString());
 
-            long sleeptimeBeforeSigkill = env.conf.getLong(
-                 "mapred.tasktracker.tasks.sleeptime-before-sigkill",
-                 ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);
-
-            ProcessTree.destroy(pid, sleeptimeBeforeSigkill,
-                     ProcessTree.isSetsidAvailable, false);
-            try {
-              LOG.info("Process exited with exit code:" + process.waitFor());
-            } catch (InterruptedException ie) {}
+              long sleeptimeBeforeSigkill = env.conf.getLong(
+                   "mapred.tasktracker.tasks.sleeptime-before-sigkill",
+                   ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);
+
+              ProcessTree.destroy(pid, sleeptimeBeforeSigkill,
+                       ProcessTree.isSetsidAvailable, false);
+              try {
+                LOG.info("Process exited with exit code:" + process.waitFor());
+              } catch (InterruptedException ie) {}
+            }
           }
         }
         removeJvm(jvmId);

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java Thu Mar 26 10:24:22 2009
@@ -50,6 +50,7 @@
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Shell;
 
 /** Base class for tasks. */
 abstract class Task implements Writable, Configurable {
@@ -313,7 +314,9 @@
     out.writeBoolean(jobSetup);
     out.writeBoolean(writeSkipRecs);
     out.writeBoolean(taskCleanup);  
-    Text.writeString(out, pidFile);
+    if(!Shell.WINDOWS) {
+      Text.writeString(out, pidFile);
+    }
   }
   
   public void readFields(DataInput in) throws IOException {
@@ -333,7 +336,9 @@
     if (taskCleanup) {
       setPhase(TaskStatus.Phase.CLEANUP);
     }
-    pidFile = Text.readString(in);
+    if(!Shell.WINDOWS) {
+      pidFile = Text.readString(in);
+    }
   }
 
   @Override

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java Thu Mar 26 10:24:22
2009
@@ -35,6 +35,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.util.ProcessTree;
+import org.apache.hadoop.util.Shell;
 import org.apache.log4j.Appender;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
@@ -461,7 +462,7 @@
     StringBuffer mergedCmd = new StringBuffer();
     
     // Spit out the pid to pidFileName
-    if (pidFileName != null) {
+    if (pidFileName != null && !Shell.WINDOWS) {
       mergedCmd.append("echo $$ > ");
       mergedCmd.append(pidFileName);
       mergedCmd.append(" ;");
@@ -473,7 +474,8 @@
     }
     if (tailLength > 0) {
       mergedCmd.append("(");
-    } else if(ProcessTree.isSetsidAvailable && useSetsid) {
+    } else if(ProcessTree.isSetsidAvailable && useSetsid &&
+        !Shell.WINDOWS) {
       mergedCmd.append("exec setsid ");
     } else {
       mergedCmd.append("exec ");

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Thu Mar 26 10:24:22
2009
@@ -368,11 +368,15 @@
       vargs.add(Integer.toString(address.getPort())); 
       vargs.add(taskid.toString());                      // pass task identifier
 
-      String pidFile = lDirAlloc.getLocalPathForWrite(
+      String pidFile = "";
+      if (!Shell.WINDOWS) {
+        pidFile = lDirAlloc.getLocalPathForWrite(
             (TaskTracker.getPidFile(t.getJobID().toString(),
                t.getTaskID().toString(), t.isTaskCleanupTask())),
             this.conf).toString();
-      t.setPidFile(pidFile);
+        t.setPidFile(pidFile);
+      }
+      
       tracker.addToMemoryManager(t.getTaskID(), conf, pidFile);
 
       // set memory limit using ulimit if feasible and necessary ...

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java?rev=758584&r1=758583&r2=758584&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKillSubProcesses.java Thu Mar
26 10:24:22 2009
@@ -35,6 +35,7 @@
 
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.ProcessTree;
+import org.apache.hadoop.util.Shell;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -87,7 +88,6 @@
     JobInProgress jip = jt.getJob(job.getID());
     for(TaskReport tr : mapReports) {
       TaskInProgress tip = jip.getTaskInProgress(tr.getTaskID());
-      assertTrue(tip.isRunning());
 
       // for this tip, get active tasks of all attempts
       while(tip.getActiveTasks().size() == 0) {
@@ -240,6 +240,12 @@
   }
 
   public void testJobKill() throws IOException {
+    if (Shell.WINDOWS) {
+      System.out.println(
+             "setsid doesn't work on WINDOWS as expected. Not testing");
+      return;
+    }
+    
     JobConf conf=null;
     try {
       mr = new MiniMRCluster(1, "file:///", 1);



Mime
View raw message