hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r938385 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JvmManager.java src/java/org/apache/hadoop/mapred/TaskRunner.java src/java/org/apache/hadoop/mapred/TaskTracker.java
Date Tue, 27 Apr 2010 09:55:50 GMT
Author: vinodkv
Date: Tue Apr 27 09:55:49 2010
New Revision: 938385

URL: http://svn.apache.org/viewvc?rev=938385&view=rev
Log:
MAPREDUCE-1397. NullPointerException observed during task failures. Contributed by Amareshwari
Sriramadasu.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JvmManager.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=938385&r1=938384&r2=938385&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Apr 27 09:55:49 2010
@@ -596,6 +596,9 @@ Trunk (unreleased changes)
     MAPREDUCE-1612. job conf file is not accessible from job history web page.
     (Ravi Gummadi and Sreekanth Ramakrishnan via vinodkv)
 
+    MAPREDUCE-1397. NullPointerException observed during task failures.
+    (Amareshwari Sriramadasu via vinodkv)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JvmManager.java?rev=938385&r1=938384&r2=938385&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JvmManager.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JvmManager.java Tue Apr 27 09:55:49
2010
@@ -32,6 +32,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapred.TaskController.TaskControllerContext;
 import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 import org.apache.hadoop.util.StringUtils;
@@ -42,9 +43,9 @@ class JvmManager {
   public static final Log LOG =
     LogFactory.getLog(JvmManager.class);
 
-  JvmManagerForType mapJvmManager;
+  private JvmManagerForType mapJvmManager;
 
-  JvmManagerForType reduceJvmManager;
+  private JvmManagerForType reduceJvmManager;
   
   public JvmEnv constructJvmEnv(List<String> setup, Vector<String>vargs,
       File stdout,File stderr,long logSize, File workDir, 
@@ -58,6 +59,15 @@ class JvmManager {
     reduceJvmManager = new JvmManagerForType(tracker.getMaxCurrentReduceTasks(),
         false, tracker);
   }
+
+  JvmManagerForType getJvmManagerForType(TaskType type) {
+    if (type.equals(TaskType.MAP)) {
+      return mapJvmManager;
+    } else if (type.equals(TaskType.REDUCE)) {
+      return reduceJvmManager;
+    }
+    return null;
+  }
   
   public void stop() {
     mapJvmManager.stop();
@@ -168,7 +178,7 @@ class JvmManager {
           tracker.getTaskController()));
   }
 
-  private static class JvmManagerForType {
+  static class JvmManagerForType {
     //Mapping from the JVM IDs to running Tasks
     Map <JVMId,TaskRunner> jvmToRunningTask = 
       new HashMap<JVMId, TaskRunner>();
@@ -263,10 +273,15 @@ class JvmManager {
     synchronized public void killJvm(JVMId jvmId) {
       JvmRunner jvmRunner;
       if ((jvmRunner = jvmIdToRunner.get(jvmId)) != null) {
-        jvmRunner.kill();
+        killJvmRunner(jvmRunner);
       }
     }
     
+    private synchronized void killJvmRunner(JvmRunner jvmRunner) {
+      jvmRunner.kill();
+      removeJvm(jvmRunner.jvmId);
+    }
+
     synchronized void dumpStack(TaskRunner tr) {
       JVMId jvmId = runningTaskToJvm.get(tr);
       if (null != jvmId) {
@@ -286,7 +301,7 @@ class JvmManager {
       List <JvmRunner> list = new ArrayList<JvmRunner>();
       list.addAll(jvmIdToRunner.values());
       for (JvmRunner jvm : list) {
-        jvm.kill();
+        killJvmRunner(jvm);
       }
     }
     
@@ -350,7 +365,7 @@ class JvmManager {
       if (spawnNewJvm) {
         if (runnerToKill != null) {
           LOG.info("Killing JVM: " + runnerToKill.jvmId);
-          runnerToKill.kill();
+          killJvmRunner(runnerToKill);
         }
         spawnNewJvm(jobId, env, t);
         return;
@@ -412,7 +427,7 @@ class JvmManager {
       }
     }
 
-    private class JvmRunner extends Thread {
+    class JvmRunner extends Thread {
       JvmEnv env;
       volatile boolean killed = false;
       volatile int numTasksRan;
@@ -472,9 +487,8 @@ class JvmManager {
        * Kills the process. Also kills its subprocesses if the process(root of subtree
        * of processes) is created using setsid.
        */
-      public void kill() {
+      synchronized void kill() {
         if (!killed) {
-          killed = true;
           TaskController controller = tracker.getTaskController();
           // Check inital context before issuing a kill to prevent situations
           // where kill is issued before task is launched.
@@ -490,7 +504,7 @@ class JvmManager {
             LOG.info(String.format("JVM Not killed %s but just removed", jvmId
                 .toString()));
           }
-          removeJvm(jvmId);
+          killed = true;
         }
       }
 

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=938385&r1=938384&r2=938385&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java Tue Apr 27 09:55:49
2010
@@ -226,14 +226,7 @@ abstract class TaskRunner extends Thread
       errorInfo = getVMEnvironment(errorInfo, workDir, conf, env,
                                    taskid, logSize);
 
-      jvmManager.launchJvm(this, 
-          jvmManager.constructJvmEnv(setup,vargs,stdout,stderr,logSize, 
-              workDir, env, conf));
-      synchronized (lock) {
-        while (!done) {
-          lock.wait();
-        }
-      }
+      launchJvmAndWait(setup, vargs, stdout, stderr, logSize, workDir, env);
       tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID());
       if (exitCodeSet) {
         if (!killed && exitCode != 0) {
@@ -277,6 +270,18 @@ abstract class TaskRunner extends Thread
     }
   }
 
+  void launchJvmAndWait(List<String> setup, Vector<String> vargs, File stdout,
+      File stderr, long logSize, File workDir, Map<String, String> env)
+      throws InterruptedException {
+    jvmManager.launchJvm(this, jvmManager.constructJvmEnv(setup, vargs, stdout,
+        stderr, logSize, workDir, env, conf));
+    synchronized (lock) {
+      while (!done) {
+        lock.wait();
+      }
+    }
+  }
+
   /**
    * Prepare the log files for the task
    * 

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=938385&r1=938384&r2=938385&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Tue Apr 27 09:55:49
2010
@@ -258,7 +258,7 @@ public class TaskTracker 
   private MapEventsFetcherThread mapEventsFetcher;
   int workerThreads;
   CleanupQueue directoryCleanupThread;
-  volatile JvmManager jvmManager;
+  private volatile JvmManager jvmManager;
   UserLogCleaner taskLogCleanupThread;
   private TaskMemoryManagerThread taskMemoryManager;
   private boolean taskMemoryManagerEnabled = true;
@@ -2064,7 +2064,12 @@ public class TaskTracker 
   public JvmManager getJvmManagerInstance() {
     return jvmManager;
   }
-  
+
+  // called from unit test  
+  void setJvmManagerInstance(JvmManager jvmManager) {
+    this.jvmManager = jvmManager;
+  }
+
   private void addToTaskQueue(LaunchTaskAction action) {
     if (action.getTask().isMapTask()) {
       mapLauncher.addToTaskQueue(action);
@@ -3721,6 +3726,16 @@ public class TaskTracker 
     return maxReduceSlots;
   }
 
+  //called from unit test
+  synchronized void setMaxMapSlots(int mapSlots) {
+    maxMapSlots = mapSlots;
+  }
+
+  //called from unit test
+  synchronized void setMaxReduceSlots(int reduceSlots) {
+    maxReduceSlots = reduceSlots;
+  }
+
   /**
    * Is the TaskMemoryManager Enabled on this system?
    * @return true if enabled, false otherwise.



Mime
View raw message