hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r746228 - in /hadoop/core/branches/branch-0.20: ./ src/mapred/org/apache/hadoop/mapred/
Date Fri, 20 Feb 2009 13:17:21 GMT
Author: ddas
Date: Fri Feb 20 13:17:20 2009
New Revision: 746228

URL: http://svn.apache.org/viewvc?rev=746228&view=rev
Log:
Merge -r 746226:746227 from trunk onto 0.20. Fixes HADOOP-5233.

Modified:
    hadoop/core/branches/branch-0.20/   (props changed)
    hadoop/core/branches/branch-0.20/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java

Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 20 13:17:20 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227

Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Fri Feb 20 13:17:20 2009
@@ -620,6 +620,11 @@
     Scheduler accesses the tasktrackers stored by the JobTracker.
     (Rahul Kumar Singh via yhemanth)
 
+    HADOOP-5233. Addresses the three issues - Race condition in updating
+    status, NPE in TaskTracker task localization when the conf file is missing
+    (HADOOP-5234) and NPE in handling KillTaskAction of a cleanup task (HADOOP-5235).
+    (Amareshwari Sriramadasu via ddas)
+
 Release 0.19.1 - Unreleased
 
   IMPROVEMENTS

Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 20 13:17:20 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Feb 20 13:17:20 2009
@@ -815,6 +815,8 @@
         } else {
           reduceCleanupTasks.add(taskid);
         }
+        // Remove the task entry from jobtracker
+        jobtracker.removeTaskEntry(taskid);
       }
       //For a failed task update the JT datastructures. 
       else if (state == TaskStatus.State.FAILED ||

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java Fri Feb
20 13:17:20 2009
@@ -514,8 +514,7 @@
           if (sendProgress) {
             // we need to send progress update
             updateCounters();
-            taskStatus.statusUpdate(getState(),
-                                    taskProgress.get(),
+            taskStatus.statusUpdate(taskProgress.get(),
                                     taskProgress.toString(), 
                                     counters);
             taskFound = umbilical.statusUpdate(taskId, taskStatus);
@@ -702,8 +701,7 @@
   private void sendLastUpdate(TaskUmbilicalProtocol umbilical) 
   throws IOException {
     // send a final status report
-    taskStatus.statusUpdate(getState(),
-                            taskProgress.get(),
+    taskStatus.statusUpdate(taskProgress.get(),
                             taskProgress.toString(), 
                             counters);
     statusUpdate(umbilical);

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
(original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
Fri Feb 20 13:17:20 2009
@@ -915,7 +915,6 @@
       t.setTaskCleanupTask();
       t.setState(taskStatuses.get(taskid).getRunState());
       cleanupTasks.put(taskid, taskTracker);
-      jobtracker.removeTaskEntry(taskid);
     }
     t.setConf(conf);
     LOG.debug("Launching task with skipRanges:"+failedRanges.getSkipRanges());

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java Fri
Feb 20 13:17:20 2009
@@ -54,7 +54,7 @@
   private long finishTime; 
   private long outputSize;
     
-  private Phase phase = Phase.STARTING; 
+  private volatile Phase phase = Phase.STARTING; 
   private Counters counters;
   private boolean includeCounters;
   private SortedRanges.Range nextRecordRange = new SortedRanges.Range();
@@ -267,16 +267,15 @@
   /**
    * Update the status of the task.
    * 
-   * @param runstate
+   * This update is done by ping thread before sending the status. 
+   * 
    * @param progress
    * @param state
    * @param counters
    */
-  synchronized void statusUpdate(State runState, 
-                                 float progress,
+  synchronized void statusUpdate(float progress,
                                  String state, 
                                  Counters counters) {
-    setRunState(runState);
     setProgress(progress);
     setStateString(state);
     setCounters(counters);

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Fri Feb 20 13:17:20 2009
@@ -2020,10 +2020,16 @@
       if (this.done || 
           (this.taskStatus.getRunState() != TaskStatus.State.RUNNING &&
           this.taskStatus.getRunState() != TaskStatus.State.COMMIT_PENDING &&
-          !isCleaningup())) {
+          !isCleaningup()) ||
+          ((this.taskStatus.getRunState() == TaskStatus.State.COMMIT_PENDING ||
+           this.taskStatus.getRunState() == TaskStatus.State.FAILED_UNCLEAN ||
+           this.taskStatus.getRunState() == TaskStatus.State.KILLED_UNCLEAN) &&
+           taskStatus.getRunState() == TaskStatus.State.RUNNING)) {
         //make sure we ignore progress messages after a task has 
         //invoked TaskUmbilicalProtocol.done() or if the task has been
-        //KILLED/FAILED
+        //KILLED/FAILED/FAILED_UNCLEAN/KILLED_UNCLEAN
+        //Also ignore progress update if the state change is from 
+        //COMMIT_PENDING/FAILED_UNCLEAN/KILLED_UNCLEA to RUNNING
         LOG.info(task.getTaskID() + " Ignoring status-update since " +
                  ((this.done) ? "task is 'done'" : 
                                 ("runState: " + this.taskStatus.getRunState()))
@@ -2376,7 +2382,10 @@
         if (wasFailure) {
           failures += 1;
         }
-        runner.kill();
+        // runner could be null if task-cleanup attempt is not localized yet
+        if (runner != null) {
+          runner.kill();
+        }
         setTaskFailState(wasFailure);
       } else if (taskStatus.getRunState() == TaskStatus.State.UNASSIGNED) {
         if (wasFailure) {
@@ -2455,6 +2464,11 @@
       }
       synchronized (this) {
         try {
+          // localJobConf could be null if localization has not happened
+          // then no cleanup will be required.
+          if (localJobConf == null) {
+            return;
+          }
           String taskDir = getLocalTaskDir(task.getJobID().toString(),
                              taskId.toString(), task.isTaskCleanupTask());
           if (needCleanup) {
@@ -2591,7 +2605,8 @@
   public synchronized void commitPending(TaskAttemptID taskid,
                                          TaskStatus taskStatus) 
   throws IOException {
-    LOG.info("Task " + taskid + " is in COMMIT_PENDING");
+    LOG.info("Task " + taskid + " is in commit-pending," +"" +
+             " task state:" +taskStatus.getRunState());
     statusUpdate(taskid, taskStatus);
     reportTaskFinished(taskid, true);
   }



Mime
View raw message