hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject svn commit: r985803 - in /hadoop/mapreduce/branches/branch-0.21: ./ src/java/org/apache/hadoop/mapreduce/jobhistory/ src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/
Date Mon, 16 Aug 2010 06:22:40 GMT
Author: amareshwari
Date: Mon Aug 16 06:22:39 2010
New Revision: 985803

URL: http://svn.apache.org/viewvc?rev=985803&view=rev
Log:
MAPREDUCE-1980. Merge -r 985797:985798 from trunk

Modified:
    hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
    hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java

Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=985803&r1=985802&r2=985803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Mon Aug 16 06:22:39 2010
@@ -1509,3 +1509,7 @@ Release 0.21.0 - 2010-08-13
     MAPREDUCE-2012. Some contrib tests fail in branch 0.21 and trunk.
     (Amareshwari Sriramadasu via tomwhite)
 
+    MAPREDUCE-1980. Fixes TaskAttemptUnsuccessfulCompletionEvent and
+    TaskAttemptFinishedEvent to correctly log event type for all task types.
+    (Amar Kamat via amareshwari)
+

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java?rev=985803&r1=985802&r2=985803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
Mon Aug 16 06:22:39 2010
@@ -92,7 +92,11 @@ public class TaskAttemptFinishedEvent  i
   Counters getCounters() { return EventReader.fromAvro(datum.counters); }
   /** Get the event type */
   public EventType getEventType() {
-    return EventType.MAP_ATTEMPT_FINISHED;
+    // Note that the task type can be setup/map/reduce/cleanup but the 
+    // attempt-type can only be map/reduce.
+    return getTaskId().getTaskType() == TaskType.MAP 
+           ? EventType.MAP_ATTEMPT_FINISHED
+           : EventType.REDUCE_ATTEMPT_FINISHED;
   }
 
 }

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java?rev=985803&r1=985802&r2=985803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
Mon Aug 16 06:22:39 2010
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.TaskStatus;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskID;
 import org.apache.hadoop.mapreduce.TaskType;
@@ -87,7 +88,17 @@ public class TaskAttemptUnsuccessfulComp
   public String getTaskStatus() { return datum.status.toString(); }
   /** Get the event type */
   public EventType getEventType() {
-    return EventType.MAP_ATTEMPT_KILLED;
+    // Note that the task type can be setup/map/reduce/cleanup but the 
+    // attempt-type can only be map/reduce.
+    // find out if the task failed or got killed
+    boolean failed = TaskStatus.State.FAILED.toString().equals(getTaskStatus());
+    return getTaskId().getTaskType() == TaskType.MAP 
+           ? (failed 
+              ? EventType.MAP_ATTEMPT_FAILED
+              : EventType.MAP_ATTEMPT_KILLED)
+           : (failed
+              ? EventType.REDUCE_ATTEMPT_FAILED
+              : EventType.REDUCE_ATTEMPT_KILLED);
   }
 
 }

Modified: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java?rev=985803&r1=985802&r2=985803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java
Mon Aug 16 06:22:39 2010
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.mapreduce.jobhistory;
 
+import org.apache.hadoop.mapred.TaskStatus;
+import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskType;
 
@@ -27,38 +29,124 @@ import junit.framework.TestCase;
  */
 public class TestJobHistoryEvents extends TestCase {
   /**
-   * Test TaskAttemptStartedEvent.
+   * Test {@link TaskAttemptStartedEvent} for various task types.
+   */
+  private static void testAttemptStartedEventForTypes(EventType expected, 
+                                                      TaskAttemptID id,
+                                                      TaskType[] types) {
+    for (TaskType t : types) {
+      TaskAttemptStartedEvent tase = 
+        new TaskAttemptStartedEvent(id, t, 0L, "", 0);
+      assertEquals(expected, tase.getEventType());
+    }
+  }
+  
+  /**
+   * Test {@link TaskAttemptStartedEvent}.
    */
   public void testTaskAttemptStartedEvent() {
     EventType expected = EventType.MAP_ATTEMPT_STARTED;
     TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1);
     
-    // check jobsetup type
-    TaskAttemptStartedEvent tase = 
-      new TaskAttemptStartedEvent(fakeId, TaskType.JOB_SETUP, 0L, "", 0);
-    assertEquals(expected, tase.getEventType());
-    
-    // check jobcleanup type
-    tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_CLEANUP, 0L, "", 0);
-    assertEquals(expected, tase.getEventType());
-    
-    // check map type
-    tase = new TaskAttemptStartedEvent(fakeId, TaskType.MAP, 0L, "", 0);
-    assertEquals(expected, tase.getEventType());
+    // check the events for job-setup, job-cleanup and map task-types
+    testAttemptStartedEventForTypes(expected, fakeId,
+                                    new TaskType[] {TaskType.JOB_SETUP, 
+                                                    TaskType.JOB_CLEANUP, 
+                                                    TaskType.MAP});
     
     expected = EventType.REDUCE_ATTEMPT_STARTED;
     fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1);
     
-    // check jobsetup type
-    tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_SETUP, 0L, "", 0);
-    assertEquals(expected, tase.getEventType());
-    
-    // check jobcleanup type
-    tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_CLEANUP, 0L, "", 0);
-    assertEquals(expected, tase.getEventType());
-    
-    // check reduce type
-    tase = new TaskAttemptStartedEvent(fakeId, TaskType.REDUCE, 0L, "", 0);
-    assertEquals(expected, tase.getEventType());
+    // check the events for job-setup, job-cleanup and reduce task-types
+    testAttemptStartedEventForTypes(expected, fakeId,
+                                    new TaskType[] {TaskType.JOB_SETUP, 
+                                                    TaskType.JOB_CLEANUP, 
+                                                    TaskType.REDUCE});
+  }
+  
+  /**
+   * Test {@link TaskAttemptUnsuccessfulCompletionEvent} for various task types.
+   */
+  private static void testFailedKilledEventsForTypes(EventType expected, 
+                                                     TaskAttemptID id,
+                                                     TaskType[] types,
+                                                     String state) {
+    for (TaskType t : types) {
+      TaskAttemptUnsuccessfulCompletionEvent tauce = 
+        new TaskAttemptUnsuccessfulCompletionEvent(id, t, state, 0L, "", "");
+      assertEquals(expected, tauce.getEventType());
+    }
+  }
+  
+  /**
+   * Test {@link TaskAttemptUnsuccessfulCompletionEvent} for killed/failed task.
+   */
+  public void testTaskAttemptUnsuccessfulCompletionEvent() {
+    TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1);
+    
+    // check killed events for job-setup, job-cleanup and map task-types
+    testFailedKilledEventsForTypes(EventType.MAP_ATTEMPT_KILLED, fakeId,
+                                   new TaskType[] {TaskType.JOB_SETUP, 
+                                                   TaskType.JOB_CLEANUP, 
+                                                   TaskType.MAP},
+                                   TaskStatus.State.KILLED.toString());
+    // check failed events for job-setup, job-cleanup and map task-types
+    testFailedKilledEventsForTypes(EventType.MAP_ATTEMPT_FAILED, fakeId,
+                                   new TaskType[] {TaskType.JOB_SETUP, 
+                                                   TaskType.JOB_CLEANUP, 
+                                                   TaskType.MAP},
+                                   TaskStatus.State.FAILED.toString());
+    
+    fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1);
+    
+    // check killed events for job-setup, job-cleanup and reduce task-types
+    testFailedKilledEventsForTypes(EventType.REDUCE_ATTEMPT_KILLED, fakeId,
+                                   new TaskType[] {TaskType.JOB_SETUP, 
+                                                   TaskType.JOB_CLEANUP, 
+                                                   TaskType.REDUCE},
+                                   TaskStatus.State.KILLED.toString());
+    // check failed events for job-setup, job-cleanup and reduce task-types
+    testFailedKilledEventsForTypes(EventType.REDUCE_ATTEMPT_FAILED, fakeId,
+                                   new TaskType[] {TaskType.JOB_SETUP, 
+                                                   TaskType.JOB_CLEANUP, 
+                                                   TaskType.REDUCE},
+                                   TaskStatus.State.FAILED.toString());
+  }
+  
+  /**
+   * Test {@link TaskAttemptFinishedEvent} for various task types.
+   */
+  private static void testFinishedEventsForTypes(EventType expected, 
+                                                 TaskAttemptID id,
+                                                 TaskType[] types) {
+    for (TaskType t : types) {
+      TaskAttemptFinishedEvent tafe = 
+        new TaskAttemptFinishedEvent(id, t, 
+            TaskStatus.State.SUCCEEDED.toString(), 0L, "", "", new Counters());
+      assertEquals(expected, tafe.getEventType());
+    }
+  }
+  
+  /**
+   * Test {@link TaskAttemptFinishedEvent} for finished task.
+   */
+  public void testTaskAttemptFinishedEvent() {
+    EventType expected = EventType.MAP_ATTEMPT_FINISHED;
+    TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1);
+    
+    // check the events for job-setup, job-cleanup and map task-types
+    testFinishedEventsForTypes(expected, fakeId,
+                               new TaskType[] {TaskType.JOB_SETUP, 
+                                               TaskType.JOB_CLEANUP, 
+                                               TaskType.MAP});
+    
+    expected = EventType.REDUCE_ATTEMPT_FINISHED;
+    fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1);
+    
+    // check the events for job-setup, job-cleanup and reduce task-types
+    testFinishedEventsForTypes(expected, fakeId,
+                               new TaskType[] {TaskType.JOB_SETUP, 
+                                               TaskType.JOB_CLEANUP, 
+                                               TaskType.REDUCE});
   }
 }



Mime
View raw message