Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 89219 invoked from network); 16 Aug 2010 06:24:01 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 16 Aug 2010 06:24:01 -0000 Received: (qmail 76855 invoked by uid 500); 16 Aug 2010 06:24:01 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 76769 invoked by uid 500); 16 Aug 2010 06:23:59 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 76690 invoked by uid 99); 16 Aug 2010 06:23:58 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 16 Aug 2010 06:23:58 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 16 Aug 2010 06:23:57 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 3078E2388A67; Mon, 16 Aug 2010 06:22:40 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r985803 - in /hadoop/mapreduce/branches/branch-0.21: ./ src/java/org/apache/hadoop/mapreduce/jobhistory/ src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/ Date: Mon, 16 Aug 2010 06:22:40 -0000 To: mapreduce-commits@hadoop.apache.org From: amareshwari@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100816062240.3078E2388A67@eris.apache.org> Author: amareshwari Date: Mon Aug 16 06:22:39 2010 New Revision: 985803 URL: http://svn.apache.org/viewvc?rev=985803&view=rev Log: MAPREDUCE-1980. Merge -r 985797:985798 from trunk Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=985803&r1=985802&r2=985803&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original) +++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Mon Aug 16 06:22:39 2010 @@ -1509,3 +1509,7 @@ Release 0.21.0 - 2010-08-13 MAPREDUCE-2012. Some contrib tests fail in branch 0.21 and trunk. (Amareshwari Sriramadasu via tomwhite) + MAPREDUCE-1980. Fixes TaskAttemptUnsuccessfulCompletionEvent and + TaskAttemptFinishedEvent to correctly log event type for all task types. + (Amar Kamat via amareshwari) + Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java?rev=985803&r1=985802&r2=985803&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java (original) +++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java Mon Aug 16 06:22:39 2010 @@ -92,7 +92,11 @@ public class TaskAttemptFinishedEvent i Counters getCounters() { return EventReader.fromAvro(datum.counters); } /** Get the event type */ public EventType getEventType() { - return EventType.MAP_ATTEMPT_FINISHED; + // Note that the task type can be setup/map/reduce/cleanup but the + // attempt-type can only be map/reduce. + return getTaskId().getTaskType() == TaskType.MAP + ? EventType.MAP_ATTEMPT_FINISHED + : EventType.REDUCE_ATTEMPT_FINISHED; } } Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java?rev=985803&r1=985802&r2=985803&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java (original) +++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java Mon Aug 16 06:22:39 2010 @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.mapred.TaskStatus; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TaskType; @@ -87,7 +88,17 @@ public class TaskAttemptUnsuccessfulComp public String getTaskStatus() { return datum.status.toString(); } /** Get the event type */ public EventType getEventType() { - return EventType.MAP_ATTEMPT_KILLED; + // Note that the task type can be setup/map/reduce/cleanup but the + // attempt-type can only be map/reduce. + // find out if the task failed or got killed + boolean failed = TaskStatus.State.FAILED.toString().equals(getTaskStatus()); + return getTaskId().getTaskType() == TaskType.MAP + ? (failed + ? EventType.MAP_ATTEMPT_FAILED + : EventType.MAP_ATTEMPT_KILLED) + : (failed + ? EventType.REDUCE_ATTEMPT_FAILED + : EventType.REDUCE_ATTEMPT_KILLED); } } Modified: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java?rev=985803&r1=985802&r2=985803&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java (original) +++ hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java Mon Aug 16 06:22:39 2010 @@ -17,6 +17,8 @@ */ package org.apache.hadoop.mapreduce.jobhistory; +import org.apache.hadoop.mapred.TaskStatus; +import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskType; @@ -27,38 +29,124 @@ import junit.framework.TestCase; */ public class TestJobHistoryEvents extends TestCase { /** - * Test TaskAttemptStartedEvent. + * Test {@link TaskAttemptStartedEvent} for various task types. + */ + private static void testAttemptStartedEventForTypes(EventType expected, + TaskAttemptID id, + TaskType[] types) { + for (TaskType t : types) { + TaskAttemptStartedEvent tase = + new TaskAttemptStartedEvent(id, t, 0L, "", 0); + assertEquals(expected, tase.getEventType()); + } + } + + /** + * Test {@link TaskAttemptStartedEvent}. */ public void testTaskAttemptStartedEvent() { EventType expected = EventType.MAP_ATTEMPT_STARTED; TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1); - // check jobsetup type - TaskAttemptStartedEvent tase = - new TaskAttemptStartedEvent(fakeId, TaskType.JOB_SETUP, 0L, "", 0); - assertEquals(expected, tase.getEventType()); - - // check jobcleanup type - tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_CLEANUP, 0L, "", 0); - assertEquals(expected, tase.getEventType()); - - // check map type - tase = new TaskAttemptStartedEvent(fakeId, TaskType.MAP, 0L, "", 0); - assertEquals(expected, tase.getEventType()); + // check the events for job-setup, job-cleanup and map task-types + testAttemptStartedEventForTypes(expected, fakeId, + new TaskType[] {TaskType.JOB_SETUP, + TaskType.JOB_CLEANUP, + TaskType.MAP}); expected = EventType.REDUCE_ATTEMPT_STARTED; fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1); - // check jobsetup type - tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_SETUP, 0L, "", 0); - assertEquals(expected, tase.getEventType()); - - // check jobcleanup type - tase = new TaskAttemptStartedEvent(fakeId, TaskType.JOB_CLEANUP, 0L, "", 0); - assertEquals(expected, tase.getEventType()); - - // check reduce type - tase = new TaskAttemptStartedEvent(fakeId, TaskType.REDUCE, 0L, "", 0); - assertEquals(expected, tase.getEventType()); + // check the events for job-setup, job-cleanup and reduce task-types + testAttemptStartedEventForTypes(expected, fakeId, + new TaskType[] {TaskType.JOB_SETUP, + TaskType.JOB_CLEANUP, + TaskType.REDUCE}); + } + + /** + * Test {@link TaskAttemptUnsuccessfulCompletionEvent} for various task types. + */ + private static void testFailedKilledEventsForTypes(EventType expected, + TaskAttemptID id, + TaskType[] types, + String state) { + for (TaskType t : types) { + TaskAttemptUnsuccessfulCompletionEvent tauce = + new TaskAttemptUnsuccessfulCompletionEvent(id, t, state, 0L, "", ""); + assertEquals(expected, tauce.getEventType()); + } + } + + /** + * Test {@link TaskAttemptUnsuccessfulCompletionEvent} for killed/failed task. + */ + public void testTaskAttemptUnsuccessfulCompletionEvent() { + TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1); + + // check killed events for job-setup, job-cleanup and map task-types + testFailedKilledEventsForTypes(EventType.MAP_ATTEMPT_KILLED, fakeId, + new TaskType[] {TaskType.JOB_SETUP, + TaskType.JOB_CLEANUP, + TaskType.MAP}, + TaskStatus.State.KILLED.toString()); + // check failed events for job-setup, job-cleanup and map task-types + testFailedKilledEventsForTypes(EventType.MAP_ATTEMPT_FAILED, fakeId, + new TaskType[] {TaskType.JOB_SETUP, + TaskType.JOB_CLEANUP, + TaskType.MAP}, + TaskStatus.State.FAILED.toString()); + + fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1); + + // check killed events for job-setup, job-cleanup and reduce task-types + testFailedKilledEventsForTypes(EventType.REDUCE_ATTEMPT_KILLED, fakeId, + new TaskType[] {TaskType.JOB_SETUP, + TaskType.JOB_CLEANUP, + TaskType.REDUCE}, + TaskStatus.State.KILLED.toString()); + // check failed events for job-setup, job-cleanup and reduce task-types + testFailedKilledEventsForTypes(EventType.REDUCE_ATTEMPT_FAILED, fakeId, + new TaskType[] {TaskType.JOB_SETUP, + TaskType.JOB_CLEANUP, + TaskType.REDUCE}, + TaskStatus.State.FAILED.toString()); + } + + /** + * Test {@link TaskAttemptFinishedEvent} for various task types. + */ + private static void testFinishedEventsForTypes(EventType expected, + TaskAttemptID id, + TaskType[] types) { + for (TaskType t : types) { + TaskAttemptFinishedEvent tafe = + new TaskAttemptFinishedEvent(id, t, + TaskStatus.State.SUCCEEDED.toString(), 0L, "", "", new Counters()); + assertEquals(expected, tafe.getEventType()); + } + } + + /** + * Test {@link TaskAttemptFinishedEvent} for finished task. + */ + public void testTaskAttemptFinishedEvent() { + EventType expected = EventType.MAP_ATTEMPT_FINISHED; + TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1); + + // check the events for job-setup, job-cleanup and map task-types + testFinishedEventsForTypes(expected, fakeId, + new TaskType[] {TaskType.JOB_SETUP, + TaskType.JOB_CLEANUP, + TaskType.MAP}); + + expected = EventType.REDUCE_ATTEMPT_FINISHED; + fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1); + + // check the events for job-setup, job-cleanup and reduce task-types + testFinishedEventsForTypes(expected, fakeId, + new TaskType[] {TaskType.JOB_SETUP, + TaskType.JOB_CLEANUP, + TaskType.REDUCE}); } }