Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 924FE9DFF for ; Mon, 25 Jun 2012 17:56:38 +0000 (UTC) Received: (qmail 65546 invoked by uid 500); 25 Jun 2012 17:56:38 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 65485 invoked by uid 500); 25 Jun 2012 17:56:38 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 65477 invoked by uid 99); 25 Jun 2012 17:56:38 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 25 Jun 2012 17:56:38 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 25 Jun 2012 17:56:34 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A74A52388978; Mon, 25 Jun 2012 17:56:13 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1353685 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/ja... Date: Mon, 25 Jun 2012 17:56:12 -0000 To: mapreduce-commits@hadoop.apache.org From: sseth@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120625175613.A74A52388978@eris.apache.org> Author: sseth Date: Mon Jun 25 17:56:11 2012 New Revision: 1353685 URL: http://svn.apache.org/viewvc?rev=1353685&view=rev Log: Merge MAPREDUCE-4290 from trunk. Fix Yarn Applicaiton Status to MR JobState conversion. (Contributed by Devaraj K) Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1353685&r1=1353684&r2=1353685&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Mon Jun 25 17:56:11 2012 @@ -63,6 +63,9 @@ Release 2.0.1-alpha - UNRELEASED MAPREDUCE-4336. Distributed Shell fails when used with the CapacityScheduler (ahmed via tucu) + MAPREDUCE-4290. Fix Yarn Applicaiton Status to MR JobState conversion. + (Devaraj K via sseth) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java?rev=1353685&r1=1353684&r2=1353685&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java Mon Jun 25 17:56:11 2012 @@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.v2.ut import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; @@ -376,22 +377,27 @@ public class TypeConverter { } return reports; } - - public static JobStatus.State fromYarn(YarnApplicationState state) { - switch (state) { + + public static State fromYarn(YarnApplicationState yarnApplicationState, + FinalApplicationStatus finalApplicationStatus) { + switch (yarnApplicationState) { case NEW: case SUBMITTED: return State.PREP; case RUNNING: return State.RUNNING; case FINISHED: - return State.SUCCEEDED; + if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) { + return State.SUCCEEDED; + } else if (finalApplicationStatus == FinalApplicationStatus.KILLED) { + return State.KILLED; + } case FAILED: return State.FAILED; case KILLED: return State.KILLED; } - throw new YarnException("Unrecognized application state: " + state); + throw new YarnException("Unrecognized application state: " + yarnApplicationState); } private static final String TT_NAME_PREFIX = "tracker_"; @@ -417,7 +423,7 @@ public class TypeConverter { new JobStatus( TypeConverter.fromYarn(application.getApplicationId()), 0.0f, 0.0f, 0.0f, 0.0f, - TypeConverter.fromYarn(application.getYarnApplicationState()), + TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()), org.apache.hadoop.mapreduce.JobPriority.NORMAL, application.getUser(), application.getName(), application.getQueue(), jobFile, trackingUrl, false Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java?rev=1353685&r1=1353684&r2=1353685&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java Mon Jun 25 17:56:11 2012 @@ -27,6 +27,7 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; @@ -45,7 +46,7 @@ public class TestTypeConverter { @Test public void testEnums() throws Exception { for (YarnApplicationState applicationState : YarnApplicationState.values()) { - TypeConverter.fromYarn(applicationState); + TypeConverter.fromYarn(applicationState, FinalApplicationStatus.FAILED); } for (TaskType taskType : TaskType.values()) { @@ -63,8 +64,6 @@ public class TestTypeConverter { for (TaskState taskState : TaskState.values()) { TypeConverter.fromYarn(taskState); } - - } @Test Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java?rev=1353685&r1=1353684&r2=1353685&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java Mon Jun 25 17:56:11 2012 @@ -19,13 +19,26 @@ package org.apache.hadoop.mapred; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; +import org.apache.hadoop.mapreduce.JobStatus; +import org.apache.hadoop.mapreduce.JobStatus.State; import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.Records; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -35,7 +48,7 @@ public class TestResourceMgrDelegate { /** * Tests that getRootQueues makes a request for the (recursive) child queues */ -@Test + @Test public void testGetRootQueues() throws IOException, InterruptedException { ClientRMProtocol applicationsManager = Mockito.mock(ClientRMProtocol.class); GetQueueInfoResponse response = Mockito.mock(GetQueueInfoResponse.class); @@ -60,4 +73,56 @@ public class TestResourceMgrDelegate { argument.getValue().getRecursive()); } + @Test + public void tesAllJobs() throws Exception { + ClientRMProtocol applicationsManager = Mockito.mock(ClientRMProtocol.class); + GetAllApplicationsResponse allApplicationsResponse = Records + .newRecord(GetAllApplicationsResponse.class); + List applications = new ArrayList(); + applications.add(getApplicationReport(YarnApplicationState.FINISHED, + FinalApplicationStatus.FAILED)); + applications.add(getApplicationReport(YarnApplicationState.FINISHED, + FinalApplicationStatus.SUCCEEDED)); + applications.add(getApplicationReport(YarnApplicationState.FINISHED, + FinalApplicationStatus.KILLED)); + applications.add(getApplicationReport(YarnApplicationState.FAILED, + FinalApplicationStatus.FAILED)); + allApplicationsResponse.setApplicationList(applications); + Mockito.when( + applicationsManager.getAllApplications(Mockito + .any(GetAllApplicationsRequest.class))).thenReturn( + allApplicationsResponse); + ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( + new YarnConfiguration(), applicationsManager); + JobStatus[] allJobs = resourceMgrDelegate.getAllJobs(); + + Assert.assertEquals(State.FAILED, allJobs[0].getState()); + Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState()); + Assert.assertEquals(State.KILLED, allJobs[2].getState()); + Assert.assertEquals(State.FAILED, allJobs[3].getState()); + } + + private ApplicationReport getApplicationReport( + YarnApplicationState yarnApplicationState, + FinalApplicationStatus finalApplicationStatus) { + ApplicationReport appReport = Mockito.mock(ApplicationReport.class); + ApplicationResourceUsageReport appResources = Mockito + .mock(ApplicationResourceUsageReport.class); + Mockito.when(appReport.getApplicationId()).thenReturn( + Records.newRecord(ApplicationId.class)); + Mockito.when(appResources.getNeededResources()).thenReturn( + Records.newRecord(Resource.class)); + Mockito.when(appResources.getReservedResources()).thenReturn( + Records.newRecord(Resource.class)); + Mockito.when(appResources.getUsedResources()).thenReturn( + Records.newRecord(Resource.class)); + Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn( + appResources); + Mockito.when(appReport.getYarnApplicationState()).thenReturn( + yarnApplicationState); + Mockito.when(appReport.getFinalApplicationStatus()).thenReturn( + finalApplicationStatus); + + return appReport; + } }