Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 775D510923 for ; Mon, 10 Mar 2014 20:37:16 +0000 (UTC) Received: (qmail 72451 invoked by uid 500); 10 Mar 2014 20:37:03 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 72411 invoked by uid 500); 10 Mar 2014 20:37:01 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 72186 invoked by uid 99); 10 Mar 2014 20:36:57 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Mar 2014 20:36:57 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Mar 2014 20:36:55 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 376CC2388A02; Mon, 10 Mar 2014 20:36:35 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1576072 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-serve... Date: Mon, 10 Mar 2014 20:36:35 -0000 To: yarn-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140310203635.376CC2388A02@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Mon Mar 10 20:36:34 2014 New Revision: 1576072 URL: http://svn.apache.org/r1576072 Log: YARN-1788. Fixed a bug in ResourceManager to set the apps-completed and apps-killed metrics correctly for killed applications. Contributed by Varun Vasudev. Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1576072&r1=1576071&r2=1576072&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Mon Mar 10 20:36:34 2014 @@ -436,6 +436,10 @@ Release 2.4.0 - UNRELEASED YARN-1793. Fixed ClientRMService#forceKillApplication not killing unmanaged application. (Karthik Kambatla via jianhe) + YARN-1788. Fixed a bug in ResourceManager to set the apps-completed and + apps-killed metrics correctly for killed applications. (Varun Vasudev via + vinodkv) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1576072&r1=1576071&r2=1576072&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Mon Mar 10 20:36:34 2014 @@ -1043,8 +1043,8 @@ public class RMAppImpl implements RMApp, if (app.finishTime == 0 ) { app.finishTime = System.currentTimeMillis(); } - app.handler.handle(new AppRemovedSchedulerEvent(app.applicationId, app - .getState())); + app.handler.handle(new AppRemovedSchedulerEvent(app.applicationId, + finalState)); app.handler.handle( new RMAppManagerEvent(app.applicationId, RMAppManagerEventType.APP_COMPLETED)); Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java?rev=1576072&r1=1576071&r2=1576072&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java Mon Mar 10 20:36:34 2014 @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.log4j.Level; import org.apache.log4j.LogManager; @@ -519,7 +520,13 @@ public class TestRM { } }; + // test metrics + QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics(); + int appsKilled = metrics.getAppsKilled(); + int appsSubmitted = metrics.getAppsSubmitted(); + rm.start(); + MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm1.registerNode(); @@ -552,6 +559,11 @@ public class TestRM { new RMAppEvent(application.getApplicationId(), RMAppEventType.ATTEMPT_KILLED)); rm.waitForState(application.getApplicationId(), RMAppState.KILLED); + + // test metrics + metrics = rm.getResourceScheduler().getRootQueueMetrics(); + Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled()); + Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted()); } public static void main(String[] args) throws Exception { Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1576072&r1=1576071&r2=1576072&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Mon Mar 10 20:36:34 2014 @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; @@ -87,6 +88,7 @@ public class TestRMAppTransitions { private RMStateStore store; private RMApplicationHistoryWriter writer; private YarnScheduler scheduler; + private TestSchedulerEventDispatcher schedulerDispatcher; // ignore all the RM application attempt events private static final class TestApplicationAttemptEventDispatcher implements @@ -148,8 +150,11 @@ public class TestRMAppTransitions { // handle all the scheduler events - same as in ResourceManager.java private static final class TestSchedulerEventDispatcher implements EventHandler { + public SchedulerEvent lastSchedulerEvent; + @Override public void handle(SchedulerEvent event) { + lastSchedulerEvent = event; } } @@ -201,8 +206,9 @@ public class TestRMAppTransitions { rmDispatcher.register(RMAppManagerEventType.class, new TestApplicationManagerEventDispatcher()); + schedulerDispatcher = new TestSchedulerEventDispatcher(); rmDispatcher.register(SchedulerEventType.class, - new TestSchedulerEventDispatcher()); + schedulerDispatcher); rmDispatcher.init(conf); rmDispatcher.start(); @@ -502,6 +508,7 @@ public class TestRMAppTransitions { assertKilled(application); assertAppFinalStateNotSaved(application); verifyApplicationFinished(RMAppState.KILLED); + verifyAppRemovedSchedulerEvent(RMAppState.KILLED); } @Test @@ -534,6 +541,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); verifyApplicationFinished(RMAppState.KILLED); + verifyAppRemovedSchedulerEvent(RMAppState.KILLED); } @Test (timeout = 30000) @@ -583,6 +591,7 @@ public class TestRMAppTransitions { assertKilled(application); assertAppFinalStateSaved(application); verifyApplicationFinished(RMAppState.KILLED); + verifyAppRemovedSchedulerEvent(RMAppState.KILLED); } @Test @@ -640,6 +649,7 @@ public class TestRMAppTransitions { assertKilled(application); assertAppFinalStateSaved(application); verifyApplicationFinished(RMAppState.KILLED); + verifyAppRemovedSchedulerEvent(RMAppState.KILLED); } @Test @@ -663,6 +673,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); verifyApplicationFinished(RMAppState.KILLED); + verifyAppRemovedSchedulerEvent(RMAppState.KILLED); } @Test @@ -868,4 +879,15 @@ public class TestRMAppTransitions { verify(writer).applicationFinished(any(RMApp.class), finalState.capture()); Assert.assertEquals(state, finalState.getValue()); } + + private void verifyAppRemovedSchedulerEvent(RMAppState finalState) { + Assert.assertEquals(SchedulerEventType.APP_REMOVED, + schedulerDispatcher.lastSchedulerEvent.getType()); + if(schedulerDispatcher.lastSchedulerEvent instanceof + AppRemovedSchedulerEvent) { + AppRemovedSchedulerEvent appRemovedEvent = + (AppRemovedSchedulerEvent) schedulerDispatcher.lastSchedulerEvent; + Assert.assertEquals(finalState, appRemovedEvent.getFinalState()); + } + } }