Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BEB7A114B7 for ; Fri, 12 Sep 2014 18:46:54 +0000 (UTC) Received: (qmail 54031 invoked by uid 500); 12 Sep 2014 18:46:46 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 53884 invoked by uid 500); 12 Sep 2014 18:46:46 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 53699 invoked by uid 99); 12 Sep 2014 18:46:45 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 12 Sep 2014 18:46:45 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id C477CA11F89; Fri, 12 Sep 2014 18:46:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: arp@apache.org To: common-commits@hadoop.apache.org Date: Fri, 12 Sep 2014 18:47:01 -0000 Message-Id: In-Reply-To: <30645674436a4925adf47a74fed60e1d@git.apache.org> References: <30645674436a4925adf47a74fed60e1d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [18/41] YARN-415. Capture aggregate memory allocation at the app-level for chargeback. Contributed by Eric Payne & Andrey Klochkov http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java new file mode 100644 index 0000000..d4ecc89 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -0,0 +1,401 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; + +import org.apache.commons.lang.time.DateUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestContainerResourceUsage { + + private YarnConfiguration conf; + + @Before + public void setup() throws UnknownHostException { + Logger rootLogger = LogManager.getRootLogger(); + rootLogger.setLevel(Level.DEBUG); + conf = new YarnConfiguration(); + UserGroupInformation.setConfiguration(conf); + conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + } + + @After + public void tearDown() { + } + + @Test (timeout = 60000) + public void testUsageWithOneAttemptAndOneContainer() throws Exception { + MockRM rm = new MockRM(conf); + rm.start(); + + MockNM nm = + new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); + nm.registerNode(); + + RMApp app0 = rm.submitApp(200); + + RMAppMetrics rmAppMetrics = app0.getRMAppMetrics(); + Assert.assertTrue( + "Before app submittion, memory seconds should have been 0 but was " + + rmAppMetrics.getMemorySeconds(), + rmAppMetrics.getMemorySeconds() == 0); + Assert.assertTrue( + "Before app submission, vcore seconds should have been 0 but was " + + rmAppMetrics.getVcoreSeconds(), + rmAppMetrics.getVcoreSeconds() == 0); + + RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); + + nm.nodeHeartbeat(true); + MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId()); + am0.registerAppAttempt(); + + RMContainer rmContainer = + rm.getResourceScheduler() + .getRMContainer(attempt0.getMasterContainer().getId()); + + // Allow metrics to accumulate. + Thread.sleep(1000); + + rmAppMetrics = app0.getRMAppMetrics(); + Assert.assertTrue( + "While app is running, memory seconds should be >0 but is " + + rmAppMetrics.getMemorySeconds(), + rmAppMetrics.getMemorySeconds() > 0); + Assert.assertTrue( + "While app is running, vcore seconds should be >0 but is " + + rmAppMetrics.getVcoreSeconds(), + rmAppMetrics.getVcoreSeconds() > 0); + + MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0); + + AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer); + rmAppMetrics = app0.getRMAppMetrics(); + + Assert.assertEquals("Unexcpected MemorySeconds value", + ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds()); + Assert.assertEquals("Unexpected VcoreSeconds value", + ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds()); + + rm.stop(); + } + + @Test (timeout = 60000) + public void testUsageWithMultipleContainersAndRMRestart() throws Exception { + // Set max attempts to 1 so that when the first attempt fails, the app + // won't try to start a new one. + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + + MockRM rm0 = new MockRM(conf, memStore); + rm0.start(); + MockNM nm = + new MockNM("127.0.0.1:1234", 65536, rm0.getResourceTrackerService()); + nm.registerNode(); + + RMApp app0 = rm0.submitApp(200); + + rm0.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED); + RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); + ApplicationAttemptId attemptId0 = attempt0.getAppAttemptId(); + rm0.waitForState(attemptId0, RMAppAttemptState.SCHEDULED); + + nm.nodeHeartbeat(true); + rm0.waitForState(attemptId0, RMAppAttemptState.ALLOCATED); + MockAM am0 = rm0.sendAMLaunched(attempt0.getAppAttemptId()); + am0.registerAppAttempt(); + + int NUM_CONTAINERS = 2; + am0.allocate("127.0.0.1" , 1000, NUM_CONTAINERS, + new ArrayList()); + nm.nodeHeartbeat(true); + List conts = am0.allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers(); + while (conts.size() != NUM_CONTAINERS) { + nm.nodeHeartbeat(true); + conts.addAll(am0.allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers()); + Thread.sleep(500); + } + + // launch the 2nd and 3rd containers. + for (Container c : conts) { + nm.nodeHeartbeat(attempt0.getAppAttemptId(), + c.getId().getId(), ContainerState.RUNNING); + rm0.waitForState(nm, c.getId(), RMContainerState.RUNNING); + } + + // Get the RMContainers for all of the live containers, to be used later + // for metrics calculations and comparisons. + Collection rmContainers = + rm0.scheduler + .getSchedulerAppInfo(attempt0.getAppAttemptId()) + .getLiveContainers(); + + // Give the metrics time to accumulate. + Thread.sleep(1000); + + // Stop all non-AM containers + for (Container c : conts) { + if (c.getId().getId() == 1) continue; + nm.nodeHeartbeat(attempt0.getAppAttemptId(), + c.getId().getId(), ContainerState.COMPLETE); + rm0.waitForState(nm, c.getId(), RMContainerState.COMPLETED); + } + + // After all other containers have completed, manually complete the master + // container in order to trigger a save to the state store of the resource + // usage metrics. This will cause the attempt to fail, and, since the max + // attempt retries is 1, the app will also fail. This is intentional so + // that all containers will complete prior to saving. + ContainerId cId = ContainerId.newInstance(attempt0.getAppAttemptId(), 1); + nm.nodeHeartbeat(attempt0.getAppAttemptId(), + cId.getId(), ContainerState.COMPLETE); + rm0.waitForState(nm, cId, RMContainerState.COMPLETED); + + // Check that the container metrics match those from the app usage report. + long memorySeconds = 0; + long vcoreSeconds = 0; + for (RMContainer c : rmContainers) { + AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); + memorySeconds += ru.getMemorySeconds(); + vcoreSeconds += ru.getVcoreSeconds(); + } + + RMAppMetrics metricsBefore = app0.getRMAppMetrics(); + Assert.assertEquals("Unexcpected MemorySeconds value", + memorySeconds, metricsBefore.getMemorySeconds()); + Assert.assertEquals("Unexpected VcoreSeconds value", + vcoreSeconds, metricsBefore.getVcoreSeconds()); + + // create new RM to represent RM restart. Load up the state store. + MockRM rm1 = new MockRM(conf, memStore); + rm1.start(); + RMApp app0After = + rm1.getRMContext().getRMApps().get(app0.getApplicationId()); + + // Compare container resource usage metrics from before and after restart. + RMAppMetrics metricsAfter = app0After.getRMAppMetrics(); + Assert.assertEquals("Vcore seconds were not the same after RM Restart", + metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds()); + Assert.assertEquals("Memory seconds were not the same after RM Restart", + metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds()); + + rm0.stop(); + rm0.close(); + rm1.stop(); + rm1.close(); + } + + @Test(timeout = 60000) + public void testUsageAfterAMRestartWithMultipleContainers() throws Exception { + amRestartTests(false); + } + + @Test(timeout = 60000) + public void testUsageAfterAMRestartKeepContainers() throws Exception { + amRestartTests(true); + } + + private void amRestartTests(boolean keepRunningContainers) + throws Exception { + MockRM rm = new MockRM(conf); + rm.start(); + + RMApp app = + rm.submitApp(200, "name", "user", + new HashMap(), false, "default", -1, + null, "MAPREDUCE", false, keepRunningContainers); + MockNM nm = + new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService()); + nm.registerNode(); + + MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm); + int NUM_CONTAINERS = 1; + // allocate NUM_CONTAINERS containers + am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, + new ArrayList()); + nm.nodeHeartbeat(true); + + // wait for containers to be allocated. + List containers = + am0.allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers(); + while (containers.size() != NUM_CONTAINERS) { + nm.nodeHeartbeat(true); + containers.addAll(am0.allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers()); + Thread.sleep(200); + } + + // launch the 2nd container. + ContainerId containerId2 = + ContainerId.newInstance(am0.getApplicationAttemptId(), 2); + nm.nodeHeartbeat(am0.getApplicationAttemptId(), + containerId2.getId(), ContainerState.RUNNING); + rm.waitForState(nm, containerId2, RMContainerState.RUNNING); + + // Capture the containers here so the metrics can be calculated after the + // app has completed. + Collection rmContainers = + rm.scheduler + .getSchedulerAppInfo(am0.getApplicationAttemptId()) + .getLiveContainers(); + + // fail the first app attempt by sending CONTAINER_FINISHED event without + // registering. + ContainerId amContainerId = + app.getCurrentAppAttempt().getMasterContainer().getId(); + nm.nodeHeartbeat(am0.getApplicationAttemptId(), + amContainerId.getId(), ContainerState.COMPLETE); + am0.waitForState(RMAppAttemptState.FAILED); + + long memorySeconds = 0; + long vcoreSeconds = 0; + + // Calculate container usage metrics for first attempt. + if (keepRunningContainers) { + // Only calculate the usage for the one container that has completed. + for (RMContainer c : rmContainers) { + if (c.getContainerId().equals(amContainerId)) { + AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); + memorySeconds += ru.getMemorySeconds(); + vcoreSeconds += ru.getVcoreSeconds(); + } else { + // The remaining container should be RUNNING. + Assert.assertTrue("After first attempt failed, remaining container " + + "should still be running. ", + c.getContainerState().equals(ContainerState.RUNNING)); + } + } + } else { + // If keepRunningContainers is false, all live containers should now + // be completed. Calculate the resource usage metrics for all of them. + for (RMContainer c : rmContainers) { + AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); + memorySeconds += ru.getMemorySeconds(); + vcoreSeconds += ru.getVcoreSeconds(); + } + } + + // wait for app to start a new attempt. + rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); + + // assert this is a new AM. + RMAppAttempt attempt2 = app.getCurrentAppAttempt(); + Assert.assertFalse(attempt2.getAppAttemptId() + .equals(am0.getApplicationAttemptId())); + + // launch the new AM + nm.nodeHeartbeat(true); + MockAM am1 = rm.sendAMLaunched(attempt2.getAppAttemptId()); + am1.registerAppAttempt(); + + // allocate NUM_CONTAINERS containers + am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, + new ArrayList()); + nm.nodeHeartbeat(true); + + // wait for containers to be allocated. + containers = + am1.allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers(); + while (containers.size() != NUM_CONTAINERS) { + nm.nodeHeartbeat(true); + containers.addAll(am1.allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers()); + Thread.sleep(200); + } + + rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); + + // Capture running containers for later use by metrics calculations. + rmContainers = rm.scheduler.getSchedulerAppInfo(attempt2.getAppAttemptId()) + .getLiveContainers(); + + // complete container by sending the container complete event which has + // earlier attempt's attemptId + amContainerId = app.getCurrentAppAttempt().getMasterContainer().getId(); + nm.nodeHeartbeat(am0.getApplicationAttemptId(), + amContainerId.getId(), ContainerState.COMPLETE); + + MockRM.finishAMAndVerifyAppState(app, rm, nm, am1); + + // Calculate container usage metrics for second attempt. + for (RMContainer c : rmContainers) { + AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); + memorySeconds += ru.getMemorySeconds(); + vcoreSeconds += ru.getVcoreSeconds(); + } + + RMAppMetrics rmAppMetrics = app.getRMAppMetrics(); + + Assert.assertEquals("Unexcpected MemorySeconds value", + memorySeconds, rmAppMetrics.getMemorySeconds()); + Assert.assertEquals("Unexpected VcoreSeconds value", + vcoreSeconds, rmAppMetrics.getVcoreSeconds()); + + rm.stop(); + return; + } + + private AggregateAppResourceUsage calculateContainerResourceMetrics( + RMContainer rmContainer) { + Resource resource = rmContainer.getContainer().getResource(); + long usedMillis = + rmContainer.getFinishTime() - rmContainer.getCreationTime(); + long memorySeconds = resource.getMemory() + * usedMillis / DateUtils.MILLIS_PER_SECOND; + long vcoreSeconds = resource.getVirtualCores() + * usedMillis / DateUtils.MILLIS_PER_SECOND; + return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index ff60fcd..15e45c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -170,7 +171,7 @@ public abstract class MockAsm extends MockApps { @Override public RMAppMetrics getRMAppMetrics() { - return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0); + return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0); } } @@ -259,6 +260,22 @@ public abstract class MockAsm extends MockApps { public Set getApplicationTags() { return null; } + + @Override + public ApplicationReport createAndGetApplicationReport( + String clientUserName, boolean allowAccess) { + ApplicationResourceUsageReport usageReport = + ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, + 0, 0); + ApplicationReport report = ApplicationReport.newInstance( + getApplicationId(), appAttemptId, getUser(), getQueue(), + getName(), null, 0, null, null, getDiagnostics().toString(), + getTrackingUrl(), getStartTime(), getFinishTime(), + getFinalApplicationStatus(), usageReport , null, getProgress(), + type, null); + return report; + } + }; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 620ba9f..2621dff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -64,7 +64,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMSta import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; @@ -152,6 +154,8 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{ SecretKey clientTokenMasterKey, TestDispatcher dispatcher) throws Exception { + RMAppAttemptMetrics mockRmAppAttemptMetrics = + mock(RMAppAttemptMetrics.class); Container container = new ContainerPBImpl(); container.setId(ConverterUtils.toContainerId(containerIdStr)); RMAppAttempt mockAttempt = mock(RMAppAttempt.class); @@ -160,6 +164,10 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{ when(mockAttempt.getAMRMToken()).thenReturn(appToken); when(mockAttempt.getClientTokenMasterKey()) .thenReturn(clientTokenMasterKey); + when(mockAttempt.getRMAppAttemptMetrics()) + .thenReturn(mockRmAppAttemptMetrics); + when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) + .thenReturn(new AggregateAppResourceUsage(0,0)); dispatcher.attemptId = attemptId; store.storeNewApplicationAttempt(mockAttempt); waitNotify(dispatcher); @@ -224,6 +232,8 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{ "container_1352994193343_0002_01_000001", null, null, dispatcher); RMApp mockRemovedApp = mock(RMApp.class); + RMAppAttemptMetrics mockRmAppAttemptMetrics = + mock(RMAppAttemptMetrics.class); HashMap attempts = new HashMap(); ApplicationSubmissionContext context = @@ -234,6 +244,10 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{ when(mockRemovedApp.getAppAttempts()).thenReturn(attempts); RMAppAttempt mockRemovedAttempt = mock(RMAppAttempt.class); when(mockRemovedAttempt.getAppAttemptId()).thenReturn(attemptIdRemoved); + when(mockRemovedAttempt.getRMAppAttemptMetrics()) + .thenReturn(mockRmAppAttemptMetrics); + when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) + .thenReturn(new AggregateAppResourceUsage(0,0)); attempts.put(attemptIdRemoved, mockRemovedAttempt); store.removeApplication(mockRemovedApp); @@ -304,7 +318,7 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{ oldAttemptState.getAppAttemptCredentials(), oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", - FinalApplicationStatus.SUCCEEDED, 100); + FinalApplicationStatus.SUCCEEDED, 100, 0, 0); store.updateApplicationAttemptState(newAttemptState); // test updating the state of an app/attempt whose initial state was not @@ -327,7 +341,7 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{ oldAttemptState.getAppAttemptCredentials(), oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", - FinalApplicationStatus.SUCCEEDED, 111); + FinalApplicationStatus.SUCCEEDED, 111, 0, 0); store.updateApplicationAttemptState(dummyAttempt); // let things settle down http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 3c871df..5874b5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -19,8 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -38,6 +40,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.yarn.MockApps; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -61,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; @@ -74,6 +78,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; @RunWith(value = Parameterized.class) @@ -189,7 +194,7 @@ public class TestRMAppTransitions { AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class); store = mock(RMStateStore.class); writer = mock(RMApplicationHistoryWriter.class); - this.rmContext = + RMContext realRMContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, new AMRMTokenSecretManager(conf, this.rmContext), @@ -197,7 +202,14 @@ public class TestRMAppTransitions { new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), writer); - ((RMContextImpl)rmContext).setStateStore(store); + ((RMContextImpl)realRMContext).setStateStore(store); + + this.rmContext = spy(realRMContext); + + ResourceScheduler resourceScheduler = mock(ResourceScheduler.class); + doReturn(null).when(resourceScheduler) + .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()); + doReturn(resourceScheduler).when(rmContext).getScheduler(); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher(this.rmContext)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index efcecd9..ae318b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -49,6 +50,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -87,6 +89,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -107,6 +111,8 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; +import org.mockito.Mockito; @RunWith(value = Parameterized.class) public class TestRMAppAttemptTransitions { @@ -120,7 +126,9 @@ public class TestRMAppAttemptTransitions { private boolean isSecurityEnabled; private RMContext rmContext; + private RMContext spyRMContext; private YarnScheduler scheduler; + private ResourceScheduler resourceScheduler; private ApplicationMasterService masterService; private ApplicationMasterLauncher applicationMasterLauncher; private AMLivelinessMonitor amLivelinessMonitor; @@ -262,7 +270,20 @@ public class TestRMAppAttemptTransitions { ApplicationId applicationId = MockApps.newAppID(appId++); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0); - + + resourceScheduler = mock(ResourceScheduler.class); + + ApplicationResourceUsageReport appResUsgRpt = + mock(ApplicationResourceUsageReport.class); + when(appResUsgRpt.getMemorySeconds()).thenReturn(0L); + when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L); + when(resourceScheduler + .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any())) + .thenReturn(appResUsgRpt); + spyRMContext = spy(rmContext); + Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler(); + + final String user = MockApps.newUserName(); final String queue = MockApps.newQueue(); submissionContext = mock(ApplicationSubmissionContext.class); @@ -278,17 +299,18 @@ public class TestRMAppAttemptTransitions { application = mock(RMAppImpl.class); applicationAttempt = - new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler, + new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), false); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); - + spyRMContext.getRMApps().put(application.getApplicationId(), application); + testAppAttemptNewState(); } @After public void tearDown() throws Exception { - ((AsyncDispatcher)this.rmContext.getDispatcher()).stop(); + ((AsyncDispatcher)this.spyRMContext.getDispatcher()).stop(); } @@ -699,6 +721,46 @@ public class TestRMAppAttemptTransitions { } @Test + public void testUsageReport() { + // scheduler has info on running apps + ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId(); + ApplicationResourceUsageReport appResUsgRpt = + mock(ApplicationResourceUsageReport.class); + when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L); + when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L); + when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class))) + .thenReturn(appResUsgRpt); + + // start and finish the attempt + Container amContainer = allocateApplicationAttempt(); + launchApplicationAttempt(amContainer); + runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); + applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId, + "", FinalApplicationStatus.SUCCEEDED, "")); + + // expect usage stats to come from the scheduler report + ApplicationResourceUsageReport report = + applicationAttempt.getApplicationResourceUsageReport(); + Assert.assertEquals(123456L, report.getMemorySeconds()); + Assert.assertEquals(55544L, report.getVcoreSeconds()); + + // finish app attempt and remove it from scheduler + when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L); + when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L); + sendAttemptUpdateSavedEvent(applicationAttempt); + applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( + attemptId, + ContainerStatus.newInstance( + amContainer.getId(), ContainerState.COMPLETE, "", 0))); + + when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null); + + report = applicationAttempt.getApplicationResourceUsageReport(); + Assert.assertEquals(223456, report.getMemorySeconds()); + Assert.assertEquals(75544, report.getVcoreSeconds()); + } + + @Test public void testUnmanagedAMUnexpectedRegistration() { unmanagedAM = true; when(submissionContext.getUnmanagedAM()).thenReturn(true); @@ -1243,7 +1305,7 @@ public class TestRMAppAttemptTransitions { public void testContainersCleanupForLastAttempt() { // create a failed attempt. applicationAttempt = - new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), rmContext, + new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), true); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 44f8381..9862cfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -23,10 +23,13 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -59,6 +62,8 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; +import org.mockito.Mockito; @SuppressWarnings({ "unchecked", "rawtypes" }) public class TestRMContainerImpl { @@ -86,12 +91,18 @@ public class TestRMContainerImpl { Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); + ConcurrentMap rmApps = + spy(new ConcurrentHashMap()); + RMApp rmApp = mock(RMApp.class); + when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null); + Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any()); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); + when(rmContext.getRMApps()).thenReturn(rmApps); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index d5eb933..083cb71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -39,12 +39,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.junit.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -59,6 +62,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -79,6 +83,8 @@ import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -90,6 +96,7 @@ public class TestLeafQueue { RecordFactoryProvider.getRecordFactory(null); RMContext rmContext; + RMContext spyRMContext; CapacityScheduler cs; CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; @@ -107,6 +114,14 @@ public class TestLeafQueue { CapacityScheduler spyCs = new CapacityScheduler(); cs = spy(spyCs); rmContext = TestUtils.getMockRMContext(); + spyRMContext = spy(rmContext); + + ConcurrentMap spyApps = + spy(new ConcurrentHashMap()); + RMApp rmApp = mock(RMApp.class); + when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null); + Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any()); + when(spyRMContext.getRMApps()).thenReturn(spyApps); csConf = new CapacitySchedulerConfiguration(); @@ -143,7 +158,7 @@ public class TestLeafQueue { queues, queues, TestUtils.spyHook); - cs.setRMContext(rmContext); + cs.setRMContext(spyRMContext); cs.init(csConf); cs.start(); } @@ -280,14 +295,14 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_1, user_0); // same user @@ -329,14 +344,14 @@ public class TestLeafQueue { final ApplicationAttemptId appAttemptId_0 = TestUtils .getMockApplicationAttemptId(0, 1); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_d, d, null, - rmContext); + spyRMContext); d.submitApplicationAttempt(app_0, user_d); // Attempt the same application again final ApplicationAttemptId appAttemptId_1 = TestUtils .getMockApplicationAttemptId(0, 2); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_d, d, null, - rmContext); + spyRMContext); d.submitApplicationAttempt(app_1, user_d); // same user } @@ -373,7 +388,7 @@ public class TestLeafQueue { final ApplicationAttemptId appAttemptId_1 = TestUtils .getMockApplicationAttemptId(0, 2); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, null, - rmContext); + spyRMContext); a.submitApplicationAttempt(app_1, user_0); // same user assertEquals(1, a.getMetrics().getAppsSubmitted()); @@ -411,14 +426,14 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_1, user_0); // same user @@ -545,21 +560,21 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_1, user_0); // same user final ApplicationAttemptId appAttemptId_2 = TestUtils.getMockApplicationAttemptId(2, 0); FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_1, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_2, user_1); // Setup some nodes @@ -639,21 +654,21 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_1, user_0); // same user final ApplicationAttemptId appAttemptId_2 = TestUtils.getMockApplicationAttemptId(2, 0); FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_1, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_2, user_1); // Setup some nodes @@ -750,28 +765,28 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_1, user_0); // same user final ApplicationAttemptId appAttemptId_2 = TestUtils.getMockApplicationAttemptId(2, 0); FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_1, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_2, user_1); final ApplicationAttemptId appAttemptId_3 = TestUtils.getMockApplicationAttemptId(3, 0); FiCaSchedulerApp app_3 = new FiCaSchedulerApp(appAttemptId_3, user_2, a, - a.getActiveUsersManager(), rmContext); + a.getActiveUsersManager(), spyRMContext); a.submitApplicationAttempt(app_3, user_2); // Setup some nodes @@ -935,14 +950,14 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_1, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_1, user_1); // Setup some nodes @@ -1043,14 +1058,14 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_1, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_1, user_1); // Setup some nodes @@ -1150,14 +1165,14 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_1, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_1, user_1); // Setup some nodes @@ -1277,7 +1292,7 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext)); + mock(ActiveUsersManager.class), spyRMContext)); a.submitApplicationAttempt(app_0, user_0); // Setup some nodes and racks @@ -1418,7 +1433,7 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext)); + mock(ActiveUsersManager.class), spyRMContext)); a.submitApplicationAttempt(app_0, user_0); // Setup some nodes and racks @@ -1549,7 +1564,7 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext)); + mock(ActiveUsersManager.class), spyRMContext)); a.submitApplicationAttempt(app_0, user_0); // Setup some nodes and racks @@ -1652,21 +1667,21 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_e, e, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); e.submitApplicationAttempt(app_0, user_e); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_e, e, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); e.submitApplicationAttempt(app_1, user_e); // same user final ApplicationAttemptId appAttemptId_2 = TestUtils.getMockApplicationAttemptId(2, 0); FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_e, e, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); e.submitApplicationAttempt(app_2, user_e); // same user // before reinitialization @@ -1730,21 +1745,21 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_e, e, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); e.submitApplicationAttempt(app_0, user_e); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_e, e, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); e.submitApplicationAttempt(app_1, user_e); // same user final ApplicationAttemptId appAttemptId_2 = TestUtils.getMockApplicationAttemptId(2, 0); FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_e, e, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); e.submitApplicationAttempt(app_2, user_e); // same user // before updating cluster resource @@ -1807,14 +1822,14 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext)); + mock(ActiveUsersManager.class), spyRMContext)); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = spy(new FiCaSchedulerApp(appAttemptId_1, user_0, a, - mock(ActiveUsersManager.class), rmContext)); + mock(ActiveUsersManager.class), spyRMContext)); a.submitApplicationAttempt(app_1, user_0); // Setup some nodes and racks @@ -2062,14 +2077,14 @@ public class TestLeafQueue { TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_0, user_0); final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, - mock(ActiveUsersManager.class), rmContext); + mock(ActiveUsersManager.class), spyRMContext); a.submitApplicationAttempt(app_1, user_0); // same user // Setup some nodes http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index 61def87..bd7f1bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -162,7 +162,7 @@ public class FairSchedulerTestBase { RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt); when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn( - new RMAppAttemptMetrics(id)); + new RMAppAttemptMetrics(id, resourceManager.getRMContext())); resourceManager.getRMContext().getRMApps() .put(id.getApplicationId(), rmApp); return id; @@ -183,7 +183,7 @@ public class FairSchedulerTestBase { RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt); when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn( - new RMAppAttemptMetrics(id)); + new RMAppAttemptMetrics(id,resourceManager.getRMContext())); resourceManager.getRMContext().getRMApps() .put(id.getApplicationId(), rmApp); return id; http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java index 111bf47..f07cb8d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java @@ -136,7 +136,7 @@ public class TestRMWebAppFairScheduler { MockRMApp app = new MockRMApp(i, i, state) { @Override public RMAppMetrics getRMAppMetrics() { - return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0); + return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0); } @Override public YarnApplicationState createApplicationState() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index b009bfa..3701dd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1322,7 +1322,7 @@ public class TestRMWebServicesApps extends JerseyTest { Exception { // 28 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 24, info.length()); + assertEquals("incorrect number of elements", 26, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 9609ba3..0fd5b24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1197,7 +1197,9 @@ ResourceManager REST API's. "queue" : "default", "allocatedMB" : 0, "allocatedVCores" : 0, - "runningContainers" : 0 + "runningContainers" : 0, + "memorySeconds" : 151730, + "vcoreSeconds" : 103 }, { "finishedTime" : 1326815789546, @@ -1218,7 +1220,9 @@ ResourceManager REST API's. "queue" : "default", "allocatedMB" : 0, "allocatedVCores" : 0, - "runningContainers" : 1 + "runningContainers" : 1, + "memorySeconds" : 640064, + "vcoreSeconds" : 442 } ] } @@ -1271,6 +1275,8 @@ _01_000001 0 0 0 + 151730 + 103 application_1326815542473_0002 @@ -1293,6 +1299,8 @@ _01_000001 0 0 0 + 640064 + 442 +---+ @@ -1491,6 +1499,10 @@ _01_000001 +---------------------------------------------------------------+ | runningContainers | int | The number of containers currently running for the application | +---------------------------------------------------------------+ +| memorySeconds | long | The amount of memory the application has allocated (megabyte-seconds) | +*---------------+--------------+--------------------------------+ +| vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) | +*---------------+--------------+--------------------------------+ ** Response Examples @@ -1532,7 +1544,9 @@ _01_000001 "elapsedTime" : 446748, "diagnostics" : "", "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5", - "queue" : "a1" + "queue" : "a1", + "memorySeconds" : 151730, + "vcoreSeconds" : 103 } } +---+ @@ -1576,6 +1590,8 @@ _01_000001 446748 http://host.domain.com:8042/node/containerlogs/container_1326821518301_0005_01_000001 host.domain.com:8042 + 151730 + 103 +---+