Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3310A17AE5 for ; Fri, 6 Feb 2015 21:46:37 +0000 (UTC) Received: (qmail 53757 invoked by uid 500); 6 Feb 2015 21:46:33 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 53440 invoked by uid 500); 6 Feb 2015 21:46:33 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 53423 invoked by uid 99); 6 Feb 2015 21:46:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 06 Feb 2015 21:46:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1DC21E0243; Fri, 6 Feb 2015 21:46:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zhz@apache.org To: common-commits@hadoop.apache.org Date: Fri, 06 Feb 2015 21:46:34 -0000 Message-Id: <7e88d9a2b3b4496b88919cda6e981b3b@git.apache.org> In-Reply-To: <45882ce1749b480984b67e2725c7f039@git.apache.org> References: <45882ce1749b480984b67e2725c7f039@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [02/50] [abbrv] hadoop git commit: YARN-2543. Made resource usage be published to the timeline server too. Contributed by Naganarasimha G R. YARN-2543. Made resource usage be published to the timeline server too. Contributed by Naganarasimha G R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42242977 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42242977 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42242977 Branch: refs/heads/HDFS-EC Commit: 422429778fb46a1eed3de80cfb7c7d90eddea571 Parents: b6c8a0f Author: Zhijie Shen Authored: Tue Feb 3 17:33:53 2015 -0800 Committer: Zhe Zhang Committed: Fri Feb 6 13:45:46 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../ApplicationHistoryManagerOnTimelineStore.java | 12 +++++++++++- .../TestApplicationHistoryClientService.java | 4 ++++ ...stApplicationHistoryManagerOnTimelineStore.java | 9 +++++++++ .../metrics/ApplicationMetricsConstants.java | 6 ++++++ .../metrics/ApplicationFinishedEvent.java | 9 ++++++++- .../metrics/SystemMetricsPublisher.java | 10 +++++++++- .../metrics/TestSystemMetricsPublisher.java | 17 +++++++++++++++-- 8 files changed, 65 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/42242977/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 30f5a1e..33e98d1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -470,6 +470,9 @@ Release 2.7.0 - UNRELEASED YARN-3056. Add verification for containerLaunchDuration in TestNodeManagerMetrics. (zhihai xu via xgong) + YARN-2543. Made resource usage be published to the timeline server too. + (Naganarasimha G R via zjshen) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/42242977/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index c88cccc..22418a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; @@ -222,6 +223,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService String diagnosticsInfo = null; FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED; YarnApplicationState state = null; + ApplicationResourceUsageReport appResources = null; Map appViewACLs = new HashMap(); Map entityInfo = entity.getOtherInfo(); @@ -260,6 +262,14 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) .toString(); } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { + long vcoreSeconds=Long.parseLong(entityInfo.get( + ApplicationMetricsConstants.APP_CPU_METRICS).toString()); + long memorySeconds=Long.parseLong(entityInfo.get( + ApplicationMetricsConstants.APP_MEM_METRICS).toString()); + appResources=ApplicationResourceUsageReport + .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); + } } List events = entity.getEvents(); if (events != null) { @@ -310,7 +320,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, + diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42242977/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index 2a38d0a..32d011e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -80,6 +80,10 @@ public class TestApplicationHistoryClientService { clientService.getClientHandler().getApplicationReport(request); ApplicationReport appReport = response.getApplicationReport(); Assert.assertNotNull(appReport); + Assert.assertEquals(123, appReport.getApplicationResourceUsageReport() + .getMemorySeconds()); + Assert.assertEquals(345, appReport.getApplicationResourceUsageReport() + .getVcoreSeconds()); Assert.assertEquals("application_0_0001", appReport.getApplicationId() .toString()); Assert.assertEquals("test app type", http://git-wip-us.apache.org/repos/asf/hadoop/blob/42242977/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index a093f19..50a15f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -201,6 +202,12 @@ public class TestApplicationHistoryManagerOnTimelineStore { app.getOriginalTrackingUrl()); Assert.assertEquals("test diagnostics info", app.getDiagnostics()); } + ApplicationResourceUsageReport applicationResourceUsageReport = + app.getApplicationResourceUsageReport(); + Assert.assertEquals(123, + applicationResourceUsageReport.getMemorySeconds()); + Assert + .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); Assert.assertEquals(FinalApplicationStatus.UNDEFINED, app.getFinalApplicationStatus()); Assert.assertEquals(YarnApplicationState.FINISHED, @@ -416,6 +423,8 @@ public class TestApplicationHistoryManagerOnTimelineStore { entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue"); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); + entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123); + entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS,345); if (emptyACLs) { entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, ""); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/42242977/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index ee34c49..df8eecb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -63,6 +63,12 @@ public class ApplicationMetricsConstants { public static final String STATE_EVENT_INFO = "YARN_APPLICATION_STATE"; + + public static final String APP_CPU_METRICS = + "YARN_APPLICATION_CPU_METRIC"; + + public static final String APP_MEM_METRICS = + "YARN_APPLICATION_MEM_METRIC"; public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/42242977/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java index c457710..8d75f92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; public class ApplicationFinishedEvent extends SystemMetricsEvent { @@ -31,6 +32,7 @@ public class ApplicationFinishedEvent extends private FinalApplicationStatus appStatus; private YarnApplicationState state; private ApplicationAttemptId latestAppAttemptId; + private RMAppMetrics appMetrics; public ApplicationFinishedEvent( ApplicationId appId, @@ -38,13 +40,15 @@ public class ApplicationFinishedEvent extends FinalApplicationStatus appStatus, YarnApplicationState state, ApplicationAttemptId latestAppAttemptId, - long finishedTime) { + long finishedTime, + RMAppMetrics appMetrics) { super(SystemMetricsEventType.APP_FINISHED, finishedTime); this.appId = appId; this.diagnosticsInfo = diagnosticsInfo; this.appStatus = appStatus; this.latestAppAttemptId = latestAppAttemptId; this.state = state; + this.appMetrics=appMetrics; } @Override @@ -72,4 +76,7 @@ public class ApplicationFinishedEvent extends return latestAppAttemptId; } + public RMAppMetrics getAppMetrics() { + return appMetrics; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42242977/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index d8e7ef6..3adf519 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -120,7 +121,8 @@ public class SystemMetricsPublisher extends CompositeService { RMServerUtils.createApplicationState(state), app.getCurrentAppAttempt() == null ? null : app.getCurrentAppAttempt().getAppAttemptId(), - finishedTime)); + finishedTime, + app.getRMAppMetrics())); } } @@ -276,6 +278,12 @@ public class SystemMetricsPublisher extends CompositeService { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, event.getLatestApplicationAttemptId().toString()); } + RMAppMetrics appMetrics = event.getAppMetrics(); + entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS, + appMetrics.getVcoreSeconds()); + entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, + appMetrics.getMemorySeconds()); + tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); http://git-wip-us.apache.org/repos/asf/hadoop/blob/42242977/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 946d526..9f02721 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -148,8 +149,18 @@ public class TestSystemMetricsPublisher { entity.getOtherInfo().get( ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); } else { - Assert.assertEquals("", entity.getOtherInfo().get( - ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); + Assert.assertEquals( + "", + entity.getOtherInfo().get( + ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); + Assert.assertEquals( + app.getRMAppMetrics().getMemorySeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString())); + Assert.assertEquals( + app.getRMAppMetrics().getVcoreSeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString())); } boolean hasCreatedEvent = false; boolean hasFinishedEvent = false; @@ -340,6 +351,8 @@ public class TestSystemMetricsPublisher { when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); + when(app.getRMAppMetrics()).thenReturn( + new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); return app; }