From common-commits-return-92275-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Sat Jan 5 00:21:06 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 7E97818072F for ; Sat, 5 Jan 2019 00:21:03 +0100 (CET) Received: (qmail 50476 invoked by uid 500); 4 Jan 2019 23:20:57 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 50084 invoked by uid 99); 4 Jan 2019 23:20:56 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Jan 2019 23:20:56 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 734B3821C5; Fri, 4 Jan 2019 23:20:55 +0000 (UTC) Date: Fri, 04 Jan 2019 23:21:04 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] 10/46: YARN-6232. Update resource usage and preempted resource calculations to take into account all resource types. Contributed by Varun Vasudev. MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: jhung@apache.org In-Reply-To: <154664405408.18045.9135713536105672735@gitbox.apache.org> References: <154664405408.18045.9135713536105672735@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/YARN-8200 X-Git-Reftype: branch X-Git-Rev: 0d277ce3c633107c108326ea89de9a63147f99d3 X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20190104232055.734B3821C5@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. jhung pushed a commit to branch YARN-8200 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 0d277ce3c633107c108326ea89de9a63147f99d3 Author: Sunil G AuthorDate: Mon Mar 6 11:34:20 2017 +0530 YARN-6232. Update resource usage and preempted resource calculations to take into account all resource types. Contributed by Varun Vasudev. (cherry picked from commit dae65f3bef8ffa34d02a37041f1dfdfeeee91845) Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java --- .../records/ApplicationResourceUsageReport.java | 58 +++++++- .../src/main/proto/yarn_protos.proto | 7 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 35 ++--- .../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 16 ++- .../pb/ApplicationResourceUsageReportPBImpl.java | 151 ++++++++++++++++++--- .../yarn/api/records/impl/pb/ProtoUtils.java | 34 +++++ .../org/apache/hadoop/yarn/util/StringHelper.java | 36 +++++ .../hadoop/yarn/api/BasePBImplRecordsTest.java | 12 ++ .../apache/hadoop/yarn/api/TestPBImplRecords.java | 4 + .../ApplicationHistoryManagerOnTimelineStore.java | 18 ++- .../hadoop/yarn/server/utils/BuilderUtils.java | 16 +-- .../yarn/server/resourcemanager/RMAppManager.java | 8 +- .../yarn/server/resourcemanager/RMServerUtils.java | 10 +- .../resourcemanager/recovery/RMStateStore.java | 7 +- .../records/ApplicationAttemptStateData.java | 89 ++++++++++-- .../impl/pb/ApplicationAttemptStateDataPBImpl.java | 50 +++++++ .../server/resourcemanager/rmapp/RMAppImpl.java | 43 +++--- .../server/resourcemanager/rmapp/RMAppMetrics.java | 41 ++++-- .../rmapp/attempt/AggregateAppResourceUsage.java | 34 ++--- .../rmapp/attempt/RMAppAttemptImpl.java | 32 ++--- .../rmapp/attempt/RMAppAttemptMetrics.java | 106 ++++++++++++--- .../rmcontainer/RMContainerImpl.java | 16 +-- .../scheduler/SchedulerApplicationAttempt.java | 47 ++++--- .../server/resourcemanager/webapp/RMAppBlock.java | 14 +- .../server/resourcemanager/webapp/dao/AppInfo.java | 28 ++++ .../resourcemanager/webapp/dao/ResourceInfo.java | 36 ++++- .../resourcemanager/webapp/dao/SchedulerInfo.java | 2 +- .../yarn_server_resourcemanager_recovery.proto | 2 + .../server/resourcemanager/TestAppManager.java | 7 +- .../TestContainerResourceUsage.java | 7 +- .../applicationsmanager/MockAsm.java | 9 +- .../metrics/TestSystemMetricsPublisher.java | 15 +- .../metrics/TestSystemMetricsPublisherForV2.java | 22 ++- .../recovery/RMStateStoreTestBase.java | 8 +- .../recovery/TestZKRMStateStore.java | 23 ++-- .../server/resourcemanager/webapp/TestAppPage.java | 8 +- .../webapp/TestRMWebAppFairScheduler.java | 5 +- 37 files changed, 803 insertions(+), 253 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index 3cf8f3d..f9c8975 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -24,6 +24,9 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; +import java.util.HashMap; +import java.util.Map; + /** * Contains various scheduling metrics to be reported by UI and CLI. */ @@ -35,9 +38,9 @@ public abstract class ApplicationResourceUsageReport { @Unstable public static ApplicationResourceUsageReport newInstance( int numUsedContainers, int numReservedContainers, Resource usedResources, - Resource reservedResources, Resource neededResources, long memorySeconds, - long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc, - long preemptedMemorySeconds, long preemptedVcoresSeconds) { + Resource reservedResources, Resource neededResources, + Map resourceSecondsMap, float queueUsagePerc, + float clusterUsagePerc, Map preemtedResourceSecondsMap) { ApplicationResourceUsageReport report = Records.newRecord(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); @@ -45,12 +48,10 @@ public abstract class ApplicationResourceUsageReport { report.setUsedResources(usedResources); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); - report.setMemorySeconds(memorySeconds); - report.setVcoreSeconds(vcoreSeconds); + report.setResourceSecondsMap(resourceSecondsMap); report.setQueueUsagePercentage(queueUsagePerc); report.setClusterUsagePercentage(clusterUsagePerc); - report.setPreemptedMemorySeconds(preemptedMemorySeconds); - report.setPreemptedVcoreSeconds(preemptedVcoresSeconds); + report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap); return report; } @@ -229,4 +230,47 @@ public abstract class ApplicationResourceUsageReport { @Public @Unstable public abstract long getPreemptedVcoreSeconds(); + + /** + * Get the aggregated number of resources that the application has + * allocated times the number of seconds the application has been running. + * @return map containing the resource name and aggregated resource-seconds + */ + @Public + @Unstable + public abstract Map getResourceSecondsMap(); + + /** + * Set the aggregated number of resources that the application has + * allocated times the number of seconds the application has been running. + * @param resourceSecondsMap map containing the resource name and aggregated + * resource-seconds + */ + @Private + @Unstable + public abstract void setResourceSecondsMap( + Map resourceSecondsMap); + + + /** + * Get the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * @return map containing the resource name and aggregated preempted + * resource-seconds + */ + @Public + @Unstable + public abstract Map getPreemptedResourceSecondsMap(); + + /** + * Set the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * @param preemptedResourceSecondsMap map containing the resource name and + * aggregated preempted resource-seconds + */ + @Private + @Unstable + public abstract void setPreemptedResourceSecondsMap( + Map preemptedResourceSecondsMap); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index a7bf414..4f4c3e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -217,6 +217,11 @@ message LocalResourceProto { optional bool should_be_uploaded_to_shared_cache = 7; } +message StringLongMapProto { + required string key = 1; + required int64 value = 2; +} + message ApplicationResourceUsageReportProto { optional int32 num_used_containers = 1; optional int32 num_reserved_containers = 2; @@ -229,6 +234,8 @@ message ApplicationResourceUsageReportProto { optional float cluster_usage_percentage = 9; optional int64 preempted_memory_seconds = 10; optional int64 preempted_vcore_seconds = 11; + repeated StringLongMapProto application_resource_usage_map = 12; + repeated StringLongMapProto application_preempted_resource_usage_map = 13; } message ApplicationReportProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 5f6b300..2a9b3bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -62,6 +62,8 @@ import org.apache.hadoop.yarn.util.Times; import com.google.common.annotations.VisibleForTesting; +import static org.apache.hadoop.yarn.util.StringHelper.getResourceSecondsString; + @Private @Unstable public class ApplicationCLI extends YarnCLI { @@ -711,24 +713,9 @@ public class ApplicationCLI extends YarnCLI { appReportStr.println(appReport.getRpcPort()); appReportStr.print("\tAM Host : "); appReportStr.println(appReport.getHost()); - appReportStr.print("\tAggregate Resource Allocation : "); - ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport(); - if (usageReport != null) { - //completed app report in the timeline server doesn't have usage report - appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, "); - appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds"); - appReportStr.print("\tAggregate Resource Preempted : "); - appReportStr.print(usageReport.getPreemptedMemorySeconds() + - " MB-seconds, "); - appReportStr.println(usageReport.getPreemptedVcoreSeconds() + - " vcore-seconds"); - } else { - appReportStr.println("N/A"); - appReportStr.print("\tAggregate Resource Preempted : "); - appReportStr.println("N/A"); - } + printResourceUsage(appReportStr, usageReport); appReportStr.print("\tLog Aggregation Status : "); appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A" : appReport.getLogAggregationStatus()); @@ -759,6 +746,22 @@ public class ApplicationCLI extends YarnCLI { return 0; } + private void printResourceUsage(PrintWriter appReportStr, + ApplicationResourceUsageReport usageReport) { + appReportStr.print("\tAggregate Resource Allocation : "); + if (usageReport != null) { + appReportStr.println( + getResourceSecondsString(usageReport.getResourceSecondsMap())); + appReportStr.print("\tAggregate Resource Preempted : "); + appReportStr.println(getResourceSecondsString( + usageReport.getPreemptedResourceSecondsMap())); + } else { + appReportStr.println("N/A"); + appReportStr.print("\tAggregate Resource Preempted : "); + appReportStr.println("N/A"); + } + } + private String getAllValidApplicationStates() { StringBuilder sb = new StringBuilder(); sb.append("The valid application state can be" + " one of the following: "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 0f0c9bf..eee3af1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -39,8 +39,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -69,6 +71,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; @@ -118,9 +121,18 @@ public class TestYarnCLI { for (int i = 0; i < 2; ++i) { ApplicationCLI cli = createAndGetAppCLI(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + Map resourceSecondsMap = new HashMap<>(); + Map preemptedResoureSecondsMap = new HashMap<>(); + resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L); + resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L); + preemptedResoureSecondsMap + .put(ResourceInformation.MEMORY_MB.getName(), 1111L); + preemptedResoureSecondsMap + .put(ResourceInformation.VCORES.getName(), 2222L); ApplicationResourceUsageReport usageReport = i == 0 ? null : - ApplicationResourceUsageReport.newInstance( - 2, 0, null, null, null, 123456, 4567, 0, 0, 1111, 2222); + ApplicationResourceUsageReport + .newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0, + preemptedResoureSecondsMap); ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java index 1c85e28..2f94a24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java @@ -22,12 +22,16 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import com.google.protobuf.TextFormat; +import java.util.HashMap; +import java.util.Map; + @Private @Unstable public class ApplicationResourceUsageReportPBImpl @@ -41,6 +45,9 @@ extends ApplicationResourceUsageReport { Resource reservedResources; Resource neededResources; + private Map resourceSecondsMap; + private Map preemptedResourceSecondsMap; + public ApplicationResourceUsageReportPBImpl() { builder = ApplicationResourceUsageReportProto.newBuilder(); } @@ -49,6 +56,8 @@ extends ApplicationResourceUsageReport { ApplicationResourceUsageReportProto proto) { this.proto = proto; viaProto = true; + getResourceSecondsMap(); + getPreemptedResourceSecondsMap(); } public synchronized ApplicationResourceUsageReportProto getProto() { @@ -89,6 +98,23 @@ extends ApplicationResourceUsageReport { if (this.neededResources != null) { builder.setNeededResources(convertToProtoFormat(this.neededResources)); } + builder.clearApplicationResourceUsageMap(); + builder.clearApplicationPreemptedResourceUsageMap(); + + if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap + .isEmpty()) { + builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils + .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap)); + } + if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) { + builder.addAllApplicationResourceUsageMap( + ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); + } + + builder.setMemorySeconds(this.getMemorySeconds()); + builder.setVcoreSeconds(this.getVcoreSeconds()); + builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds()); + builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds()); } private void mergeLocalToProto() { @@ -196,54 +222,64 @@ extends ApplicationResourceUsageReport { @Override public synchronized void setMemorySeconds(long memory_seconds) { - maybeInitBuilder(); - builder.setMemorySeconds(memory_seconds); + getResourceSecondsMap() + .put(ResourceInformation.MEMORY_MB.getName(), memory_seconds); } - + @Override public synchronized long getMemorySeconds() { - ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; - return p.getMemorySeconds(); + Map tmp = getResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { + return tmp.get(ResourceInformation.MEMORY_MB.getName()); + } + return 0; } @Override public synchronized void setVcoreSeconds(long vcore_seconds) { - maybeInitBuilder(); - builder.setVcoreSeconds(vcore_seconds); + getResourceSecondsMap() + .put(ResourceInformation.VCORES.getName(), vcore_seconds); } @Override public synchronized long getVcoreSeconds() { - ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; - return (p.getVcoreSeconds()); + Map tmp = getResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.VCORES.getName())) { + return tmp.get(ResourceInformation.VCORES.getName()); + } + return 0; } @Override public synchronized void setPreemptedMemorySeconds( long preemptedMemorySeconds) { - maybeInitBuilder(); - builder.setPreemptedMemorySeconds(preemptedMemorySeconds); + getPreemptedResourceSecondsMap() + .put(ResourceInformation.MEMORY_MB.getName(), preemptedMemorySeconds); } @Override public synchronized long getPreemptedMemorySeconds() { - ApplicationResourceUsageReportProtoOrBuilder p = - viaProto ? proto : builder; - return p.getPreemptedMemorySeconds(); + Map tmp = getPreemptedResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { + return tmp.get(ResourceInformation.MEMORY_MB.getName()); + } + return 0; } @Override public synchronized void setPreemptedVcoreSeconds( long vcoreSeconds) { - maybeInitBuilder(); - builder.setPreemptedVcoreSeconds(vcoreSeconds); + getPreemptedResourceSecondsMap() + .put(ResourceInformation.VCORES.getName(), vcoreSeconds); } @Override public synchronized long getPreemptedVcoreSeconds() { - ApplicationResourceUsageReportProtoOrBuilder p = - viaProto ? proto : builder; - return (p.getPreemptedVcoreSeconds()); + Map tmp = getPreemptedResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.VCORES.getName())) { + return tmp.get(ResourceInformation.VCORES.getName()); + } + return 0; } private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { @@ -277,4 +313,81 @@ extends ApplicationResourceUsageReport { maybeInitBuilder(); builder.setClusterUsagePercentage((clusterUsagePerc)); } + + @Override + public synchronized void setResourceSecondsMap( + Map resourceSecondsMap) { + this.resourceSecondsMap = resourceSecondsMap; + if (resourceSecondsMap == null) { + return; + } + if (!resourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.setMemorySeconds(0L); + } + if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) { + this.setVcoreSeconds(0L); + } + } + + @Override + public synchronized Map getResourceSecondsMap() { + if (this.resourceSecondsMap != null) { + return this.resourceSecondsMap; + } + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + this.resourceSecondsMap = ProtoUtils + .convertStringLongMapProtoListToMap( + p.getApplicationResourceUsageMapList()); + if (!this.resourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.setMemorySeconds(p.getMemorySeconds()); + } + if (!this.resourceSecondsMap + .containsKey(ResourceInformation.VCORES.getName())) { + this.setVcoreSeconds(p.getVcoreSeconds()); + } + this.setMemorySeconds(p.getMemorySeconds()); + this.setVcoreSeconds(p.getVcoreSeconds()); + return this.resourceSecondsMap; + } + + @Override + public synchronized void setPreemptedResourceSecondsMap( + Map preemptedResourceSecondsMap) { + this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; + if (preemptedResourceSecondsMap == null) { + return; + } + if (!preemptedResourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.setPreemptedMemorySeconds(0L); + } + if (!preemptedResourceSecondsMap + .containsKey(ResourceInformation.VCORES.getName())) { + this.setPreemptedVcoreSeconds(0L); + } + } + + @Override + public synchronized Map getPreemptedResourceSecondsMap() { + if (this.preemptedResourceSecondsMap != null) { + return this.preemptedResourceSecondsMap; + } + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + this.preemptedResourceSecondsMap = ProtoUtils + .convertStringLongMapProtoListToMap( + p.getApplicationPreemptedResourceUsageMapList()); + if (!this.preemptedResourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.setPreemptedMemorySeconds(p.getPreemptedMemorySeconds()); + } + if (!this.preemptedResourceSecondsMap + .containsKey(ResourceInformation.VCORES.getName())) { + this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds()); + } + this.setPreemptedMemorySeconds(p.getPreemptedMemorySeconds()); + this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds()); + return this.preemptedResourceSecondsMap; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index 662db5e..e8b0594 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -19,6 +19,10 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -46,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; @@ -462,6 +467,35 @@ public class ProtoUtils { public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { return ResourceTypes.valueOf(e.name()); } + + public static Map convertStringLongMapProtoListToMap( + List pList) { + Resource tmp = Resource.newInstance(0, 0); + Map ret = new HashMap<>(); + for (Map.Entry entry : tmp.getResources() + .entrySet()) { + ret.put(entry.getKey(), 0L); + } + if (pList != null) { + for (YarnProtos.StringLongMapProto p : pList) { + ret.put(p.getKey(), p.getValue()); + } + } + return ret; + } + + public static List convertMapToStringLongMapProtoList( + Map map) { + List ret = new ArrayList<>(); + for (Map.Entry entry : map.entrySet()) { + YarnProtos.StringLongMapProto.Builder tmp = + YarnProtos.StringLongMapProto.newBuilder(); + tmp.setKey(entry.getKey()); + tmp.setValue(entry.getValue()); + ret.add(tmp.build()); + } + return ret; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java index 0240fbc..01e8951 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java @@ -20,9 +20,15 @@ package org.apache.hadoop.yarn.util; import com.google.common.base.Joiner; import com.google.common.base.Splitter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** * Common string manipulation helpers @@ -174,4 +180,34 @@ public final class StringHelper { } sb.append(part); } + + public static String getResourceSecondsString(Map targetMap) { + List strings = new ArrayList<>(targetMap.size()); + //completed app report in the timeline server doesn't have usage report + Long memorySeconds = 0L; + Long vcoreSeconds = 0L; + if (targetMap.containsKey(ResourceInformation.MEMORY_MB.getName())) { + memorySeconds = targetMap.get(ResourceInformation.MEMORY_MB.getName()); + } + if (targetMap.containsKey(ResourceInformation.VCORES.getName())) { + vcoreSeconds = targetMap.get(ResourceInformation.VCORES.getName()); + } + strings.add(memorySeconds + " MB-seconds"); + strings.add(vcoreSeconds + " vcore-seconds"); + Map tmp = ResourceUtils.getResourceTypes(); + if (targetMap.size() > 2) { + for (Map.Entry entry : targetMap.entrySet()) { + if (!entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) + && !entry.getKey().equals(ResourceInformation.VCORES.getName())) { + String units = ""; + if (tmp.containsKey(entry.getKey())) { + units = tmp.get(entry.getKey()).getUnits(); + } + strings.add(entry.getValue() + " " + entry.getKey() + "-" + units + + "seconds"); + } + } + } + return Joiner.on(", ").join(strings); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java index 82170b3..8694651 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -37,6 +37,9 @@ public class BasePBImplRecordsTest { @SuppressWarnings("checkstyle:visibilitymodifier") protected static HashMap typeValueCache = new HashMap(); + @SuppressWarnings("checkstyle:visibilitymodifier") + protected static HashMap> excludedPropertiesMap = + new HashMap<>(); private static Random rand = new Random(); private static byte [] bytes = new byte[] {'1', '2', '3', '4'}; @@ -167,6 +170,10 @@ public class BasePBImplRecordsTest { private Map getGetSetPairs(Class recordClass) throws Exception { Map ret = new HashMap(); + List excluded = null; + if (excludedPropertiesMap.containsKey(recordClass.getClass())) { + excluded = excludedPropertiesMap.get(recordClass.getClass()); + } Method [] methods = recordClass.getDeclaredMethods(); // get all get methods for (int i = 0; i < methods.length; i++) { @@ -224,6 +231,11 @@ public class BasePBImplRecordsTest { (gsp.setMethod == null)) { LOG.info(String.format("Exclude potential property: %s\n", gsp.propertyName)); itr.remove(); + } else if ((excluded != null && excluded.contains(gsp.propertyName))) { + LOG.info(String.format( + "Excluding potential property(present in exclusion list): %s\n", + gsp.propertyName)); + itr.remove(); } else { LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type)); gsp.testValue = genTypeValue(gsp.type); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 1aed065..f71cd3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -336,6 +336,8 @@ import org.junit.Ignore; import org.junit.Test; import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.Arrays; /** * Test class for YARN API protocol records. @@ -738,6 +740,8 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { @Test public void testApplicationResourceUsageReportPBImpl() throws Exception { + excludedPropertiesMap.put(ApplicationResourceUsageReportPBImpl.class.getClass(), + Arrays.asList("PreemptedResourceSecondsMap", "ResourceSecondsMap")); validatePBImplRecord(ApplicationResourceUsageReportPBImpl.class, ApplicationResourceUsageReportProto.class); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index b4c91f9..a35f038 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -338,9 +339,20 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS); long preemptedVcoreSeconds = parseLong(entityInfo, ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS); - appResources = ApplicationResourceUsageReport.newInstance(0, 0, null, - null, null, memorySeconds, vcoreSeconds, 0, 0, - preemptedMemorySeconds, preemptedVcoreSeconds); + Map resourceSecondsMap = new HashMap<>(); + Map preemptedResoureSecondsMap = new HashMap<>(); + resourceSecondsMap + .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); + resourceSecondsMap + .put(ResourceInformation.VCORES.getName(), vcoreSeconds); + preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), + preemptedMemorySeconds); + preemptedResoureSecondsMap + .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds); + + appResources = ApplicationResourceUsageReport + .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0, + preemptedResoureSecondsMap); } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index cd146a1..66d95bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -65,8 +65,6 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; @@ -435,12 +433,12 @@ public class BuilderUtils { queue, priority, amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts, resource, null); } - + public static ApplicationResourceUsageReport newApplicationResourceUsageReport( int numUsedContainers, int numReservedContainers, Resource usedResources, - Resource reservedResources, Resource neededResources, long memorySeconds, - long vcoreSeconds, long preemptedMemorySeconds, - long preemptedVcoreSeconds) { + Resource reservedResources, Resource neededResources, + Map resourceSecondsMap, + Map preemptedResourceSecondsMap) { ApplicationResourceUsageReport report = recordFactory.newRecordInstance(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); @@ -448,10 +446,8 @@ public class BuilderUtils { report.setUsedResources(usedResources); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); - report.setMemorySeconds(memorySeconds); - report.setVcoreSeconds(vcoreSeconds); - report.setPreemptedMemorySeconds(preemptedMemorySeconds); - report.setPreemptedVcoreSeconds(preemptedVcoreSeconds); + report.setResourceSecondsMap(resourceSecondsMap); + report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 2ffdbc3..7dbf57b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.util.Times; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; +import org.apache.hadoop.yarn.util.StringHelper; /** * This class manages the list of applications for the resource manager. @@ -188,7 +189,12 @@ public class RMAppManager implements EventHandler, .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) - .add("applicationType", app.getApplicationType()); + .add("applicationType", app.getApplicationType()) + .add("resourceSeconds", StringHelper + .getResourceSecondsString(metrics.getResourceSecondsMap())) + .add("preemptedResourceSeconds", StringHelper + .getResourceSecondsString( + metrics.getPreemptedResourceSecondsMap())); return summary; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 35b0c98..f802b8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -478,7 +478,7 @@ public class RMServerUtils { DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = BuilderUtils.newApplicationResourceUsageReport(-1, -1, Resources.createResource(-1, -1), Resources.createResource(-1, -1), - Resources.createResource(-1, -1), 0, 0, 0, 0); + Resources.createResource(-1, -1), new HashMap(), new HashMap()); /** @@ -622,4 +622,12 @@ public class RMServerUtils { return labelsToNodes.get(label); } } + + public static Long getOrDefault(Map map, String key, + Long defaultValue) { + if (map.containsKey(key)) { + return map.get(key); + } + return defaultValue; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index e8ed0b7..f0ab324 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -853,11 +853,8 @@ public abstract class RMStateStore extends AbstractService { appAttempt.getAppAttemptId(), appAttempt.getMasterContainer(), credentials, appAttempt.getStartTime(), - resUsage.getMemorySeconds(), - resUsage.getVcoreSeconds(), - attempMetrics.getPreemptedMemory(), - attempMetrics.getPreemptedVcore() - ); + resUsage.getResourceUsageSecondsMap(), + attempMetrics.getPreemptedResourceSecondsMap()); getRMStateStoreEventHandler().handle( new RMStateStoreAppAttemptEvent(attemptState)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java index 67aaf94..2de071a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java @@ -25,23 +25,28 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.Records; +import java.util.Map; + /* * Contains the state data that needs to be persisted for an ApplicationAttempt */ @Public @Unstable public abstract class ApplicationAttemptStateData { + public static ApplicationAttemptStateData newInstance( ApplicationAttemptId attemptId, Container container, Credentials attemptTokens, long startTime, RMAppAttemptState finalState, String finalTrackingUrl, String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus, - long finishTime, long memorySeconds, long vcoreSeconds, - long preemptedMemorySeconds, long preemptedVcoreSeconds) { + long finishTime, Map resourceSecondsMap, + Map preemptedResourceSecondsMap) { ApplicationAttemptStateData attemptStateData = Records.newRecord(ApplicationAttemptStateData.class); attemptStateData.setAttemptId(attemptId); @@ -54,23 +59,33 @@ public abstract class ApplicationAttemptStateData { attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus); attemptStateData.setAMContainerExitStatus(exitStatus); attemptStateData.setFinishTime(finishTime); - attemptStateData.setMemorySeconds(memorySeconds); - attemptStateData.setVcoreSeconds(vcoreSeconds); - attemptStateData.setPreemptedMemorySeconds(preemptedMemorySeconds); - attemptStateData.setPreemptedVcoreSeconds(preemptedVcoreSeconds); + attemptStateData.setMemorySeconds(RMServerUtils + .getOrDefault(resourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L)); + attemptStateData.setVcoreSeconds(RMServerUtils + .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), + 0L)); + attemptStateData.setPreemptedMemorySeconds(RMServerUtils + .getOrDefault(preemptedResourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L)); + attemptStateData.setPreemptedVcoreSeconds(RMServerUtils + .getOrDefault(preemptedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L)); + attemptStateData.setResourceSecondsMap(resourceSecondsMap); + attemptStateData + .setPreemptedResourceSecondsMap(preemptedResourceSecondsMap); return attemptStateData; } public static ApplicationAttemptStateData newInstance( ApplicationAttemptId attemptId, Container masterContainer, - Credentials attemptTokens, long startTime, long memorySeconds, - long vcoreSeconds, long preemptedMemorySeconds, - long preemptedVcoreSeconds) { - return newInstance(attemptId, masterContainer, attemptTokens, - startTime, null, "N/A", "", null, ContainerExitStatus.INVALID, 0, - memorySeconds, vcoreSeconds, - preemptedMemorySeconds, preemptedVcoreSeconds); - } + Credentials attemptTokens, long startTime, + Map resourceSeondsMap, + Map preemptedResourceSecondsMap) { + return newInstance(attemptId, masterContainer, attemptTokens, startTime, + null, "N/A", "", null, ContainerExitStatus.INVALID, 0, + resourceSeondsMap, preemptedResourceSecondsMap); + } public abstract ApplicationAttemptStateDataProto getProto(); @@ -215,4 +230,50 @@ public abstract class ApplicationAttemptStateData { @Public @Unstable public abstract void setPreemptedVcoreSeconds(long vcoreSeconds); + + /** + * Get the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * + * @return map containing the resource name and aggregated preempted + * resource-seconds + */ + @Public + @Unstable + public abstract Map getResourceSecondsMap(); + + /** + * Set the aggregated number of resources that the application has + * allocated times the number of seconds the application has been running. + * + * @param resourceSecondsMap map containing the resource name and aggregated + * resource-seconds + */ + @Public + @Unstable + public abstract void setResourceSecondsMap( + Map resourceSecondsMap); + + /** + * Get the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * + * @return map containing the resource name and aggregated preempted + * resource-seconds + */ + @Public + @Unstable + public abstract Map getPreemptedResourceSecondsMap(); + + /** + * Set the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * + * @param preemptedResourceSecondsMap map containing the resource name and + * aggregated preempted resource-seconds + */ + @Public + @Unstable + public abstract void setPreemptedResourceSecondsMap( + Map preemptedResourceSecondsMap); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java index e89726f..ed71ea2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -55,6 +56,9 @@ public class ApplicationAttemptStateDataPBImpl extends private Container masterContainer = null; private ByteBuffer appAttemptTokens = null; + private Map resourceSecondsMap; + private Map preemptedResourceSecondsMap; + public ApplicationAttemptStateDataPBImpl() { builder = ApplicationAttemptStateDataProto.newBuilder(); } @@ -404,4 +408,50 @@ public class ApplicationAttemptStateDataPBImpl extends IOUtils.closeStream(dibb); } } + + @Override + public Map getResourceSecondsMap() { + if (this.resourceSecondsMap != null) { + return this.resourceSecondsMap; + } + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap( + p.getApplicationResourceUsageMapList()); + return this.resourceSecondsMap; + } + + @Override + public void setResourceSecondsMap(Map resourceSecondsMap) { + maybeInitBuilder(); + builder.clearApplicationResourceUsageMap(); + this.resourceSecondsMap = resourceSecondsMap; + if (resourceSecondsMap != null) { + builder.addAllApplicationResourceUsageMap( + ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); + } + } + + @Override + public Map getPreemptedResourceSecondsMap() { + if (this.preemptedResourceSecondsMap != null) { + return this.preemptedResourceSecondsMap; + } + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + this.preemptedResourceSecondsMap = ProtoUtils + .convertStringLongMapProtoListToMap( + p.getApplicationResourceUsageMapList()); + return this.preemptedResourceSecondsMap; + } + + @Override + public void setPreemptedResourceSecondsMap( + Map preemptedResourceSecondsMap) { + maybeInitBuilder(); + builder.clearPreemptedResourceUsageMap(); + this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; + if (preemptedResourceSecondsMap != null) { + builder.addAllPreemptedResourceUsageMap(ProtoUtils + .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap)); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index af5ef5e..e152782 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -749,14 +749,10 @@ public class RMAppImpl implements RMApp, Recoverable { } RMAppMetrics rmAppMetrics = getRMAppMetrics(); - appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds()); - appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds()); - appUsageReport. - setPreemptedMemorySeconds(rmAppMetrics. - getPreemptedMemorySeconds()); - appUsageReport. - setPreemptedVcoreSeconds(rmAppMetrics. - getPreemptedVcoreSeconds()); + appUsageReport + .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap()); + appUsageReport.setPreemptedResourceSecondsMap( + rmAppMetrics.getPreemptedResourceSecondsMap()); } if (currentApplicationAttemptId == null) { @@ -1617,10 +1613,9 @@ public class RMAppImpl implements RMApp, Recoverable { Resource resourcePreempted = Resource.newInstance(0, 0); int numAMContainerPreempted = 0; int numNonAMContainerPreempted = 0; - long memorySeconds = 0; - long vcoreSeconds = 0; - long preemptedMemorySeconds = 0; - long preemptedVcoreSeconds = 0; + Map resourceSecondsMap = new HashMap<>(); + Map preemptedSecondsMap = new HashMap<>(); + this.readLock.lock(); try { for (RMAppAttempt attempt : attempts.values()) { @@ -1636,20 +1631,28 @@ public class RMAppImpl implements RMApp, Recoverable { // for both running and finished containers. AggregateAppResourceUsage resUsage = attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage(); - memorySeconds += resUsage.getMemorySeconds(); - vcoreSeconds += resUsage.getVcoreSeconds(); - preemptedMemorySeconds += attemptMetrics.getPreemptedMemory(); - preemptedVcoreSeconds += attemptMetrics.getPreemptedVcore(); + for (Map.Entry entry : resUsage + .getResourceUsageSecondsMap().entrySet()) { + long value = RMServerUtils + .getOrDefault(resourceSecondsMap, entry.getKey(), 0L); + value += entry.getValue(); + resourceSecondsMap.put(entry.getKey(), value); + } + for (Map.Entry entry : attemptMetrics + .getPreemptedResourceSecondsMap().entrySet()) { + long value = RMServerUtils + .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L); + value += entry.getValue(); + preemptedSecondsMap.put(entry.getKey(), value); + } } } } finally { this.readLock.unlock(); } - return new RMAppMetrics(resourcePreempted, - numNonAMContainerPreempted, numAMContainerPreempted, - memorySeconds, vcoreSeconds, - preemptedMemorySeconds, preemptedVcoreSeconds); + return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted, + numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap); } @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java index fa068ea..2bb7fd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java @@ -19,27 +19,27 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; + +import java.util.Map; public class RMAppMetrics { final Resource resourcePreempted; final int numNonAMContainersPreempted; final int numAMContainersPreempted; - final long memorySeconds; - final long vcoreSeconds; - private final long preemptedMemorySeconds; - private final long preemptedVcoreSeconds; + private final Map resourceSecondsMap; + private final Map preemptedResourceSecondsMap; public RMAppMetrics(Resource resourcePreempted, int numNonAMContainersPreempted, int numAMContainersPreempted, - long memorySeconds, long vcoreSeconds, long preemptedMemorySeconds, - long preemptedVcoreSeconds) { + Map resourceSecondsMap, + Map preemptedResourceSecondsMap) { this.resourcePreempted = resourcePreempted; this.numNonAMContainersPreempted = numNonAMContainersPreempted; this.numAMContainersPreempted = numAMContainersPreempted; - this.memorySeconds = memorySeconds; - this.vcoreSeconds = vcoreSeconds; - this.preemptedMemorySeconds = preemptedMemorySeconds; - this.preemptedVcoreSeconds = preemptedVcoreSeconds; + this.resourceSecondsMap = resourceSecondsMap; + this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; } public Resource getResourcePreempted() { @@ -55,19 +55,32 @@ public class RMAppMetrics { } public long getMemorySeconds() { - return memorySeconds; + return RMServerUtils.getOrDefault(resourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); } public long getVcoreSeconds() { - return vcoreSeconds; + return RMServerUtils + .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), + 0L); } public long getPreemptedMemorySeconds() { - return preemptedMemorySeconds; + return RMServerUtils.getOrDefault(preemptedResourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); } public long getPreemptedVcoreSeconds() { - return preemptedVcoreSeconds; + return RMServerUtils.getOrDefault(preemptedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); + } + + public Map getResourceSecondsMap() { + return resourceSecondsMap; + } + + public Map getPreemptedResourceSecondsMap() { + return preemptedResourceSecondsMap; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java index f0c2b34..b858712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java @@ -19,42 +19,38 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; + +import java.util.HashMap; +import java.util.Map; @Private public class AggregateAppResourceUsage { - long memorySeconds; - long vcoreSeconds; + private Map resourceSecondsMap = new HashMap<>(); - public AggregateAppResourceUsage(long memorySeconds, long vcoreSeconds) { - this.memorySeconds = memorySeconds; - this.vcoreSeconds = vcoreSeconds; + public AggregateAppResourceUsage(Map resourceSecondsMap) { + this.resourceSecondsMap.putAll(resourceSecondsMap); } /** * @return the memorySeconds */ public long getMemorySeconds() { - return memorySeconds; - } - - /** - * @param memorySeconds the memorySeconds to set - */ - public void setMemorySeconds(long memorySeconds) { - this.memorySeconds = memorySeconds; + return RMServerUtils.getOrDefault(resourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); } /** * @return the vcoreSeconds */ public long getVcoreSeconds() { - return vcoreSeconds; + return RMServerUtils + .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), + 0L); } - /** - * @param vcoreSeconds the vcoreSeconds to set - */ - public void setVcoreSeconds(long vcoreSeconds) { - this.vcoreSeconds = vcoreSeconds; + public Map getResourceUsageSecondsMap() { + return resourceSecondsMap; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 0b56f09..a02a6e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -939,12 +939,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } AggregateAppResourceUsage resUsage = this.attemptMetrics.getAggregateAppResourceUsage(); - report.setMemorySeconds(resUsage.getMemorySeconds()); - report.setVcoreSeconds(resUsage.getVcoreSeconds()); - report.setPreemptedMemorySeconds( - this.attemptMetrics.getPreemptedMemory()); - report.setPreemptedVcoreSeconds( - this.attemptMetrics.getPreemptedVcore()); + report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap()); + report.setPreemptedResourceSecondsMap( + this.attemptMetrics.getPreemptedResourceSecondsMap()); return report; } finally { this.readLock.unlock(); @@ -981,11 +978,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.finalStatus = attemptState.getFinalApplicationStatus(); this.startTime = attemptState.getStartTime(); this.finishTime = attemptState.getFinishTime(); - this.attemptMetrics.updateAggregateAppResourceUsage( - attemptState.getMemorySeconds(), attemptState.getVcoreSeconds()); + this.attemptMetrics + .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap()); this.attemptMetrics.updateAggregatePreemptedAppResourceUsage( - attemptState.getPreemptedMemorySeconds(), - attemptState.getPreemptedVcoreSeconds()); + attemptState.getPreemptedResourceSecondsMap()); } public void transferStateFromAttempt(RMAppAttempt attempt) { @@ -1361,16 +1357,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { RMStateStore rmStore = rmContext.getStateStore(); setFinishTime(System.currentTimeMillis()); - ApplicationAttemptStateData attemptState = - ApplicationAttemptStateData.newInstance( - applicationAttemptId, getMasterContainer(), - rmStore.getCredentialsFromAppAttempt(this), - startTime, stateToBeStored, finalTrackingUrl, diags.toString(), - finalStatus, exitStatus, - getFinishTime(), resUsage.getMemorySeconds(), - resUsage.getVcoreSeconds(), - this.attemptMetrics.getPreemptedMemory(), - this.attemptMetrics.getPreemptedVcore()); + ApplicationAttemptStateData attemptState = ApplicationAttemptStateData + .newInstance(applicationAttemptId, getMasterContainer(), + rmStore.getCredentialsFromAppAttempt(this), startTime, + stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, + getFinishTime(), resUsage.getResourceUsageSecondsMap(), + this.attemptMetrics.getPreemptedResourceSecondsMap()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index 0655609..8ef6d99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -25,11 +27,13 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -49,10 +53,8 @@ public class RMAppAttemptMetrics { private ReadLock readLock; private WriteLock writeLock; - private AtomicLong finishedMemorySeconds = new AtomicLong(0); - private AtomicLong finishedVcoreSeconds = new AtomicLong(0); - private AtomicLong preemptedMemorySeconds = new AtomicLong(0); - private AtomicLong preemptedVcoreSeconds = new AtomicLong(0); + private Map resourceUsageMap = new HashMap<>(); + private Map preemptedResourceMap = new HashMap<>(); private RMContext rmContext; private int[][] localityStatistics = @@ -102,11 +104,16 @@ public class RMAppAttemptMetrics { } public long getPreemptedMemory() { - return preemptedMemorySeconds.get(); + return preemptedResourceMap.get(ResourceInformation.MEMORY_MB.getName()) + .get(); } public long getPreemptedVcore() { - return preemptedVcoreSeconds.get(); + return preemptedResourceMap.get(ResourceInformation.VCORES.getName()).get(); + } + + public Map getPreemptedResourceSecondsMap() { + return convertAtomicLongMaptoLongMap(preemptedResourceMap); } public int getNumNonAMContainersPreempted() { @@ -122,35 +129,90 @@ public class RMAppAttemptMetrics { } public AggregateAppResourceUsage getAggregateAppResourceUsage() { - long memorySeconds = finishedMemorySeconds.get(); - long vcoreSeconds = finishedVcoreSeconds.get(); + Map resourcesUsed = + convertAtomicLongMaptoLongMap(resourceUsageMap); // Only add in the running containers if this is the active attempt. RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId()); - if (null != rmApp) { - RMAppAttempt currentAttempt = rmApp.getCurrentAppAttempt(); + if (rmApp != null) { + RMAppAttempt currentAttempt = rmContext.getRMApps().get(attemptId.getApplicationId()).getCurrentAppAttempt(); if (currentAttempt.getAppAttemptId().equals(attemptId)) { - ApplicationResourceUsageReport appResUsageReport = rmContext - .getScheduler().getAppResourceUsageReport(attemptId); + ApplicationResourceUsageReport appResUsageReport = + rmContext.getScheduler().getAppResourceUsageReport(attemptId); if (appResUsageReport != null) { - memorySeconds += appResUsageReport.getMemorySeconds(); - vcoreSeconds += appResUsageReport.getVcoreSeconds(); + Map tmp = appResUsageReport.getResourceSecondsMap(); + for (Map.Entry entry : tmp.entrySet()) { + if (resourcesUsed.containsKey(entry.getKey())) { + Long value = resourcesUsed.get(entry.getKey()); + value += entry.getValue(); + resourcesUsed.put(entry.getKey(), value); + } else{ + resourcesUsed.put(entry.getKey(), entry.getValue()); + } + } } } } - return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds); + return new AggregateAppResourceUsage(resourcesUsed); + } + + public void updateAggregateAppResourceUsage(Resource allocated, + long deltaUsedMillis) { + updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap); + } + + public void updateAggregatePreemptedAppResourceUsage(Resource allocated, + long deltaUsedMillis) { + updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap); } - public void updateAggregateAppResourceUsage(long finishedMemorySeconds, - long finishedVcoreSeconds) { - this.finishedMemorySeconds.addAndGet(finishedMemorySeconds); - this.finishedVcoreSeconds.addAndGet(finishedVcoreSeconds); + public void updateAggregateAppResourceUsage( + Map resourceSecondsMap) { + updateUsageMap(resourceSecondsMap, resourceUsageMap); } public void updateAggregatePreemptedAppResourceUsage( - long preemptedMemorySeconds, long preemptedVcoreSeconds) { - this.preemptedMemorySeconds.addAndGet(preemptedMemorySeconds); - this.preemptedVcoreSeconds.addAndGet(preemptedVcoreSeconds); + Map preemptedResourceSecondsMap) { + updateUsageMap(preemptedResourceSecondsMap, preemptedResourceMap); + } + + private void updateUsageMap(Resource allocated, long deltaUsedMillis, + Map targetMap) { + for (Map.Entry entry : allocated.getResources() + .entrySet()) { + AtomicLong resourceUsed; + if (!targetMap.containsKey(entry.getKey())) { + resourceUsed = new AtomicLong(0); + targetMap.put(entry.getKey(), resourceUsed); + + } + resourceUsed = targetMap.get(entry.getKey()); + resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis) + / DateUtils.MILLIS_PER_SECOND); + } + } + + private void updateUsageMap(Map sourceMap, + Map targetMap) { + for (Map.Entry entry : sourceMap.entrySet()) { + AtomicLong resourceUsed; + if (!targetMap.containsKey(entry.getKey())) { + resourceUsed = new AtomicLong(0); + targetMap.put(entry.getKey(), resourceUsed); + + } + resourceUsed = targetMap.get(entry.getKey()); + resourceUsed.set(entry.getValue()); + } + } + + private Map convertAtomicLongMaptoLongMap( + Map source) { + Map ret = new HashMap<>(); + for (Map.Entry entry : source.entrySet()) { + ret.put(entry.getKey(), entry.getValue().get()); + } + return ret; } public void incNumAllocatedContainers(NodeType containerType, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index b185495..76933f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -25,7 +25,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; -import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -716,20 +715,15 @@ public class RMContainerImpl implements RMContainer { if (rmAttempt != null) { long usedMillis = container.finishTime - container.creationTime; - long memorySeconds = resource.getMemorySize() - * usedMillis / DateUtils.MILLIS_PER_SECOND; - long vcoreSeconds = resource.getVirtualCores() - * usedMillis / DateUtils.MILLIS_PER_SECOND; rmAttempt.getRMAppAttemptMetrics() - .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds); + .updateAggregateAppResourceUsage(resource, usedMillis); // If this is a preempted container, update preemption metrics if (ContainerExitStatus.PREEMPTED == container.finishedStatus - .getExitStatus()) { - rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource, - container); + .getExitStatus()) { rmAttempt.getRMAppAttemptMetrics() - .updateAggregatePreemptedAppResourceUsage(memorySeconds, - vcoreSeconds); + .updatePreemptionInfo(resource, container); + rmAttempt.getRMAppAttemptMetrics() + .updateAggregatePreemptedAppResourceUsage(resource, usedMillis); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 05dc834..1785dfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -55,11 +55,13 @@ import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.ContainerType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; @@ -107,9 +109,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000; protected long lastMemoryAggregateAllocationUpdateTime = 0; - private long lastMemorySeconds = 0; - private long lastVcoreSeconds = 0; - + private Map lastResourceSecondsMap = new HashMap<>(); protected final AppSchedulingInfo appSchedulingInfo; protected ApplicationAttemptId attemptId; protected Map liveContainers = @@ -1002,22 +1002,24 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { // recently. if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) { - long memorySeconds = 0; - long vcoreSeconds = 0; + Map resourceSecondsMap = new HashMap<>(); for (RMContainer rmContainer : this.liveContainers.values()) { long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); Resource resource = rmContainer.getContainer().getResource(); - memorySeconds += resource.getMemorySize() * usedMillis / - DateUtils.MILLIS_PER_SECOND; - vcoreSeconds += resource.getVirtualCores() * usedMillis - / DateUtils.MILLIS_PER_SECOND; + for (Map.Entry entry : resource + .getResources().entrySet()) { + long value = RMServerUtils + .getOrDefault(resourceSecondsMap, entry.getKey(), 0L); + value += entry.getValue().getValue() * usedMillis + / DateUtils.MILLIS_PER_SECOND; + resourceSecondsMap.put(entry.getKey(), value); + } } lastMemoryAggregateAllocationUpdateTime = currentTimeMillis; - lastMemorySeconds = memorySeconds; - lastVcoreSeconds = vcoreSeconds; + lastResourceSecondsMap = resourceSecondsMap; } - return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds); + return new AggregateAppResourceUsage(lastResourceSecondsMap); } public ApplicationResourceUsageReport getResourceUsageReport() { @@ -1032,6 +1034,11 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { Resource cluster = rmContext.getScheduler().getClusterResource(); ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); + Map preemptedResourceSecondsMaps = new HashMap<>(); + preemptedResourceSecondsMaps + .put(ResourceInformation.MEMORY_MB.getName(), 0L); + preemptedResourceSecondsMaps + .put(ResourceInformation.VCORES.getName(), 0L); float queueUsagePerc = 0.0f; float clusterUsagePerc = 0.0f; if (!calc.isInvalidDivisor(cluster)) { @@ -1041,15 +1048,15 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { queueUsagePerc = calc.divide(cluster, usedResourceClone, Resources.multiply(cluster, queueCapacityPerc)) * 100; } - clusterUsagePerc = calc.divide(cluster, usedResourceClone, cluster) - * 100; + clusterUsagePerc = + calc.divide(cluster, usedResourceClone, cluster) * 100; } - return ApplicationResourceUsageReport.newInstance(liveContainers.size(), - reservedContainers.size(), usedResourceClone, reservedResourceClone, - Resources.add(usedResourceClone, reservedResourceClone), - runningResourceUsage.getMemorySeconds(), - runningResourceUsage.getVcoreSeconds(), queueUsagePerc, - clusterUsagePerc, 0, 0); + return ApplicationResourceUsageReport + .newInstance(liveContainers.size(), reservedContainers.size(), + usedResourceClone, reservedResourceClone, + Resources.add(usedResourceClone, reservedResourceClone), + runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc, + clusterUsagePerc, preemptedResourceSecondsMaps); } finally { writeLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java index 3f774e5..bbc35fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.AppBlock; +import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; @@ -106,15 +107,12 @@ public class RMAppBlock extends AppBlock{ attemptResourcePreempted) ._("Number of Non-AM Containers Preempted from Current Attempt:", attemptNumNonAMContainerPreempted) - ._("Aggregate Resource Allocation:", - String.format("%d MB-seconds, %d vcore-seconds", - appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(), - appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())) + ._("Aggregate Resource Allocation:", appMetrics == null ? "N/A" : + StringHelper + .getResourceSecondsString(appMetrics.getResourceSecondsMap())) ._("Aggregate Preempted Resource Allocation:", - String.format("%d MB-seconds, %d vcore-seconds", - appMetrics == null ? "N/A" : appMetrics.getPreemptedMemorySeconds(), - appMetrics == null ? "N/A" : - appMetrics.getPreemptedVcoreSeconds())); + appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString( + appMetrics.getPreemptedResourceSecondsMap())); pdiv._(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 7fed3df..bff8612 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -101,6 +101,7 @@ public class AppInfo { private long vcoreSeconds; protected float queueUsagePercentage; protected float clusterUsagePercentage; + protected Map resourceSecondsMap; // preemption info fields private long preemptedResourceMB; @@ -109,6 +110,7 @@ public class AppInfo { private int numAMContainerPreempted; private long preemptedMemorySeconds; private long preemptedVcoreSeconds; + protected Map preemptedResourceSecondsMap; // list of resource requests @XmlElement(name = "resourceRequests") @@ -234,8 +236,10 @@ public class AppInfo { appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); + resourceSecondsMap = appMetrics.getResourceSecondsMap(); preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); + preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap(); ApplicationSubmissionContext appSubmissionContext = app.getApplicationSubmissionContext(); unmanagedApplication = appSubmissionContext.getUnmanagedAM(); @@ -454,6 +458,22 @@ public class AppInfo { return this.reservedVCores; } + public long getPreemptedMB() { + return preemptedResourceMB; + } + + public long getPreemptedVCores() { + return preemptedResourceVCores; + } + + public int getNumNonAMContainersPreempted() { + return numNonAMContainerPreempted; + } + + public int getNumAMContainersPreempted() { + return numAMContainerPreempted; + } + public long getMemorySeconds() { return memorySeconds; } @@ -462,6 +482,10 @@ public class AppInfo { return vcoreSeconds; } + public Map getResourceSecondsMap() { + return resourceSecondsMap; + } + public long getPreemptedMemorySeconds() { return preemptedMemorySeconds; } @@ -470,6 +494,10 @@ public class AppInfo { return preemptedVcoreSeconds; } + public Map getPreemptedResourceSecondsMap() { + return preemptedResourceSecondsMap; + } + public List getResourceRequests() { return this.resourceRequests; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 5083943..e13980a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -20,46 +20,68 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; @XmlRootElement -@XmlAccessorType(XmlAccessType.FIELD) +@XmlAccessorType(XmlAccessType.NONE) public class ResourceInfo { + + @XmlElement long memory; + @XmlElement int vCores; - + + private Resource resources; + public ResourceInfo() { } public ResourceInfo(Resource res) { memory = res.getMemorySize(); vCores = res.getVirtualCores(); + resources = Resources.clone(res); } public long getMemorySize() { - return memory; + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + return resources.getMemorySize(); } public int getvCores() { - return vCores; + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + return resources.getVirtualCores(); } - + @Override public String toString() { - return ""; + return resources.toString(); } public void setMemory(int memory) { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } this.memory = memory; + resources.setMemorySize(memory); } public void setvCores(int vCores) { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } this.vCores = vCores; + resources.setVirtualCores(vCores); } public Resource getResource() { - return Resource.newInstance(memory, vCores); + return Resource.newInstance(resources); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java index cf93edd..887b854 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java @@ -73,7 +73,7 @@ public class SchedulerInfo { } public String getSchedulerResourceTypes() { - return this.schedulingResourceTypes.toString(); + return minAllocResource.getResource().getResources().keySet().toString(); } public int getMaxClusterLevelAppPriority() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 247cd21..39a56a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -87,6 +87,8 @@ message ApplicationAttemptStateDataProto { optional int64 finish_time = 12; optional int64 preempted_memory_seconds = 13; optional int64 preempted_vcore_seconds = 14; + repeated StringLongMapProto application_resource_usage_map = 15; + repeated StringLongMapProto preempted_resource_usage_map = 16; } message EpochProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index ace4ed9..3a959b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentMap; import org.apache.commons.logging.Log; @@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -827,9 +829,12 @@ public class TestAppManager{ when(app.getState()).thenReturn(RMAppState.RUNNING); when(app.getApplicationType()).thenReturn("MAPREDUCE"); when(app.getSubmitTime()).thenReturn(1000L); + Map resourceSecondsMap = new HashMap<>(); + resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L); + resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L); RMAppMetrics metrics = new RMAppMetrics(Resource.newInstance(1234, 56), - 10, 1, 16384, 64, 0, 0); + 10, 1, resourceSecondsMap, new HashMap()); when(app.getRMAppMetrics()).thenReturn(metrics); RMAppManager.ApplicationSummary.SummaryBuilder summary = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java index 11fe056..3508ab4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.lang.time.DateUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -424,6 +426,9 @@ public class TestContainerResourceUsage { * usedMillis / DateUtils.MILLIS_PER_SECOND; long vcoreSeconds = resource.getVirtualCores() * usedMillis / DateUtils.MILLIS_PER_SECOND; - return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds); + Map map = new HashMap<>(); + map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); + map.put(ResourceInformation.VCORES.getName(), vcoreSeconds); + return new AggregateAppResourceUsage(map); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index f826631..cdb3ba1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -189,7 +190,8 @@ public abstract class MockAsm extends MockApps { @Override public RMAppMetrics getRMAppMetrics() { - return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0); + return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap(), + new HashMap()); } @Override @@ -337,8 +339,9 @@ public abstract class MockAsm extends MockApps { public ApplicationReport createAndGetApplicationReport( String clientUserName, boolean allowAccess) { ApplicationResourceUsageReport usageReport = - ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, - 0, 0, 0, 0, 0, 0); + ApplicationResourceUsageReport + .newInstance(0, 0, null, null, null, new HashMap(), 0, 0, + new HashMap()); ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 5321916..36e35d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.when; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -505,9 +507,16 @@ public class TestSystemMetricsPublisher { when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); - when(app.getRMAppMetrics()).thenReturn( - new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, - Integer.MAX_VALUE, Long.MAX_VALUE)); + Map resourceMap = new HashMap<>(); + resourceMap + .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); + resourceMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); + Map preemptedMap = new HashMap<>(); + preemptedMap + .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); + preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); + when(app.getRMAppMetrics()) + .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap)); Set appTags = new HashSet(); appTags.add("test"); appTags.add("tags"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index ad71f6e..3aeafc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -29,6 +29,8 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -46,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; @@ -358,15 +361,20 @@ public class TestSystemMetricsPublisherForV2 { when(app.getDiagnostics()).thenReturn( new StringBuilder("test diagnostics info")); RMAppAttempt appAttempt = mock(RMAppAttempt.class); - when(appAttempt.getAppAttemptId()).thenReturn( - ApplicationAttemptId.newInstance(appId, 1)); + when(appAttempt.getAppAttemptId()) + .thenReturn(ApplicationAttemptId.newInstance(appId, 1)); when(app.getCurrentAppAttempt()).thenReturn(appAttempt); - when(app.getFinalApplicationStatus()).thenReturn( - FinalApplicationStatus.UNDEFINED); + when(app.getFinalApplicationStatus()) + .thenReturn(FinalApplicationStatus.UNDEFINED); + Map resourceSecondsMap = new HashMap<>(); + resourceSecondsMap + .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); + resourceSecondsMap + .put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(app.getRMAppMetrics()).thenReturn( - new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, Integer.MAX_VALUE, - Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE)); - when(app.getApplicationTags()).thenReturn(Collections. emptySet()); + new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap, + new HashMap())); + when(app.getApplicationTags()).thenReturn(Collections.emptySet()); ApplicationSubmissionContext appSubmissionContext = mock(ApplicationSubmissionContext.class); when(appSubmissionContext.getPriority()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index fdac89c..a1339f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -194,7 +194,7 @@ public class RMStateStoreTestBase { when(mockAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(0, 0)); + .thenReturn(new AggregateAppResourceUsage(new HashMap())); dispatcher.attemptId = attemptId; store.storeNewApplicationAttempt(mockAttempt); waitNotify(dispatcher); @@ -292,7 +292,7 @@ public class RMStateStoreTestBase { when(mockRemovedAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(0,0)); + .thenReturn(new AggregateAppResourceUsage(new HashMap())); attempts.put(attemptIdRemoved, mockRemovedAttempt); store.removeApplication(mockRemovedApp); @@ -369,7 +369,7 @@ public class RMStateStoreTestBase { oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 100, - oldAttemptState.getFinishTime(), 0, 0, 0, 0); + oldAttemptState.getFinishTime(), new HashMap(), new HashMap()); store.updateApplicationAttemptState(newAttemptState); // test updating the state of an app/attempt whose initial state was not @@ -393,7 +393,7 @@ public class RMStateStoreTestBase { oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 111, - oldAttemptState.getFinishTime(), 0, 0, 0, 0); + oldAttemptState.getFinishTime(), new HashMap(), new HashMap()); store.updateApplicationAttemptState(dummyAttempt); // let things settle down diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index fbacac0..5d8f1e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -34,12 +34,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.service.Service; import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.conf.HAUtil; @@ -554,7 +549,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { when(mockAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(0,0)); + .thenReturn(new AggregateAppResourceUsage(new HashMap())); store.storeNewApplicationAttempt(mockAttempt); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); @@ -566,7 +561,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { store.getCredentialsFromAppAttempt(mockAttempt), startTime, RMAppAttemptState.FINISHED, "testUrl", "test", FinalApplicationStatus.SUCCEEDED, 100, - finishTime, 0, 0, 0, 0); + finishTime, new HashMap(), new HashMap()); store.updateApplicationAttemptState(newAttemptState); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); @@ -803,10 +798,20 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { private static ApplicationAttemptStateData createFinishedAttempt( ApplicationAttemptId attemptId, Container container, long startTime, int amExitStatus) { + Map resourceSecondsMap = new HashMap<>(); + Map preemptedResoureSecondsMap = new HashMap<>(); + resourceSecondsMap + .put(ResourceInformation.MEMORY_MB.getName(), 0L); + resourceSecondsMap + .put(ResourceInformation.VCORES.getName(), 0L); + preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), + 0L); + preemptedResoureSecondsMap + .put(ResourceInformation.VCORES.getName(), 0L); return ApplicationAttemptStateData.newInstance(attemptId, container, null, startTime, RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, - amExitStatus, 0, 0, 0, 0, 0); + amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap); } private ApplicationAttemptId storeAttempt(RMStateStore store, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java index d9ed073..e67e1cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.HashMap; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -62,9 +63,10 @@ public class TestAppPage { when(app.getStartTime()).thenReturn(0L); when(app.getFinishTime()).thenReturn(0L); when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED); - - RMAppMetrics appMetrics = new RMAppMetrics( - Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0); + + RMAppMetrics appMetrics = + new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap(), + new HashMap()); when(app.getRMAppMetrics()).thenReturn(appMetrics); // initialize RM Context, and create RMApp, without creating RMAppAttempt diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java index 8c00b39..6bd78c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java @@ -51,6 +51,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.concurrent.ConcurrentMap; @@ -136,8 +137,8 @@ public class TestRMWebAppFairScheduler { MockRMApp app = new MockRMApp(i, i, state) { @Override public RMAppMetrics getRMAppMetrics() { - return new RMAppMetrics(Resource.newInstance(0, 0), - 0, 0, 0, 0, 0, 0); + return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, + new HashMap(), new HashMap()); } @Override public YarnApplicationState createApplicationState() { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org