Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C195F200C40 for ; Wed, 8 Mar 2017 11:47:40 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id C0698160B83; Wed, 8 Mar 2017 10:47:40 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 72CE5160B86 for ; Wed, 8 Mar 2017 11:47:39 +0100 (CET) Received: (qmail 24289 invoked by uid 500); 8 Mar 2017 10:47:30 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 21931 invoked by uid 99); 8 Mar 2017 10:47:29 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Mar 2017 10:47:29 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E00F9F322A; Wed, 8 Mar 2017 10:47:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Wed, 08 Mar 2017 10:48:03 -0000 Message-Id: <998da95bf42c469cb49902643509416a@git.apache.org> In-Reply-To: <9bbb8a187d214aa39965fa2d6a53d784@git.apache.org> References: <9bbb8a187d214aa39965fa2d6a53d784@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [36/50] [abbrv] hadoop git commit: YARN-6232. Update resource usage and preempted resource calculations to take into account all resource types. Contributed by Varun Vasudev. archived-at: Wed, 08 Mar 2017 10:47:40 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java index e5d6c16..a5fb5d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.AppBlock; +import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; @@ -98,15 +99,12 @@ public class RMAppBlock extends AppBlock{ attemptResourcePreempted) ._("Number of Non-AM Containers Preempted from Current Attempt:", attemptNumNonAMContainerPreempted) - ._("Aggregate Resource Allocation:", - String.format("%d MB-seconds, %d vcore-seconds", - appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(), - appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())) + ._("Aggregate Resource Allocation:", appMetrics == null ? "N/A" : + StringHelper + .getResourceSecondsString(appMetrics.getResourceSecondsMap())) ._("Aggregate Preempted Resource Allocation:", - String.format("%d MB-seconds, %d vcore-seconds", - appMetrics == null ? "N/A" : appMetrics.getPreemptedMemorySeconds(), - appMetrics == null ? "N/A" : - appMetrics.getPreemptedVcoreSeconds())); + appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString( + appMetrics.getPreemptedResourceSecondsMap())); pdiv._(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 4e85b67..eeaffc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -96,6 +96,7 @@ public class AppInfo { protected long vcoreSeconds; protected float queueUsagePercentage; protected float clusterUsagePercentage; + protected Map resourceSecondsMap; // preemption info fields protected long preemptedResourceMB; @@ -104,6 +105,7 @@ public class AppInfo { protected int numAMContainerPreempted; private long preemptedMemorySeconds; private long preemptedVcoreSeconds; + protected Map preemptedResourceSecondsMap; // list of resource requests @XmlElement(name = "resourceRequests") @@ -220,8 +222,10 @@ public class AppInfo { appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); + resourceSecondsMap = appMetrics.getResourceSecondsMap(); preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); + preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap(); ApplicationSubmissionContext appSubmissionContext = app.getApplicationSubmissionContext(); unmanagedApplication = @@ -406,7 +410,7 @@ public class AppInfo { public int getNumAMContainersPreempted() { return numAMContainerPreempted; } - + public long getMemorySeconds() { return memorySeconds; } @@ -415,6 +419,10 @@ public class AppInfo { return vcoreSeconds; } + public Map getResourceSecondsMap() { + return resourceSecondsMap; + } + public long getPreemptedMemorySeconds() { return preemptedMemorySeconds; } @@ -422,6 +430,11 @@ public class AppInfo { public long getPreemptedVcoreSeconds() { return preemptedVcoreSeconds; } + + public Map getPreemptedResourceSecondsMap() { + return preemptedResourceSecondsMap; + } + public List getResourceRequests() { return this.resourceRequests; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 5083943..e13980a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -20,46 +20,68 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; @XmlRootElement -@XmlAccessorType(XmlAccessType.FIELD) +@XmlAccessorType(XmlAccessType.NONE) public class ResourceInfo { + + @XmlElement long memory; + @XmlElement int vCores; - + + private Resource resources; + public ResourceInfo() { } public ResourceInfo(Resource res) { memory = res.getMemorySize(); vCores = res.getVirtualCores(); + resources = Resources.clone(res); } public long getMemorySize() { - return memory; + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + return resources.getMemorySize(); } public int getvCores() { - return vCores; + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + return resources.getVirtualCores(); } - + @Override public String toString() { - return ""; + return resources.toString(); } public void setMemory(int memory) { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } this.memory = memory; + resources.setMemorySize(memory); } public void setvCores(int vCores) { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } this.vCores = vCores; + resources.setVirtualCores(vCores); } public Resource getResource() { - return Resource.newInstance(memory, vCores); + return Resource.newInstance(resources); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java index cf93edd..887b854 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java @@ -73,7 +73,7 @@ public class SchedulerInfo { } public String getSchedulerResourceTypes() { - return this.schedulingResourceTypes.toString(); + return minAllocResource.getResource().getResources().keySet().toString(); } public int getMaxClusterLevelAppPriority() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 247cd21..39a56a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -87,6 +87,8 @@ message ApplicationAttemptStateDataProto { optional int64 finish_time = 12; optional int64 preempted_memory_seconds = 13; optional int64 preempted_vcore_seconds = 14; + repeated StringLongMapProto application_resource_usage_map = 15; + repeated StringLongMapProto preempted_resource_usage_map = 16; } message EpochProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 2b08689..cc49ed4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentMap; import org.apache.commons.logging.Log; @@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -669,9 +671,12 @@ public class TestAppManager{ when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName"); when(app.getState()).thenReturn(RMAppState.RUNNING); when(app.getApplicationType()).thenReturn("MAPREDUCE"); + Map resourceSecondsMap = new HashMap<>(); + resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L); + resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L); RMAppMetrics metrics = new RMAppMetrics(Resource.newInstance(1234, 56), - 10, 1, 16384, 64, 0, 0); + 10, 1, resourceSecondsMap, new HashMap<>()); when(app.getRMAppMetrics()).thenReturn(metrics); RMAppManager.ApplicationSummary.SummaryBuilder summary = http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java index 3db00a2..c87c919 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.lang.time.DateUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -424,6 +426,9 @@ public class TestContainerResourceUsage { * usedMillis / DateUtils.MILLIS_PER_SECOND; long vcoreSeconds = resource.getVirtualCores() * usedMillis / DateUtils.MILLIS_PER_SECOND; - return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds); + Map map = new HashMap<>(); + map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); + map.put(ResourceInformation.VCORES.getName(), vcoreSeconds); + return new AggregateAppResourceUsage(map); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 9be52c6..f411a28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -195,7 +196,8 @@ public abstract class MockAsm extends MockApps { @Override public RMAppMetrics getRMAppMetrics() { - return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0); + return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(), + new HashMap<>()); } @Override @@ -338,8 +340,9 @@ public abstract class MockAsm extends MockApps { public ApplicationReport createAndGetApplicationReport( String clientUserName, boolean allowAccess) { ApplicationResourceUsageReport usageReport = - ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, - 0, 0, 0, 0, 0, 0); + ApplicationResourceUsageReport + .newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0, + new HashMap<>()); ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 55e93c1..d692392 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.when; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -506,9 +508,16 @@ public class TestSystemMetricsPublisher { when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); - when(app.getRMAppMetrics()).thenReturn( - new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, - Integer.MAX_VALUE, Long.MAX_VALUE)); + Map resourceMap = new HashMap<>(); + resourceMap + .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); + resourceMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); + Map preemptedMap = new HashMap<>(); + preemptedMap + .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); + preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); + when(app.getRMAppMetrics()) + .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap)); Set appTags = new HashSet(); appTags.add("test"); appTags.add("tags"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 2d40c91..2da273b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -29,6 +29,8 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -46,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; @@ -348,15 +351,20 @@ public class TestSystemMetricsPublisherForV2 { when(app.getDiagnostics()).thenReturn( new StringBuilder("test diagnostics info")); RMAppAttempt appAttempt = mock(RMAppAttempt.class); - when(appAttempt.getAppAttemptId()).thenReturn( - ApplicationAttemptId.newInstance(appId, 1)); + when(appAttempt.getAppAttemptId()) + .thenReturn(ApplicationAttemptId.newInstance(appId, 1)); when(app.getCurrentAppAttempt()).thenReturn(appAttempt); - when(app.getFinalApplicationStatus()).thenReturn( - FinalApplicationStatus.UNDEFINED); + when(app.getFinalApplicationStatus()) + .thenReturn(FinalApplicationStatus.UNDEFINED); + Map resourceSecondsMap = new HashMap<>(); + resourceSecondsMap + .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); + resourceSecondsMap + .put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(app.getRMAppMetrics()).thenReturn( - new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, Integer.MAX_VALUE, - Long.MAX_VALUE, 0, 0)); - when(app.getApplicationTags()).thenReturn(Collections. emptySet()); + new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap, + new HashMap<>())); + when(app.getApplicationTags()).thenReturn(Collections.emptySet()); ApplicationSubmissionContext appSubmissionContext = mock(ApplicationSubmissionContext.class); when(appSubmissionContext.getPriority()) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 514e9a0..927020f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -193,7 +193,7 @@ public class RMStateStoreTestBase { when(mockAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(0, 0)); + .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); dispatcher.attemptId = attemptId; store.storeNewApplicationAttempt(mockAttempt); waitNotify(dispatcher); @@ -284,7 +284,7 @@ public class RMStateStoreTestBase { when(mockRemovedAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(0,0)); + .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); attempts.put(attemptIdRemoved, mockRemovedAttempt); store.removeApplication(mockRemovedApp); @@ -361,7 +361,7 @@ public class RMStateStoreTestBase { oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 100, - oldAttemptState.getFinishTime(), 0, 0, 0, 0); + oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(newAttemptState); // test updating the state of an app/attempt whose initial state was not @@ -385,7 +385,7 @@ public class RMStateStoreTestBase { oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 111, - oldAttemptState.getFinishTime(), 0, 0, 0, 0); + oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(dummyAttempt); // let things settle down http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index f71cf25..5d7e6ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -67,6 +67,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.HashMap; import java.util.List; import javax.crypto.SecretKey; @@ -428,7 +429,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { when(mockAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(0,0)); + .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); store.storeNewApplicationAttempt(mockAttempt); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); @@ -440,7 +441,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { store.getCredentialsFromAppAttempt(mockAttempt), startTime, RMAppAttemptState.FINISHED, "testUrl", "test", FinalApplicationStatus.SUCCEEDED, 100, - finishTime, 0, 0, 0, 0); + finishTime, new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(newAttemptState); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java index a009bc0..1010c0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.HashMap; import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -63,9 +64,10 @@ public class TestAppPage { when(app.getStartTime()).thenReturn(0L); when(app.getFinishTime()).thenReturn(0L); when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED); - - RMAppMetrics appMetrics = new RMAppMetrics( - Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0); + + RMAppMetrics appMetrics = + new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(), + new HashMap<>()); when(app.getRMAppMetrics()).thenReturn(appMetrics); // initialize RM Context, and create RMApp, without creating RMAppAttempt http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java index 78fadef..1a40e61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java @@ -52,6 +52,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.concurrent.ConcurrentMap; @@ -142,8 +143,8 @@ public class TestRMWebAppFairScheduler { MockRMApp app = new MockRMApp(i, i, state) { @Override public RMAppMetrics getRMAppMetrics() { - return new RMAppMetrics(Resource.newInstance(0, 0), - 0, 0, 0, 0, 0, 0); + return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, + new HashMap<>(), new HashMap<>()); } @Override public YarnApplicationState createApplicationState() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dc4ca0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 30f25e9..6157b14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1419,7 +1419,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - int expectedNumberOfElements = 35; + int expectedNumberOfElements = 37; String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org