Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D9061200BE5 for ; Sat, 10 Dec 2016 01:42:27 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id D7AE2160B1E; Sat, 10 Dec 2016 00:42:27 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8A3C8160B1D for ; Sat, 10 Dec 2016 01:42:26 +0100 (CET) Received: (qmail 29784 invoked by uid 500); 10 Dec 2016 00:42:25 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 29775 invoked by uid 99); 10 Dec 2016 00:42:25 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 10 Dec 2016 00:42:25 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 90D54E040F; Sat, 10 Dec 2016 00:42:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: asuresh@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-5982. Simplify opportunistic container parameters and metrics. (Konstantinos Karanasos via asuresh) Date: Sat, 10 Dec 2016 00:42:25 +0000 (UTC) archived-at: Sat, 10 Dec 2016 00:42:28 -0000 Repository: hadoop Updated Branches: refs/heads/trunk 55f5886ea -> b0aace21b YARN-5982. Simplify opportunistic container parameters and metrics. (Konstantinos Karanasos via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0aace21 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0aace21 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0aace21 Branch: refs/heads/trunk Commit: b0aace21b1ef3436ba9d516186208fee9a9ceef2 Parents: 55f5886 Author: Arun Suresh Authored: Fri Dec 9 16:41:25 2016 -0800 Committer: Arun Suresh Committed: Fri Dec 9 16:41:25 2016 -0800 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 41 ------------ .../src/main/resources/yarn-default.xml | 57 ----------------- .../scheduler/ContainerScheduler.java | 10 +-- .../nodemanager/metrics/NodeManagerMetrics.java | 65 ++++++++++---------- ...pportunisticContainerAllocatorAMService.java | 60 ++++-------------- .../scheduler/SchedulerNode.java | 6 +- .../resourcemanager/webapp/NodesPage.java | 6 +- .../resourcemanager/webapp/dao/NodeInfo.java | 10 +-- .../webapp/TestRMWebServicesNodes.java | 10 +-- 9 files changed, 68 insertions(+), 197 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 4934964..69c7b00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -323,47 +323,6 @@ public class YarnConfiguration extends Configuration { public static final boolean OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED_DEFAULT = false; - /** Minimum memory (in MB) used for allocating an opportunistic container. */ - public static final String OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB = - YARN_PREFIX + "opportunistic-containers.min-memory-mb"; - public static final int OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT = 512; - - /** Minimum virtual CPU cores used for allocating an opportunistic container. - * */ - public static final String OPPORTUNISTIC_CONTAINERS_MIN_VCORES = - YARN_PREFIX + "opportunistic-containers.min-vcores"; - public static final int OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT = 1; - - /** Maximum memory (in MB) used for allocating an opportunistic container. */ - public static final String OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB = - YARN_PREFIX + "opportunistic-containers.max-memory-mb"; - public static final int OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT = 2048; - - /** Maximum virtual CPU cores used for allocating an opportunistic container. - * */ - public static final String OPPORTUNISTIC_CONTAINERS_MAX_VCORES = - YARN_PREFIX + "opportunistic-containers.max-vcores"; - public static final int OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT = 4; - - /** Incremental memory (in MB) used for allocating an opportunistic container. - * */ - public static final String OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB = - YARN_PREFIX + "opportunistic-containers.incr-memory-mb"; - public static final int OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT = - 512; - - /** Incremental virtual CPU cores used for allocating an opportunistic - * container. */ - public static final String OPPORTUNISTIC_CONTAINERS_INCR_VCORES = - YARN_PREFIX + "opportunistic-containers.incr-vcores"; - public static final int OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT = 1; - - /** Container token expiry for opportunistic containers. */ - public static final String OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS = - YARN_PREFIX + "opportunistic-containers.container-token-expiry-ms"; - public static final int OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT = - 600000; - /** Number of nodes to be used by the Opportunistic Container allocator for * dispatching containers during container allocation. */ public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED = http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 47d12d1..c8c4edd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2766,63 +2766,6 @@ - Minimum memory (in MB) used for allocating an opportunistic container. - - yarn.opportunistic-containers.min-memory-mb - 512 - - - - - Minimum virtual CPU cores used for allocating an opportunistic container. - - yarn.opportunistic-containers.min-vcores - 1 - - - - - Maximum memory (in MB) used for allocating an opportunistic container. - - yarn.opportunistic-containers.max-memory-mb - 2048 - - - - - Maximum virtual CPU cores used for allocating an opportunistic container. - - yarn.opportunistic-containers.max-vcores - 4 - - - - - Incremental memory (in MB) used for allocating an opportunistic container. - - yarn.opportunistic-containers.incr-memory-mb - 512 - - - - - Incremental virtual CPU cores used for allocating an opportunistic - container. - - yarn.opportunistic-containers.incr-vcores - 1 - - - - - Container token expiry for opportunistic containers. - - yarn.opportunistic-containers.container-token-expiry-ms - 600000 - - - - Number of nodes to be used by the Opportunistic Container Allocator for dispatching containers during container allocation. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java index 0c2b1ac..753bf04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java @@ -170,11 +170,11 @@ public class ContainerScheduler extends AbstractService implements this.opportunisticContainersStatus.setWaitQueueLength( getNumQueuedContainers()); this.opportunisticContainersStatus.setOpportMemoryUsed( - metrics.getOpportMemoryUsed()); + metrics.getAllocatedOpportunisticGB()); this.opportunisticContainersStatus.setOpportCoresUsed( - metrics.getOpportCoresUsed()); + metrics.getAllocatedOpportunisticVCores()); this.opportunisticContainersStatus.setRunningOpportContainers( - metrics.getRunningOpportContainers()); + metrics.getRunningOpportunisticContainers()); return this.opportunisticContainersStatus; } @@ -196,7 +196,7 @@ public class ContainerScheduler extends AbstractService implements this.utilizationTracker.subtractContainerResource(container); if (container.getContainerTokenIdentifier().getExecutionType() == ExecutionType.OPPORTUNISTIC) { - this.metrics.opportunisticContainerCompleted(container); + this.metrics.completeOpportunisticContainer(container.getResource()); } startPendingContainers(); } @@ -298,7 +298,7 @@ public class ContainerScheduler extends AbstractService implements this.utilizationTracker.addContainerResources(container); if (container.getContainerTokenIdentifier().getExecutionType() == ExecutionType.OPPORTUNISTIC) { - this.metrics.opportunisticContainerStarted(container); + this.metrics.startOpportunisticContainer(container.getResource()); } container.sendLaunchEvent(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index b001b63..291b488 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -29,8 +29,6 @@ import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.yarn.api.records.Resource; import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.container - .Container; @Metrics(about="Metrics for node manager", context="yarn") public class NodeManagerMetrics { @@ -64,12 +62,12 @@ public class NodeManagerMetrics { @Metric("Disk utilization % on good log dirs") MutableGaugeInt goodLogDirsDiskUtilizationPerc; - @Metric("Memory used by Opportunistic Containers in MB") - MutableGaugeLong opportMemoryUsed; - @Metric("# of Virtual Cores used by opportunistic containers") - MutableGaugeInt opportCoresUsed; + @Metric("Current allocated memory by opportunistic containers in GB") + MutableGaugeLong allocatedOpportunisticGB; + @Metric("Current allocated Virtual Cores by opportunistic containers") + MutableGaugeInt allocatedOpportunisticVCores; @Metric("# of running opportunistic containers") - MutableGaugeInt runningOpportContainers; + MutableGaugeInt runningOpportunisticContainers; // CHECKSTYLE:ON:VisibilityModifier @@ -77,6 +75,7 @@ public class NodeManagerMetrics { private long allocatedMB; private long availableMB; + private long allocatedOpportunisticMB; public NodeManagerMetrics(JvmMetrics jvmMetrics) { this.jvmMetrics = jvmMetrics; @@ -141,30 +140,6 @@ public class NodeManagerMetrics { containersReIniting.decr(); } - public long getOpportMemoryUsed() { - return opportMemoryUsed.value(); - } - - public int getOpportCoresUsed() { - return opportCoresUsed.value(); - } - - public int getRunningOpportContainers() { - return runningOpportContainers.value(); - } - - public void opportunisticContainerCompleted(Container container) { - opportMemoryUsed.decr(container.getResource().getMemorySize()); - opportCoresUsed.decr(container.getResource().getVirtualCores()); - runningOpportContainers.decr(); - } - - public void opportunisticContainerStarted(Container container) { - opportMemoryUsed.incr(container.getResource().getMemorySize()); - opportCoresUsed.incr(container.getResource().getVirtualCores()); - runningOpportContainers.incr(); - } - public void allocateContainer(Resource res) { allocatedContainers.incr(); allocatedMB = allocatedMB + res.getMemorySize(); @@ -196,6 +171,22 @@ public class NodeManagerMetrics { availableVCores.decr(deltaVCores); } + public void startOpportunisticContainer(Resource res) { + runningOpportunisticContainers.incr(); + allocatedOpportunisticMB = allocatedOpportunisticMB + res.getMemorySize(); + allocatedOpportunisticGB + .set((int) Math.ceil(allocatedOpportunisticMB / 1024d)); + allocatedOpportunisticVCores.incr(res.getVirtualCores()); + } + + public void completeOpportunisticContainer(Resource res) { + runningOpportunisticContainers.decr(); + allocatedOpportunisticMB = allocatedOpportunisticMB - res.getMemorySize(); + allocatedOpportunisticGB + .set((int) Math.ceil(allocatedOpportunisticMB / 1024d)); + allocatedOpportunisticVCores.decr(res.getVirtualCores()); + } + public void addResource(Resource res) { availableMB = availableMB + res.getMemorySize(); availableGB.incr((int)Math.floor(availableMB/1024d)); @@ -272,4 +263,16 @@ public class NodeManagerMetrics { public int getContainersRolledbackOnFailure() { return containersRolledBackOnFailure.value(); } + + public long getAllocatedOpportunisticGB() { + return allocatedOpportunisticGB.value(); + } + + public int getAllocatedOpportunisticVCores() { + return allocatedOpportunisticVCores.value(); + } + + public int getRunningOpportunisticContainers() { + return runningOpportunisticContainers.value(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java index 7814b84..7b7963f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRespons import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; @@ -199,11 +198,12 @@ public class OpportunisticContainerAllocatorAMService } }); int tokenExpiryInterval = getConfig() - .getInt(YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT); - opCtx.updateAllocationParams(createMinContainerResource(), - createMaxContainerResource(), createIncrContainerResource(), + .getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); + opCtx.updateAllocationParams( + rmContext.getScheduler().getMinimumResourceCapability(), + rmContext.getScheduler().getMaximumResourceCapability(), + rmContext.getScheduler().getMinimumResourceCapability(), tokenExpiryInterval); appAttempt.setOpportunisticContainerContext(opCtx); } @@ -273,14 +273,14 @@ public class OpportunisticContainerAllocatorAMService RegisterDistributedSchedulingAMResponse dsResp = recordFactory .newRecordInstance(RegisterDistributedSchedulingAMResponse.class); dsResp.setRegisterResponse(response); - dsResp.setMinContainerResource(createMinContainerResource()); - dsResp.setMaxContainerResource(createMaxContainerResource()); - dsResp.setIncrContainerResource(createIncrContainerResource()); + dsResp.setMinContainerResource( + rmContext.getScheduler().getMinimumResourceCapability()); + dsResp.setMaxContainerResource( + rmContext.getScheduler().getMaximumResourceCapability()); dsResp.setContainerTokenExpiryInterval( getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT)); + YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS)); dsResp.setContainerIdStart( this.rmContext.getEpoch() << ResourceManager.EPOCH_BIT_SHIFT); @@ -384,18 +384,6 @@ public class OpportunisticContainerAllocatorAMService return nodeMonitor.getThresholdCalculator(); } - private Resource createIncrContainerResource() { - return Resource.newInstance( - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT), - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT) - ); - } - private synchronized List getLeastLoadedNodes() { long currTime = System.currentTimeMillis(); if ((currTime - lastCacheUpdateTime > cacheRefreshInterval) @@ -425,30 +413,6 @@ public class OpportunisticContainerAllocatorAMService : null; } - private Resource createMaxContainerResource() { - return Resource.newInstance( - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB, - YarnConfiguration - .OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT), - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT) - ); - } - - private Resource createMinContainerResource() { - return Resource.newInstance( - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT), - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT) - ); - } - private static ApplicationAttemptId getAppAttemptId() throws YarnException { AMRMTokenIdentifier amrmTokenIdentifier = YarnServerSecurityUtils.authorizeRequest(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 995a7b0..6744c2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -149,10 +149,10 @@ public abstract class SchedulerNode { */ public synchronized void allocateContainer(RMContainer rmContainer) { Container container = rmContainer.getContainer(); - if (rmContainer.getExecutionType() != ExecutionType.OPPORTUNISTIC) { + if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { deductUnallocatedResource(container.getResource()); + ++numContainers; } - ++numContainers; launchedContainers.put(container.getId(), rmContainer); @@ -251,8 +251,8 @@ public abstract class SchedulerNode { Container container) { if (container.getExecutionType() == ExecutionType.GUARANTEED) { addUnallocatedResource(container.getResource()); + --numContainers; } - --numContainers; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index 7b68b84..c03df63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -49,6 +49,7 @@ class NodesPage extends RmView { static class NodesBlock extends HtmlBlock { final ResourceManager rm; private static final long BYTES_IN_MB = 1024 * 1024; + private static final long BYTES_IN_GB = 1024 * 1024 * 1024; private static boolean opportunisticContainersEnabled; @Inject @@ -181,8 +182,9 @@ class NodesPage extends RmView { nodeTableData .append(String.valueOf(info.getNumRunningOpportContainers())) .append("\",\"").append("
") - .append(StringUtils.byteDesc(info.getUsedMemoryOpport())) + .append(String.valueOf(info.getUsedMemoryOpportGB())).append("'>") + .append(StringUtils.byteDesc( + info.getUsedMemoryOpportGB() * BYTES_IN_GB)) .append("\",\"") .append(String.valueOf(info.getUsedVirtualCoresOpport())) .append("\",\"") http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index aee7717..3416e52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -51,7 +51,7 @@ public class NodeInfo { protected long usedVirtualCores; protected long availableVirtualCores; private int numRunningOpportContainers; - private long usedMemoryOpport; // Memory in bytes. + private long usedMemoryOpportGB; private long usedVirtualCoresOpport; private int numQueuedContainers; protected ArrayList nodeLabels = new ArrayList(); @@ -85,7 +85,7 @@ public class NodeInfo { // Status of opportunistic containers. this.numRunningOpportContainers = 0; - this.usedMemoryOpport = 0; + this.usedMemoryOpportGB = 0; this.usedVirtualCoresOpport = 0; this.numQueuedContainers = 0; OpportunisticContainersStatus opportStatus = @@ -93,7 +93,7 @@ public class NodeInfo { if (opportStatus != null) { this.numRunningOpportContainers = opportStatus.getRunningOpportContainers(); - this.usedMemoryOpport = opportStatus.getOpportMemoryUsed(); + this.usedMemoryOpportGB = opportStatus.getOpportMemoryUsed(); this.usedVirtualCoresOpport = opportStatus.getOpportCoresUsed(); this.numQueuedContainers = opportStatus.getQueuedOpportContainers(); } @@ -165,8 +165,8 @@ public class NodeInfo { return numRunningOpportContainers; } - public long getUsedMemoryOpport() { - return usedMemoryOpport; + public long getUsedMemoryOpportGB() { + return usedMemoryOpportGB; } public long getUsedVirtualCoresOpport() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index 10aa92a..fb597fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -726,7 +726,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase { "aggregatedContainersVirtualMemoryMB"), WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"), WebServicesTestUtils.getXmlInt(element, "numRunningOpportContainers"), - WebServicesTestUtils.getXmlLong(element, "usedMemoryOpport"), + WebServicesTestUtils.getXmlLong(element, "usedMemoryOpportGB"), WebServicesTestUtils.getXmlInt(element, "usedVirtualCoresOpport"), WebServicesTestUtils.getXmlInt(element, "numQueuedContainers")); } @@ -753,7 +753,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase { resourceInfo.getInt("aggregatedContainersVirtualMemoryMB"), resourceInfo.getDouble("containersCPUUsage"), nodeInfo.getInt("numRunningOpportContainers"), - nodeInfo.getLong("usedMemoryOpport"), + nodeInfo.getLong("usedMemoryOpportGB"), nodeInfo.getInt("usedVirtualCoresOpport"), nodeInfo.getInt("numQueuedContainers")); } @@ -766,7 +766,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase { int nodePhysicalMemoryMB, int nodeVirtualMemoryMB, double nodeCPUUsage, int containersPhysicalMemoryMB, int containersVirtualMemoryMB, double containersCPUUsage, int numRunningOpportContainers, - long usedMemoryOpport, int usedVirtualCoresOpport, + long usedMemoryOpportGB, int usedVirtualCoresOpport, int numQueuedContainers) throws JSONException, Exception { @@ -827,8 +827,8 @@ public class TestRMWebServicesNodes extends JerseyTestBase { numRunningOpportContainers, opportunisticStatus.getRunningOpportContainers(), numRunningOpportContainers); - assertEquals("usedMemoryOpport doesn't match: " + usedMemoryOpport, - opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpport); + assertEquals("usedMemoryOpportGB doesn't match: " + usedMemoryOpportGB, + opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpportGB); assertEquals( "usedVirtualCoresOpport doesn't match: " + usedVirtualCoresOpport, opportunisticStatus.getOpportCoresUsed(), usedVirtualCoresOpport); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org