Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id BE1B52004A0 for ; Wed, 16 Aug 2017 19:07:12 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id BC4E516919C; Wed, 16 Aug 2017 17:07:12 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B43AB169196 for ; Wed, 16 Aug 2017 19:07:11 +0200 (CEST) Received: (qmail 64763 invoked by uid 500); 16 Aug 2017 17:06:57 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 61797 invoked by uid 99); 16 Aug 2017 17:06:55 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Aug 2017 17:06:55 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B6C53DFB86; Wed, 16 Aug 2017 17:06:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: haibochen@apache.org To: common-commits@hadoop.apache.org Date: Wed, 16 Aug 2017 17:07:41 -0000 Message-Id: <7f29295ad0a44c47ac6848acb79fe9cb@git.apache.org> In-Reply-To: <72328aab7afd4d1eab061a19c2206cdb@git.apache.org> References: <72328aab7afd4d1eab061a19c2206cdb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [49/50] [abbrv] hadoop git commit: YARN-6670 Add separate NM overallocation thresholds for cpu and memory (Haibo Chen) archived-at: Wed, 16 Aug 2017 17:07:12 -0000 YARN-6670 Add separate NM overallocation thresholds for cpu and memory (Haibo Chen) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5baae1be Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5baae1be Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5baae1be Branch: refs/heads/YARN-1011 Commit: 5baae1bee02fa3f65fd1f8aff174820034d9ceab Parents: 89dbd480 Author: Haibo Chen Authored: Mon Jul 10 09:55:42 2017 -0700 Committer: Haibo Chen Committed: Wed Aug 16 10:02:43 2017 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 36 +++++++++-- .../src/main/resources/yarn-default.xml | 42 ++++++++++-- .../server/api/records/ResourceThresholds.java | 11 +++- .../monitor/ContainersMonitorImpl.java | 67 +++++++++++++++----- 4 files changed, 124 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5baae1be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9e0a2a4..a54bd11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1614,17 +1614,39 @@ public class YarnConfiguration extends Configuration { public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS = 3000; - /** Overallocation (= allocation based on utilization) configs. */ - public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD = - NM_PREFIX + "overallocation.allocation-threshold"; - public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD - = 0f; + /** + * General overallocation threshold if no resource-type-specific + * threshold is provided. + */ + public static final String NM_OVERALLOCATION_GENERAL_THRESHOLD = + NM_PREFIX + "overallocation.general-utilization-threshold"; + public static final float + DEFAULT_NM_OVERALLOCATION_GENERAL_THRESHOLD = -1.0f; + /** + * The maximum value of utilization threshold for all resource types + * up to which the scheduler allocates OPPORTUNISTIC containers. + */ @Private - public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f; + public static final float MAX_NM_OVERALLOCATION_THRESHOLD = 0.95f; + + /** + * NM CPU utilization threshold up to which the scheduler allocates + * OPPORTUNISTIC containers after the node's capacity is fully allocated. + */ + public static final String NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD = + NM_PREFIX + "overallocation.cpu-utilization-threshold"; + + /** + * NM memory utilization threshold up to which the scheduler allocates + * OPPORTUNISTIC containers after the node's capacity is fully allocated. + */ + public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD = + NM_PREFIX + "overallocation.memory-utilization-threshold"; + public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD = NM_PREFIX + "overallocation.preemption-threshold"; public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD - = 0f; + = 0.96f; /** * Interval of time the linux container executor should try cleaning up http://git-wip-us.apache.org/repos/asf/hadoop/blob/5baae1be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c87c200..d76e13e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1546,14 +1546,44 @@ The extent of over-allocation (container-allocation based on + current utilization instead of prior allocation) allowed on this node that + applies to all resource types (expressed as a float between 0 and 0.95). + By default, over-allocation is turned off (value = -1). When turned on, + the node allows running OPPORTUNISTIC containers when the aggregate + utilization for each resource type is under the value specified here + multiplied by the node's advertised capacity. Note for each resource type, + it can be overridden by the type specific setting: + yarn.nodemanager.overallocation.cpu-utilization-threshold for CPU + yarn.nodemanager.overallocation.memory-utilization-threshold for memory + + yarn.nodemanager.overallocation.general-utilization-threshold + -1.0 + + + + The extent of over-allocation (container-allocation based on current utilization instead of prior allocation) allowed on this node, - expressed as a float between 0 and 0.95. By default, over-allocation is - turned off (value = 0). When turned on, the node allows running - OPPORTUNISTIC containers when the aggregate utilization is under the + in terms of the percentage of overall NM memory capacity utilized ( + expressed as a float between 0 and 0.95). By default, over-allocation is + turned off (value = -1). When turned on, the node allows running + OPPORTUNISTIC containers only when the aggregate utilization is under the value specified here multiplied by the node's advertised capacity. - yarn.nodemanager.overallocation.allocation-threshold - 0f + yarn.nodemanager.overallocation.memory-utilization-threshold + ${yarn.nodemanager.overallocation.general-utilization-threshold} + + + + The extent of over-allocation (container-allocation based on + current utilization instead of prior allocation) allowed on this node, + in terms of the percentage of overall NM CPU capacity utilized ( + expressed as a float between 0 and 0.95). By default, over-allocation is + turned off (value = -1). When turned on, the node allows running + OPPORTUNISTIC containers only when the aggregate utilization is under the + value specified here multiplied by the node's advertised capacity. + + yarn.nodemanager.overallocation.cpu-utilization-threshold + ${yarn.nodemanager.overallocation.general-utilization-threshold} @@ -1562,7 +1592,7 @@ beyond which OPPORTUNISTIC containers should start getting preempted. yarn.nodemanager.overallocation.preemption-threshold - 1 + 0.96 http://git-wip-us.apache.org/repos/asf/hadoop/blob/5baae1be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java index d57706a..c81e405 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java @@ -28,10 +28,15 @@ import org.apache.hadoop.yarn.server.api.records.impl.pb.ResourceThresholdsPBImp @InterfaceAudience.Private @InterfaceStability.Evolving public abstract class ResourceThresholds { - public static ResourceThresholds newInstance(float threshold) { + public static ResourceThresholds newInstance(float overallThreshold) { + return newInstance(overallThreshold, overallThreshold); + } + + public static ResourceThresholds newInstance(float cpuThreshold, + float memoryThreshold) { ResourceThresholds thresholds = new ResourceThresholdsPBImpl(); - thresholds.setMemoryThreshold(threshold); - thresholds.setCpuThreshold(threshold); + thresholds.setCpuThreshold(cpuThreshold); + thresholds.setMemoryThreshold(memoryThreshold); return thresholds; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5baae1be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 34f6ab1..3b56f99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -224,25 +224,60 @@ public class ContainersMonitorImpl extends AbstractService implements } private void initializeOverAllocation(Configuration conf) { - float overAllocationTreshold = conf.getFloat( - YarnConfiguration.NM_OVERALLOCATION_ALLOCATION_THRESHOLD, - YarnConfiguration.DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD); - overAllocationTreshold = Math.min(overAllocationTreshold, - YarnConfiguration.MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD); - overAllocationTreshold = Math.max(0, overAllocationTreshold); - - if (overAllocationTreshold > 0f) { - ((NodeManager.NMContext) context).setOverAllocationInfo( - OverAllocationInfo.newInstance( - ResourceThresholds.newInstance(overAllocationTreshold))); + float generalResourceOverAllocationThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_GENERAL_THRESHOLD, + YarnConfiguration.DEFAULT_NM_OVERALLOCATION_GENERAL_THRESHOLD); + + float overAllocationMemoryUtilizationThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD, + generalResourceOverAllocationThreshold); + overAllocationMemoryUtilizationThreshold = Math.min( + overAllocationMemoryUtilizationThreshold, + YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD); + if (overAllocationMemoryUtilizationThreshold <= 0) { + LOG.info("NodeManager oversubscription is disabled because the memory " + + "utilization threshold is no larger than zero."); + return; + } - float preemptionThreshold = conf.getFloat( - YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD, - YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD); + float overAllocationCpuUtilizationThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD, + generalResourceOverAllocationThreshold); + overAllocationCpuUtilizationThreshold = Math.min( + overAllocationCpuUtilizationThreshold, + YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD); + if (overAllocationCpuUtilizationThreshold <= 0) { + LOG.info("NodeManager oversubscription is disabled because the CPU " + + "utilization threshold is no larger than zero."); + return; + } - this.overAllocationPreemptionThresholds = - ResourceThresholds.newInstance(preemptionThreshold); + float preemptionThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD, + YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD); + if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) { + LOG.info("NodeManager oversubscription is disabled because preemption" + + "threshold is no larger than the cpu utilization threshold."); + return; + } + if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) { + LOG.info("NodeManager oversubscription is disabled because preemption" + + "threshold is no larger than the memory utilization threshold."); + return; } + + ResourceThresholds resourceThresholds = ResourceThresholds.newInstance( + overAllocationCpuUtilizationThreshold, + overAllocationMemoryUtilizationThreshold); + ((NodeManager.NMContext) context).setOverAllocationInfo( + OverAllocationInfo.newInstance(resourceThresholds)); + this.overAllocationPreemptionThresholds = + ResourceThresholds.newInstance(preemptionThreshold); + + LOG.info("NodeManager oversubscription enabled with overallocation " + + "thresholds (memory:" + overAllocationMemoryUtilizationThreshold + + ", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" + + " threshold: " + preemptionThreshold); } private boolean isResourceCalculatorAvailable() { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org