Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6EA4117BDA for ; Mon, 9 Mar 2015 20:25:51 +0000 (UTC) Received: (qmail 70669 invoked by uid 500); 9 Mar 2015 20:25:29 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 70033 invoked by uid 500); 9 Mar 2015 20:25:29 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 67146 invoked by uid 99); 9 Mar 2015 20:25:28 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 09 Mar 2015 20:25:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8AA65E1834; Mon, 9 Mar 2015 20:25:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jing9@apache.org To: common-commits@hadoop.apache.org Date: Mon, 09 Mar 2015 20:26:01 -0000 Message-Id: <0d76569dbacf48158bad3169b1b637c2@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [35/50] [abbrv] hadoop git commit: YARN-3275. CapacityScheduler: Preemption happening on non-preemptable queues. Contributed by Eric Payne YARN-3275. CapacityScheduler: Preemption happening on non-preemptable queues. Contributed by Eric Payne Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dfc015f2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dfc015f2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dfc015f2 Branch: refs/heads/HDFS-7285 Commit: dfc015f295d92286ad570667556e837bd1d30134 Parents: fcae120 Author: Jason Lowe Authored: Fri Mar 6 22:36:18 2015 +0000 Committer: Jing Zhao Committed: Mon Mar 9 13:11:26 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/util/resource/Resources.java | 5 ++++ .../ProportionalCapacityPreemptionPolicy.java | 27 ++++++++++++++++---- ...estProportionalCapacityPreemptionPolicy.java | 24 +++++++++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfc015f2/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c2aa2ef..250fc1c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -719,6 +719,9 @@ Release 2.7.0 - UNRELEASED YARN-3227. Timeline renew delegation token fails when RM user's TGT is expired (Zhijie Shen via xgong) + YARN-3275. CapacityScheduler: Preemption happening on non-preemptable + queues (Eric Payne via jlowe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfc015f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index a205bd1..bcb0421 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -260,4 +260,9 @@ public class Resources { return createResource(Math.min(lhs.getMemory(), rhs.getMemory()), Math.min(lhs.getVirtualCores(), rhs.getVirtualCores())); } + + public static Resource componentwiseMax(Resource lhs, Resource rhs) { + return createResource(Math.max(lhs.getMemory(), rhs.getMemory()), + Math.max(lhs.getVirtualCores(), rhs.getVirtualCores())); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfc015f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 738f527..87a2a00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -527,6 +527,17 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic List skippedAMContainerlist = new ArrayList(); for (TempQueue qT : queues) { + if (qT.preemptionDisabled && qT.leafQueue != null) { + if (LOG.isDebugEnabled()) { + if (Resources.greaterThan(rc, clusterResource, + qT.toBePreempted, Resource.newInstance(0, 0))) { + LOG.debug("Tried to preempt the following " + + "resources from non-preemptable queue: " + + qT.queueName + " - Resources: " + qT.toBePreempted); + } + } + continue; + } // we act only if we are violating balance by more than // maxIgnoredOverCapacity if (Resources.greaterThan(rc, clusterResource, qT.current, @@ -734,6 +745,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic float absUsed = root.getAbsoluteUsedCapacity(); float absCap = root.getAbsoluteCapacity(); float absMaxCap = root.getAbsoluteMaximumCapacity(); + boolean preemptionDisabled = root.getPreemptionDisabled(); Resource current = Resources.multiply(clusterResources, absUsed); Resource guaranteed = Resources.multiply(clusterResources, absCap); @@ -747,8 +759,8 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic LeafQueue l = (LeafQueue) root; Resource pending = l.getTotalResourcePending(); ret = new TempQueue(queueName, current, pending, guaranteed, - maxCapacity); - if (root.getPreemptionDisabled()) { + maxCapacity, preemptionDisabled); + if (preemptionDisabled) { ret.untouchableExtra = extra; } else { ret.preemptableExtra = extra; @@ -757,7 +769,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic } else { Resource pending = Resource.newInstance(0, 0); ret = new TempQueue(root.getQueueName(), current, pending, guaranteed, - maxCapacity); + maxCapacity, false); Resource childrensPreemptable = Resource.newInstance(0, 0); for (CSQueue c : root.getChildQueues()) { TempQueue subq = cloneQueues(c, clusterResources); @@ -816,9 +828,10 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic final ArrayList children; LeafQueue leafQueue; + boolean preemptionDisabled; TempQueue(String queueName, Resource current, Resource pending, - Resource guaranteed, Resource maxCapacity) { + Resource guaranteed, Resource maxCapacity, boolean preemptionDisabled) { this.queueName = queueName; this.current = current; this.pending = pending; @@ -831,6 +844,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic this.children = new ArrayList(); this.untouchableExtra = Resource.newInstance(0, 0); this.preemptableExtra = Resource.newInstance(0, 0); + this.preemptionDisabled = preemptionDisabled; } public void setLeafQueue(LeafQueue l){ @@ -862,10 +876,13 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic // the unused ones Resource offer(Resource avail, ResourceCalculator rc, Resource clusterResource) { + Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax( + Resources.subtract(maxCapacity, idealAssigned), + Resource.newInstance(0, 0)); // remain = avail - min(avail, (max - assigned), (current + pending - assigned)) Resource accepted = Resources.min(rc, clusterResource, - Resources.subtract(maxCapacity, idealAssigned), + absMaxCapIdealAssignedDelta, Resources.min(rc, clusterResource, avail, Resources.subtract( Resources.add(current, pending), idealAssigned))); Resource remain = Resources.subtract(avail, accepted); http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfc015f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index 696b9bb..8f5237e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -532,6 +532,30 @@ public class TestProportionalCapacityPreemptionPolicy { } @Test + public void testPerQueueDisablePreemptionOverAbsMaxCapacity() { + int[][] qData = new int[][] { + // / A D + // B C E F + {1000, 725, 360, 365, 275, 17, 258 }, // absCap + {1000,1000,1000,1000, 550, 109,1000 }, // absMaxCap + {1000, 741, 396, 345, 259, 110, 149 }, // used + { 40, 20, 0, 20, 20, 20, 0 }, // pending + { 0, 0, 0, 0, 0, 0, 0 }, // reserved + // appA appB appC appD + { 4, 2, 1, 1, 2, 1, 1 }, // apps + { -1, -1, 1, 1, -1, 1, 1 }, // req granulrity + { 2, 2, 0, 0, 2, 0, 0 }, // subqueues + }; + // QueueE inherits non-preemption from QueueD + schedConf.setPreemptionDisabled("root.queueD", true); + ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); + policy.editSchedule(); + // appC is running on QueueE. QueueE is over absMaxCap, but is not + // preemptable. Therefore, appC resources should not be preempted. + verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appC))); + } + + @Test public void testOverCapacityImbalance() { int[][] qData = new int[][]{ // / A B C