Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6C72310A6F for ; Mon, 17 Nov 2014 06:30:22 +0000 (UTC) Received: (qmail 13459 invoked by uid 500); 17 Nov 2014 06:30:05 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 13189 invoked by uid 500); 17 Nov 2014 06:30:05 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 11827 invoked by uid 99); 17 Nov 2014 06:30:04 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Nov 2014 06:30:04 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 5188393D387; Mon, 17 Nov 2014 06:30:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinayakumarb@apache.org To: common-commits@hadoop.apache.org Date: Mon, 17 Nov 2014 06:30:41 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [39/45] hadoop git commit: YARN-2811. In Fair Scheduler, reservation fulfillments shouldn't ignore max share (Siqi Li via Sandy Ryza) YARN-2811. In Fair Scheduler, reservation fulfillments shouldn't ignore max share (Siqi Li via Sandy Ryza) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a47f890 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a47f890 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a47f890 Branch: refs/heads/HDFS-EC Commit: 1a47f890ba3cb22b6262f47c1f1af2990559bb89 Parents: 49c3889 Author: Sandy Ryza Authored: Fri Nov 14 15:18:56 2014 -0800 Committer: Sandy Ryza Committed: Fri Nov 14 15:18:56 2014 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FairScheduler.java | 18 +++- .../scheduler/fair/TestFairScheduler.java | 88 +++++++++++++++++++- 3 files changed, 103 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a47f890/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1f8af55..cfd9459 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -225,6 +225,9 @@ Release 2.6.0 - 2014-11-18 YARN-2505. Supported get/add/remove/change labels in RM REST API. (Craig Welch via zjshen) + YARN-2811. In Fair Scheduler, reservation fulfillments shouldn't ignore max + share (Siqi Li via Sandy Ryza) + IMPROVEMENTS YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a47f890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 3fc3019..94fb849 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1029,7 +1029,10 @@ public class FairScheduler extends FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable(); if (reservedAppSchedulable != null) { Priority reservedPriority = node.getReservedContainer().getReservedPriority(); - if (!reservedAppSchedulable.hasContainerForNode(reservedPriority, node)) { + FSQueue queue = reservedAppSchedulable.getQueue(); + + if (!reservedAppSchedulable.hasContainerForNode(reservedPriority, node) + || !fitInMaxShare(queue)) { // Don't hold the reservation if app can no longer use it LOG.info("Releasing reservation that cannot be satisfied for application " + reservedAppSchedulable.getApplicationAttemptId() @@ -1043,7 +1046,6 @@ public class FairScheduler extends + reservedAppSchedulable.getApplicationAttemptId() + " on node: " + node); } - node.getReservedAppSchedulable().assignReservedContainer(node); } } @@ -1065,6 +1067,18 @@ public class FairScheduler extends updateRootQueueMetrics(); } + private boolean fitInMaxShare(FSQueue queue) { + if (Resources.fitsIn(queue.getResourceUsage(), queue.getMaxShare())) { + return false; + } + + FSQueue parentQueue = queue.getParent(); + if (parentQueue != null) { + return fitInMaxShare(parentQueue); + } + return true; + } + public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) { return super.getApplicationAttempt(appAttemptId); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a47f890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 61cbdc1..f223458 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -722,6 +722,85 @@ public class TestFairScheduler extends FairSchedulerTestBase { } + @Test (timeout = 5000) + public void testContainerReservationNotExceedingQueueMax() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("1024mb,5vcores"); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println("1024mb,5vcores"); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + // Add a node + RMNode node1 = + MockNodes + .newNodeInfo(1, Resources.createResource(3072, 5), 1, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Queue 1 requests full capacity of the queue + createSchedulingRequest(2048, "queue1", "user1", 1); + scheduler.update(); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); + scheduler.handle(updateEvent); + + // Make sure queue 1 is allocated app capacity + assertEquals(2048, scheduler.getQueueManager().getQueue("queue1"). + getResourceUsage().getMemory()); + + // Now queue 2 requests likewise + ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user2", 1); + scheduler.update(); + scheduler.handle(updateEvent); + + // Make sure queue 2 is allocated app capacity + assertEquals(1024, scheduler.getQueueManager().getQueue("queue2"). + getResourceUsage().getMemory()); + + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1); + scheduler.update(); + scheduler.handle(updateEvent); + + // Make sure queue 1 is waiting with a reservation + assertEquals(1024, scheduler.getSchedulerApp(attId1) + .getCurrentReservation().getMemory()); + + // Now remove app of queue2 + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent( + attId, RMAppAttemptState.FINISHED, false); + scheduler.update(); + scheduler.handle(appRemovedEvent1); + + // Queue should have no apps + assertEquals(0, scheduler.getQueueManager().getQueue("queue2"). + getResourceUsage().getMemory()); + + createSchedulingRequest(1024, "queue2", "user2", 1); + scheduler.handle(updateEvent); + // Make sure allocated memory of queue1 doesn't exceed its maximum + assertEquals(2048, scheduler.getQueueManager().getQueue("queue1"). + getResourceUsage().getMemory()); + //the reservation of queue1 should be reclaim + assertEquals(0, scheduler.getSchedulerApp(attId1). + getCurrentReservation().getMemory()); + assertEquals(1024, scheduler.getQueueManager().getQueue("queue2"). + getResourceUsage().getMemory()); + } + @Test public void testUserAsDefaultQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); @@ -2076,9 +2155,10 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); - // Reserved container should still be at lower priority + // Reserved container should will be at higher priority, + // since old reservation cannot be satisfied for (RMContainer container : app.getReservedContainers()) { - assertEquals(2, container.getReservedPriority().getPriority()); + assertEquals(1, container.getReservedPriority().getPriority()); } // Complete container @@ -2091,11 +2171,11 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.update(); scheduler.handle(updateEvent); - // Reserved container (at lower priority) should be run + // Reserved container (at higher priority) should be run Collection liveContainers = app.getLiveContainers(); assertEquals(1, liveContainers.size()); for (RMContainer liveContainer : liveContainers) { - Assert.assertEquals(2, liveContainer.getContainer().getPriority().getPriority()); + Assert.assertEquals(1, liveContainer.getContainer().getPriority().getPriority()); } assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());