Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6FAF2200B34 for ; Sat, 18 Jun 2016 07:27:16 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 6E4A7160A62; Sat, 18 Jun 2016 05:27:16 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 470EA160A61 for ; Sat, 18 Jun 2016 07:27:15 +0200 (CEST) Received: (qmail 81631 invoked by uid 500); 18 Jun 2016 05:27:14 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 81622 invoked by uid 99); 18 Jun 2016 05:27:14 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 18 Jun 2016 05:27:14 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 30A8BE049D; Sat, 18 Jun 2016 05:27:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kasha@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-5077. Fix FSLeafQueue#getFairShare() for queues with zero fairshare. (Yufei Gu via kasha) Date: Sat, 18 Jun 2016 05:27:14 +0000 (UTC) archived-at: Sat, 18 Jun 2016 05:27:16 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2 e9c115523 -> 25be8105c YARN-5077. Fix FSLeafQueue#getFairShare() for queues with zero fairshare. (Yufei Gu via kasha) (cherry picked from commit 20f2799938c5cad59f59617cdb0f86e137c3c2c0) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25be8105 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25be8105 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25be8105 Branch: refs/heads/branch-2 Commit: 25be8105cd048201f19e573f130b96cada0e9ede Parents: e9c1155 Author: Karthik Kambatla Authored: Fri Jun 17 22:24:42 2016 -0700 Committer: Karthik Kambatla Committed: Fri Jun 17 22:25:04 2016 -0700 ---------------------------------------------------------------------- .../scheduler/fair/FSLeafQueue.java | 33 ++++----- .../resourcemanager/scheduler/fair/FSQueue.java | 19 ----- .../scheduler/fair/SchedulingPolicy.java | 11 --- .../DominantResourceFairnessPolicy.java | 5 -- .../fair/policies/FairSharePolicy.java | 5 -- .../scheduler/fair/policies/FifoPolicy.java | 5 -- .../scheduler/fair/TestFairScheduler.java | 78 ++++++++++++++++++++ 7 files changed, 94 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/25be8105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index d8b51f7..6723fd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.util.resource.Resources; @Private @@ -482,8 +481,7 @@ public class FSLeafQueue extends FSQueue { /** * Check whether this queue can run this application master under the - * maxAMShare limit. For FIFO and FAIR policies, check if the VCore usage - * takes up the entire cluster or maxResources for the queue. + * maxAMShare limit. * @param amResource * @return true if this queue can run */ @@ -493,24 +491,25 @@ public class FSLeafQueue extends FSQueue { if (Math.abs(maxAMShare - -1.0f) < 0.0001) { return true; } - Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare); - Resource ifRunAMResource = Resources.add(amResourceUsage, amResource); - boolean overMaxAMShareLimit = policy - .checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource); + // If FairShare is zero, use min(maxShare, available resource) to compute + // maxAMResource + Resource maxResource = Resources.clone(getFairShare()); + if (maxResource.getMemorySize() == 0) { + maxResource.setMemory( + Math.min(scheduler.getRootQueueMetrics().getAvailableMB(), + getMaxShare().getMemorySize())); + } - // For fair policy and fifo policy which doesn't check VCore usages, - // additionally check if the AM takes all available VCores or - // over maxResource to avoid deadlock. - if (!overMaxAMShareLimit && !policy.equals( - SchedulingPolicy.getInstance(DominantResourceFairnessPolicy.class))) { - overMaxAMShareLimit = - isVCoresOverMaxResource(ifRunAMResource.getVirtualCores()) || - ifRunAMResource.getVirtualCores() >= - scheduler.getRootQueueMetrics().getAvailableVirtualCores(); + if (maxResource.getVirtualCoresSize() == 0) { + maxResource.setVirtualCores(Math.min( + scheduler.getRootQueueMetrics().getAvailableVirtualCores(), + getMaxShare().getVirtualCoresSize())); } - return !overMaxAMShareLimit; + Resource maxAMResource = Resources.multiply(maxResource, maxAMShare); + Resource ifRunAMResource = Resources.add(amResourceUsage, amResource); + return Resources.fitsIn(ifRunAMResource, maxAMResource); } public void addAMResourceUsage(Resource amResource) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/25be8105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index f50c358..25554dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -311,25 +311,6 @@ public abstract class FSQueue implements Queue, Schedulable { } /** - * Helper method to check if requested VCores are over maxResource. - * @param requestedVCores the number of VCores requested - * @return true if the number of VCores requested is over the maxResource; - * false otherwise - */ - protected boolean isVCoresOverMaxResource(int requestedVCores) { - if (requestedVCores >= scheduler.getAllocationConfiguration(). - getMaxResources(getName()).getVirtualCores()) { - return true; - } - - if (getParent() == null) { - return false; - } - - return getParent().isVCoresOverMaxResource(requestedVCores); - } - - /** * Returns true if queue has at least one app running. */ public boolean isActive() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/25be8105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java index 160ba4b..9eda46c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java @@ -178,17 +178,6 @@ public abstract class SchedulingPolicy { Resource usage, Resource fairShare); /** - * Check if a leaf queue's AM resource usage over its limit under this policy - * - * @param usage {@link Resource} the resource used by application masters - * @param maxAMResource {@link Resource} the maximum allowed resource for - * application masters - * @return true if AM resource usage is over the limit - */ - public abstract boolean checkIfAMResourceUsageOverLimit( - Resource usage, Resource maxAMResource); - - /** * Get headroom by calculating the min of clusterAvailable and * (queueFairShare - queueUsage) resources that are * applicable to this policy. For eg if only memory then leave other http://git-wip-us.apache.org/repos/asf/hadoop/blob/25be8105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 623437a..ad41b11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -94,11 +94,6 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy { } @Override - public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) { - return !Resources.fitsIn(usage, maxAMResource); - } - - @Override public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, Resource maxAvailable) { long queueAvailableMemory = http://git-wip-us.apache.org/repos/asf/hadoop/blob/25be8105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java index 42d0420..6aa8405 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java @@ -151,11 +151,6 @@ public class FairSharePolicy extends SchedulingPolicy { } @Override - public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) { - return usage.getMemorySize() > maxAMResource.getMemorySize(); - } - - @Override public byte getApplicableDepth() { return SchedulingPolicy.DEPTH_ANY; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/25be8105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java index c277df1..d3fdcf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java @@ -114,11 +114,6 @@ public class FifoPolicy extends SchedulingPolicy { } @Override - public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) { - return usage.getMemorySize() > maxAMResource.getMemorySize(); - } - - @Override public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, Resource maxAvailable) { long queueAvailableMemory = Math.max( http://git-wip-us.apache.org/repos/asf/hadoop/blob/25be8105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index beb0541..61d9322 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1081,6 +1081,84 @@ public class TestFairScheduler extends FairSchedulerTestBase { getCurrentReservation().getMemorySize()); } + /** + * The test verifies that zero-FairShare queues (because of zero/tiny + * weight) can get resources for the AM. + */ + @Test + public void testRequestAMResourceInZeroFairShareQueue() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0.0"); + out.println("4096mb,10vcores"); + out.println("0.5"); + out.println(""); + out.println(""); + out.println("2.0"); + out.println(""); + out.println(""); + out.println("0.000001"); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node = + MockNodes.newNodeInfo(1, Resources.createResource(8192, 20), + 0, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.update(); + + //create request for non-zero weight queue + createSchedulingRequest(1024, "root.queue2", "user2", 1); + scheduler.update(); + scheduler.handle(updateEvent); + + // A managed AM which need 3G memory will not get resource, + // since it request more than the maxAMShare (4G * 0.5 = 2G). + Resource amResource = Resource.newInstance(1024, 1); + int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(); + ApplicationAttemptId attId1 = createAppAttemptId(1, 1); + createApplicationWithAMResource(attId1, "root.queue1", "user1", amResource); + createSchedulingRequestExistingApplication(3 * 1024, 1, amPriority, attId1); + FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application 1 should not be running", + 0, app1.getLiveContainers().size()); + + // A managed AM which need 2G memory will get resource, + // since it request no more than the maxAMShare (4G * 0.5 = 2G). + ApplicationAttemptId attId2 = createAppAttemptId(2, 1); + createApplicationWithAMResource(attId2, "root.queue1", "user1", amResource); + createSchedulingRequestExistingApplication(2 * 1024, 1, amPriority, attId2); + FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application 2 should be running", + 1, app2.getLiveContainers().size()); + + // A managed AM which need 1G memory will get resource, even thought its + // fair share is 0 because its weight is tiny(0.000001). + ApplicationAttemptId attId3 = createAppAttemptId(3, 1); + createApplicationWithAMResource(attId3, "root.queue3", "user1", amResource); + createSchedulingRequestExistingApplication(1024, 1, amPriority, attId3); + FSAppAttempt app3 = scheduler.getSchedulerApp(attId3); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application 3 should be running", + 1, app3.getLiveContainers().size()); + } + @Test (timeout = 500000) public void testContainerReservationNotExceedingQueueMax() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org