Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 92C74200D11 for ; Mon, 18 Sep 2017 06:22:59 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 913E91609D9; Mon, 18 Sep 2017 04:22:59 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 0FB721609D8 for ; Mon, 18 Sep 2017 06:22:57 +0200 (CEST) Received: (qmail 25600 invoked by uid 500); 18 Sep 2017 04:22:57 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 25591 invoked by uid 99); 18 Sep 2017 04:22:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 18 Sep 2017 04:22:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 09BB3E95BD; Mon, 18 Sep 2017 04:22:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Message-Id: <2ce6266d69f147a7970665dba5ec4882@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-7172. ResourceCalculator.fitsIn() should not take a cluster resource parameter. (Sen Zhao via wangda) Date: Mon, 18 Sep 2017 04:22:57 +0000 (UTC) archived-at: Mon, 18 Sep 2017 04:22:59 -0000 Repository: hadoop Updated Branches: refs/heads/trunk 8d7cc22ac -> e81596d06 YARN-7172. ResourceCalculator.fitsIn() should not take a cluster resource parameter. (Sen Zhao via wangda) Change-Id: Icc3670c9381ce7591ca69ec12da5aa52d3612d34 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e81596d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e81596d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e81596d0 Branch: refs/heads/trunk Commit: e81596d06d226f1cfa44b2390ce3095ed4dee621 Parents: 8d7cc22 Author: Wangda Tan Authored: Sun Sep 17 21:20:43 2017 -0700 Committer: Wangda Tan Committed: Sun Sep 17 21:20:43 2017 -0700 ---------------------------------------------------------------------- .../resource/DefaultResourceCalculator.java | 3 +- .../resource/DominantResourceCalculator.java | 2 +- .../yarn/util/resource/ResourceCalculator.java | 3 +- .../hadoop/yarn/util/resource/Resources.java | 4 +-- .../util/resource/TestResourceCalculator.java | 24 +++++++------- .../server/resourcemanager/RMServerUtils.java | 3 +- .../CapacitySchedulerPreemptionUtils.java | 4 +-- ...QueuePriorityContainerCandidateSelector.java | 5 ++- .../ReservedContainerCandidatesSelector.java | 34 +++++++++----------- .../scheduler/capacity/AbstractCSQueue.java | 2 +- .../allocator/RegularContainerAllocator.java | 8 ++--- .../scheduler/common/fica/FiCaSchedulerApp.java | 21 +++++------- .../scheduler/capacity/TestReservations.java | 20 +++++------- 13 files changed, 55 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index bdf60bd..7f155e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -123,8 +123,7 @@ public class DefaultResourceCalculator extends ResourceCalculator { } @Override - public boolean fitsIn(Resource cluster, - Resource smaller, Resource bigger) { + public boolean fitsIn(Resource smaller, Resource bigger) { return smaller.getMemorySize() <= bigger.getMemorySize(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index d64f03e..ca828a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -538,7 +538,7 @@ public class DominantResourceCalculator extends ResourceCalculator { } @Override - public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { + public boolean fitsIn(Resource smaller, Resource bigger) { int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation sResourceInformation = smaller http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 398dac5..d59560f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -225,8 +225,7 @@ public abstract class ResourceCalculator { /** * Check if a smaller resource can be contained by bigger resource. */ - public abstract boolean fitsIn(Resource cluster, - Resource smaller, Resource bigger); + public abstract boolean fitsIn(Resource smaller, Resource bigger); /** * Check if resource has any major resource types (which are all NodeManagers http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 325bce4..793aebf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -437,9 +437,9 @@ public class Resources { return true; } - public static boolean fitsIn(ResourceCalculator rc, Resource cluster, + public static boolean fitsIn(ResourceCalculator rc, Resource smaller, Resource bigger) { - return rc.fitsIn(cluster, smaller, bigger); + return rc.fitsIn(smaller, bigger); } public static Resource componentwiseMin(Resource lhs, Resource rhs) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java index 5b4155c..5f3ed19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java @@ -24,14 +24,13 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) public class TestResourceCalculator { @@ -64,29 +63,28 @@ public class TestResourceCalculator { @Test(timeout = 10000) public void testFitsIn() { - Resource cluster = Resource.newInstance(1024, 1); if (resourceCalculator instanceof DefaultResourceCalculator) { - assertTrue(resourceCalculator.fitsIn(cluster, + Assert.assertTrue(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(2, 1))); - assertTrue(resourceCalculator.fitsIn(cluster, + Assert.assertTrue(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(2, 2))); - assertTrue(resourceCalculator.fitsIn(cluster, + Assert.assertTrue(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(1, 2))); - assertTrue(resourceCalculator.fitsIn(cluster, + Assert.assertTrue(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(1, 1))); - assertFalse(resourceCalculator.fitsIn(cluster, + Assert.assertFalse(resourceCalculator.fitsIn( Resource.newInstance(2, 1), Resource.newInstance(1, 2))); } else if (resourceCalculator instanceof DominantResourceCalculator) { - assertFalse(resourceCalculator.fitsIn(cluster, + Assert.assertFalse(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(2, 1))); - assertTrue(resourceCalculator.fitsIn(cluster, + Assert.assertTrue(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(2, 2))); - assertTrue(resourceCalculator.fitsIn(cluster, + Assert.assertTrue(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(1, 2))); - assertFalse(resourceCalculator.fitsIn(cluster, + Assert.assertFalse(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(1, 1))); - assertFalse(resourceCalculator.fitsIn(cluster, + Assert.assertFalse(resourceCalculator.fitsIn( Resource.newInstance(2, 1), Resource.newInstance(1, 2))); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 2aae3a5..4934243 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -300,8 +300,7 @@ public class RMServerUtils { // Target resource of the increase request is more than NM can offer ResourceScheduler scheduler = rmContext.getScheduler(); RMNode rmNode = request.getSchedulerNode().getRMNode(); - if (!Resources.fitsIn(scheduler.getResourceCalculator(), - scheduler.getClusterResource(), targetResource, + if (!Resources.fitsIn(scheduler.getResourceCalculator(), targetResource, rmNode.getTotalCapability())) { String msg = "Target resource=" + targetResource + " of containerId=" + containerId + " is more than node's total resource=" http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java index 0ae3ef0..f097e9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java @@ -156,8 +156,8 @@ public class CapacitySchedulerPreemptionUtils { if (null != toObtainByPartition && Resources.greaterThan(rc, clusterResource, toObtainByPartition, Resources.none()) - && Resources.fitsIn(rc, clusterResource, - rmContainer.getAllocatedResource(), totalPreemptionAllowed) + && Resources.fitsIn(rc, rmContainer.getAllocatedResource(), + totalPreemptionAllowed) && !Resources.isAnyMajorResourceZero(rc, toObtainByPartition)) { Resources.subtractFrom(toObtainByPartition, rmContainer.getAllocatedResource()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java index c730a2d..7b7404c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java @@ -229,8 +229,7 @@ public class QueuePriorityContainerCandidateSelector // If we already can allocate the reserved container after preemption, // skip following steps - if (Resources.fitsIn(rc, clusterResource, lacking, - Resources.none())) { + if (Resources.fitsIn(rc, lacking, Resources.none())) { return true; } @@ -270,7 +269,7 @@ public class QueuePriorityContainerCandidateSelector } // Lacking <= 0 means we can allocate the reserved container - if (Resources.fitsIn(rc, clusterResource, lacking, Resources.none())) { + if (Resources.fitsIn(rc, lacking, Resources.none())) { return true; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java index de23d0a..ff100d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java @@ -87,8 +87,8 @@ public class ReservedContainerCandidatesSelector // Get list of nodes for preemption, ordered by preemption cost List nodesForPreemption = getNodesForPreemption( - clusterResource, queueToPreemptableResourceByPartition, - selectedCandidates, totalPreemptedResourceAllowed); + queueToPreemptableResourceByPartition, selectedCandidates, + totalPreemptedResourceAllowed); for (NodeForPreemption nfp : nodesForPreemption) { RMContainer reservedContainer = nfp.schedulerNode.getReservedContainer(); @@ -97,9 +97,8 @@ public class ReservedContainerCandidatesSelector } NodeForPreemption preemptionResult = getPreemptionCandidatesOnNode( - nfp.schedulerNode, clusterResource, - queueToPreemptableResourceByPartition, selectedCandidates, - totalPreemptedResourceAllowed, false); + nfp.schedulerNode, queueToPreemptableResourceByPartition, + selectedCandidates, totalPreemptedResourceAllowed, false); if (null != preemptionResult) { for (RMContainer c : preemptionResult.selectedContainers) { ApplicationAttemptId appId = c.getApplicationAttemptId(); @@ -135,8 +134,7 @@ public class ReservedContainerCandidatesSelector return preemptable; } - private boolean tryToPreemptFromQueue(Resource cluster, String queueName, - String partitionName, + private boolean tryToPreemptFromQueue(String queueName, String partitionName, Map> queueToPreemptableResourceByPartition, Resource required, Resource totalPreemptionAllowed, boolean readOnly) { Resource preemptable = getPreemptableResource(queueName, partitionName, @@ -145,11 +143,11 @@ public class ReservedContainerCandidatesSelector return false; } - if (!Resources.fitsIn(rc, cluster, required, preemptable)) { + if (!Resources.fitsIn(rc, required, preemptable)) { return false; } - if (!Resources.fitsIn(rc, cluster, required, totalPreemptionAllowed)) { + if (!Resources.fitsIn(rc, required, totalPreemptionAllowed)) { return false; } @@ -165,7 +163,6 @@ public class ReservedContainerCandidatesSelector /** * Try to check if we can preempt resources for reserved container in given node * @param node - * @param cluster * @param queueToPreemptableResourceByPartition it's a map of * > * @param readOnly do we want to modify preemptable resource after we selected @@ -174,7 +171,7 @@ public class ReservedContainerCandidatesSelector * to satisfy reserved resource */ private NodeForPreemption getPreemptionCandidatesOnNode( - FiCaSchedulerNode node, Resource cluster, + FiCaSchedulerNode node, Map> queueToPreemptableResourceByPartition, Map> selectedCandidates, Resource totalPreemptionAllowed, boolean readOnly) { @@ -204,8 +201,7 @@ public class ReservedContainerCandidatesSelector String partition = node.getPartition(); // Avoid preempt any container if required <= available + killable - if (Resources.fitsIn(rc, cluster, reservedContainer.getReservedResource(), - cur)) { + if (Resources.fitsIn(rc, reservedContainer.getReservedResource(), cur)) { return null; } @@ -232,9 +228,9 @@ public class ReservedContainerCandidatesSelector // Can we preempt container c? // Check if we have quota to preempt this container - boolean canPreempt = tryToPreemptFromQueue(cluster, containerQueueName, - partition, queueToPreemptableResourceByPartition, - c.getAllocatedResource(), totalPreemptionAllowed, readOnly); + boolean canPreempt = tryToPreemptFromQueue(containerQueueName, partition, + queueToPreemptableResourceByPartition, c.getAllocatedResource(), + totalPreemptionAllowed, readOnly); // If we can, add to selected container, and change resource accordingly. if (canPreempt) { @@ -246,7 +242,7 @@ public class ReservedContainerCandidatesSelector Resources.addTo(totalSelected, c.getAllocatedResource()); } Resources.addTo(cur, c.getAllocatedResource()); - if (Resources.fitsIn(rc, cluster, + if (Resources.fitsIn(rc, reservedContainer.getReservedResource(), cur)) { canAllocateReservedContainer = true; break; @@ -282,7 +278,7 @@ public class ReservedContainerCandidatesSelector return nfp; } - private List getNodesForPreemption(Resource cluster, + private List getNodesForPreemption( Map> queueToPreemptableResourceByPartition, Map> selectedCandidates, Resource totalPreemptionAllowed) { @@ -292,7 +288,7 @@ public class ReservedContainerCandidatesSelector for (FiCaSchedulerNode node : preemptionContext.getScheduler() .getAllNodes()) { if (node.getReservedContainer() != null) { - NodeForPreemption nfp = getPreemptionCandidatesOnNode(node, cluster, + NodeForPreemption nfp = getPreemptionCandidatesOnNode(node, queueToPreemptableResourceByPartition, selectedCandidates, totalPreemptionAllowed, true); if (null != nfp) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index d7c452a..250f4e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -909,7 +909,7 @@ public abstract class AbstractCSQueue implements CSQueue { maxResourceLimit = labelManager.getResourceByLabel( schedulerContainer.getNodePartition(), cluster); } - if (!Resources.fitsIn(resourceCalculator, cluster, + if (!Resources.fitsIn(resourceCalculator, Resources.add(queueUsage.getUsed(partition), netAllocated), maxResourceLimit)) { if (LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index f753d31..72dfbdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -548,10 +548,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator { toKillContainers.add(killableContainer); Resources.addTo(availableAndKillable, killableContainer.getAllocatedResource()); - if (Resources.fitsIn(rc, - clusterResource, - capability, - availableAndKillable)) { + if (Resources.fitsIn(rc, capability, availableAndKillable)) { // Stop if we find enough spaces availableContainers = 1; break; @@ -579,8 +576,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator { // under the limit. resourceNeedToUnReserve = capability; } - unreservedContainer = - application.findNodeToUnreserve(clusterResource, node, + unreservedContainer = application.findNodeToUnreserve(node, schedulerKey, resourceNeedToUnReserve); // When (minimum-unreserved-resource > 0 OR we cannot allocate // new/reserved http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 17bb104..a12c5ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -314,7 +314,6 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { } private boolean commonCheckContainerAllocation( - Resource cluster, ContainerAllocationProposal allocation, SchedulerContainer schedulerContainer) { // Make sure node is not reserved by anyone else @@ -355,8 +354,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { } } } - if (!Resources.fitsIn(rc, cluster, - allocation.getAllocatedOrReservedResource(), + if (!Resources.fitsIn(rc, allocation.getAllocatedOrReservedResource(), availableResource)) { if (LOG.isDebugEnabled()) { LOG.debug("Node doesn't have enough available resource, asked=" @@ -419,8 +417,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { // Common part of check container allocation regardless if it is a // increase container or regular container - commonCheckContainerAllocation(cluster, allocation, - schedulerContainer); + commonCheckContainerAllocation(allocation, schedulerContainer); } else { // Container reserved first time will be NEW, after the container // accepted & confirmed, it will become RESERVED state @@ -721,9 +718,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { } @VisibleForTesting - public NodeId getNodeIdToUnreserve( - SchedulerRequestKey schedulerKey, Resource resourceNeedUnreserve, - ResourceCalculator rc, Resource clusterResource) { + public NodeId getNodeIdToUnreserve(SchedulerRequestKey schedulerKey, + Resource resourceNeedUnreserve, ResourceCalculator resourceCalculator) { // first go around make this algorithm simple and just grab first // reservation that has enough resources Map reservedContainers = this.reservedContainers.get( @@ -738,7 +734,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { // make sure we unreserve one with at least the same amount of // resources, otherwise could affect capacity limits - if (Resources.fitsIn(rc, clusterResource, resourceNeedUnreserve, + if (Resources.fitsIn(resourceCalculator, resourceNeedUnreserve, reservedResource)) { if (LOG.isDebugEnabled()) { LOG.debug( @@ -806,14 +802,13 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { } @VisibleForTesting - public RMContainer findNodeToUnreserve(Resource clusterResource, - FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, - Resource minimumUnreservedResource) { + public RMContainer findNodeToUnreserve(FiCaSchedulerNode node, + SchedulerRequestKey schedulerKey, Resource minimumUnreservedResource) { try { readLock.lock(); // need to unreserve some other container first NodeId idToUnreserve = getNodeIdToUnreserve(schedulerKey, - minimumUnreservedResource, rc, clusterResource); + minimumUnreservedResource, rc); if (idToUnreserve == null) { if (LOG.isDebugEnabled()) { LOG.debug("checked to see if could unreserve for app but nothing " http://git-wip-us.apache.org/repos/asf/hadoop/blob/e81596d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 5e6548b..32f022f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -896,8 +896,7 @@ public class TestReservations { String host_1 = "host_1"; FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, 8 * GB); - - Resource clusterResource = Resources.createResource(2 * 8 * GB); + // Setup resource-requests Priority p = TestUtils.createMockPriority(5); @@ -933,28 +932,27 @@ public class TestReservations { node_0.getNodeID(), "user", rmContext); // no reserved containers - NodeId unreserveId = - app_0.getNodeIdToUnreserve(priorityMap, capability, - cs.getResourceCalculator(), clusterResource); + NodeId unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, + cs.getResourceCalculator()); assertEquals(null, unreserveId); // no reserved containers - reserve then unreserve app_0.reserve(node_0, priorityMap, rmContainer_1, container_1); app_0.unreserve(priorityMap, node_0, rmContainer_1); unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, - cs.getResourceCalculator(), clusterResource); + cs.getResourceCalculator()); assertEquals(null, unreserveId); // no container large enough is reserved app_0.reserve(node_0, priorityMap, rmContainer_1, container_1); unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, - cs.getResourceCalculator(), clusterResource); + cs.getResourceCalculator()); assertEquals(null, unreserveId); // reserve one that is now large enough app_0.reserve(node_1, priorityMap, rmContainer, container); unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, - cs.getResourceCalculator(), clusterResource); + cs.getResourceCalculator()); assertEquals(node_1.getNodeID(), unreserveId); } @@ -1001,16 +999,14 @@ public class TestReservations { node_1.getNodeID(), "user", rmContext); // nothing reserved - RMContainer toUnreserveContainer = - app_0.findNodeToUnreserve(csContext.getClusterResource(), node_1, + RMContainer toUnreserveContainer = app_0.findNodeToUnreserve(node_1, priorityMap, capability); assertTrue(toUnreserveContainer == null); // reserved but scheduler doesn't know about that node. app_0.reserve(node_1, priorityMap, rmContainer, container); node_1.reserveResource(app_0, priorityMap, rmContainer); - toUnreserveContainer = - app_0.findNodeToUnreserve(csContext.getClusterResource(), node_1, + toUnreserveContainer = app_0.findNodeToUnreserve(node_1, priorityMap, capability); assertTrue(toUnreserveContainer == null); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org