Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DD0451836C for ; Mon, 4 May 2015 20:04:24 +0000 (UTC) Received: (qmail 13440 invoked by uid 500); 4 May 2015 20:04:21 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 13234 invoked by uid 500); 4 May 2015 20:04:20 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 12785 invoked by uid 99); 4 May 2015 20:04:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 May 2015 20:04:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6DFA3E0A36; Mon, 4 May 2015 20:04:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zjshen@apache.org To: common-commits@hadoop.apache.org Date: Mon, 04 May 2015 20:04:27 -0000 Message-Id: <4d49d5a38085431aa043278ab466884c@git.apache.org> In-Reply-To: <6c861728d79543b6973a8facd17148c6@git.apache.org> References: <6c861728d79543b6973a8facd17148c6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [08/50] [abbrv] hadoop git commit: YARN-3485. FairScheduler headroom calculation doesn't consider maxResources for Fifo and FairShare policies. (kasha) YARN-3485. FairScheduler headroom calculation doesn't consider maxResources for Fifo and FairShare policies. (kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19dd6d93 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19dd6d93 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19dd6d93 Branch: refs/heads/YARN-2928 Commit: 19dd6d93513d77e8be3b56cc7f6ba6774a2a6915 Parents: a742c10 Author: Karthik Kambatla Authored: Tue Apr 28 21:00:35 2015 -0700 Committer: Zhijie Shen Committed: Mon May 4 12:58:53 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FSAppAttempt.java | 20 ++++-- .../scheduler/fair/SchedulingPolicy.java | 4 +- .../DominantResourceFairnessPolicy.java | 6 +- .../fair/policies/FairSharePolicy.java | 6 +- .../scheduler/fair/policies/FifoPolicy.java | 6 +- .../scheduler/fair/TestFSAppAttempt.java | 66 ++++++++++++++------ 7 files changed, 76 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/19dd6d93/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0f81fbc..7d90477 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -372,6 +372,9 @@ Release 2.7.1 - UNRELEASED YARN-3464. Race condition in LocalizerRunner kills localizer before localizing all resources. (Zhihai Xu via kasha) + + YARN-3485. FairScheduler headroom calculation doesn't consider + maxResources for Fifo and FairShare policies. (kasha) Release 2.7.0 - 2015-04-20 http://git-wip-us.apache.org/repos/asf/hadoop/blob/19dd6d93/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index f0d1ed1..6287deb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -172,6 +172,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt + this.attemptResourceUsage.getReserved()); } + /** + * Headroom depends on resources in the cluster, current usage of the + * queue, queue's fair-share and queue's max-resources. + */ @Override public Resource getHeadroom() { final FSQueue queue = (FSQueue) this.queue; @@ -182,18 +186,22 @@ public class FSAppAttempt extends SchedulerApplicationAttempt Resource clusterResource = this.scheduler.getClusterResource(); Resource clusterUsage = this.scheduler.getRootQueueMetrics() .getAllocatedResources(); - Resource clusterAvailableResource = Resources.subtract(clusterResource, - clusterUsage); + + Resource clusterAvailableResources = + Resources.subtract(clusterResource, clusterUsage); + Resource queueMaxAvailableResources = + Resources.subtract(queue.getMaxShare(), queueUsage); + Resource maxAvailableResource = Resources.componentwiseMin( + clusterAvailableResources, queueMaxAvailableResources); + Resource headroom = policy.getHeadroom(queueFairShare, - queueUsage, clusterAvailableResource); + queueUsage, maxAvailableResource); if (LOG.isDebugEnabled()) { LOG.debug("Headroom calculation for " + this.getName() + ":" + "Min(" + "(queueFairShare=" + queueFairShare + " - queueUsage=" + queueUsage + ")," + - " clusterAvailableResource=" + clusterAvailableResource + - "(clusterResource=" + clusterResource + - " - clusterUsage=" + clusterUsage + ")" + + " maxAvailableResource=" + maxAvailableResource + "Headroom=" + headroom); } return headroom; http://git-wip-us.apache.org/repos/asf/hadoop/blob/19dd6d93/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java index bf2a25b..abdc834 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java @@ -185,10 +185,10 @@ public abstract class SchedulingPolicy { * * @param queueFairShare fairshare in the queue * @param queueUsage resources used in the queue - * @param clusterAvailable available resource in cluster + * @param maxAvailable available resource in cluster for this queue * @return calculated headroom */ public abstract Resource getHeadroom(Resource queueFairShare, - Resource queueUsage, Resource clusterAvailable); + Resource queueUsage, Resource maxAvailable); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19dd6d93/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 3f6cbd1..86d503b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -90,15 +90,15 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy { @Override public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, - Resource clusterAvailable) { + Resource maxAvailable) { int queueAvailableMemory = Math.max(queueFairShare.getMemory() - queueUsage.getMemory(), 0); int queueAvailableCPU = Math.max(queueFairShare.getVirtualCores() - queueUsage .getVirtualCores(), 0); Resource headroom = Resources.createResource( - Math.min(clusterAvailable.getMemory(), queueAvailableMemory), - Math.min(clusterAvailable.getVirtualCores(), + Math.min(maxAvailable.getMemory(), queueAvailableMemory), + Math.min(maxAvailable.getVirtualCores(), queueAvailableCPU)); return headroom; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19dd6d93/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java index 97669cb..918db9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java @@ -116,12 +116,12 @@ public class FairSharePolicy extends SchedulingPolicy { @Override public Resource getHeadroom(Resource queueFairShare, - Resource queueUsage, Resource clusterAvailable) { + Resource queueUsage, Resource maxAvailable) { int queueAvailableMemory = Math.max( queueFairShare.getMemory() - queueUsage.getMemory(), 0); Resource headroom = Resources.createResource( - Math.min(clusterAvailable.getMemory(), queueAvailableMemory), - clusterAvailable.getVirtualCores()); + Math.min(maxAvailable.getMemory(), queueAvailableMemory), + maxAvailable.getVirtualCores()); return headroom; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19dd6d93/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java index a2e17ec..7d88933 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java @@ -109,12 +109,12 @@ public class FifoPolicy extends SchedulingPolicy { @Override public Resource getHeadroom(Resource queueFairShare, - Resource queueUsage, Resource clusterAvailable) { + Resource queueUsage, Resource maxAvailable) { int queueAvailableMemory = Math.max( queueFairShare.getMemory() - queueUsage.getMemory(), 0); Resource headroom = Resources.createResource( - Math.min(clusterAvailable.getMemory(), queueAvailableMemory), - clusterAvailable.getVirtualCores()); + Math.min(maxAvailable.getMemory(), queueAvailableMemory), + maxAvailable.getVirtualCores()); return headroom; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19dd6d93/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java index f560690..43fe186 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java @@ -198,18 +198,24 @@ public class TestFSAppAttempt extends FairSchedulerTestBase { Mockito.when(mockScheduler.getClock()).thenReturn(scheduler.getClock()); final FSLeafQueue mockQueue = Mockito.mock(FSLeafQueue.class); - final Resource queueFairShare = Resources.createResource(4096, 4); - final Resource queueUsage = Resource.newInstance(1024, 1); + + final Resource queueMaxResources = Resource.newInstance(5 * 1024, 3); + final Resource queueFairShare = Resources.createResource(4096, 2); + final Resource queueUsage = Resource.newInstance(2048, 2); + + final Resource queueStarvation = + Resources.subtract(queueFairShare, queueUsage); + final Resource queueMaxResourcesAvailable = + Resources.subtract(queueMaxResources, queueUsage); + final Resource clusterResource = Resources.createResource(8192, 8); - final Resource clusterUsage = Resources.createResource(6144, 2); - final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class); + final Resource clusterUsage = Resources.createResource(2048, 2); + final Resource clusterAvailable = + Resources.subtract(clusterResource, clusterUsage); - ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); - RMContext rmContext = resourceManager.getRMContext(); - FSAppAttempt schedulerApp = - new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue , - null, rmContext); + final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class); + Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources); Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare); Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage); Mockito.when(mockScheduler.getClusterResource()).thenReturn @@ -219,27 +225,51 @@ public class TestFSAppAttempt extends FairSchedulerTestBase { Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn (fakeRootQueueMetrics); - int minClusterAvailableMemory = 2048; - int minClusterAvailableCPU = 6; - int minQueueAvailableCPU = 3; + ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); + RMContext rmContext = resourceManager.getRMContext(); + FSAppAttempt schedulerApp = + new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue , + null, rmContext); // Min of Memory and CPU across cluster and queue is used in // DominantResourceFairnessPolicy Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(DominantResourceFairnessPolicy.class)); - verifyHeadroom(schedulerApp, minClusterAvailableMemory, - minQueueAvailableCPU); + verifyHeadroom(schedulerApp, + min(queueStarvation.getMemory(), + clusterAvailable.getMemory(), + queueMaxResourcesAvailable.getMemory()), + min(queueStarvation.getVirtualCores(), + clusterAvailable.getVirtualCores(), + queueMaxResourcesAvailable.getVirtualCores()) + ); // Fair and Fifo ignore CPU of queue, so use cluster available CPU Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(FairSharePolicy.class)); - verifyHeadroom(schedulerApp, minClusterAvailableMemory, - minClusterAvailableCPU); + verifyHeadroom(schedulerApp, + min(queueStarvation.getMemory(), + clusterAvailable.getMemory(), + queueMaxResourcesAvailable.getMemory()), + Math.min( + clusterAvailable.getVirtualCores(), + queueMaxResourcesAvailable.getVirtualCores()) + ); Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy .getInstance(FifoPolicy.class)); - verifyHeadroom(schedulerApp, minClusterAvailableMemory, - minClusterAvailableCPU); + verifyHeadroom(schedulerApp, + min(queueStarvation.getMemory(), + clusterAvailable.getMemory(), + queueMaxResourcesAvailable.getMemory()), + Math.min( + clusterAvailable.getVirtualCores(), + queueMaxResourcesAvailable.getVirtualCores()) + ); + } + + private static int min(int value1, int value2, int value3) { + return Math.min(Math.min(value1, value2), value3); } protected void verifyHeadroom(FSAppAttempt schedulerApp,