Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CF3121759D for ; Wed, 18 Mar 2015 03:34:12 +0000 (UTC) Received: (qmail 99061 invoked by uid 500); 18 Mar 2015 03:34:00 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 98696 invoked by uid 500); 18 Mar 2015 03:34:00 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 89938 invoked by uid 99); 18 Mar 2015 03:33:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Mar 2015 03:33:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A2B3EE18BA; Wed, 18 Mar 2015 03:33:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zjshen@apache.org To: common-commits@hadoop.apache.org Date: Wed, 18 Mar 2015 03:34:36 -0000 Message-Id: <2d70b66393b64fd5ae50e05a30d060ce@git.apache.org> In-Reply-To: <325bc1d993444655ac6a57376101f5d3@git.apache.org> References: <325bc1d993444655ac6a57376101f5d3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [44/50] hadoop git commit: YARN-3305. Normalize AM resource request on app submission. Contributed by Rohith Sharmaks YARN-3305. Normalize AM resource request on app submission. Contributed by Rohith Sharmaks Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/968425e9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/968425e9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/968425e9 Branch: refs/heads/YARN-2928 Commit: 968425e9f7b850ff9c2ab8ca37a64c3fdbe77dbf Parents: 32b4330 Author: Jian He Authored: Tue Mar 17 13:49:59 2015 -0700 Committer: Jian He Committed: Tue Mar 17 13:49:59 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 7 +++-- .../server/resourcemanager/RMAppManager.java | 6 ++++- .../server/resourcemanager/TestAppManager.java | 5 ++++ .../resourcemanager/TestClientRMService.java | 5 ++++ .../capacity/TestCapacityScheduler.java | 27 ++++++++++++++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/968425e9/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fee0ce0..bb752ab 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -66,8 +66,11 @@ Release 2.8.0 - UNRELEASED BUG FIXES - YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena - via devaraj) + YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena + via devaraj) + + YARN-3305. Normalize AM resource request on app submission. (Rohith Sharmaks + via jianhe) Release 2.7.0 - UNRELEASED http://git-wip-us.apache.org/repos/asf/hadoop/blob/968425e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 8dcfe67..9197630 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -390,7 +390,11 @@ public class RMAppManager implements EventHandler, + " for application " + submissionContext.getApplicationId(), e); throw e; } - + SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(), + scheduler.getClusterResource(), + scheduler.getMinimumResourceCapability(), + scheduler.getMaximumResourceCapability(), + scheduler.getMinimumResourceCapability()); return amReq; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/968425e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index d2ac4ef..5ebc68c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -604,6 +605,10 @@ public class TestAppManager{ when(scheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); + + ResourceCalculator rs = mock(ResourceCalculator.class); + when(scheduler.getResourceCalculator()).thenReturn(rs); + return scheduler; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/968425e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index dd2b3f8..430763c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -146,6 +146,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.UTCClock; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.AfterClass; import org.junit.Assert; @@ -1289,6 +1290,10 @@ public class TestClientRMService { Arrays.asList(getApplicationAttemptId(103))); ApplicationAttemptId attemptId = getApplicationAttemptId(1); when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null); + + ResourceCalculator rs = mock(ResourceCalculator.class); + when(yarnScheduler.getResourceCalculator()).thenReturn(rs); + return yarnScheduler; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/968425e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 7a265dc..052749f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2543,6 +2543,33 @@ public class TestCapacityScheduler { + "queue-a's max capacity will be violated if container allocated"); } + // Test verifies AM Used resource for LeafQueue when AM ResourceRequest is + // lesser than minimumAllocation + @Test(timeout = 30000) + public void testAMUsedResource() throws Exception { + MockRM rm = setUpMove(); + Configuration conf = rm.getConfig(); + int minAllocMb = + conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + int amMemory = 50; + assertTrue("AM memory is greater than or equql to minAllocation", + amMemory < minAllocMb); + Resource minAllocResource = Resource.newInstance(minAllocMb, 1); + String queueName = "a1"; + RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName); + + assertEquals("RMApp does not containes minimum allocation", + minAllocResource, rmApp.getAMResourceRequest().getCapability()); + + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + LeafQueue queueA = + (LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName); + assertEquals("Minimum Resource for AM is incorrect", minAllocResource, + queueA.getUser("user_0").getResourceUsage().getAMUsed()); + rm.stop(); + } + private void setMaxAllocMb(Configuration conf, int maxAllocMb) { conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, maxAllocMb);