Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7B1859B1E for ; Sat, 4 Feb 2012 22:45:22 +0000 (UTC) Received: (qmail 43643 invoked by uid 500); 4 Feb 2012 22:45:22 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 43485 invoked by uid 500); 4 Feb 2012 22:45:21 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 43477 invoked by uid 99); 4 Feb 2012 22:45:21 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Feb 2012 22:45:21 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Feb 2012 22:45:18 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0B38D2388B46; Sat, 4 Feb 2012 22:44:57 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1240634 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/ hadoop-ya... Date: Sat, 04 Feb 2012 22:44:56 -0000 To: mapreduce-commits@hadoop.apache.org From: mahadev@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120204224457.0B38D2388B46@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mahadev Date: Sat Feb 4 22:44:43 2012 New Revision: 1240634 URL: http://svn.apache.org/viewvc?rev=1240634&view=rev Log: MAPREDUCE-3765. FifoScheduler does not respect yarn.scheduler.fifo.minimum-allocation-mb setting (Hitesh Shah via mahadev) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1240634&r1=1240633&r2=1240634&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sat Feb 4 22:44:43 2012 @@ -702,6 +702,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3775. Change MiniYarnCluster to escape special chars in testname. (Hitesh Shah via mahadev) + MAPREDUCE-3765. FifoScheduler does not respect yarn.scheduler.fifo.minimum- + allocation-mb setting (Hitesh Shah via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1240634&r1=1240633&r2=1240634&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/dev-support/findbugs-exclude.xml Sat Feb 4 22:44:43 2012 @@ -161,6 +161,12 @@ + + + + + + Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1240634&r1=1240633&r2=1240634&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Sat Feb 4 22:44:43 2012 @@ -230,7 +230,7 @@ public class FifoScheduler implements Re } // Sanity check - SchedulerUtils.normalizeRequests(ask, MINIMUM_MEMORY); + SchedulerUtils.normalizeRequests(ask, minimumAllocation.getMemory()); // Release containers for (ContainerId releasedContainer : release) { @@ -592,7 +592,7 @@ public class FifoScheduler implements Re minimumAllocation)) { LOG.debug("Node heartbeat " + rmNode.getNodeID() + " available resource = " + node.getAvailableResource()); - + assignContainers(node); LOG.debug("Node after allocation " + rmNode.getNodeID() + " resource = " Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1240634&r1=1240633&r2=1240634&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Sat Feb 4 22:44:43 2012 @@ -24,38 +24,22 @@ import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.AMResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; -import org.junit.After; -import org.junit.Before; import org.junit.Test; public class TestFifoScheduler { private static final Log LOG = LogFactory.getLog(TestFifoScheduler.class); - private ResourceManager resourceManager = null; - - @Before - public void setUp() throws Exception { - Store store = StoreFactory.getStore(new Configuration()); - resourceManager = new ResourceManager(store); - resourceManager.init(new Configuration()); - } - - @After - public void tearDown() throws Exception { - } + private final int GB = 1024; @Test public void test() throws Exception { @@ -63,7 +47,6 @@ public class TestFifoScheduler { rootLogger.setLevel(Level.DEBUG); MockRM rm = new MockRM(); rm.start(); - int GB = 1024; MockNM nm1 = rm.registerNode("h1:1234", 6 * GB); MockNM nm2 = rm.registerNode("h2:5678", 4 * GB); @@ -146,8 +129,48 @@ public class TestFifoScheduler { rm.stop(); } + private void testMinimumAllocation(YarnConfiguration conf) + throws Exception { + MockRM rm = new MockRM(conf); + rm.start(); + + // Register node1 + MockNM nm1 = rm.registerNode("h1:1234", 6 * GB); + + // Submit an application + RMApp app1 = rm.submitApp(256); + + // kick the scheduling + nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); + am1.registerAppAttempt(); + SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( + nm1.getNodeId()); + + int checkAlloc = + conf.getInt("yarn.scheduler.fifo.minimum-allocation-mb", GB); + Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemory()); + + rm.stop(); + } + + @Test + public void testDefaultMinimumAllocation() throws Exception { + testMinimumAllocation(new YarnConfiguration()); + } + + @Test + public void testNonDefaultMinimumAllocation() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt("yarn.scheduler.fifo.minimum-allocation-mb", 512); + testMinimumAllocation(conf); + } + public static void main(String[] args) throws Exception { TestFifoScheduler t = new TestFifoScheduler(); t.test(); + t.testDefaultMinimumAllocation(); + t.testNonDefaultMinimumAllocation(); } }