Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 894959586 for ; Fri, 13 Jul 2012 20:54:05 +0000 (UTC) Received: (qmail 45995 invoked by uid 500); 13 Jul 2012 20:54:05 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 45955 invoked by uid 500); 13 Jul 2012 20:54:05 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 45944 invoked by uid 99); 13 Jul 2012 20:54:05 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 13 Jul 2012 20:54:05 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 13 Jul 2012 20:54:03 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 3E7072388860; Fri, 13 Jul 2012 20:53:44 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1361397 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/ hadoop-yarn/hadoop-yarn-server/had... Date: Fri, 13 Jul 2012 20:53:44 -0000 To: mapreduce-commits@hadoop.apache.org From: bobby@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120713205344.3E7072388860@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: bobby Date: Fri Jul 13 20:53:43 2012 New Revision: 1361397 URL: http://svn.apache.org/viewvc?rev=1361397&view=rev Log: MAPREDUCE-4299. Terasort hangs with MR2 FifoScheduler (Tom White via bobby) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1361397&r1=1361396&r2=1361397&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Jul 13 20:53:43 2012 @@ -685,6 +685,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4419. ./mapred queue -info -showJobs displays all the jobs irrespective of (Devaraj K via bobby) + MAPREDUCE-4299. Terasort hangs with MR2 FifoScheduler (Tom White via + bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1361397&r1=1361396&r2=1361397&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Jul 13 20:53:43 2012 @@ -372,16 +372,20 @@ public class FifoScheduler implements Re } } - application.setHeadroom(clusterResource); - LOG.debug("post-assignContainers"); application.showRequests(); // Done if (Resources.lessThan(node.getAvailableResource(), minimumAllocation)) { - return; + break; } } + + // Update the applications' headroom to correctly take into + // account the containers assigned in this update. + for (SchedulerApp application : applications.values()) { + application.setHeadroom(Resources.subtract(clusterResource, usedResource)); + } } private int getMaxAllocatableContainers(SchedulerApp application, Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1361397&r1=1361396&r2=1361397&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Fri Jul 13 20:53:43 2012 @@ -19,26 +19,39 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.AMResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -207,6 +220,68 @@ public class TestFifoScheduler { Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB()); } + + @Test + public void testHeadroom() throws Exception { + + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, + ResourceScheduler.class); + MockRM rm = new MockRM(conf); + rm.start(); + FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler(); + + // Add a node + RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); + fs.handle(new NodeAddedSchedulerEvent(n1)); + + // Add two applications + ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1); + ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( + appId1, 1); + SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue", + "user"); + fs.handle(event1); + + ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2); + ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId( + appId2, 1); + SchedulerEvent event2 = new AppAddedSchedulerEvent(appAttemptId2, "queue", + "user"); + fs.handle(event2); + + List emptyStatus = new ArrayList(); + List emptyId = new ArrayList(); + List emptyAsk = new ArrayList(); + + // Set up resource requests + + // Ask for a 1 GB container for app 1 + List ask1 = new ArrayList(); + ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", + BuilderUtils.newResource(GB), 1)); + fs.allocate(appAttemptId1, ask1, emptyId); + + // Ask for a 2 GB container for app 2 + List ask2 = new ArrayList(); + ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", + BuilderUtils.newResource(2 * GB), 1)); + fs.allocate(appAttemptId2, ask2, emptyId); + + // Trigger container assignment + fs.handle(new NodeUpdateSchedulerEvent(n1, emptyStatus, emptyStatus)); + + // Get the allocation for the applications and verify headroom + Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId); + Assert.assertEquals("Allocation headroom", 1 * GB, + allocation1.getResourceLimit().getMemory()); + + Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId); + Assert.assertEquals("Allocation headroom", 1 * GB, + allocation2.getResourceLimit().getMemory()); + + } + public static void main(String[] args) throws Exception { TestFifoScheduler t = new TestFifoScheduler();