Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CE2FE10B53 for ; Tue, 22 Oct 2013 23:58:40 +0000 (UTC) Received: (qmail 81737 invoked by uid 500); 22 Oct 2013 23:58:40 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 81714 invoked by uid 500); 22 Oct 2013 23:58:40 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 81706 invoked by uid 99); 22 Oct 2013 23:58:40 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 22 Oct 2013 23:58:40 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 22 Oct 2013 23:58:37 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id DCB152388A32; Tue, 22 Oct 2013 23:58:15 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1534862 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ hadoop-yarn/hadoop-yarn-ser... Date: Tue, 22 Oct 2013 23:58:15 -0000 To: yarn-commits@hadoop.apache.org From: sandy@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131022235815.DCB152388A32@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: sandy Date: Tue Oct 22 23:58:15 2013 New Revision: 1534862 URL: http://svn.apache.org/r1534862 Log: YARN-1330. Fair Scheduler: defaultQueueSchedulingPolicy does not take effect (Sandy Ryza) Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1534862&r1=1534861&r2=1534862&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Tue Oct 22 23:58:15 2013 @@ -118,6 +118,9 @@ Release 2.2.1 - UNRELEASED YARN-1331. yarn.cmd exits with NoClassDefFoundError trying to run rmadmin or logs. (cnauroth) + YARN-1330. Fair Scheduler: defaultQueueSchedulingPolicy does not take effect + (Sandy Ryza) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1534862&r1=1534861&r2=1534862&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Tue Oct 22 23:58:15 2013 @@ -378,22 +378,24 @@ public class QueueManager { queueMaxAppsDefault, defaultSchedPolicy, minSharePreemptionTimeouts, queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout); - // Update metrics + // Make sure all queues exist + for (String name: queueNamesInAllocFile) { + getLeafQueue(name, true); + } + for (FSQueue queue : queues.values()) { + // Update queue metrics FSQueueMetrics queueMetrics = queue.getMetrics(); queueMetrics.setMinShare(queue.getMinShare()); queueMetrics.setMaxShare(queue.getMaxShare()); + // Set scheduling policies + if (queuePolicies.containsKey(queue.getName())) { + queue.setPolicy(queuePolicies.get(queue.getName())); + } else { + queue.setPolicy(SchedulingPolicy.getDefault()); + } } - // Create all queus - for (String name: queueNamesInAllocFile) { - getLeafQueue(name, true); - } - - // Set custom policies as specified - for (Map.Entry entry : queuePolicies.entrySet()) { - queues.get(entry.getKey()).setPolicy(entry.getValue()); - } } } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1534862&r1=1534861&r2=1534862&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Tue Oct 22 23:58:15 2013 @@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; @@ -807,6 +808,7 @@ public class TestFairScheduler { out.println(""); out.println("2048mb,0vcores"); out.println("alice,bob admins"); + out.println("fair"); out.println(""); // Give queue C no minimum out.println(""); @@ -833,6 +835,8 @@ public class TestFairScheduler { + ""); // Set fair share preemption timeout to 5 minutes out.println("300"); + // Set default scheduling policy to DRF + out.println("drf"); out.println(""); out.close(); @@ -894,6 +898,18 @@ public class TestFairScheduler { assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE")); assertEquals(300000, queueManager.getFairSharePreemptionTimeout()); + + // Verify existing queues have default scheduling policy + assertEquals(DominantResourceFairnessPolicy.NAME, + queueManager.getQueue("root").getPolicy().getName()); + assertEquals(DominantResourceFairnessPolicy.NAME, + queueManager.getQueue("root.queueA").getPolicy().getName()); + // Verify default is overriden if specified explicitly + assertEquals(FairSharePolicy.NAME, + queueManager.getQueue("root.queueB").getPolicy().getName()); + // Verify new queue gets default scheduling policy + assertEquals(DominantResourceFairnessPolicy.NAME, + queueManager.getLeafQueue("root.newqueue", true).getPolicy().getName()); } @Test