Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A447A108AC for ; Thu, 3 Oct 2013 01:18:15 +0000 (UTC) Received: (qmail 34409 invoked by uid 500); 3 Oct 2013 01:18:15 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 34386 invoked by uid 500); 3 Oct 2013 01:18:15 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 34378 invoked by uid 99); 3 Oct 2013 01:18:15 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 03 Oct 2013 01:18:15 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 03 Oct 2013 01:18:10 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E39CA23888E4; Thu, 3 Oct 2013 01:17:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1528696 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ hadoop-yarn/hadoop-yarn-server/hadoop-y... Date: Thu, 03 Oct 2013 01:17:48 -0000 To: yarn-commits@hadoop.apache.org From: sandy@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131003011748.E39CA23888E4@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: sandy Date: Thu Oct 3 01:17:48 2013 New Revision: 1528696 URL: http://svn.apache.org/r1528696 Log: YARN-1213. Restore config to ban submitting to undeclared pools in the Fair Scheduler. (Sandy Ryza) Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1528696&r1=1528695&r2=1528696&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Oct 3 01:17:48 2013 @@ -92,6 +92,9 @@ Release 2.1.2 - UNRELEASED YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza) + YARN-1213. Restore config to ban submitting to undeclared pools in the + Fair Scheduler. (Sandy Ryza) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1528696&r1=1528695&r2=1528696&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Thu Oct 3 01:17:48 2013 @@ -677,10 +677,11 @@ public class FairScheduler implements Re queueName = user; } - FSLeafQueue queue = queueMgr.getLeafQueue(queueName); + FSLeafQueue queue = queueMgr.getLeafQueue(queueName, + conf.getAllowUndeclaredPools()); if (queue == null) { // queue is not an existing or createable leaf queue - queue = queueMgr.getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); + queue = queueMgr.getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, false); } if (rmApp != null) { @@ -726,7 +727,7 @@ public class FairScheduler implements Re // Inform the queue FSLeafQueue queue = queueMgr.getLeafQueue(application.getQueue() - .getQueueName()); + .getQueueName(), false); queue.removeApp(application); // Remove from our data-structure Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1528696&r1=1528695&r2=1528696&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java Thu Oct 3 01:17:48 2013 @@ -54,6 +54,11 @@ public class FairSchedulerConfiguration protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; protected static final String EVENT_LOG_DIR = "eventlog.dir"; + /** Whether pools can be created that were not specified in the FS configuration file + */ + protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools"; + protected static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true; + /** Whether to use the user name as the queue name (instead of "default") if * the request does not specify a queue. */ protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + "user-as-default-queue"; @@ -141,6 +146,10 @@ public class FairSchedulerConfiguration DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); return Resources.createResource(incrementMemory, incrementCores); } + + public boolean getAllowUndeclaredPools() { + return getBoolean(ALLOW_UNDECLARED_POOLS, DEFAULT_ALLOW_UNDECLARED_POOLS); + } public boolean getUserAsDefaultQueue() { return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE); Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1528696&r1=1528695&r2=1528696&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Thu Oct 3 01:17:48 2013 @@ -109,26 +109,27 @@ public class QueueManager { lastSuccessfulReload = scheduler.getClock().getTime(); lastReloadAttempt = scheduler.getClock().getTime(); // Create the default queue - getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); + getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true); } /** - * Get a queue by name, creating it if necessary. If the queue - * is not or can not be a leaf queue, i.e. it already exists as a parent queue, - * or one of the parents in its name is already a leaf queue, null is returned. + * Get a queue by name, creating it if the create param is true and is necessary. + * If the queue is not or can not be a leaf queue, i.e. it already exists as a + * parent queue, or one of the parents in its name is already a leaf queue, + * null is returned. * * The root part of the name is optional, so a queue underneath the root * named "queue1" could be referred to as just "queue1", and a queue named * "queue2" underneath a parent named "parent1" that is underneath the root * could be referred to as just "parent1.queue2". */ - public FSLeafQueue getLeafQueue(String name) { + public FSLeafQueue getLeafQueue(String name, boolean create) { if (!name.startsWith(ROOT_QUEUE + ".")) { name = ROOT_QUEUE + "." + name; } synchronized (queues) { FSQueue queue = queues.get(name); - if (queue == null) { + if (queue == null && create) { FSLeafQueue leafQueue = createLeafQueue(name); if (leafQueue == null) { return null; @@ -224,13 +225,6 @@ public class QueueManager { } /** - * Get the queue for a given AppSchedulable. - */ - public FSLeafQueue getQueueForApp(AppSchedulable app) { - return getLeafQueue(app.getApp().getQueueName()); - } - - /** * Reload allocations file if it hasn't been loaded in a while */ public void reloadAllocsIfNecessary() { @@ -384,7 +378,7 @@ public class QueueManager { // Create all queus for (String name: queueNamesInAllocFile) { - getLeafQueue(name); + getLeafQueue(name, true); } // Set custom policies as specified Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1528696&r1=1528695&r2=1528696&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Thu Oct 3 01:17:48 2013 @@ -436,9 +436,9 @@ public class TestFairScheduler { Collection queues = queueManager.getLeafQueues(); assertEquals(3, queues.size()); - FSLeafQueue queue1 = queueManager.getLeafQueue("default"); - FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2"); - FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3"); + FSLeafQueue queue1 = queueManager.getLeafQueue("default", true); + FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2", true); + FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3", true); assertEquals(capacity / 2, queue1.getFairShare().getMemory()); assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue2.getFairShare().getMemory()); @@ -450,20 +450,20 @@ public class TestFairScheduler { @Test public void testHierarchicalQueuesSimilarParents() { QueueManager queueManager = scheduler.getQueueManager(); - FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child"); + FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true); Assert.assertEquals(2, queueManager.getLeafQueues().size()); Assert.assertNotNull(leafQueue); Assert.assertEquals("root.parent.child", leafQueue.getName()); - FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent"); + FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true); Assert.assertNull(leafQueue2); Assert.assertEquals(2, queueManager.getLeafQueues().size()); - FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild"); + FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild", true); Assert.assertNull(leafQueue3); Assert.assertEquals(2, queueManager.getLeafQueues().size()); - FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister"); + FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true); Assert.assertNotNull(leafQueue4); Assert.assertEquals("root.parent.sister", leafQueue4.getName()); Assert.assertEquals(3, queueManager.getLeafQueues().size()); @@ -612,9 +612,9 @@ public class TestFairScheduler { AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( createAppAttemptId(1, 1), "default", "user1"); scheduler.handle(appAddedEvent); - assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getAppSchedulables().size()); - assertEquals(0, scheduler.getQueueManager().getLeafQueue("default") + assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) .getAppSchedulables().size()); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); @@ -622,11 +622,11 @@ public class TestFairScheduler { AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent( createAppAttemptId(2, 1), "default", "user2"); scheduler.handle(appAddedEvent2); - assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getAppSchedulables().size()); - assertEquals(1, scheduler.getQueueManager().getLeafQueue("default") + assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true) .getAppSchedulables().size()); - assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2") + assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true) .getAppSchedulables().size()); } @@ -772,7 +772,7 @@ public class TestFairScheduler { assertEquals(2, scheduler.getQueueManager().getLeafQueues().size()); // That queue should have one app - assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getAppSchedulables().size()); AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( @@ -782,7 +782,7 @@ public class TestFairScheduler { scheduler.handle(appRemovedEvent1); // Queue should have no apps - assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1") + assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) .getAppSchedulables().size()); } @@ -919,10 +919,10 @@ public class TestFairScheduler { Collection leafQueues = queueManager.getLeafQueues(); Assert.assertEquals(4, leafQueues.size()); - Assert.assertNotNull(queueManager.getLeafQueue("queueA")); - Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC")); - Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD")); - Assert.assertNotNull(queueManager.getLeafQueue("default")); + Assert.assertNotNull(queueManager.getLeafQueue("queueA", true)); + Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC", true)); + Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD", true)); + Assert.assertNotNull(queueManager.getLeafQueue("default", true)); // Make sure querying for queues didn't create any new ones: Assert.assertEquals(4, leafQueues.size()); } @@ -1423,9 +1423,9 @@ public class TestFairScheduler { scheduler.update(); FSLeafQueue schedC = - scheduler.getQueueManager().getLeafQueue("queueC"); + scheduler.getQueueManager().getLeafQueue("queueC", true); FSLeafQueue schedD = - scheduler.getQueueManager().getLeafQueue("queueD"); + scheduler.getQueueManager().getLeafQueue("queueD", true); assertTrue(Resources.equals( Resources.none(), scheduler.resToPreempt(schedC, clock.getTime()))); @@ -1688,7 +1688,7 @@ public class TestFairScheduler { FSSchedulerApp app1 = scheduler.applications.get(attId1); FSSchedulerApp app2 = scheduler.applications.get(attId2); - FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1"); + FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1", true); queue1.setPolicy(new FifoPolicy()); scheduler.update(); @@ -1716,7 +1716,7 @@ public class TestFairScheduler { public void testMaxAssign() throws AllocationConfigurationException { // set required scheduler configs scheduler.assignMultiple = true; - scheduler.getQueueManager().getLeafQueue("root.default") + scheduler.getQueueManager().getLeafQueue("root.default", true) .setPolicy(SchedulingPolicy.getDefault()); RMNode node = @@ -1793,7 +1793,7 @@ public class TestFairScheduler { FSSchedulerApp app3 = scheduler.applications.get(attId3); FSSchedulerApp app4 = scheduler.applications.get(attId4); - scheduler.getQueueManager().getLeafQueue(fifoQueue) + scheduler.getQueueManager().getLeafQueue(fifoQueue, true) .setPolicy(SchedulingPolicy.parse("fifo")); scheduler.update(); @@ -2405,4 +2405,46 @@ public class TestFairScheduler { //expected } } + + @Test + public void testDontAllowUndeclaredPools() throws Exception{ + Configuration conf = createConfiguration(); + conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + FSLeafQueue jerryQueue = queueManager.getLeafQueue("jerry", false); + FSLeafQueue defaultQueue = queueManager.getLeafQueue("default", false); + + // Should get put into jerry + createSchedulingRequest(1024, "jerry", "someuser"); + assertEquals(1, jerryQueue.getAppSchedulables().size()); + + // Should get forced into default + createSchedulingRequest(1024, "newqueue", "someuser"); + assertEquals(1, jerryQueue.getAppSchedulables().size()); + assertEquals(1, defaultQueue.getAppSchedulables().size()); + + // Would get put into someuser because of user-as-default-queue, but should + // be forced into default + createSchedulingRequest(1024, "default", "someuser"); + assertEquals(1, jerryQueue.getAppSchedulables().size()); + assertEquals(2, defaultQueue.getAppSchedulables().size()); + + // Should get put into jerry because of user-as-default-queue + createSchedulingRequest(1024, "default", "jerry"); + assertEquals(2, jerryQueue.getAppSchedulables().size()); + assertEquals(2, defaultQueue.getAppSchedulables().size()); + } }