Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3DDEB11AC7 for ; Sat, 7 Jun 2014 16:30:15 +0000 (UTC) Received: (qmail 37531 invoked by uid 500); 7 Jun 2014 16:30:15 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 37490 invoked by uid 500); 7 Jun 2014 16:30:15 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 37480 invoked by uid 99); 7 Jun 2014 16:30:15 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 07 Jun 2014 16:30:15 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 07 Jun 2014 16:30:08 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 087E32388B43; Sat, 7 Jun 2014 16:29:25 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: svn commit: r1601151 [5/5] - in /hadoop/common/branches/HDFS-5442/hadoop-yarn-project: ./ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yar... Date: Sat, 07 Jun 2014 16:29:20 -0000 To: yarn-commits@hadoop.apache.org From: vinayakumarb@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140607162925.087E32388B43@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1601151&r1=1601150&r2=1601151&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Sat Jun 7 16:29:10 2014 @@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.exceptions import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; @@ -73,12 +72,12 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; @@ -120,6 +119,8 @@ public class TestFairScheduler extends F // to initialize the master key resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); + + scheduler.setRMContext(resourceManager.getRMContext()); } @After @@ -133,12 +134,12 @@ public class TestFairScheduler extends F @Test (timeout = 30000) public void testConfValidation() throws Exception { - ResourceScheduler scheduler = new FairScheduler(); + FairScheduler scheduler = new FairScheduler(); Configuration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); try { - scheduler.reinitialize(conf, null); + scheduler.serviceInit(conf); fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation."); } catch (YarnRuntimeException e) { @@ -152,7 +153,7 @@ public class TestFairScheduler extends F conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1); try { - scheduler.reinitialize(conf, null); + scheduler.serviceInit(conf); fail("Exception is expected because the min vcores allocation is" + " larger than the max vcores allocation."); } catch (YarnRuntimeException e) { @@ -184,6 +185,8 @@ public class TestFairScheduler extends F conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); Assert.assertEquals(true, scheduler.assignMultiple); Assert.assertEquals(3, scheduler.maxAssign); @@ -211,6 +214,7 @@ public class TestFairScheduler extends F FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); conf.setInt( FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); + fs.init(conf); fs.reinitialize(conf, null); Assert.assertEquals(256, fs.getMinimumResourceCapability().getMemory()); Assert.assertEquals(1, fs.getMinimumResourceCapability().getVirtualCores()); @@ -228,8 +232,9 @@ public class TestFairScheduler extends F FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); conf.setInt( FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); - fs.reinitialize(conf, null); - Assert.assertEquals(0, fs.getMinimumResourceCapability().getMemory()); + fs.init(conf); + fs.reinitialize(conf, null); + Assert.assertEquals(0, fs.getMinimumResourceCapability().getMemory()); Assert.assertEquals(0, fs.getMinimumResourceCapability().getVirtualCores()); Assert.assertEquals(512, fs.getIncrementResourceCapability().getMemory()); Assert.assertEquals(2, fs.getIncrementResourceCapability().getVirtualCores()); @@ -237,6 +242,8 @@ public class TestFairScheduler extends F @Test public void testAggregateCapacityTracking() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node @@ -262,6 +269,8 @@ public class TestFairScheduler extends F @Test public void testSimpleFairShareCalculation() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) @@ -289,6 +298,8 @@ public class TestFairScheduler extends F @Test public void testSimpleHierarchicalFairShareCalculation() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) @@ -322,6 +333,8 @@ public class TestFairScheduler extends F @Test public void testHierarchicalQueuesSimilarParents() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); @@ -346,6 +359,8 @@ public class TestFairScheduler extends F @Test public void testSchedulerRootQueueMetrics() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node @@ -385,6 +400,8 @@ public class TestFairScheduler extends F @Test (timeout = 5000) public void testSimpleContainerAllocation() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node @@ -433,6 +450,8 @@ public class TestFairScheduler extends F @Test (timeout = 5000) public void testSimpleContainerReservation() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node @@ -487,48 +506,27 @@ public class TestFairScheduler extends F @Test public void testUserAsDefaultQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - RMContext rmContext = resourceManager.getRMContext(); - Map appsMap = rmContext.getRMApps(); ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); - RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, - null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null, null); - appsMap.put(appAttemptId.getApplicationId(), rmApp); - - AppAddedSchedulerEvent appAddedEvent = - new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "default", - "user1"); - scheduler.handle(appAddedEvent); - AppAttemptAddedSchedulerEvent attempAddedEvent = - new AppAttemptAddedSchedulerEvent(appAttemptId, false); - scheduler.handle(attempAddedEvent); + createApplicationWithAMResource(appAttemptId, "default", "user1", null); assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) .getRunnableAppSchedulables().size()); - assertEquals("root.user1", rmApp.getQueue()); + assertEquals("root.user1", resourceManager.getRMContext().getRMApps() + .get(appAttemptId.getApplicationId()).getQueue()); } @Test public void testNotUserAsDefaultQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - RMContext rmContext = resourceManager.getRMContext(); - Map appsMap = rmContext.getRMApps(); ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); - RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, - null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null, null); - appsMap.put(appAttemptId.getApplicationId(), rmApp); - - AppAddedSchedulerEvent appAddedEvent = - new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "default", - "user2"); - scheduler.handle(appAddedEvent); - AppAttemptAddedSchedulerEvent attempAddedEvent = - new AppAttemptAddedSchedulerEvent(appAttemptId, false); - scheduler.handle(attempAddedEvent); + createApplicationWithAMResource(appAttemptId, "default", "user2", null); assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true) @@ -539,6 +537,8 @@ public class TestFairScheduler extends F @Test public void testEmptyQueueName() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // only default queue @@ -559,8 +559,10 @@ public class TestFairScheduler extends F @Test public void testAssignToQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); @@ -577,6 +579,8 @@ public class TestFairScheduler extends F @Test public void testAssignToNonLeafQueueReturnsNull() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true); @@ -594,6 +598,8 @@ public class TestFairScheduler extends F public void testQueuePlacementWithPolicy() throws Exception { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId appId; @@ -654,7 +660,9 @@ public class TestFairScheduler extends F out.println(""); out.println(""); out.close(); - + + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) @@ -703,6 +711,8 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); @@ -735,6 +745,8 @@ public class TestFairScheduler extends F RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); int capacity = 16 * 1024; @@ -769,6 +781,8 @@ public class TestFairScheduler extends F */ @Test public void testQueueDemandCalculation() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId id11 = createAppAttemptId(1, 1); @@ -819,6 +833,8 @@ public class TestFairScheduler extends F @Test public void testAppAdditionAndRemoval() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId attemptId =createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default", @@ -869,6 +885,8 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); @@ -901,7 +919,9 @@ public class TestFairScheduler extends F out.println(""); out.println(""); out.close(); - + + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); @@ -928,6 +948,8 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) @@ -985,8 +1007,10 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1, @@ -1057,7 +1081,9 @@ public class TestFairScheduler extends F out.println(""); out.println(""); out.close(); - + + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Create two nodes @@ -1224,6 +1250,8 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Create four nodes @@ -1321,6 +1349,8 @@ public class TestFairScheduler extends F @Test (timeout = 5000) public void testMultipleContainersWaitingForReservation() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node @@ -1363,8 +1393,10 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + // Add a node RMNode node1 = MockNodes @@ -1404,6 +1436,8 @@ public class TestFairScheduler extends F @Test (timeout = 5000) public void testReservationWhileMultiplePriorities() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node @@ -1484,8 +1518,10 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "norealuserhasthisname", 1); ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1", @@ -1499,6 +1535,8 @@ public class TestFairScheduler extends F @Test (timeout = 5000) public void testMultipleNodesSingleRackRequest() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = @@ -1548,6 +1586,8 @@ public class TestFairScheduler extends F @Test (timeout = 5000) public void testFifoWithinQueue() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = @@ -1592,6 +1632,8 @@ public class TestFairScheduler extends F @Test(timeout = 3000) public void testMaxAssign() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = @@ -1635,6 +1677,8 @@ public class TestFairScheduler extends F */ @Test(timeout = 5000) public void testAssignContainer() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); final String user = "user1"; @@ -1718,9 +1762,11 @@ public class TestFairScheduler extends F out.println(""); out.println(""); out.close(); - + + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + int appId = this.APP_ID++; String user = "usernotallow"; String queue = "queue1"; @@ -1769,6 +1815,8 @@ public class TestFairScheduler extends F @Test public void testReservationThatDoesntFit() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = @@ -1797,6 +1845,8 @@ public class TestFairScheduler extends F @Test public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); @@ -1825,6 +1875,8 @@ public class TestFairScheduler extends F @Test public void testStrictLocality() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); @@ -1865,6 +1917,8 @@ public class TestFairScheduler extends F @Test public void testCancelStrictLocality() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); @@ -1915,6 +1969,8 @@ public class TestFairScheduler extends F */ @Test public void testReservationsStrictLocality() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); @@ -1955,6 +2011,8 @@ public class TestFairScheduler extends F @Test public void testNoMoreCpuOnNode() throws IOException { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), @@ -1976,6 +2034,8 @@ public class TestFairScheduler extends F @Test public void testBasicDRFAssignment() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 5)); @@ -2016,6 +2076,8 @@ public class TestFairScheduler extends F */ @Test public void testBasicDRFWithQueues() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 7), @@ -2052,6 +2114,8 @@ public class TestFairScheduler extends F @Test public void testDRFHierarchicalQueues() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(12288, 12), @@ -2120,7 +2184,9 @@ public class TestFairScheduler extends F public void testHostPortNodeName() throws Exception { conf.setBoolean(YarnConfiguration .RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true); - scheduler.reinitialize(conf, + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1", 1); @@ -2200,9 +2266,11 @@ public class TestFairScheduler extends F out.println(""); out.println(""); out.close(); - + + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + // exceeds no limits ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1"); verifyAppRunnable(attId1, true); @@ -2236,6 +2304,186 @@ public class TestFairScheduler extends F } @Test + public void testQueueMaxAMShare() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0.2"); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node = + MockNodes.newNodeInfo(1, Resources.createResource(20480, 20), + 0, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.update(); + + FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1", true); + assertEquals("Queue queue1's fair share should be 10240", + 10240, queue1.getFairShare().getMemory()); + + Resource amResource1 = Resource.newInstance(1024, 1); + Resource amResource2 = Resource.newInstance(2048, 2); + Resource amResource3 = Resource.newInstance(1860, 2); + int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(); + // Exceeds no limits + ApplicationAttemptId attId1 = createAppAttemptId(1, 1); + createApplicationWithAMResource(attId1, "queue1", "user1", amResource1); + createSchedulingRequestExistingApplication(1024, 1, amPriority, attId1); + FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application1's AM requests 1024 MB memory", + 1024, app1.getAMResource().getMemory()); + assertEquals("Application1's AM should be running", + 1, app1.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 1024 MB memory", + 1024, queue1.getAmResourceUsage().getMemory()); + + // Exceeds no limits + ApplicationAttemptId attId2 = createAppAttemptId(2, 1); + createApplicationWithAMResource(attId2, "queue1", "user1", amResource1); + createSchedulingRequestExistingApplication(1024, 1, amPriority, attId2); + FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application2's AM requests 1024 MB memory", + 1024, app2.getAMResource().getMemory()); + assertEquals("Application2's AM should be running", + 1, app2.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Exceeds queue limit + ApplicationAttemptId attId3 = createAppAttemptId(3, 1); + createApplicationWithAMResource(attId3, "queue1", "user1", amResource1); + createSchedulingRequestExistingApplication(1024, 1, amPriority, attId3); + FSSchedulerApp app3 = scheduler.getSchedulerApp(attId3); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application3's AM requests 1024 MB memory", + 1024, app3.getAMResource().getMemory()); + assertEquals("Application3's AM should not be running", + 0, app3.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Still can run non-AM container + createSchedulingRequestExistingApplication(1024, 1, attId1); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application1 should have two running containers", + 2, app1.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Remove app1, app3's AM should become running + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent(attId1, RMAppAttemptState.FINISHED, false); + scheduler.update(); + scheduler.handle(appRemovedEvent1); + scheduler.handle(updateEvent); + assertEquals("Application1's AM should be finished", + 0, app1.getLiveContainers().size()); + assertEquals("Application3's AM should be running", + 1, app3.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Exceeds queue limit + ApplicationAttemptId attId4 = createAppAttemptId(4, 1); + createApplicationWithAMResource(attId4, "queue1", "user1", amResource2); + createSchedulingRequestExistingApplication(2048, 2, amPriority, attId4); + FSSchedulerApp app4 = scheduler.getSchedulerApp(attId4); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application4's AM requests 2048 MB memory", + 2048, app4.getAMResource().getMemory()); + assertEquals("Application4's AM should not be running", + 0, app4.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Exceeds queue limit + ApplicationAttemptId attId5 = createAppAttemptId(5, 1); + createApplicationWithAMResource(attId5, "queue1", "user1", amResource2); + createSchedulingRequestExistingApplication(2048, 2, amPriority, attId5); + FSSchedulerApp app5 = scheduler.getSchedulerApp(attId5); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application5's AM requests 2048 MB memory", + 2048, app5.getAMResource().getMemory()); + assertEquals("Application5's AM should not be running", + 0, app5.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Remove un-running app doesn't affect others + AppAttemptRemovedSchedulerEvent appRemovedEvent4 = + new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.KILLED, false); + scheduler.handle(appRemovedEvent4); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application5's AM should not be running", + 0, app5.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Remove app2 and app3, app5's AM should become running + AppAttemptRemovedSchedulerEvent appRemovedEvent2 = + new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED, false); + AppAttemptRemovedSchedulerEvent appRemovedEvent3 = + new AppAttemptRemovedSchedulerEvent(attId3, RMAppAttemptState.FINISHED, false); + scheduler.handle(appRemovedEvent2); + scheduler.handle(appRemovedEvent3); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application2's AM should be finished", + 0, app2.getLiveContainers().size()); + assertEquals("Application3's AM should be finished", + 0, app3.getLiveContainers().size()); + assertEquals("Application5's AM should be running", + 1, app5.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Check amResource normalization + ApplicationAttemptId attId6 = createAppAttemptId(6, 1); + createApplicationWithAMResource(attId6, "queue1", "user1", amResource3); + createSchedulingRequestExistingApplication(1860, 2, amPriority, attId6); + FSSchedulerApp app6 = scheduler.getSchedulerApp(attId6); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application6's AM should not be running", + 0, app6.getLiveContainers().size()); + assertEquals("Application6's AM requests 2048 MB memory", + 2048, app6.getAMResource().getMemory()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Remove all apps + AppAttemptRemovedSchedulerEvent appRemovedEvent5 = + new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.FINISHED, false); + AppAttemptRemovedSchedulerEvent appRemovedEvent6 = + new AppAttemptRemovedSchedulerEvent(attId6, RMAppAttemptState.FINISHED, false); + scheduler.handle(appRemovedEvent5); + scheduler.handle(appRemovedEvent6); + scheduler.update(); + assertEquals("Queue1's AM resource usage should be 0", + 0, queue1.getAmResourceUsage().getMemory()); + } + + @Test public void testMaxRunningAppsHierarchicalQueues() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); MockClock clock = new MockClock(); @@ -2254,9 +2502,11 @@ public class TestFairScheduler extends F out.println(""); out.println(""); out.close(); - + + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + // exceeds no limits ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1"); verifyAppRunnable(attId1, true); @@ -2316,6 +2566,9 @@ public class TestFairScheduler extends F Configuration conf = createConfiguration(); conf.setBoolean(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, true); + fs.setRMContext(resourceManager.getRMContext()); + fs.init(conf); + fs.start(); fs.reinitialize(conf, resourceManager.getRMContext()); Assert.assertTrue("Continuous scheduling should be enabled.", fs.isContinuousSchedulingEnabled()); @@ -2396,6 +2649,8 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); @@ -2439,6 +2694,8 @@ public class TestFairScheduler extends F out.println(""); out.close(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); List rules = scheduler.allocConf.placementPolicy @@ -2455,6 +2712,8 @@ public class TestFairScheduler extends F @SuppressWarnings("resource") @Test public void testBlacklistNodes() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); final int GB = 1024; @@ -2507,6 +2766,8 @@ public class TestFairScheduler extends F @Test public void testGetAppsInQueue() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId appAttId1 = @@ -2552,8 +2813,10 @@ public class TestFairScheduler extends F @Test public void testMoveRunnableApp() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); @@ -2591,8 +2854,10 @@ public class TestFairScheduler extends F @Test public void testMoveNonRunnableApp() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); @@ -2611,8 +2876,10 @@ public class TestFairScheduler extends F @Test public void testMoveMakesAppRunnable() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); @@ -2638,8 +2905,10 @@ public class TestFairScheduler extends F @Test (expected = YarnException.class) public void testMoveWouldViolateMaxAppsConstraints() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + QueueManager queueMgr = scheduler.getQueueManager(); queueMgr.getLeafQueue("queue2", true); scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); @@ -2652,8 +2921,10 @@ public class TestFairScheduler extends F @Test (expected = YarnException.class) public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - + QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); queueMgr.getLeafQueue("queue2", true); @@ -2675,6 +2946,8 @@ public class TestFairScheduler extends F @Test (expected = YarnException.class) public void testMoveToNonexistentQueue() throws Exception { + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.getQueueManager().getLeafQueue("queue1", true); Modified: hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java?rev=1601151&r1=1601150&r2=1601151&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java Sat Jun 7 16:29:10 2014 @@ -51,6 +51,8 @@ public class TestFairSchedulerEventLog { resourceManager = new ResourceManager(); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); } @@ -69,7 +71,13 @@ public class TestFairSchedulerEventLog { public void tearDown() { logFile.delete(); logFile.getParentFile().delete(); // fairscheduler/ - scheduler = null; - resourceManager = null; + if (scheduler != null) { + scheduler.stop(); + scheduler = null; + } + if (resourceManager != null) { + resourceManager.stop(); + resourceManager = null; + } } } Modified: hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1601151&r1=1601150&r2=1601151&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Sat Jun 7 16:29:10 2014 @@ -145,9 +145,13 @@ public class TestFifoScheduler { RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, writer); - FifoScheduler schedular = new FifoScheduler(); - schedular.reinitialize(new Configuration(), rmContext); - QueueMetrics metrics = schedular.getRootQueueMetrics(); + FifoScheduler scheduler = new FifoScheduler(); + Configuration conf = new Configuration(); + scheduler.setRMContext(rmContext); + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, rmContext); + QueueMetrics metrics = scheduler.getRootQueueMetrics(); int beforeAppsSubmitted = metrics.getAppsSubmitted(); ApplicationId appId = BuilderUtils.newApplicationId(200, 1); @@ -155,18 +159,19 @@ public class TestFifoScheduler { appId, 1); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user"); - schedular.handle(appEvent); + scheduler.handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false); - schedular.handle(attemptEvent); + scheduler.handle(attemptEvent); appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2); SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false); - schedular.handle(attemptEvent2); + scheduler.handle(attemptEvent2); int afterAppsSubmitted = metrics.getAppsSubmitted(); Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted); + scheduler.stop(); } @Test(timeout=2000) @@ -184,6 +189,9 @@ public class TestFifoScheduler { null, containerTokenSecretManager, nmTokenSecretManager, null, writer); FifoScheduler scheduler = new FifoScheduler(); + scheduler.setRMContext(rmContext); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(new Configuration(), rmContext); RMNode node0 = MockNodes.newNodeInfo(1, @@ -232,6 +240,7 @@ public class TestFifoScheduler { //Also check that the containers were scheduled SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId); Assert.assertEquals(3, info.getLiveContainers().size()); + scheduler.stop(); } @Test(timeout=2000) @@ -254,6 +263,9 @@ public class TestFifoScheduler { return nodes; } }; + scheduler.setRMContext(rmContext); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(new Configuration(), rmContext); RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 4), 1, "127.0.0.1"); Modified: hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1601151&r1=1601150&r2=1601151&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Sat Jun 7 16:29:10 2014 @@ -203,10 +203,11 @@ public class TestRMWebApp { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); + cs.init(conf); return cs; } @@ -269,19 +270,21 @@ public class TestRMWebApp { ResourceManager rm = mock(ResourceManager.class); RMContext rmContext = mockRMContext(apps, racks, nodes, mbsPerNode); - ResourceScheduler rs = mockFifoScheduler(); + ResourceScheduler rs = mockFifoScheduler(rmContext); when(rm.getResourceScheduler()).thenReturn(rs); when(rm.getRMContext()).thenReturn(rmContext); return rm; } - public static FifoScheduler mockFifoScheduler() throws Exception { + public static FifoScheduler mockFifoScheduler(RMContext rmContext) + throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupFifoQueueConfiguration(conf); FifoScheduler fs = new FifoScheduler(); fs.setConf(new YarnConfiguration()); - fs.reinitialize(conf, null); + fs.setRMContext(rmContext); + fs.init(conf); return fs; } Modified: hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java?rev=1601151&r1=1601150&r2=1601151&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java Sat Jun 7 16:29:10 2014 @@ -25,9 +25,12 @@ import org.apache.hadoop.service.Service import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.net.InetSocketAddress; + public class TestWebAppProxyServer { private WebAppProxyServer webAppProxy = null; private final String proxyAddress = "0.0.0.0:8888"; @@ -56,4 +59,14 @@ public class TestWebAppProxyServer { } assertEquals(STATE.STARTED, webAppProxy.getServiceState()); } + + @Test + public void testBindAddress() { + YarnConfiguration conf = new YarnConfiguration(); + + InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf); + Assert.assertEquals("Web Proxy default bind address port is incorrect", + YarnConfiguration.DEFAULT_PROXY_PORT, + defaultBindAddress.getPort()); + } } Modified: hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1601151&r1=1601150&r2=1601151&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-5442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Sat Jun 7 16:29:10 2014 @@ -237,6 +237,11 @@ Allocation file format * maxRunningApps: limit the number of apps from the queue to run at once + * maxAMShare: limit the fraction of the queue's fair share that can be used + to run application masters. This property can only be used for leaf queues. + Default value is 1.0f, which means AMs in the leaf queue can take up to 100% + of both the memory and CPU fair share. + * weight: to share the cluster non-proportionally with other queues. Weights default to 1, and a queue with weight 2 should receive approximately twice as many resources as a queue with the default weight. @@ -279,6 +284,9 @@ Allocation file format * <>, which sets the default running app limit for queues; overriden by maxRunningApps element in each queue. + * <>, which sets the default AM resource + limit for queue; overriden by maxAMShare element in each queue. + * <>, which sets the default scheduling policy for queues; overriden by the schedulingPolicy element in each queue if specified. Defaults to "fair". @@ -328,6 +336,7 @@ Allocation file format 10000 mb,0vcores 90000 mb,0vcores 50 + 0.1 2.0 fair @@ -336,6 +345,8 @@ Allocation file format + 0.5 +