Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 40A05200C30 for ; Tue, 31 Jan 2017 02:57:19 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 3D497160B66; Tue, 31 Jan 2017 01:57:19 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id EBCFA160B61 for ; Tue, 31 Jan 2017 02:57:17 +0100 (CET) Received: (qmail 77829 invoked by uid 500); 31 Jan 2017 01:57:14 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 77116 invoked by uid 99); 31 Jan 2017 01:57:14 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Jan 2017 01:57:14 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0F95EDFF0F; Tue, 31 Jan 2017 01:57:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: fabbri@apache.org To: common-commits@hadoop.apache.org Date: Tue, 31 Jan 2017 01:57:22 -0000 Message-Id: <44a0608e591441c98d34ba68c4c3c00e@git.apache.org> In-Reply-To: <7672b345db3c489b9ab514860550baca@git.apache.org> References: <7672b345db3c489b9ab514860550baca@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [10/50] [abbrv] hadoop git commit: YARN-5864. Capacity Scheduler - Queue Priorities. (wangda) archived-at: Tue, 31 Jan 2017 01:57:19 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index bb400ef..dd6b25b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -776,4 +776,115 @@ public class TestContainerAllocation { Resources.createResource(20 * GB), "", true).getMemorySize()); rm1.close(); } + + + @Test(timeout = 60000) + public void testQueuePriorityOrdering() throws Exception { + CapacitySchedulerConfiguration newConf = + (CapacitySchedulerConfiguration) TestUtils + .getConfigurationWithMultipleQueues(conf); + + // Set ordering policy + newConf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT, + CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY); + + // Set maximum capacity of A to 20 + newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 20); + newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1); + newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".b", 2); + newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".a", 3); + + MockRM rm1 = new MockRM(newConf); + + rm1.getRMContext().setNodeLabelManager(mgr); + rm1.start(); + MockNM nm1 = rm1.registerNode("h1:1234", 100 * GB); + + // launch an app to queue A, AM container should be launched in nm1 + RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + + // launch an app to queue B, AM container should be launched in nm1 + RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "b"); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); + + // launch an app to queue C, AM container should be launched in nm1 + RMApp app3 = rm1.submitApp(2 * GB, "app", "user", null, "c"); + MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); + + // Each application asks 10 * 5GB containers + am1.allocate("*", 5 * GB, 10, null); + am2.allocate("*", 5 * GB, 10, null); + am3.allocate("*", 5 * GB, 10, null); + + CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); + RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); + + FiCaSchedulerApp schedulerApp1 = + cs.getApplicationAttempt(am1.getApplicationAttemptId()); + FiCaSchedulerApp schedulerApp2 = + cs.getApplicationAttempt(am2.getApplicationAttemptId()); + FiCaSchedulerApp schedulerApp3 = + cs.getApplicationAttempt(am3.getApplicationAttemptId()); + + // container will be allocated to am1 + // App1 will get 2 container allocated (plus AM container) + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + Assert.assertEquals(2, schedulerApp1.getLiveContainers().size()); + Assert.assertEquals(1, schedulerApp2.getLiveContainers().size()); + Assert.assertEquals(1, schedulerApp3.getLiveContainers().size()); + + // container will be allocated to am1 again, + // App1 will get 3 container allocated (plus AM container) + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + Assert.assertEquals(3, schedulerApp1.getLiveContainers().size()); + Assert.assertEquals(1, schedulerApp2.getLiveContainers().size()); + Assert.assertEquals(1, schedulerApp3.getLiveContainers().size()); + + // (Now usages of queues: a=12G (satisfied), b=2G, c=2G) + + // container will be allocated to am2 (since app1 reaches its guaranteed + // capacity) + // App2 will get 2 container allocated (plus AM container) + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + Assert.assertEquals(3, schedulerApp1.getLiveContainers().size()); + Assert.assertEquals(2, schedulerApp2.getLiveContainers().size()); + Assert.assertEquals(1, schedulerApp3.getLiveContainers().size()); + + // Do this 3 times + // container will be allocated to am2 (since app1 reaches its guaranteed + // capacity) + // App2 will get 2 container allocated (plus AM container) + for (int i = 0; i < 3; i++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + } + Assert.assertEquals(3, schedulerApp1.getLiveContainers().size()); + Assert.assertEquals(5, schedulerApp2.getLiveContainers().size()); + Assert.assertEquals(1, schedulerApp3.getLiveContainers().size()); + + // (Now usages of queues: a=12G (satisfied), b=22G (satisfied), c=2G)) + + // Do this 10 times + for (int i = 0; i < 10; i++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + } + Assert.assertEquals(3, schedulerApp1.getLiveContainers().size()); + Assert.assertEquals(5, schedulerApp2.getLiveContainers().size()); + Assert.assertEquals(11, schedulerApp3.getLiveContainers().size()); + + // (Now usages of queues: a=12G (satisfied), b=22G (satisfied), + // c=52G (satisfied and no pending)) + + // Do this 20 times, we can only allocate 3 containers, 1 to A and 3 to B + for (int i = 0; i < 20; i++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + } + Assert.assertEquals(4, schedulerApp1.getLiveContainers().size()); + Assert.assertEquals(6, schedulerApp2.getLiveContainers().size()); + Assert.assertEquals(11, schedulerApp3.getLiveContainers().size()); + + // (Now usages of queues: a=17G (satisfied), b=27G (satisfied), c=52G)) + + rm1.close(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index f1396b6..bd038e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -175,9 +175,6 @@ public class TestLeafQueue { thenReturn(Resources.createResource(16*GB, 32)); when(csContext.getClusterResource()). thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32)); - when(csContext.getNonPartitionedQueueComparator()). - thenReturn( - CapacitySchedulerQueueManager.NON_PARTITIONED_QUEUE_COMPARATOR); when(csContext.getResourceCalculator()). thenReturn(resourceCalculator); when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); @@ -414,7 +411,7 @@ public class TestLeafQueue { "testPolicyRoot" + System.currentTimeMillis(); OrderingPolicy comPol = - testConf.getOrderingPolicy(tproot); + testConf.getAppOrderingPolicy(tproot); } @@ -489,16 +486,16 @@ public class TestLeafQueue { "testPolicyRoot" + System.currentTimeMillis(); OrderingPolicy schedOrder = - testConf.getOrderingPolicy(tproot); + testConf.getAppOrderingPolicy(tproot); //override default to fair String policyType = CapacitySchedulerConfiguration.PREFIX + tproot + "." + CapacitySchedulerConfiguration.ORDERING_POLICY; testConf.set(policyType, - CapacitySchedulerConfiguration.FAIR_ORDERING_POLICY); + CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY); schedOrder = - testConf.getOrderingPolicy(tproot); + testConf.getAppOrderingPolicy(tproot); FairOrderingPolicy fop = (FairOrderingPolicy) schedOrder; assertFalse(fop.getSizeBasedWeight()); @@ -508,7 +505,7 @@ public class TestLeafQueue { FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT; testConf.set(sbwConfig, "true"); schedOrder = - testConf.getOrderingPolicy(tproot); + testConf.getAppOrderingPolicy(tproot); fop = (FairOrderingPolicy) schedOrder; assertTrue(fop.getSizeBasedWeight()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 1348f51..11fea82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -97,9 +97,6 @@ public class TestParentQueue { Resources.createResource(16*GB, 32)); when(csContext.getClusterResource()). thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32)); - when(csContext.getNonPartitionedQueueComparator()). - thenReturn( - CapacitySchedulerQueueManager.NON_PARTITIONED_QUEUE_COMPARATOR); when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); when(csContext.getResourceCalculator()). thenReturn(resourceComparator); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 3a154b2..5e6548b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -133,8 +133,6 @@ public class TestReservations { Resources.createResource(16 * GB, 12)); when(csContext.getClusterResource()).thenReturn( Resources.createResource(100 * 16 * GB, 100 * 12)); - when(csContext.getNonPartitionedQueueComparator()).thenReturn( - CapacitySchedulerQueueManager.NON_PARTITIONED_QUEUE_COMPARATOR); when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); when(csContext.getRMContext()).thenReturn(rmContext); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java new file mode 100644 index 0000000..e3c108a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableTable; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestPriorityUtilizationQueueOrderingPolicy { + private List mockCSQueues(String[] queueNames, int[] priorities, + float[] utilizations, String partition) { + // sanity check + assert queueNames != null && priorities != null && utilizations != null + && queueNames.length > 0 && queueNames.length == priorities.length + && priorities.length == utilizations.length; + + List list = new ArrayList<>(); + for (int i = 0; i < queueNames.length; i++) { + CSQueue q = mock(CSQueue.class); + when(q.getQueueName()).thenReturn(queueNames[i]); + + QueueCapacities qc = new QueueCapacities(false); + qc.setUsedCapacity(partition, utilizations[i]); + + when(q.getQueueCapacities()).thenReturn(qc); + when(q.getPriority()).thenReturn(Priority.newInstance(priorities[i])); + + list.add(q); + } + + return list; + } + + private void verifyOrder(QueueOrderingPolicy orderingPolicy, String partition, + String[] expectedOrder) { + Iterator iter = orderingPolicy.getAssignmentIterator(partition); + int i = 0; + while (iter.hasNext()) { + CSQueue q = iter.next(); + Assert.assertEquals(expectedOrder[i], q.getQueueName()); + i++; + } + + assert i == expectedOrder.length; + } + + @Test + public void testUtilizationOrdering() { + PriorityUtilizationQueueOrderingPolicy policy = + new PriorityUtilizationQueueOrderingPolicy(false); + + // Case 1, one queue + policy.setQueues(mockCSQueues(new String[] { "a" }, new int[] { 0 }, + new float[] { 0.1f }, "")); + verifyOrder(policy, "", new String[] { "a" }); + + // Case 2, 2 queues + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 0, 0 }, + new float[] { 0.1f, 0.0f }, "")); + verifyOrder(policy, "", new String[] { "b", "a" }); + + // Case 3, 3 queues + policy.setQueues( + mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 0, 0, 0 }, + new float[] { 0.1f, 0.0f, 0.2f }, "")); + verifyOrder(policy, "", new String[] { "b", "a", "c" }); + + // Case 4, 3 queues, ignore priority + policy.setQueues( + mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 }, + new float[] { 0.1f, 0.0f, 0.2f }, "")); + verifyOrder(policy, "", new String[] { "b", "a", "c" }); + + // Case 5, 3 queues, look at partition (default) + policy.setQueues( + mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 }, + new float[] { 0.1f, 0.0f, 0.2f }, "x")); + verifyOrder(policy, "", new String[] { "a", "b", "c" }); + + // Case 5, 3 queues, look at partition (x) + policy.setQueues( + mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 }, + new float[] { 0.1f, 0.0f, 0.2f }, "x")); + verifyOrder(policy, "x", new String[] { "b", "a", "c" }); + + // Case 6, 3 queues, with different accessibility to partition + List queues = mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 }, + new float[] { 0.1f, 0.0f, 0.2f }, "x"); + // a can access "x" + when(queues.get(0).getAccessibleNodeLabels()).thenReturn(ImmutableSet.of("x", "y")); + // c can access "x" + when(queues.get(2).getAccessibleNodeLabels()).thenReturn(ImmutableSet.of("x", "y")); + policy.setQueues(queues); + verifyOrder(policy, "x", new String[] { "a", "c", "b" }); + } + + @Test + public void testPriorityUtilizationOrdering() { + PriorityUtilizationQueueOrderingPolicy policy = + new PriorityUtilizationQueueOrderingPolicy(true); + + // Case 1, one queue + policy.setQueues(mockCSQueues(new String[] { "a" }, new int[] { 1 }, + new float[] { 0.1f }, "")); + verifyOrder(policy, "", new String[] { "a" }); + + // Case 2, 2 queues, both under utilized, same priority + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 }, + new float[] { 0.2f, 0.1f }, "")); + verifyOrder(policy, "", new String[] { "b", "a" }); + + // Case 3, 2 queues, both over utilized, same priority + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 }, + new float[] { 1.1f, 1.2f }, "")); + verifyOrder(policy, "", new String[] { "a", "b" }); + + // Case 4, 2 queues, one under and one over, same priority + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 }, + new float[] { 0.1f, 1.2f }, "")); + verifyOrder(policy, "", new String[] { "a", "b" }); + + // Case 5, 2 queues, both over utilized, different priority + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 }, + new float[] { 1.1f, 1.2f }, "")); + verifyOrder(policy, "", new String[] { "b", "a" }); + + // Case 6, 2 queues, both under utilized, different priority + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 }, + new float[] { 0.1f, 0.2f }, "")); + verifyOrder(policy, "", new String[] { "b", "a" }); + + // Case 7, 2 queues, one under utilized and one over utilized, + // different priority (1) + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 }, + new float[] { 0.1f, 1.2f }, "")); + verifyOrder(policy, "", new String[] { "a", "b" }); + + // Case 8, 2 queues, one under utilized and one over utilized, + // different priority (1) + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 2, 1 }, + new float[] { 0.1f, 1.2f }, "")); + verifyOrder(policy, "", new String[] { "a", "b" }); + + // Case 9, 2 queues, one under utilized and one meet, different priority (1) + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 }, + new float[] { 0.1f, 1.0f }, "")); + verifyOrder(policy, "", new String[] { "a", "b" }); + + // Case 10, 2 queues, one under utilized and one meet, different priority (2) + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 2, 1 }, + new float[] { 0.1f, 1.0f }, "")); + verifyOrder(policy, "", new String[] { "a", "b" }); + + // Case 11, 2 queues, one under utilized and one meet, same priority + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 }, + new float[] { 0.1f, 1.0f }, "")); + verifyOrder(policy, "", new String[] { "a", "b" }); + + // Case 12, 2 queues, both meet, different priority + policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 }, + new float[] { 1.0f, 1.0f }, "")); + verifyOrder(policy, "", new String[] { "b", "a" }); + + // Case 13, 5 queues, different priority + policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" }, + new int[] { 1, 2, 0, 0, 3 }, + new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "")); + verifyOrder(policy, "", new String[] { "e", "c", "b", "a", "d" }); + + // Case 14, 5 queues, different priority, partition default; + policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" }, + new int[] { 1, 2, 0, 0, 3 }, + new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x")); + verifyOrder(policy, "", new String[] { "e", "b", "a", "c", "d" }); + + // Case 15, 5 queues, different priority, partition x; + policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" }, + new int[] { 1, 2, 0, 0, 3 }, + new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x")); + verifyOrder(policy, "x", new String[] { "e", "c", "b", "a", "d" }); + + // Case 16, 5 queues, different priority, partition x; and different + // accessibility + List queues = mockCSQueues(new String[] { "a", "b", "c", "d", "e" }, + new int[] { 1, 2, 0, 0, 3 }, + new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x"); + // Only a/d has access to x + when(queues.get(0).getAccessibleNodeLabels()).thenReturn( + ImmutableSet.of("x")); + when(queues.get(3).getAccessibleNodeLabels()).thenReturn( + ImmutableSet.of("x")); + policy.setQueues(queues); + verifyOrder(policy, "x", new String[] { "a", "d", "e", "c", "b" }); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java index 98cfdab..37fc3b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java @@ -20,23 +20,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; import java.util.*; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeLabel; -import org.apache.hadoop.yarn.server.resourcemanager.MockAM; -import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.junit.Assert; import org.junit.Test; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -157,7 +148,7 @@ public class TestFairOrderingPolicy { // Define top-level queues String queuePath = CapacitySchedulerConfiguration.ROOT + ".default"; - csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_ORDERING_POLICY); + csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY); csConf.setOrderingPolicyParameter(queuePath, FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true"); csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org