hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fab...@apache.org
Subject [10/50] [abbrv] hadoop git commit: YARN-5864. Capacity Scheduler - Queue Priorities. (wangda)
Date Tue, 31 Jan 2017 01:57:22 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index bb400ef..dd6b25b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -776,4 +776,115 @@ public class TestContainerAllocation {
         Resources.createResource(20 * GB), "", true).getMemorySize());
     rm1.close();
   }
+
+
+  @Test(timeout = 60000)
+  public void testQueuePriorityOrdering() throws Exception {
+    CapacitySchedulerConfiguration newConf =
+        (CapacitySchedulerConfiguration) TestUtils
+            .getConfigurationWithMultipleQueues(conf);
+
+    // Set ordering policy
+    newConf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT,
+        CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
+
+    // Set maximum capacity of A to 20
+    newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 20);
+    newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1);
+    newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".b", 2);
+    newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".a", 3);
+
+    MockRM rm1 = new MockRM(newConf);
+
+    rm1.getRMContext().setNodeLabelManager(mgr);
+    rm1.start();
+    MockNM nm1 = rm1.registerNode("h1:1234", 100 * GB);
+
+    // launch an app to queue A, AM container should be launched in nm1
+    RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+    // launch an app to queue B, AM container should be launched in nm1
+    RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "b");
+    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+    // launch an app to queue C, AM container should be launched in nm1
+    RMApp app3 = rm1.submitApp(2 * GB, "app", "user", null, "c");
+    MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
+
+    // Each application asks 10 * 5GB containers
+    am1.allocate("*", 5 * GB, 10, null);
+    am2.allocate("*", 5 * GB, 10, null);
+    am3.allocate("*", 5 * GB, 10, null);
+
+    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+    RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+    FiCaSchedulerApp schedulerApp1 =
+        cs.getApplicationAttempt(am1.getApplicationAttemptId());
+    FiCaSchedulerApp schedulerApp2 =
+        cs.getApplicationAttempt(am2.getApplicationAttemptId());
+    FiCaSchedulerApp schedulerApp3 =
+        cs.getApplicationAttempt(am3.getApplicationAttemptId());
+
+    // container will be allocated to am1
+    // App1 will get 2 container allocated (plus AM container)
+    cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+    Assert.assertEquals(2, schedulerApp1.getLiveContainers().size());
+    Assert.assertEquals(1, schedulerApp2.getLiveContainers().size());
+    Assert.assertEquals(1, schedulerApp3.getLiveContainers().size());
+
+    // container will be allocated to am1 again,
+    // App1 will get 3 container allocated (plus AM container)
+    cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+    Assert.assertEquals(3, schedulerApp1.getLiveContainers().size());
+    Assert.assertEquals(1, schedulerApp2.getLiveContainers().size());
+    Assert.assertEquals(1, schedulerApp3.getLiveContainers().size());
+
+    // (Now usages of queues: a=12G (satisfied), b=2G, c=2G)
+
+    // container will be allocated to am2 (since app1 reaches its guaranteed
+    // capacity)
+    // App2 will get 2 container allocated (plus AM container)
+    cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+    Assert.assertEquals(3, schedulerApp1.getLiveContainers().size());
+    Assert.assertEquals(2, schedulerApp2.getLiveContainers().size());
+    Assert.assertEquals(1, schedulerApp3.getLiveContainers().size());
+
+    // Do this 3 times
+    // container will be allocated to am2 (since app1 reaches its guaranteed
+    // capacity)
+    // App2 will get 2 container allocated (plus AM container)
+    for (int i = 0; i < 3; i++) {
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+    }
+    Assert.assertEquals(3, schedulerApp1.getLiveContainers().size());
+    Assert.assertEquals(5, schedulerApp2.getLiveContainers().size());
+    Assert.assertEquals(1, schedulerApp3.getLiveContainers().size());
+
+    // (Now usages of queues: a=12G (satisfied), b=22G (satisfied), c=2G))
+
+    // Do this 10 times
+    for (int i = 0; i < 10; i++) {
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+    }
+    Assert.assertEquals(3, schedulerApp1.getLiveContainers().size());
+    Assert.assertEquals(5, schedulerApp2.getLiveContainers().size());
+    Assert.assertEquals(11, schedulerApp3.getLiveContainers().size());
+
+    // (Now usages of queues: a=12G (satisfied), b=22G (satisfied),
+    // c=52G (satisfied and no pending))
+
+    // Do this 20 times, we can only allocate 3 containers, 1 to A and 3 to B
+    for (int i = 0; i < 20; i++) {
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+    }
+    Assert.assertEquals(4, schedulerApp1.getLiveContainers().size());
+    Assert.assertEquals(6, schedulerApp2.getLiveContainers().size());
+    Assert.assertEquals(11, schedulerApp3.getLiveContainers().size());
+
+    // (Now usages of queues: a=17G (satisfied), b=27G (satisfied), c=52G))
+
+    rm1.close();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index f1396b6..bd038e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -175,9 +175,6 @@ public class TestLeafQueue {
         thenReturn(Resources.createResource(16*GB, 32));
     when(csContext.getClusterResource()).
         thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
-    when(csContext.getNonPartitionedQueueComparator()).
-        thenReturn(
-            CapacitySchedulerQueueManager.NON_PARTITIONED_QUEUE_COMPARATOR);
     when(csContext.getResourceCalculator()).
         thenReturn(resourceCalculator);
     when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
@@ -414,7 +411,7 @@ public class TestLeafQueue {
       "testPolicyRoot" + System.currentTimeMillis();
 
     OrderingPolicy<FiCaSchedulerApp> comPol =    
-      testConf.<FiCaSchedulerApp>getOrderingPolicy(tproot);
+      testConf.<FiCaSchedulerApp>getAppOrderingPolicy(tproot);
     
     
   }
@@ -489,16 +486,16 @@ public class TestLeafQueue {
       "testPolicyRoot" + System.currentTimeMillis();
 
     OrderingPolicy<FiCaSchedulerApp> schedOrder =
-      testConf.<FiCaSchedulerApp>getOrderingPolicy(tproot);
+      testConf.<FiCaSchedulerApp>getAppOrderingPolicy(tproot);
 
     //override default to fair
     String policyType = CapacitySchedulerConfiguration.PREFIX + tproot +
       "." + CapacitySchedulerConfiguration.ORDERING_POLICY;
 
     testConf.set(policyType,
-      CapacitySchedulerConfiguration.FAIR_ORDERING_POLICY);
+      CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY);
     schedOrder =
-      testConf.<FiCaSchedulerApp>getOrderingPolicy(tproot);
+      testConf.<FiCaSchedulerApp>getAppOrderingPolicy(tproot);
     FairOrderingPolicy fop = (FairOrderingPolicy<FiCaSchedulerApp>) schedOrder;
     assertFalse(fop.getSizeBasedWeight());
 
@@ -508,7 +505,7 @@ public class TestLeafQueue {
       FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT;
     testConf.set(sbwConfig, "true");
     schedOrder =
-      testConf.<FiCaSchedulerApp>getOrderingPolicy(tproot);
+      testConf.<FiCaSchedulerApp>getAppOrderingPolicy(tproot);
     fop = (FairOrderingPolicy<FiCaSchedulerApp>) schedOrder;
     assertTrue(fop.getSizeBasedWeight());
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 1348f51..11fea82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -97,9 +97,6 @@ public class TestParentQueue {
         Resources.createResource(16*GB, 32));
     when(csContext.getClusterResource()).
         thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
-    when(csContext.getNonPartitionedQueueComparator()).
-        thenReturn(
-            CapacitySchedulerQueueManager.NON_PARTITIONED_QUEUE_COMPARATOR);
     when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
     when(csContext.getResourceCalculator()).
         thenReturn(resourceComparator);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 3a154b2..5e6548b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -133,8 +133,6 @@ public class TestReservations {
         Resources.createResource(16 * GB, 12));
     when(csContext.getClusterResource()).thenReturn(
         Resources.createResource(100 * 16 * GB, 100 * 12));
-    when(csContext.getNonPartitionedQueueComparator()).thenReturn(
-        CapacitySchedulerQueueManager.NON_PARTITIONED_QUEUE_COMPARATOR);
     when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
     when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
     when(csContext.getRMContext()).thenReturn(rmContext);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
new file mode 100644
index 0000000..e3c108a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableTable;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestPriorityUtilizationQueueOrderingPolicy {
+  private List<CSQueue> mockCSQueues(String[] queueNames, int[] priorities,
+      float[] utilizations, String partition) {
+    // sanity check
+    assert queueNames != null && priorities != null && utilizations != null
+        && queueNames.length > 0 && queueNames.length == priorities.length
+        && priorities.length == utilizations.length;
+
+    List<CSQueue> list = new ArrayList<>();
+    for (int i = 0; i < queueNames.length; i++) {
+      CSQueue q = mock(CSQueue.class);
+      when(q.getQueueName()).thenReturn(queueNames[i]);
+
+      QueueCapacities qc = new QueueCapacities(false);
+      qc.setUsedCapacity(partition, utilizations[i]);
+
+      when(q.getQueueCapacities()).thenReturn(qc);
+      when(q.getPriority()).thenReturn(Priority.newInstance(priorities[i]));
+
+      list.add(q);
+    }
+
+    return list;
+  }
+
+  private void verifyOrder(QueueOrderingPolicy orderingPolicy, String partition,
+      String[] expectedOrder) {
+    Iterator<CSQueue> iter = orderingPolicy.getAssignmentIterator(partition);
+    int i = 0;
+    while (iter.hasNext()) {
+      CSQueue q = iter.next();
+      Assert.assertEquals(expectedOrder[i], q.getQueueName());
+      i++;
+    }
+
+    assert i == expectedOrder.length;
+  }
+
+  @Test
+  public void testUtilizationOrdering() {
+    PriorityUtilizationQueueOrderingPolicy policy =
+        new PriorityUtilizationQueueOrderingPolicy(false);
+
+    // Case 1, one queue
+    policy.setQueues(mockCSQueues(new String[] { "a" }, new int[] { 0 },
+        new float[] { 0.1f }, ""));
+    verifyOrder(policy, "", new String[] { "a" });
+
+    // Case 2, 2 queues
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 0, 0 },
+        new float[] { 0.1f, 0.0f }, ""));
+    verifyOrder(policy, "", new String[] { "b", "a" });
+
+    // Case 3, 3 queues
+    policy.setQueues(
+        mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 0, 0, 0 },
+            new float[] { 0.1f, 0.0f, 0.2f }, ""));
+    verifyOrder(policy, "", new String[] { "b", "a", "c" });
+
+    // Case 4, 3 queues, ignore priority
+    policy.setQueues(
+        mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 },
+            new float[] { 0.1f, 0.0f, 0.2f }, ""));
+    verifyOrder(policy, "", new String[] { "b", "a", "c" });
+
+    // Case 5, 3 queues, look at partition (default)
+    policy.setQueues(
+        mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 },
+            new float[] { 0.1f, 0.0f, 0.2f }, "x"));
+    verifyOrder(policy, "", new String[] { "a", "b", "c" });
+
+    // Case 5, 3 queues, look at partition (x)
+    policy.setQueues(
+        mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 },
+            new float[] { 0.1f, 0.0f, 0.2f }, "x"));
+    verifyOrder(policy, "x", new String[] { "b", "a", "c" });
+
+    // Case 6, 3 queues, with different accessibility to partition
+    List<CSQueue> queues = mockCSQueues(new String[] { "a", "b", "c" }, new int[] {
2, 1, 0 },
+        new float[] { 0.1f, 0.0f, 0.2f }, "x");
+    // a can access "x"
+    when(queues.get(0).getAccessibleNodeLabels()).thenReturn(ImmutableSet.of("x", "y"));
+    // c can access "x"
+    when(queues.get(2).getAccessibleNodeLabels()).thenReturn(ImmutableSet.of("x", "y"));
+    policy.setQueues(queues);
+    verifyOrder(policy, "x", new String[] { "a", "c", "b" });
+  }
+
+  @Test
+  public void testPriorityUtilizationOrdering() {
+    PriorityUtilizationQueueOrderingPolicy policy =
+        new PriorityUtilizationQueueOrderingPolicy(true);
+
+    // Case 1, one queue
+    policy.setQueues(mockCSQueues(new String[] { "a" }, new int[] { 1 },
+        new float[] { 0.1f }, ""));
+    verifyOrder(policy, "", new String[] { "a" });
+
+    // Case 2, 2 queues, both under utilized, same priority
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
+        new float[] { 0.2f, 0.1f }, ""));
+    verifyOrder(policy, "", new String[] { "b", "a" });
+
+    // Case 3, 2 queues, both over utilized, same priority
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
+        new float[] { 1.1f, 1.2f }, ""));
+    verifyOrder(policy, "", new String[] { "a", "b" });
+
+    // Case 4, 2 queues, one under and one over, same priority
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
+        new float[] { 0.1f, 1.2f }, ""));
+    verifyOrder(policy, "", new String[] { "a", "b" });
+
+    // Case 5, 2 queues, both over utilized, different priority
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
+        new float[] { 1.1f, 1.2f }, ""));
+    verifyOrder(policy, "", new String[] { "b", "a" });
+
+    // Case 6, 2 queues, both under utilized, different priority
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
+        new float[] { 0.1f, 0.2f }, ""));
+    verifyOrder(policy, "", new String[] { "b", "a" });
+
+    // Case 7, 2 queues, one under utilized and one over utilized,
+    // different priority (1)
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
+        new float[] { 0.1f, 1.2f }, ""));
+    verifyOrder(policy, "", new String[] { "a", "b" });
+
+    // Case 8, 2 queues, one under utilized and one over utilized,
+    // different priority (1)
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 2, 1 },
+        new float[] { 0.1f, 1.2f }, ""));
+    verifyOrder(policy, "", new String[] { "a", "b" });
+
+    // Case 9, 2 queues, one under utilized and one meet, different priority (1)
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
+        new float[] { 0.1f, 1.0f }, ""));
+    verifyOrder(policy, "", new String[] { "a", "b" });
+
+    // Case 10, 2 queues, one under utilized and one meet, different priority (2)
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 2, 1 },
+        new float[] { 0.1f, 1.0f }, ""));
+    verifyOrder(policy, "", new String[] { "a", "b" });
+
+    // Case 11, 2 queues, one under utilized and one meet, same priority
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
+        new float[] { 0.1f, 1.0f }, ""));
+    verifyOrder(policy, "", new String[] { "a", "b" });
+
+    // Case 12, 2 queues, both meet, different priority
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
+        new float[] { 1.0f, 1.0f }, ""));
+    verifyOrder(policy, "", new String[] { "b", "a" });
+
+    // Case 13, 5 queues, different priority
+    policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
+        new int[] { 1, 2, 0, 0, 3 },
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, ""));
+    verifyOrder(policy, "", new String[] { "e", "c", "b", "a", "d" });
+
+    // Case 14, 5 queues, different priority, partition default;
+    policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
+        new int[] { 1, 2, 0, 0, 3 },
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x"));
+    verifyOrder(policy, "", new String[] { "e", "b", "a", "c", "d" });
+
+    // Case 15, 5 queues, different priority, partition x;
+    policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
+        new int[] { 1, 2, 0, 0, 3 },
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x"));
+    verifyOrder(policy, "x", new String[] { "e", "c", "b", "a", "d" });
+
+    // Case 16, 5 queues, different priority, partition x; and different
+    // accessibility
+    List<CSQueue> queues = mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
+        new int[] { 1, 2, 0, 0, 3 },
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x");
+    // Only a/d has access to x
+    when(queues.get(0).getAccessibleNodeLabels()).thenReturn(
+        ImmutableSet.of("x"));
+    when(queues.get(3).getAccessibleNodeLabels()).thenReturn(
+        ImmutableSet.of("x"));
+    policy.setQueues(queues);
+    verifyOrder(policy, "x", new String[] { "a", "d", "e", "c", "b" });
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce832059/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
index 98cfdab..37fc3b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
@@ -20,23 +20,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
 
 import java.util.*;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
-import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.junit.Assert;
 import org.junit.Test;
 
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 
@@ -157,7 +148,7 @@ public class TestFairOrderingPolicy {
 
     // Define top-level queues
     String queuePath = CapacitySchedulerConfiguration.ROOT + ".default";
-    csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_ORDERING_POLICY);
+    csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY);
     csConf.setOrderingPolicyParameter(queuePath,
         FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true");
     csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message