hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1415815 [5/5] - in /hadoop/common/branches/branch-trunk-win/hadoop-yarn-project: ./ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/ hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-a...
Date Fri, 30 Nov 2012 19:58:44 GMT
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Fri Nov 30 19:58:09 2012
@@ -31,6 +31,10 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
+import javax.xml.parsers.ParserConfigurationException;
+
+import junit.framework.Assert;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.Clock;
@@ -47,7 +51,7 @@ import org.apache.hadoop.yarn.factories.
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -61,6 +65,7 @@ import org.apache.hadoop.yarn.server.res
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.xml.sax.SAXException;
 
 public class TestFairScheduler {
 
@@ -97,7 +102,7 @@ public class TestFairScheduler {
     Configuration conf = createConfiguration();
     // All tests assume only one assignment per node update
     conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
-    Store store = StoreFactory.getStore(conf);
+    RMStateStore store = StoreFactory.getStore(conf);
     resourceManager = new ResourceManager(store);
     resourceManager.init(conf);
     ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
@@ -195,15 +200,64 @@ public class TestFairScheduler {
 
     scheduler.update();
 
-    Collection<FSQueue> queues = scheduler.getQueueManager().getQueues();
+    Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
     assertEquals(3, queues.size());
 
-    for (FSQueue p : queues) {
-      if (p.getName() != "default") {
-        assertEquals(5120, p.getQueueSchedulable().getFairShare().getMemory());
+    for (FSLeafQueue p : queues) {
+      if (!p.getName().equals("root.default")) {
+        assertEquals(5120, p.getFairShare().getMemory());
       }
     }
   }
+  
+  @Test
+  public void testSimpleHierarchicalFairShareCalculation() {
+    // Add one big node (only care about aggregate capacity)
+    int capacity = 10 * 24;
+    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(capacity));
+    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+    scheduler.handle(nodeEvent1);
+
+    // Have two queues which want entire cluster capacity
+    createSchedulingRequest(10 * 1024, "queue1", "user1");
+    createSchedulingRequest(10 * 1024, "parent.queue2", "user1");
+    createSchedulingRequest(10 * 1024, "parent.queue3", "user1");
+
+    scheduler.update();
+
+    QueueManager queueManager = scheduler.getQueueManager();
+    Collection<FSLeafQueue> queues = queueManager.getLeafQueues();
+    assertEquals(4, queues.size());
+    
+    FSLeafQueue queue1 = queueManager.getLeafQueue("queue1");
+    FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2");
+    FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3");
+    assertEquals(capacity / 2, queue1.getFairShare().getMemory());
+    assertEquals(capacity / 4, queue2.getFairShare().getMemory());
+    assertEquals(capacity / 4, queue3.getFairShare().getMemory());
+  }
+  
+  @Test
+  public void testHierarchicalQueuesSimilarParents() {
+    QueueManager queueManager = scheduler.getQueueManager();
+    FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child");
+    Assert.assertEquals(2, queueManager.getLeafQueues().size());
+    Assert.assertNotNull(leafQueue);
+    Assert.assertEquals("root.parent.child", leafQueue.getName());
+
+    FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent");
+    Assert.assertNull(leafQueue2);
+    Assert.assertEquals(2, queueManager.getLeafQueues().size());
+    
+    FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild");
+    Assert.assertNull(leafQueue3);
+    Assert.assertEquals(2, queueManager.getLeafQueues().size());
+    
+    FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister");
+    Assert.assertNotNull(leafQueue4);
+    Assert.assertEquals("root.parent.sister", leafQueue4.getName());
+    Assert.assertEquals(3, queueManager.getLeafQueues().size());
+  }
 
   @Test
   public void testSimpleContainerAllocation() {
@@ -228,14 +282,14 @@ public class TestFairScheduler {
     // Asked for less than min_allocation.
     assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
         scheduler.getQueueManager().getQueue("queue1").
-        getQueueSchedulable().getResourceUsage().getMemory());
+        getResourceUsage().getMemory());
 
     NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2,
         new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
     scheduler.handle(updateEvent2);
 
     assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
-      getQueueSchedulable().getResourceUsage().getMemory());
+      getResourceUsage().getMemory());
   }
 
   @Test
@@ -254,7 +308,7 @@ public class TestFairScheduler {
 
     // Make sure queue 1 is allocated app capacity
     assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
-        getQueueSchedulable().getResourceUsage().getMemory());
+        getResourceUsage().getMemory());
 
     // Now queue 2 requests likewise
     ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
@@ -263,7 +317,7 @@ public class TestFairScheduler {
 
     // Make sure queue 2 is waiting with a reservation
     assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
-      getQueueSchedulable().getResourceUsage().getMemory());
+      getResourceUsage().getMemory());
     assertEquals(1024, scheduler.applications.get(attId).getCurrentReservation().getMemory());
 
     // Now another node checks in with capacity
@@ -276,7 +330,7 @@ public class TestFairScheduler {
 
     // Make sure this goes to queue 2
     assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
-        getQueueSchedulable().getResourceUsage().getMemory());
+        getResourceUsage().getMemory());
 
     // The old reservation should still be there...
     assertEquals(1024, scheduler.applications.get(attId).getCurrentReservation().getMemory());
@@ -294,17 +348,22 @@ public class TestFairScheduler {
     AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
         createAppAttemptId(1, 1), "default", "user1");
     scheduler.handle(appAddedEvent);
-    assertEquals(1, scheduler.getQueueManager().getQueue("user1").getApplications().size());
-    assertEquals(0, scheduler.getQueueManager().getQueue("default").getApplications().size());
+    assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+        .getAppSchedulables().size());
+    assertEquals(0, scheduler.getQueueManager().getLeafQueue("default")
+        .getAppSchedulables().size());
 
     conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
     scheduler.reinitialize(conf, resourceManager.getRMContext());
     AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent(
         createAppAttemptId(2, 1), "default", "user2");
     scheduler.handle(appAddedEvent2);
-    assertEquals(1, scheduler.getQueueManager().getQueue("user1").getApplications().size());
-    assertEquals(1, scheduler.getQueueManager().getQueue("default").getApplications().size());
-    assertEquals(0, scheduler.getQueueManager().getQueue("user2").getApplications().size());
+    assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+        .getAppSchedulables().size());
+    assertEquals(1, scheduler.getQueueManager().getLeafQueue("default")
+        .getAppSchedulables().size());
+    assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2")
+        .getAppSchedulables().size());
   }
 
   @Test
@@ -338,18 +397,17 @@ public class TestFairScheduler {
 
     scheduler.update();
 
-    Collection<FSQueue> queues = scheduler.getQueueManager().getQueues();
+    Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
     assertEquals(3, queues.size());
 
-    for (FSQueue p : queues) {
-      if (p.getName().equals("queueA")) {
-        assertEquals(1024, p.getQueueSchedulable().getFairShare().getMemory());
+    for (FSLeafQueue p : queues) {
+      if (p.getName().equals("root.queueA")) {
+        assertEquals(1024, p.getFairShare().getMemory());
       }
-      else if (p.getName().equals("queueB")) {
-        assertEquals(2048, p.getQueueSchedulable().getFairShare().getMemory());
+      else if (p.getName().equals("root.queueB")) {
+        assertEquals(2048, p.getFairShare().getMemory());
       }
     }
-
   }
 
   /**
@@ -358,11 +416,11 @@ public class TestFairScheduler {
   @Test
   public void testQueueDemandCalculation() throws Exception {
     ApplicationAttemptId id11 = createAppAttemptId(1, 1);
-    scheduler.addApplication(id11, "queue1", "user1");
+    scheduler.addApplication(id11, "root.queue1", "user1");
     ApplicationAttemptId id21 = createAppAttemptId(2, 1);
-    scheduler.addApplication(id21, "queue2", "user1");
+    scheduler.addApplication(id21, "root.queue2", "user1");
     ApplicationAttemptId id22 = createAppAttemptId(2, 2);
-    scheduler.addApplication(id22, "queue2", "user1");
+    scheduler.addApplication(id22, "root.queue2", "user1");
 
     int minReqSize = YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB;
     
@@ -388,10 +446,10 @@ public class TestFairScheduler {
 
     scheduler.update();
 
-    assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("queue1")
-        .getQueueSchedulable().getDemand().getMemory());
+    assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("root.queue1")
+        .getDemand().getMemory());
     assertEquals(2 * minReqSize + 2 * minReqSize + (2 * minReqSize), scheduler
-        .getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand()
+        .getQueueManager().getQueue("root.queue2").getDemand()
         .getMemory());
   }
 
@@ -401,11 +459,12 @@ public class TestFairScheduler {
         createAppAttemptId(1, 1), "default", "user1");
     scheduler.handle(appAddedEvent1);
 
-    // Scheduler should have one queue (the default)
-    assertEquals(1, scheduler.getQueueManager().getQueues().size());
+    // Scheduler should have two queues (the default and the one created for user1)
+    assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
 
     // That queue should have one app
-    assertEquals(1, scheduler.getQueueManager().getQueue("default").getApplications().size());
+    assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+        .getAppSchedulables().size());
 
     AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
         createAppAttemptId(1, 1), RMAppAttemptState.FINISHED);
@@ -413,8 +472,9 @@ public class TestFairScheduler {
     // Now remove app
     scheduler.handle(appRemovedEvent1);
 
-    // Default queue should have no apps
-    assertEquals(0, scheduler.getQueueManager().getQueue("default").getApplications().size());
+    // Queue should have no apps
+    assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1")
+        .getAppSchedulables().size());
   }
 
   @Test
@@ -466,61 +526,99 @@ public class TestFairScheduler {
     QueueManager queueManager = scheduler.getQueueManager();
     queueManager.initialize();
 
-    assertEquals(6, queueManager.getQueues().size()); // 5 in file + default queue
+    assertEquals(6, queueManager.getLeafQueues().size()); // 5 in file + default queue
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME));
+        queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME));
+        queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
 
     assertEquals(Resources.createResource(1024),
-        queueManager.getMinResources("queueA"));
+        queueManager.getMinResources("root.queueA"));
     assertEquals(Resources.createResource(2048),
-        queueManager.getMinResources("queueB"));
+        queueManager.getMinResources("root.queueB"));
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources("queueC"));
+        queueManager.getMinResources("root.queueC"));
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources("queueD"));
+        queueManager.getMinResources("root.queueD"));
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources("queueE"));
+        queueManager.getMinResources("root.queueE"));
 
-    assertEquals(15, queueManager.getQueueMaxApps(YarnConfiguration.DEFAULT_QUEUE_NAME));
-    assertEquals(15, queueManager.getQueueMaxApps("queueA"));
-    assertEquals(15, queueManager.getQueueMaxApps("queueB"));
-    assertEquals(15, queueManager.getQueueMaxApps("queueC"));
-    assertEquals(3, queueManager.getQueueMaxApps("queueD"));
-    assertEquals(15, queueManager.getQueueMaxApps("queueE"));
+    assertEquals(15, queueManager.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
+    assertEquals(15, queueManager.getQueueMaxApps("root.queueA"));
+    assertEquals(15, queueManager.getQueueMaxApps("root.queueB"));
+    assertEquals(15, queueManager.getQueueMaxApps("root.queueC"));
+    assertEquals(3, queueManager.getQueueMaxApps("root.queueD"));
+    assertEquals(15, queueManager.getQueueMaxApps("root.queueE"));
     assertEquals(10, queueManager.getUserMaxApps("user1"));
     assertEquals(5, queueManager.getUserMaxApps("user2"));
 
     // Unspecified queues should get default ACL
-    Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("queueA");
+    Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("root.queueA");
     assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
     assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
     assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
     assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
     // Queue B ACL
-    Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("queueB");
+    Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
     assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
     assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
 
     // Queue c ACL
-    Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("queueC");
+    Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
     assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
     assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout(
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." + 
         YarnConfiguration.DEFAULT_QUEUE_NAME));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA"));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueB"));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueC"));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueD"));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA"));
-    assertEquals(60000, queueManager.getMinSharePreemptionTimeout("queueE"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueB"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueC"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueD"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
+    assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE"));
     assertEquals(300000, queueManager.getFairSharePreemptionTimeout());
   }
 
   @Test
+  public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException,

+      AllocationConfigurationException, ParserConfigurationException {
+    Configuration conf = createConfiguration();
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"queueA\">");
+    out.println("<minResources>2048</minResources>");
+    out.println("</queue>");
+    out.println("<queue name=\"queueB\">");
+    out.println("<minResources>2048</minResources>");
+    out.println("<queue name=\"queueC\">");
+    out.println("<minResources>2048</minResources>");
+    out.println("</queue>");
+    out.println("<queue name=\"queueD\">");
+    out.println("<minResources>2048</minResources>");
+    out.println("</queue>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+
+    QueueManager queueManager = scheduler.getQueueManager();
+    queueManager.initialize();
+    
+    Collection<FSLeafQueue> leafQueues = queueManager.getLeafQueues();
+    Assert.assertEquals(4, leafQueues.size());
+    Assert.assertNotNull(queueManager.getLeafQueue("queueA"));
+    Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC"));
+    Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD"));
+    Assert.assertNotNull(queueManager.getLeafQueue("default"));
+    // Make sure querying for queues didn't create any new ones:
+    Assert.assertEquals(4, leafQueues.size());
+  }
+  
+  @Test
   public void testBackwardsCompatibleAllocationFileParsing() throws Exception {
     Configuration conf = createConfiguration();
     conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
@@ -569,29 +667,29 @@ public class TestFairScheduler {
     QueueManager queueManager = scheduler.getQueueManager();
     queueManager.initialize();
 
-    assertEquals(6, queueManager.getQueues().size()); // 5 in file + default queue
+    assertEquals(6, queueManager.getLeafQueues().size()); // 5 in file + default queue
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME));
+        queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME));
+        queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
 
     assertEquals(Resources.createResource(1024),
-        queueManager.getMinResources("queueA"));
+        queueManager.getMinResources("root.queueA"));
     assertEquals(Resources.createResource(2048),
-        queueManager.getMinResources("queueB"));
+        queueManager.getMinResources("root.queueB"));
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources("queueC"));
+        queueManager.getMinResources("root.queueC"));
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources("queueD"));
+        queueManager.getMinResources("root.queueD"));
     assertEquals(Resources.createResource(0),
-        queueManager.getMinResources("queueE"));
+        queueManager.getMinResources("root.queueE"));
 
-    assertEquals(15, queueManager.getQueueMaxApps(YarnConfiguration.DEFAULT_QUEUE_NAME));
-    assertEquals(15, queueManager.getQueueMaxApps("queueA"));
-    assertEquals(15, queueManager.getQueueMaxApps("queueB"));
-    assertEquals(15, queueManager.getQueueMaxApps("queueC"));
-    assertEquals(3, queueManager.getQueueMaxApps("queueD"));
-    assertEquals(15, queueManager.getQueueMaxApps("queueE"));
+    assertEquals(15, queueManager.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
+    assertEquals(15, queueManager.getQueueMaxApps("root.queueA"));
+    assertEquals(15, queueManager.getQueueMaxApps("root.queueB"));
+    assertEquals(15, queueManager.getQueueMaxApps("root.queueC"));
+    assertEquals(3, queueManager.getQueueMaxApps("root.queueD"));
+    assertEquals(15, queueManager.getQueueMaxApps("root.queueE"));
     assertEquals(10, queueManager.getUserMaxApps("user1"));
     assertEquals(5, queueManager.getUserMaxApps("user2"));
 
@@ -603,23 +701,23 @@ public class TestFairScheduler {
     assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
     // Queue B ACL
-    Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("queueB");
+    Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
     assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
     assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
 
     // Queue c ACL
-    Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("queueC");
+    Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
     assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
     assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout(
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
         YarnConfiguration.DEFAULT_QUEUE_NAME));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA"));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueB"));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueC"));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueD"));
-    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA"));
-    assertEquals(60000, queueManager.getMinSharePreemptionTimeout("queueE"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueB"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueC"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueD"));
+    assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
+    assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE"));
     assertEquals(300000, queueManager.getFairSharePreemptionTimeout());
   }
 
@@ -659,25 +757,25 @@ public class TestFairScheduler {
     // Queue B arrives and wants 1 * 1024
     createSchedulingRequest(1 * 1024, "queueB", "user1");
     scheduler.update();
-    Collection<FSQueue> queues = scheduler.getQueueManager().getQueues();
+    Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
     assertEquals(3, queues.size());
 
     // Queue A should be above min share, B below.
-    for (FSQueue p : queues) {
-      if (p.getName().equals("queueA")) {
-        assertEquals(false, scheduler.isStarvedForMinShare(p.getQueueSchedulable()));
+    for (FSLeafQueue p : queues) {
+      if (p.getName().equals("root.queueA")) {
+        assertEquals(false, scheduler.isStarvedForMinShare(p));
       }
-      else if (p.getName().equals("queueB")) {
-        assertEquals(true, scheduler.isStarvedForMinShare(p.getQueueSchedulable()));
+      else if (p.getName().equals("root.queueB")) {
+        assertEquals(true, scheduler.isStarvedForMinShare(p));
       }
     }
 
     // Node checks in again, should allocate for B
     scheduler.handle(nodeEvent2);
     // Now B should have min share ( = demand here)
-    for (FSQueue p : queues) {
-      if (p.getName().equals("queueB")) {
-        assertEquals(false, scheduler.isStarvedForMinShare(p.getQueueSchedulable()));
+    for (FSLeafQueue p : queues) {
+      if (p.getName().equals("root.queueB")) {
+        assertEquals(false, scheduler.isStarvedForMinShare(p));
       }
     }
   }
@@ -718,16 +816,16 @@ public class TestFairScheduler {
     // Queue B arrives and wants 1 * 1024
     createSchedulingRequest(1 * 1024, "queueB", "user1");
     scheduler.update();
-    Collection<FSQueue> queues = scheduler.getQueueManager().getQueues();
+    Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
     assertEquals(3, queues.size());
 
     // Queue A should be above fair share, B below.
-    for (FSQueue p : queues) {
-      if (p.getName().equals("queueA")) {
-        assertEquals(false, scheduler.isStarvedForFairShare(p.getQueueSchedulable()));
+    for (FSLeafQueue p : queues) {
+      if (p.getName().equals("root.queueA")) {
+        assertEquals(false, scheduler.isStarvedForFairShare(p));
       }
-      else if (p.getName().equals("queueB")) {
-        assertEquals(true, scheduler.isStarvedForFairShare(p.getQueueSchedulable()));
+      else if (p.getName().equals("root.queueB")) {
+        assertEquals(true, scheduler.isStarvedForFairShare(p));
       }
     }
 
@@ -735,9 +833,9 @@ public class TestFairScheduler {
     scheduler.handle(nodeEvent2);
     // B should not be starved for fair share, since entire demand is
     // satisfied.
-    for (FSQueue p : queues) {
-      if (p.getName().equals("queueB")) {
-        assertEquals(false, scheduler.isStarvedForFairShare(p.getQueueSchedulable()));
+    for (FSLeafQueue p : queues) {
+      if (p.getName().equals("root.queueB")) {
+        assertEquals(false, scheduler.isStarvedForFairShare(p));
       }
     }
   }
@@ -845,7 +943,7 @@ public class TestFairScheduler {
 
     // We should be able to claw back one container from A and B each.
     // Make sure it is lowest priority container.
-    scheduler.preemptResources(scheduler.getQueueSchedulables(),
+    scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
         Resources.createResource(2 * 1024));
     assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size());
     assertEquals(1, scheduler.applications.get(app2).getLiveContainers().size());
@@ -856,7 +954,7 @@ public class TestFairScheduler {
 
     // We should be able to claw back another container from A and B each.
     // Make sure it is lowest priority container.
-    scheduler.preemptResources(scheduler.getQueueSchedulables(),
+    scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
         Resources.createResource(2 * 1024));
     assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size());
     assertEquals(0, scheduler.applications.get(app2).getLiveContainers().size());
@@ -866,7 +964,7 @@ public class TestFairScheduler {
     assertEquals(0, scheduler.applications.get(app6).getLiveContainers().size());
 
     // Now A and B are below fair share, so preemption shouldn't do anything
-    scheduler.preemptResources(scheduler.getQueueSchedulables(),
+    scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
         Resources.createResource(2 * 1024));
     assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size());
     assertEquals(0, scheduler.applications.get(app2).getLiveContainers().size());
@@ -977,10 +1075,10 @@ public class TestFairScheduler {
 
     scheduler.update();
 
-    FSQueueSchedulable schedC =
-        scheduler.getQueueManager().getQueue("queueC").getQueueSchedulable();
-    FSQueueSchedulable schedD =
-        scheduler.getQueueManager().getQueue("queueD").getQueueSchedulable();
+    FSLeafQueue schedC =
+        scheduler.getQueueManager().getLeafQueue("queueC");
+    FSLeafQueue schedD =
+        scheduler.getQueueManager().getLeafQueue("queueD");
 
     assertTrue(Resources.equals(
         Resources.none(), scheduler.resToPreempt(schedC, clock.getTime())));

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
Fri Nov 30 19:58:09 2012
@@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.Task;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
 import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher;
@@ -59,7 +59,7 @@ public class TestFifoScheduler {
   
   @Before
   public void setUp() throws Exception {
-    Store store = StoreFactory.getStore(new Configuration());
+    RMStateStore store = StoreFactory.getStore(new Configuration());
     resourceManager = new ResourceManager(store);
     Configuration conf = new Configuration();
     conf.setClass(YarnConfiguration.RM_SCHEDULER, 
@@ -91,7 +91,7 @@ public class TestFifoScheduler {
   @Test
   public void testAppAttemptMetrics() throws Exception {
     AsyncDispatcher dispatcher = new InlineDispatcher();
-    RMContext rmContext = new RMContextImpl(null, dispatcher, null,
+    RMContext rmContext = new RMContextImpl(dispatcher, null,
         null, null, null, null, null, null);
 
     FifoScheduler schedular = new FifoScheduler();

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
Fri Nov 30 19:58:09 2012
@@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -160,7 +159,7 @@ public class TestRMWebApp {
     for (RMNode node : deactivatedNodes) {
       deactivatedNodesMap.put(node.getHostName(), node);
     }
-   return new RMContextImpl(new MemStore(), null, null, null, null,
+   return new RMContextImpl(null, null, null, null,
        null, null, null, null) {
       @Override
       public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
@@ -201,7 +200,7 @@ public class TestRMWebApp {
 
     CapacityScheduler cs = new CapacityScheduler();
     cs.setConf(new YarnConfiguration());
-    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
+    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
       null, new RMContainerTokenSecretManager(conf),
       new ClientToAMTokenSecretManagerInRM()));
     return cs;

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Fri Nov 30 19:58:09 2012
@@ -48,7 +48,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.service.CompositeService;
@@ -154,7 +154,7 @@ public class MiniYARNCluster extends Com
           getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS,
               MiniYARNCluster.getHostname() + ":0");
         }
-        Store store = StoreFactory.getStore(getConfig());
+        RMStateStore store = StoreFactory.getStore(getConfig());
         resourceManager = new ResourceManager(store) {
           @Override
           protected void doSecureLogin() throws IOException {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Fri Nov 30 19:58:09 2012
@@ -53,6 +53,22 @@ Hadoop MapReduce Next Generation - Fair 
   capacity between the running apps. queues can also be given weights to share 
   the cluster non-proportionally in the config file.
 
+  The fair scheduler supports hierarchical queues. All queues descend from a
+  queue named "root". Available resources are distributed among the children
+  of the root queue in the typical fair scheduling fashion. Then, the children
+  distribute the resources assigned to them to their children in the same
+  fashion.  Applications may only be scheduled on leaf queues. Queues can be
+  specified as children of other queues by placing them as sub-elements of 
+  their parents in the fair scheduler configuration file.
+  
+  A queue's name starts with the names of its parents, with periods as
+  separators.  So a queue named "queue1" under the root named, would be 
+  referred to as "root.queue1", and a queue named "queue2" under a queue
+  named "parent1" would be referred to as "root.parent1.queue2". When
+  referring to queues, the root part of the name is optional, so queue1 could
+  be referred to as just "queue1", and a queue2 could be referred to as just 
+  "parent1.queue2".
+
   In addition to providing fair sharing, the Fair Scheduler allows assigning 
   guaranteed minimum shares to queues, which is useful for ensuring that 
   certain users, groups or production applications always get sufficient 
@@ -163,11 +179,14 @@ Allocation file format
 <?xml version="1.0"?>
 <allocations>
   <queue name="sample_queue">
-    <minResources>100000</minResources>
-    <maxResources>900000</maxResources>
+    <minResources>10000</minResources>
+    <maxResources>90000</maxResources>
     <maxRunningApps>50</maxRunningApps>
     <weight>2.0</weight>
     <schedulingMode>fair</schedulingMode>
+    <queue name="sample_sub_queue">
+      <minResources>5000</minResources>
+    </queue>
   </queue>
   <user name="sample_user">
     <maxRunningApps>30</maxRunningApps>

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
Fri Nov 30 19:58:09 2012
@@ -56,6 +56,8 @@ History Server REST API's.
 *---------------+--------------+-------------------------------+
 || Item         || Data Type   || Description                   |
 *---------------+--------------+-------------------------------+
+| startedOn | long  | The time the history server was started (in ms since epoch)|
+*---------------+--------------+-------------------------------+
 | hadoopVersion | string  | Version of hadoop common |
 *---------------+--------------+-------------------------------+
 | hadoopBuildVersion | string  | Hadoop common build string with build version, user, and
checksum |
@@ -87,6 +89,7 @@ History Server REST API's.
 +---+
 {   
    "historyInfo" : {
+      "startedOn":1353512830963,
       "hadoopVersionBuiltOn" : "Wed Jan 11 21:18:36 UTC 2012",
       "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7",
       "hadoopVersion" : "0.23.1-SNAPSHOT"
@@ -117,6 +120,7 @@ History Server REST API's.
 +---+
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <historyInfo>
+  <startedOn>1353512830963</startedOn>
   <hadoopVersion>0.23.1-SNAPSHOT</hadoopVersion>
   <hadoopBuildVersion>0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7</hadoopBuildVersion>
   <hadoopVersionBuiltOn>Wed Jan 11 21:18:36 UTC 2012</hadoopVersionBuiltOn>

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
Fri Nov 30 19:58:09 2012
@@ -49,7 +49,7 @@ MapReduce NextGen aka YARN aka MRv2
 
   * {{{./WebApplicationProxy.html}Web Application Proxy}}
 
-  * {{{./CLIMiniCluster.html}CLI MiniCluster}}
+  * {{{../../hadoop-project-dist/hadoop-common/CLIMiniCluster.html}CLI MiniCluster}}
 
-  * {{{./EncryptedShuffle.html}Encrypted Shuffle}}
+  * {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html}Encrypted
Shuffle}}
 



Mime
View raw message