hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From umamah...@apache.org
Subject svn commit: r1593948 [2/2] - in /hadoop/common/branches/HDFS-2006/hadoop-yarn-project: ./ hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-app...
Date Mon, 12 May 2014 12:44:10 GMT
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
(original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Mon May 12 12:43:59 2014
@@ -350,8 +350,7 @@ public class TestLeafQueue {
     // Submit applications
     final ApplicationAttemptId appAttemptId_0 = TestUtils
         .getMockApplicationAttemptId(0, 1);
-    FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, null,
-        rmContext);
+
     AppAddedSchedulerEvent addAppEvent =
         new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(),
           a.getQueueName(), user_0);
@@ -687,7 +686,7 @@ public class TestLeafQueue {
     assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
     assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
     assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G
-    assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G
+    assertEquals(0*GB, app_1.getHeadroom().getMemory()); // User limit = 2G
 
     // Again one to user_0 since he hasn't exceeded user limit yet
     a.assignContainers(clusterResource, node_0);
@@ -695,7 +694,7 @@ public class TestLeafQueue {
     assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
     assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
     assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G
-    assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G
+    assertEquals(0*GB, app_1.getHeadroom().getMemory()); // 3G - 2G
     
     // Submit requests for app_1 and set max-cap
     a.setMaxCapacity(.1f);

Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
(original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
Mon May 12 12:43:59 2014
@@ -18,7 +18,6 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
 
 import java.io.File;
 import java.io.FileWriter;
@@ -28,6 +27,7 @@ import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
 import org.apache.hadoop.yarn.util.Clock;
@@ -99,9 +99,12 @@ public class TestAllocationFileLoaderSer
     assertEquals(1, rules.size());
     assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass());
     assertEquals(1, allocConf.getQueueMaxApps("root.queueA"));
-    assertEquals(2, allocConf.getQueueNames().size());
-    assertTrue(allocConf.getQueueNames().contains("root.queueA"));
-    assertTrue(allocConf.getQueueNames().contains("root.queueB"));
+    assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+        .size());
+    assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+        .contains("root.queueA"));
+    assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+        .contains("root.queueB"));
     
     confHolder.allocConf = null;
     
@@ -114,6 +117,9 @@ public class TestAllocationFileLoaderSer
     out.println("  </queue>");
     out.println("  <queuePlacementPolicy>");
     out.println("    <rule name='specified' />");
+    out.println("    <rule name='nestedUserQueue' >");  
+    out.println("         <rule name='primaryGroup' />");
+    out.println("    </rule>");
     out.println("    <rule name='default' />");
     out.println("  </queuePlacementPolicy>");
     out.println("</allocations>");
@@ -131,12 +137,18 @@ public class TestAllocationFileLoaderSer
     allocConf = confHolder.allocConf;
     policy = allocConf.getPlacementPolicy();
     rules = policy.getRules();
-    assertEquals(2, rules.size());
+    assertEquals(3, rules.size());
     assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass());
-    assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass());
+    assertEquals(QueuePlacementRule.NestedUserQueue.class, rules.get(1)
+        .getClass());
+    assertEquals(QueuePlacementRule.PrimaryGroup.class,
+        ((NestedUserQueue) (rules.get(1))).nestedRule.getClass());
+    assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass());
     assertEquals(3, allocConf.getQueueMaxApps("root.queueB"));
-    assertEquals(1, allocConf.getQueueNames().size());
-    assertTrue(allocConf.getQueueNames().contains("root.queueB"));
+    assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+        .size());
+    assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+        .contains("root.queueB"));
   }
   
   @Test
@@ -170,6 +182,14 @@ public class TestAllocationFileLoaderSer
     out.println("<queue name=\"queueE\">");
     out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
     out.println("</queue>");
+    //Make queue F a parent queue without configured leaf queues using the 'type' attribute
+    out.println("<queue name=\"queueF\" type=\"parent\" >");
+    out.println("</queue>");
+    //Create hierarchical queues G,H
+    out.println("<queue name=\"queueG\">");
+    out.println("   <queue name=\"queueH\">");
+    out.println("   </queue>");
+    out.println("</queue>");
     // Set default limit of apps per queue to 15
     out.println("<queueMaxAppsDefault>15</queueMaxAppsDefault>");
     // Set default limit of apps per user to 5
@@ -194,7 +214,7 @@ public class TestAllocationFileLoaderSer
     allocLoader.reloadAllocations();
     AllocationConfiguration queueConf = confHolder.allocConf;
     
-    assertEquals(5, queueConf.getQueueNames().size());
+    assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
     assertEquals(Resources.createResource(0),
         queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
     assertEquals(Resources.createResource(0),
@@ -250,6 +270,14 @@ public class TestAllocationFileLoaderSer
     assertEquals(60000, queueConf.getMinSharePreemptionTimeout("root.queueE"));
     assertEquals(300000, queueConf.getFairSharePreemptionTimeout());
     
+    assertTrue(queueConf.getConfiguredQueues()
+        .get(FSQueueType.PARENT)
+        .contains("root.queueF"));
+    assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT)
+        .contains("root.queueG"));
+    assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF)
+        .contains("root.queueG.queueH"));
+
     // Verify existing queues have default scheduling policy
     assertEquals(DominantResourceFairnessPolicy.NAME,
         queueConf.getSchedulingPolicy("root").getName());
@@ -315,7 +343,7 @@ public class TestAllocationFileLoaderSer
     allocLoader.reloadAllocations();
     AllocationConfiguration queueConf = confHolder.allocConf;
 
-    assertEquals(5, queueConf.getQueueNames().size());
+    assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
     assertEquals(Resources.createResource(0),
         queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
     assertEquals(Resources.createResource(0),

Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Mon May 12 12:43:59 2014
@@ -34,6 +34,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -43,7 +44,6 @@ import java.util.Set;
 import javax.xml.parsers.ParserConfigurationException;
 
 import org.junit.Assert;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
@@ -737,8 +738,11 @@ public class TestFairScheduler {
     rules.add(new QueuePlacementRule.Default().initialize(true, null));
     Set<String> queues = Sets.newHashSet("root.user1", "root.user3group",
         "root.user4subgroup1", "root.user4subgroup2" , "root.user5subgroup2");
+    Map<FSQueueType, Set<String>> configuredQueues = new HashMap<FSQueueType,
Set<String>>();
+    configuredQueues.put(FSQueueType.LEAF, queues);
+    configuredQueues.put(FSQueueType.PARENT, new HashSet<String>());
     scheduler.getAllocationConfiguration().placementPolicy =
-        new QueuePlacementPolicy(rules, queues, conf);
+        new QueuePlacementPolicy(rules, configuredQueues, conf);
     appId = createSchedulingRequest(1024, "somequeue", "user1");
     assertEquals("root.somequeue", scheduler.getSchedulerApp(appId).getQueueName());
     appId = createSchedulingRequest(1024, "default", "user1");
@@ -758,7 +762,7 @@ public class TestFairScheduler {
     rules.add(new QueuePlacementRule.Specified().initialize(true, null));
     rules.add(new QueuePlacementRule.Default().initialize(true, null));
     scheduler.getAllocationConfiguration().placementPolicy =
-        new QueuePlacementPolicy(rules, queues, conf);
+        new QueuePlacementPolicy(rules, configuredQueues, conf);
     appId = createSchedulingRequest(1024, "somequeue", "user1");
     assertEquals("root.user1", scheduler.getSchedulerApp(appId).getQueueName());
     appId = createSchedulingRequest(1024, "somequeue", "otheruser");
@@ -809,7 +813,89 @@ public class TestFairScheduler {
       }
     }
   }
+  
+  @Test
+  public void testNestedUserQueue() throws IOException {
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+    conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+        SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"user1group\" type=\"parent\">");
+    out.println("<minResources>1024mb,0vcores</minResources>");
+    out.println("</queue>");
+    out.println("<queuePlacementPolicy>");
+    out.println("<rule name=\"specified\" create=\"false\" />");
+    out.println("<rule name=\"nestedUserQueue\">");
+    out.println("     <rule name=\"primaryGroup\" create=\"false\" />");
+    out.println("</rule>");
+    out.println("<rule name=\"default\" />");
+    out.println("</queuePlacementPolicy>");
+    out.println("</allocations>");
+    out.close();
+
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+    RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
+
+    FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default",
+        "user1");
+
+    assertEquals("root.user1group.user1", user1Leaf.getName());
+  }
+
+  @Test
+  public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception {
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
 
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"parentq\" type=\"parent\">");
+    out.println("<minResources>1024mb,0vcores</minResources>");
+    out.println("</queue>");
+    out.println("<queuePlacementPolicy>");
+    out.println("<rule name=\"nestedUserQueue\">");
+    out.println("     <rule name=\"specified\" create=\"false\" />");
+    out.println("</rule>");
+    out.println("<rule name=\"default\" />");
+    out.println("</queuePlacementPolicy>");
+    out.println("</allocations>");
+    out.close();
+
+    RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
+    RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
+
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    int capacity = 16 * 1024;
+    // create node with 16 G
+    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(capacity),
+        1, "127.0.0.1");
+    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+    scheduler.handle(nodeEvent1);
+
+    // user1,user2 submit their apps to parentq and create user queues
+    scheduler.assignToQueue(rmApp1, "root.parentq", "user1");
+    scheduler.assignToQueue(rmApp2, "root.parentq", "user2");
+
+    scheduler.update();
+
+    Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager()
+        .getLeafQueues();
+
+    for (FSLeafQueue leaf : leafQueues) {
+      if (leaf.getName().equals("root.parentq.user1")
+          || leaf.getName().equals("root.parentq.user2")) {
+        // assert that the fair share is 1/4th node1's capacity
+        assertEquals(capacity / 4, leaf.getFairShare().getMemory());
+        // assert weights are equal for both the user queues
+        assertEquals(1.0, leaf.getWeights().getWeight(ResourceType.MEMORY), 0);
+      }
+    }
+  }
+  
   /**
    * Make allocation requests and ensure they are reflected in queue demand.
    */

Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
(original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
Mon May 12 12:43:59 2014
@@ -17,8 +17,7 @@
 */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.*;
 import static org.mockito.Mockito.*;
 
 import java.util.HashSet;
@@ -57,45 +56,77 @@ public class TestQueueManager {
   
   @Test
   public void testReloadTurnsLeafQueueIntoParent() throws Exception {
-    updateConfiguredQueues(queueManager, "queue1");
+    updateConfiguredLeafQueues(queueManager, "queue1");
     
     // When no apps are running in the leaf queue, should be fine turning it
     // into a parent.
-    updateConfiguredQueues(queueManager, "queue1.queue2");
+    updateConfiguredLeafQueues(queueManager, "queue1.queue2");
     assertNull(queueManager.getLeafQueue("queue1", false));
     assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
     
     // When leaf queues are empty, should be ok deleting them and
     // turning parent into a leaf.
-    updateConfiguredQueues(queueManager, "queue1");
+    updateConfiguredLeafQueues(queueManager, "queue1");
     assertNull(queueManager.getLeafQueue("queue1.queue2", false));
     assertNotNull(queueManager.getLeafQueue("queue1", false));
     
     // When apps exist in leaf queue, we shouldn't be able to create
     // children under it, but things should work otherwise.
     notEmptyQueues.add(queueManager.getLeafQueue("queue1", false));
-    updateConfiguredQueues(queueManager, "queue1.queue2");
+    updateConfiguredLeafQueues(queueManager, "queue1.queue2");
     assertNull(queueManager.getLeafQueue("queue1.queue2", false));
     assertNotNull(queueManager.getLeafQueue("queue1", false));
     
     // When apps exist in leaf queues under a parent queue, shouldn't be
     // able to turn it into a leaf queue, but things should work otherwise.
     notEmptyQueues.clear();
-    updateConfiguredQueues(queueManager, "queue1.queue2");
+    updateConfiguredLeafQueues(queueManager, "queue1.queue2");
     notEmptyQueues.add(queueManager.getQueue("root.queue1"));
-    updateConfiguredQueues(queueManager, "queue1");
+    updateConfiguredLeafQueues(queueManager, "queue1");
     assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
     assertNull(queueManager.getLeafQueue("queue1", false));
     
     // Should never to be able to create a queue under the default queue
-    updateConfiguredQueues(queueManager, "default.queue3");
+    updateConfiguredLeafQueues(queueManager, "default.queue3");
     assertNull(queueManager.getLeafQueue("default.queue3", false));
     assertNotNull(queueManager.getLeafQueue("default", false));
   }
   
-  private void updateConfiguredQueues(QueueManager queueMgr, String... confQueues) {
+  @Test
+  public void testReloadTurnsLeafToParentWithNoLeaf() {
+    AllocationConfiguration allocConf = new AllocationConfiguration(conf);
+    // Create a leaf queue1
+    allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.queue1");
+    queueManager.updateAllocationConfiguration(allocConf);
+    assertNotNull(queueManager.getLeafQueue("root.queue1", false));
+
+    // Lets say later on admin makes queue1 a parent queue by
+    // specifying "type=parent" in the alloc xml and lets say apps running in
+    // queue1
+    notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false));
+    allocConf = new AllocationConfiguration(conf);
+    allocConf.configuredQueues.get(FSQueueType.PARENT)
+        .add("root.queue1");
+
+    // When allocs are reloaded queue1 shouldn't be converter to parent
+    queueManager.updateAllocationConfiguration(allocConf);
+    assertNotNull(queueManager.getLeafQueue("root.queue1", false));
+    assertNull(queueManager.getParentQueue("root.queue1", false));
+
+    // Now lets assume apps completed and there are no apps in queue1
+    notEmptyQueues.clear();
+    // We should see queue1 transform from leaf queue to parent queue.
+    queueManager.updateAllocationConfiguration(allocConf);
+    assertNull(queueManager.getLeafQueue("root.queue1", false));
+    assertNotNull(queueManager.getParentQueue("root.queue1", false));
+    // this parent should not have any children
+    assertTrue(queueManager.getParentQueue("root.queue1", false)
+        .getChildQueues().isEmpty());
+  }
+  
+  private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues)
{
     AllocationConfiguration allocConf = new AllocationConfiguration(conf);
-    allocConf.queueNames = Sets.newHashSet(confQueues);
+    allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
     queueMgr.updateAllocationConfiguration(allocConf);
   }
 }

Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
(original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
Mon May 12 12:43:59 2014
@@ -17,8 +17,11 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
 
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
@@ -28,16 +31,15 @@ import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 
-import com.google.common.collect.Sets;
-
 public class TestQueuePlacementPolicy {
   private final static Configuration conf = new Configuration();
-  private final static Set<String> configuredQueues = Sets.newHashSet("root.someuser");
+  private Map<FSQueueType, Set<String>> configuredQueues;
   
   @BeforeClass
   public static void setup() {
@@ -45,6 +47,14 @@ public class TestQueuePlacementPolicy {
         SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
   }
   
+  @Before
+  public void initTest() {
+    configuredQueues = new HashMap<FSQueueType, Set<String>>();
+    for (FSQueueType type : FSQueueType.values()) {
+      configuredQueues.put(type, new HashSet<String>());
+    }
+  }
+
   @Test
   public void testSpecifiedUserPolicy() throws Exception {
     StringBuffer sb = new StringBuffer();
@@ -53,9 +63,12 @@ public class TestQueuePlacementPolicy {
     sb.append("  <rule name='user' />");
     sb.append("</queuePlacementPolicy>");
     QueuePlacementPolicy policy = parse(sb.toString());
-    assertEquals("root.specifiedq",policy.assignAppToQueue("specifiedq", "someuser"));
-    assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser"));
-    assertEquals("root.otheruser", policy.assignAppToQueue("default", "otheruser"));
+    assertEquals("root.specifiedq",
+        policy.assignAppToQueue("specifiedq", "someuser"));
+    assertEquals("root.someuser",
+        policy.assignAppToQueue("default", "someuser"));
+    assertEquals("root.otheruser",
+        policy.assignAppToQueue("default", "otheruser"));
   }
   
   @Test
@@ -66,6 +79,8 @@ public class TestQueuePlacementPolicy {
     sb.append("  <rule name='user' create=\"false\" />");
     sb.append("  <rule name='default' />");
     sb.append("</queuePlacementPolicy>");
+    
+    configuredQueues.get(FSQueueType.LEAF).add("root.someuser");
     QueuePlacementPolicy policy = parse(sb.toString());
     assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser"));
     assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser"));
@@ -81,7 +96,8 @@ public class TestQueuePlacementPolicy {
     sb.append("  <rule name='reject' />");
     sb.append("</queuePlacementPolicy>");
     QueuePlacementPolicy policy = parse(sb.toString());
-    assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser"));
+    assertEquals("root.specifiedq",
+        policy.assignAppToQueue("specifiedq", "someuser"));
     assertEquals(null, policy.assignAppToQueue("default", "someuser"));
   }
   
@@ -117,10 +133,188 @@ public class TestQueuePlacementPolicy {
     parse(sb.toString());
   }
   
+  @Test
+  public void testNestedUserQueueParsingErrors() {
+    // No nested rule specified in hierarchical user queue
+    StringBuffer sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='specified' />");
+    sb.append("  <rule name='nestedUserQueue'/>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    assertIfExceptionThrown(sb);
+
+    // Specified nested rule is not a QueuePlacementRule
+    sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='specified' />");
+    sb.append("  <rule name='nestedUserQueue'>");
+    sb.append("       <rule name='unknownRule'/>");
+    sb.append("  </rule>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    assertIfExceptionThrown(sb);
+  }
+
+  private void assertIfExceptionThrown(StringBuffer sb) {
+    Throwable th = null;
+    try {
+      parse(sb.toString());
+    } catch (Exception e) {
+      th = e;
+    }
+
+    assertTrue(th instanceof AllocationConfigurationException);
+  }
+
+  @Test
+  public void testNestedUserQueueParsing() throws Exception {
+    StringBuffer sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='specified' />");
+    sb.append("  <rule name='nestedUserQueue'>");
+    sb.append("       <rule name='primaryGroup'/>");
+    sb.append("  </rule>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    Throwable th = null;
+    try {
+      parse(sb.toString());
+    } catch (Exception e) {
+      th = e;
+    }
+
+    assertNull(th);
+  }
+
+  @Test
+  public void testNestedUserQueuePrimaryGroup() throws Exception {
+    StringBuffer sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='specified' create='false' />");
+    sb.append("  <rule name='nestedUserQueue'>");
+    sb.append("       <rule name='primaryGroup'/>");
+    sb.append("  </rule>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    // User queue would be created under primary group queue
+    QueuePlacementPolicy policy = parse(sb.toString());
+    assertEquals("root.user1group.user1",
+        policy.assignAppToQueue("root.default", "user1"));
+    // Other rules above and below hierarchical user queue rule should work as
+    // usual
+    configuredQueues.get(FSQueueType.LEAF).add("root.specifiedq");
+    // test if specified rule(above nestedUserQueue rule) works ok
+    assertEquals("root.specifiedq",
+        policy.assignAppToQueue("root.specifiedq", "user2"));
+
+    // test if default rule(below nestedUserQueue rule) works
+    configuredQueues.get(FSQueueType.LEAF).add("root.user3group");
+    assertEquals("root.default",
+        policy.assignAppToQueue("root.default", "user3"));
+  }
+
+  @Test
+  public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception {
+    // Primary group rule has create='false'
+    StringBuffer sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='nestedUserQueue'>");
+    sb.append("       <rule name='primaryGroup' create='false'/>");
+    sb.append("  </rule>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    QueuePlacementPolicy policy = parse(sb.toString());
+
+    // Should return root.default since primary group 'root.user1group' is not
+    // configured
+    assertEquals("root.default",
+        policy.assignAppToQueue("root.default", "user1"));
+
+    // Let's configure primary group and check if user queue is created
+    configuredQueues.get(FSQueueType.PARENT).add("root.user1group");
+    policy = parse(sb.toString());
+    assertEquals("root.user1group.user1",
+        policy.assignAppToQueue("root.default", "user1"));
+
+    // Both Primary group and nestedUserQueue rule has create='false'
+    sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='nestedUserQueue' create='false'>");
+    sb.append("       <rule name='primaryGroup' create='false'/>");
+    sb.append("  </rule>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    // Should return root.default since primary group and user queue for user 2
+    // are not configured.
+    assertEquals("root.default",
+        policy.assignAppToQueue("root.default", "user2"));
+
+    // Now configure both primary group and the user queue for user2
+    configuredQueues.get(FSQueueType.PARENT).add("root.user2group");
+    configuredQueues.get(FSQueueType.LEAF).add("root.user2group.user2");
+    policy = parse(sb.toString());
+
+    assertEquals("root.user2group.user2",
+        policy.assignAppToQueue("root.default", "user2"));
+  }
+
+  @Test
+  public void testNestedUserQueueSecondaryGroup() throws Exception {
+    StringBuffer sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='nestedUserQueue'>");
+    sb.append("       <rule name='secondaryGroupExistingQueue'/>");
+    sb.append("  </rule>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    QueuePlacementPolicy policy = parse(sb.toString());
+    // Should return root.default since secondary groups are not configured
+    assertEquals("root.default",
+        policy.assignAppToQueue("root.default", "user1"));
+
+    // configure secondary group for user1
+    configuredQueues.get(FSQueueType.PARENT).add("root.user1subgroup1");
+    policy = parse(sb.toString());
+    // user queue created should be created under secondary group
+    assertEquals("root.user1subgroup1.user1",
+        policy.assignAppToQueue("root.default", "user1"));
+  }
+
+  @Test
+  public void testNestedUserQueueSpecificRule() throws Exception {
+    // This test covers the use case where users can specify different parent
+    // queues and want user queues under those.
+    StringBuffer sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='nestedUserQueue'>");
+    sb.append("       <rule name='specified' create='false'/>");
+    sb.append("  </rule>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    // Let's create couple of parent queues
+    configuredQueues.get(FSQueueType.PARENT).add("root.parent1");
+    configuredQueues.get(FSQueueType.PARENT).add("root.parent2");
+
+    QueuePlacementPolicy policy = parse(sb.toString());
+    assertEquals("root.parent1.user1",
+        policy.assignAppToQueue("root.parent1", "user1"));
+    assertEquals("root.parent2.user2",
+        policy.assignAppToQueue("root.parent2", "user2"));
+  }
+  
   private QueuePlacementPolicy parse(String str) throws Exception {
     // Read and parse the allocations file.
-    DocumentBuilderFactory docBuilderFactory =
-      DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
+        .newInstance();
     docBuilderFactory.setIgnoringComments(true);
     DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
     Document doc = builder.parse(IOUtils.toInputStream(str));

Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
(original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
Mon May 12 12:43:59 2014
@@ -37,24 +37,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>commons-el</groupId>
-          <artifactId>commons-el</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>tomcat</groupId>
-          <artifactId>jasper-runtime</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>tomcat</groupId>
-          <artifactId>jasper-compiler</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.mortbay.jetty</groupId>
-          <artifactId>jsp-2.1-jetty</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>

Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
(original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
Mon May 12 12:43:59 2014
@@ -43,24 +43,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>commons-el</groupId>
-          <artifactId>commons-el</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>tomcat</groupId>
-          <artifactId>jasper-runtime</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>tomcat</groupId>
-          <artifactId>jasper-compiler</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.mortbay.jetty</groupId>
-          <artifactId>jsp-2.1-jetty</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>

Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Mon May 12 12:43:59 2014
@@ -206,8 +206,10 @@ Allocation file format
   The allocation file must be in XML format. The format contains five types of
   elements:
 
- * <<Queue elements>>, which represent queues. Each may contain the following
-     properties:
+ * <<Queue elements>>, which represent queues. Queue elements can take an optional
+   attribute ’type’,which when set to ‘parent’ makes it a parent queue.
This is useful
+   when we want to create a parent queue without configuring any leaf queues.
+   Each queue element may contain the following properties:
 
    * minResources: minimum resources the queue is entitled to, in the form
      "X mb, Y vcores". For the single-resource fairness policy, the vcores
@@ -299,6 +301,15 @@ Allocation file format
        that matches a secondary group of the user who submitted it. The first
        secondary group that matches a configured queue will be selected.
 
+     * nestedUserQueue : the app is placed into a queue with the name of the user
+       under the queue suggested by the nested rule. This is similar to ‘user’
+       rule,the difference being in ‘nestedUserQueue’ rule,user queues can be created

+       under any parent queue, while ‘user’ rule creates user queues only under
root queue.
+       Note that nestedUserQueue rule would be applied only if the nested rule returns a

+       parent queue.One can configure a parent queue either by setting ‘type’ attribute
of queue
+       to ‘parent’ or by configuring at least one leaf under that queue which makes
it a parent.
+       See example allocation for a sample use case. 
+
      * default: the app is placed into the queue named "default".
 
      * reject: the app is rejected.
@@ -319,6 +330,12 @@ Allocation file format
       <minResources>5000 mb,0vcores</minResources>
     </queue>
   </queue>
+
+  <!—- Queue ‘secondary_group_queue’ is a parent queue and may have
+       user queues under it -—>
+  <queue name=“secondary_group_queue” type=“parent”>
+  <weight>3.0</weight>
+  </queue>
   
   <user name="sample_user">
     <maxRunningApps>30</maxRunningApps>
@@ -328,6 +345,9 @@ Allocation file format
   <queuePlacementPolicy>
     <rule name="specified" />
     <rule name="primaryGroup" create="false" />
+    <rule name=“nestedUserQueue”>
+        <rule name=“secondaryGroupExistingQueue” create=“false” />
+    </rule>
     <rule name="default" />
   </queuePlacementPolicy>
 </allocations>



Mime
View raw message