hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sa...@apache.org
Subject svn commit: r1597204 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ hadoop-yarn/hadoop-yarn-server/hadoop-y...
Date Fri, 23 May 2014 22:38:53 GMT
Author: sandy
Date: Fri May 23 22:38:52 2014
New Revision: 1597204

URL: http://svn.apache.org/r1597204
Log:
YARN-2012. Fair Scheduler: allow default queue placement rule to take an arbitrary queue (Ashwin
Shankar via Sandy Ryza)

Modified:
    hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm

Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1597204&r1=1597203&r2=1597204&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Fri May 23 22:38:52 2014
@@ -99,6 +99,9 @@ Release 2.5.0 - UNRELEASED
     YARN-1937. Added owner-only ACLs support for Timeline Client and server.
     (Zhijie Shen via vinodkv)
 
+    YARN-2012. Fair Scheduler: allow default queue placement rule to take an
+    arbitrary queue (Ashwin Shankar via Sandy Ryza)
+
   OPTIMIZATIONS
 
   BUG FIXES 

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java?rev=1597204&r1=1597203&r2=1597204&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
Fri May 23 22:38:52 2014
@@ -276,21 +276,38 @@ public abstract class QueuePlacementRule
   }
   
   /**
-   * Places all apps in the default queue
+   * Places apps in the specified default queue. If no default queue is
+   * specified the app is placed in root.default queue.
    */
   public static class Default extends QueuePlacementRule {
+    private String defaultQueueName;
+
+    @Override
+    public void initializeFromXml(Element el)
+        throws AllocationConfigurationException {
+      defaultQueueName = el.getAttribute("queue");
+      if (defaultQueueName != null && !defaultQueueName.isEmpty()) {
+        if (!defaultQueueName.startsWith("root.")) {
+          defaultQueueName = "root." + defaultQueueName;
+        }
+      } else {
+        defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
+      }
+      super.initializeFromXml(el);
+    }
+
     @Override
     protected String getQueueForApp(String requestedQueue, String user,
         Groups groups, Map<FSQueueType, Set<String>> configuredQueues) {
-      return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
+      return defaultQueueName;
     }
-    
+
     @Override
     public boolean isTerminal() {
       return true;
     }
   }
-  
+
   /**
    * Rejects all apps
    */

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java?rev=1597204&r1=1597203&r2=1597204&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
Fri May 23 22:38:52 2014
@@ -134,6 +134,22 @@ public class TestQueuePlacementPolicy {
   }
   
   @Test
+  public void testDefaultRuleWithQueueAttribute() throws Exception {
+    // This test covers the use case where we would like default rule
+    // to point to a different queue by default rather than root.default
+    configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue");
+    StringBuffer sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='specified' create='false' />");
+    sb.append("  <rule name='default' queue='root.someDefaultQueue'/>");
+    sb.append("</queuePlacementPolicy>");
+
+    QueuePlacementPolicy policy = parse(sb.toString());
+    assertEquals("root.someDefaultQueue",
+        policy.assignAppToQueue("root.default", "user1"));
+  }
+  
+  @Test
   public void testNestedUserQueueParsingErrors() {
     // No nested rule specified in hierarchical user queue
     StringBuffer sb = new StringBuffer();
@@ -311,6 +327,25 @@ public class TestQueuePlacementPolicy {
         policy.assignAppToQueue("root.parent2", "user2"));
   }
   
+  @Test
+  public void testNestedUserQueueDefaultRule() throws Exception {
+    // This test covers the use case where we would like user queues to be
+    // created under a default parent queue
+    configuredQueues.get(FSQueueType.PARENT).add("root.parentq");
+    StringBuffer sb = new StringBuffer();
+    sb.append("<queuePlacementPolicy>");
+    sb.append("  <rule name='specified' create='false' />");
+    sb.append("  <rule name='nestedUserQueue'>");
+    sb.append("       <rule name='default' queue='root.parentq'/>");
+    sb.append("  </rule>");
+    sb.append("  <rule name='default' />");
+    sb.append("</queuePlacementPolicy>");
+
+    QueuePlacementPolicy policy = parse(sb.toString());
+    assertEquals("root.parentq.user1",
+        policy.assignAppToQueue("root.default", "user1"));
+  }
+  
   private QueuePlacementPolicy parse(String str) throws Exception {
     // Read and parse the allocations file.
     DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1597204&r1=1597203&r2=1597204&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Fri May 23 22:38:52 2014
@@ -310,7 +310,8 @@ Allocation file format
        to ‘parent’ or by configuring at least one leaf under that queue which makes
it a parent.
        See example allocation for a sample use case. 
 
-     * default: the app is placed into the queue named "default".
+     * default: the app is placed into the queue specified in the ‘queue’ attribute
of the 
+       default rule. If ‘queue’ attribute is not specified, the app is placed into
‘root.default’ queue.
 
      * reject: the app is rejected.
 
@@ -348,7 +349,7 @@ Allocation file format
     <rule name=“nestedUserQueue”>
         <rule name=“secondaryGroupExistingQueue” create=“false” />
     </rule>
-    <rule name="default" />
+    <rule name="default" queue=“sample_queue” />
   </queuePlacementPolicy>
 </allocations>
 ---



Mime
View raw message