hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sa...@apache.org
Subject svn commit: r1545157 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ hadoop-yarn/hadoop-yarn-server/hadoop-y...
Date Mon, 25 Nov 2013 06:04:26 GMT
Author: sandy
Date: Mon Nov 25 06:04:26 2013
New Revision: 1545157

URL: http://svn.apache.org/r1545157
Log:
YARN-1423. Support queue placement by secondary group in the Fair Scheduler (Ted Malaska via
Sandy Ryza)

Modified:
    hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm

Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1545157&r1=1545156&r2=1545157&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Mon Nov 25 06:04:26 2013
@@ -117,6 +117,9 @@ Release 2.3.0 - UNRELEASED
     YARN-1303. Fixed DistributedShell to not fail with multiple commands separated
     by a semi-colon as shell-command. (Xuan Gong via vinodkv)
 
+    YARN-1423. Support queue placement by secondary group in the Fair Scheduler
+    (Ted Malaska via Sandy Ryza)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java?rev=1545157&r1=1545156&r2=1545157&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
Mon Nov 25 06:04:26 2013
@@ -39,6 +39,8 @@ public class QueuePlacementPolicy {
         new HashMap<String, Class<? extends QueuePlacementRule>>();
     map.put("user", QueuePlacementRule.User.class);
     map.put("primaryGroup", QueuePlacementRule.PrimaryGroup.class);
+    map.put("secondaryGroupExistingQueue",
+        QueuePlacementRule.SecondaryGroupExistingQueue.class);
     map.put("specified", QueuePlacementRule.Specified.class);
     map.put("default", QueuePlacementRule.Default.class);
     map.put("reject", QueuePlacementRule.Reject.class);

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java?rev=1545157&r1=1545156&r2=1545157&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
Mon Nov 25 06:04:26 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
 import java.io.IOException;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.security.Groups;
@@ -58,7 +59,7 @@ public abstract class QueuePlacementRule
    */
   public String assignAppToQueue(String requestedQueue, String user,
       Groups groups, Collection<String> configuredQueues) throws IOException {
-    String queue = getQueueForApp(requestedQueue, user, groups);
+    String queue = getQueueForApp(requestedQueue, user, groups, configuredQueues);
     if (create || configuredQueues.contains(queue)) {
       return queue;
     } else {
@@ -103,7 +104,7 @@ public abstract class QueuePlacementRule
    *    continue to the next rule.
    */
   protected abstract String getQueueForApp(String requestedQueue, String user,
-      Groups groups) throws IOException;
+      Groups groups, Collection<String> configuredQueues) throws IOException;
 
   /**
    * Places apps in queues by username of the submitter
@@ -111,7 +112,7 @@ public abstract class QueuePlacementRule
   public static class User extends QueuePlacementRule {
     @Override
     protected String getQueueForApp(String requestedQueue,
-        String user, Groups groups) {
+        String user, Groups groups, Collection<String> configuredQueues) {
       return "root." + user;
     }
     
@@ -127,7 +128,8 @@ public abstract class QueuePlacementRule
   public static class PrimaryGroup extends QueuePlacementRule {
     @Override
     protected String getQueueForApp(String requestedQueue,
-        String user, Groups groups) throws IOException {
+        String user, Groups groups, 
+        Collection<String> configuredQueues) throws IOException {
       return "root." + groups.getGroups(user).get(0);
     }
     
@@ -136,6 +138,33 @@ public abstract class QueuePlacementRule
       return create;
     }
   }
+  
+  /**
+   * Places apps in queues by secondary group of the submitter
+   * 
+   * Match will be made on first secondary group that exist in
+   * queues
+   */
+  public static class SecondaryGroupExistingQueue extends QueuePlacementRule {
+    @Override
+    protected String getQueueForApp(String requestedQueue,
+        String user, Groups groups, 
+        Collection<String> configuredQueues) throws IOException {
+      List<String> groupNames = groups.getGroups(user);
+      for (int i = 1; i < groupNames.size(); i++) {
+        if (configuredQueues.contains("root." + groupNames.get(i))) {
+          return "root." + groupNames.get(i);
+        }
+      }
+      
+      return "";
+    }
+        
+    @Override
+    public boolean isTerminal() {
+      return create;
+    }
+  }
 
   /**
    * Places apps in queues by requested queue of the submitter
@@ -143,7 +172,7 @@ public abstract class QueuePlacementRule
   public static class Specified extends QueuePlacementRule {
     @Override
     protected String getQueueForApp(String requestedQueue,
-        String user, Groups groups) {
+        String user, Groups groups, Collection<String> configuredQueues) {
       if (requestedQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) {
         return "";
       } else {
@@ -166,7 +195,7 @@ public abstract class QueuePlacementRule
   public static class Default extends QueuePlacementRule {
     @Override
     protected String getQueueForApp(String requestedQueue, String user,
-        Groups groups) {
+        Groups groups, Collection<String> configuredQueues) {
       return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
     }
     
@@ -188,7 +217,7 @@ public abstract class QueuePlacementRule
     
     @Override
     protected String getQueueForApp(String requestedQueue, String user,
-        Groups groups) {
+        Groups groups, Collection<String> configuredQueues) {
       throw new UnsupportedOperationException();
     }
     

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java?rev=1545157&r1=1545156&r2=1545157&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
Mon Nov 25 06:04:26 2013
@@ -28,7 +28,7 @@ public class SimpleGroupsMapping impleme
   
   @Override
   public List<String> getGroups(String user) {
-    return Arrays.asList(user + "group");
+    return Arrays.asList(user + "group", user + "subgroup1", user + "subgroup2");
   }
 
   @Override

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1545157&r1=1545156&r2=1545157&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Mon Nov 25 06:04:26 2013
@@ -682,8 +682,10 @@ public class TestFairScheduler {
     rules.add(new QueuePlacementRule.Specified().initialize(true, null));
     rules.add(new QueuePlacementRule.User().initialize(false, null));
     rules.add(new QueuePlacementRule.PrimaryGroup().initialize(false, null));
+    rules.add(new QueuePlacementRule.SecondaryGroupExistingQueue().initialize(false, null));
     rules.add(new QueuePlacementRule.Default().initialize(true, null));
-    Set<String> queues = Sets.newHashSet("root.user1", "root.user3group");
+    Set<String> queues = Sets.newHashSet("root.user1", "root.user3group", 
+        "root.user4subgroup1", "root.user4subgroup2" , "root.user5subgroup2");
     scheduler.getQueueManager().placementPolicy = new QueuePlacementPolicy(
         rules, queues, conf);
     appId = createSchedulingRequest(1024, "somequeue", "user1");
@@ -692,6 +694,10 @@ public class TestFairScheduler {
     assertEquals("root.user1", apps.get(appId).getQueueName());
     appId = createSchedulingRequest(1024, "default", "user3");
     assertEquals("root.user3group", apps.get(appId).getQueueName());
+    appId = createSchedulingRequest(1024, "default", "user4");
+    assertEquals("root.user4subgroup1", apps.get(appId).getQueueName());
+    appId = createSchedulingRequest(1024, "default", "user5");
+    assertEquals("root.user5subgroup2", apps.get(appId).getQueueName());
     appId = createSchedulingRequest(1024, "default", "otheruser");
     assertEquals("root.default", apps.get(appId).getQueueName());
     

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1545157&r1=1545156&r2=1545157&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Mon Nov 25 06:04:26 2013
@@ -287,6 +287,10 @@ Allocation file format
      * primaryGroup: the app is placed into a queue with the name of the
        primary group of the user who submitted it.
 
+     * secondaryGroupExistingQueue: the app is placed into a queue with a name
+       that matches a secondary group of the user who submitted it. The first
+       secondary group that matches a configured queue will be selected.
+
      * default: the app is placed into the queue named "default".
 
      * reject: the app is rejected.



Mime
View raw message