hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snem...@apache.org
Subject [hadoop] branch trunk updated: YARN-9868. Validate %primary_group queue in CS queue manager. Contributed by Manikandan R
Date Mon, 13 Jan 2020 16:16:19 GMT
This is an automated email from the ASF dual-hosted git repository.

snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new b7ef8a3  YARN-9868. Validate %primary_group queue in CS queue manager. Contributed
by Manikandan R
b7ef8a3 is described below

commit b7ef8a333f6717735caeee09ae4ce67bff7285e3
Author: Szilard Nemeth <snemeth@apache.org>
AuthorDate: Mon Jan 13 17:15:09 2020 +0100

    YARN-9868. Validate %primary_group queue in CS queue manager. Contributed by Manikandan
R
---
 .../placement/UserGroupMappingPlacementRule.java   | 22 ++++++++++++++++------
 .../TestUserGroupMappingPlacementRule.java         | 21 +++++++++++++++++++++
 .../TestCapacitySchedulerAutoCreatedQueueBase.java |  4 ++--
 .../TestCapacitySchedulerQueueMappingFactory.java  |  4 ++--
 4 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index 0caa602..de80410 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -179,11 +179,15 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
           if (mapping.getParentQueue() != null
               && mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
               && mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
-            QueueMapping queueMapping =
-                new QueueMapping(mapping.getType(), mapping.getSource(),
-                    user, groups.getGroups(user).get(0));
-            validateQueueMapping(queueMapping);
-            return getPlacementContext(queueMapping, user);
+            if (this.queueManager
+                .getQueue(groups.getGroups(user).get(0)) != null) {
+              QueueMapping queueMapping = new QueueMapping(mapping.getType(),
+                  mapping.getSource(), user, groups.getGroups(user).get(0));
+              validateQueueMapping(queueMapping);
+              return getPlacementContext(queueMapping, user);
+            } else {
+              return null;
+            }
           } else if (mapping.getParentQueue() != null
               && mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING)
               && mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
@@ -203,7 +207,13 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
           } else if (mapping.queue.equals(CURRENT_USER_MAPPING)) {
             return getPlacementContext(mapping, user);
           } else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
-            return getPlacementContext(mapping, groups.getGroups(user).get(0));
+            if (this.queueManager
+                .getQueue(groups.getGroups(user).get(0)) != null) {
+              return getPlacementContext(mapping,
+                  groups.getGroups(user).get(0));
+            } else {
+              return null;
+            }
           } else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) {
             String secondaryGroup = getSecondaryGroup(user);
             if (secondaryGroup != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
index 7590756..5cd6ea1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.placement;
 
+import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -26,6 +27,7 @@ import java.util.Arrays;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.security.GroupMappingServiceProvider;
 import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.NullGroupsMapping;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -90,6 +92,7 @@ public class TestUserGroupMappingPlacementRule {
     when(queueManager.getQueue("agroup")).thenReturn(agroup);
     when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
     when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
+
     rule.setQueueManager(queueManager);
     ApplicationSubmissionContext asc = Records.newRecord(
         ApplicationSubmissionContext.class);
@@ -119,6 +122,20 @@ public class TestUserGroupMappingPlacementRule {
   }
 
   @Test
+  public void testNullGroupMapping() throws YarnException {
+    conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+        NullGroupsMapping.class, GroupMappingServiceProvider.class);
+    try {
+      verifyQueueMapping(
+          new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
+          "default");
+      fail("No Groups for user 'a'");
+    } catch (YarnException e) {
+      // Exception is expected as there are no groups for given user
+    }
+  }
+
+  @Test
   public void testMapping() throws YarnException {
 
     verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1");
@@ -131,6 +148,10 @@ public class TestUserGroupMappingPlacementRule {
     verifyQueueMapping(
         new QueueMapping(MappingType.USER, "%user", "%primary_group"), "a",
         "agroup");
+    // Queue "bgroup" is not configured, hence "default" should be used
+    verifyQueueMapping(
+        new QueueMapping(MappingType.USER, "%user", "%primary_group"), "b",
+        "default");
     verifyQueueMapping(
         new QueueMapping(MappingType.USER, "%user", "%user", "%primary_group"),
         "a", YarnConfiguration.DEFAULT_QUEUE_NAME, "a", false, "agroup");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index c422dfd..59fbb84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -339,8 +339,8 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
     // Define top-level queues
     // Set childQueue for root
     conf.setQueues(ROOT,
-        new String[] {"a", "b", "c", "d", "esubgroup1", "asubgroup2",
-            "fgroup"});
+        new String[] {"a", "b", "c", "d", "esubgroup1", "esubgroup2", "fgroup",
+            "a1group", "ggroup", "g"});
 
     conf.setCapacity(A, A_CAPACITY);
     conf.setCapacity(B, B_CAPACITY);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
index 4cec544..4f63b8f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
@@ -242,8 +242,8 @@ public class TestCapacitySchedulerQueueMappingFactory {
     testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "f");
 
     try {
-      testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "h");
-      fail("Leaf Queue 'h' doesn't exists");
+      testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "g");
+      fail("Queue 'g' exists, but type is not Leaf Queue");
     } catch (YarnException e) {
       // Exception is expected as there is no such leaf queue
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message