From common-commits-return-97871-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Mon Jan 13 16:23:28 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 9E00B18064E for ; Mon, 13 Jan 2020 17:23:28 +0100 (CET) Received: (qmail 54822 invoked by uid 500); 13 Jan 2020 16:23:27 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 54813 invoked by uid 99); 13 Jan 2020 16:23:27 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Jan 2020 16:23:27 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 6882381F11; Mon, 13 Jan 2020 16:23:27 +0000 (UTC) Date: Mon, 13 Jan 2020 16:23:27 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] branch trunk updated: YARN-9912. Capacity scheduler: support u:user2:%secondary_group queue mapping. Contributed by Manikandan R MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <157893260687.31270.9942196063197042963@gitbox.apache.org> From: snemeth@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/trunk X-Git-Reftype: branch X-Git-Oldrev: b7ef8a333f6717735caeee09ae4ce67bff7285e3 X-Git-Newrev: 621c5eac38697755cf29aa8618869c09092b6a80 X-Git-Rev: 621c5eac38697755cf29aa8618869c09092b6a80 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 621c5ea YARN-9912. Capacity scheduler: support u:user2:%secondary_group queue mapping. Contributed by Manikandan R 621c5ea is described below commit 621c5eac38697755cf29aa8618869c09092b6a80 Author: Szilard Nemeth AuthorDate: Mon Jan 13 17:23:00 2020 +0100 YARN-9912. Capacity scheduler: support u:user2:%secondary_group queue mapping. Contributed by Manikandan R --- .../placement/UserGroupMappingPlacementRule.java | 20 +++-- .../TestCapacitySchedulerAutoCreatedQueueBase.java | 9 ++- .../TestCapacitySchedulerQueueMappingFactory.java | 85 ++++++++++++++++++++++ 3 files changed, 103 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index de80410..5221ace 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -168,6 +168,11 @@ public class UserGroupMappingPlacementRule extends PlacementRule { break; } } + + if (secondaryGroup == null && LOG.isDebugEnabled()) { + LOG.debug("User {} is not associated with any Secondary " + + "Group. Hence it may use the 'default' queue", user); + } return secondaryGroup; } @@ -198,10 +203,6 @@ public class UserGroupMappingPlacementRule extends PlacementRule { validateQueueMapping(queueMapping); return getPlacementContext(queueMapping, user); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("User {} is not associated with any Secondary Group. " - + "Hence it may use the 'default' queue", user); - } return null; } } else if (mapping.queue.equals(CURRENT_USER_MAPPING)) { @@ -219,10 +220,6 @@ public class UserGroupMappingPlacementRule extends PlacementRule { if (secondaryGroup != null) { return getPlacementContext(mapping, secondaryGroup); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("User {} is not associated with any Secondary " - + "Group. Hence it may use the 'default' queue", user); - } return null; } } else { @@ -232,6 +229,13 @@ public class UserGroupMappingPlacementRule extends PlacementRule { if (user.equals(mapping.source)) { if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { return getPlacementContext(mapping, groups.getGroups(user).get(0)); + } else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) { + String secondaryGroup = getSecondaryGroup(user); + if (secondaryGroup != null) { + return getPlacementContext(mapping, secondaryGroup); + } else { + return null; + } } else { return getPlacementContext(mapping); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index 59fbb84..9527e80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -122,6 +122,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { public static final String B1 = B + ".b1"; public static final String B2 = B + ".b2"; public static final String B3 = B + ".b3"; + public static final String B4 = B + ".b4subgroup1"; public static final String ASUBGROUP1_A = ASUBGROUP1 + ".e"; public static final String AGROUP_A = AGROUP + ".f"; public static final float A_CAPACITY = 20f; @@ -135,8 +136,8 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { public static final float A2_CAPACITY = 70; public static final float B1_CAPACITY = 60f; public static final float B2_CAPACITY = 20f; - public static final float B3_CAPACITY = 20f; - + public static final float B3_CAPACITY = 10f; + public static final float B4_CAPACITY = 10f; public static final int NODE_MEMORY = 16; public static final int NODE1_VCORES = 16; @@ -356,13 +357,15 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { conf.setCapacity(A2, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); - conf.setQueues(B, new String[] { "b1", "b2", "b3" }); + conf.setQueues(B, new String[] { "b1", "b2", "b3", "b4subgroup1" }); conf.setCapacity(B1, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); conf.setCapacity(B2, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); + conf.setCapacity(B4, B4_CAPACITY); + conf.setUserLimitFactor(B4, 100.0f); conf.setQueues(ASUBGROUP1, new String[] {"e"}); conf.setCapacity(ASUBGROUP1_A, 100f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java index 4f63b8f..79b6f24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java @@ -236,8 +236,14 @@ public class TestCapacitySchedulerQueueMappingFactory { UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user", getQueueMapping("%secondary_group", "%user")); + // u:b4:%secondary_group + UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, + "b4", "%secondary_group"); queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping2); + queueMappingsForUG.add(userQueueMapping3); testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "f"); @@ -416,4 +422,83 @@ public class TestCapacitySchedulerQueueMappingFactory { } } } + + @Test + public void testFixedUserWithDynamicGroupQueue() throws Exception { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + + List queuePlacementRules = new ArrayList<>(); + queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); + conf.setQueuePlacementRules(queuePlacementRules); + + List existingMappingsForUG = + conf.getQueueMappings(); + + // set queue mapping + List queueMappingsForUG = + new ArrayList<>(); + + // u:user1:b1 + UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, + "user1", "b1"); + + // u:user2:%primary_group + UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, + "user2", "%primary_group"); + + // u:b4:%secondary_group + UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = + new UserGroupMappingPlacementRule.QueueMapping( + UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "b4", + "%secondary_group"); + + queueMappingsForUG.add(userQueueMapping1); + queueMappingsForUG.add(userQueueMapping2); + queueMappingsForUG.add(userQueueMapping3); + existingMappingsForUG.addAll(queueMappingsForUG); + conf.setQueueMappings(existingMappingsForUG); + + //override with queue mappings + conf.setOverrideWithQueueMappings(true); + + MockRM mockRM = null; + try { + mockRM = new MockRM(conf); + CapacityScheduler cs = (CapacityScheduler) mockRM.getResourceScheduler(); + cs.updatePlacementRules(); + mockRM.start(); + cs.start(); + + ApplicationSubmissionContext asc = + Records.newRecord(ApplicationSubmissionContext.class); + asc.setQueue("default"); + + List rules = + cs.getRMContext().getQueuePlacementManager().getPlacementRules(); + UserGroupMappingPlacementRule r = + (UserGroupMappingPlacementRule) rules.get(0); + + ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1"); + assertEquals("Queue", "b1", ctx.getQueue()); + + ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "user2"); + assertEquals("Queue", "user2group", ctx1.getQueue()); + + ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "b4"); + assertEquals("Queue", "b4subgroup1", ctx2.getQueue()); + } finally { + if (mockRM != null) { + mockRM.close(); + } + } + } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org