Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 45D84944C for ; Tue, 13 Dec 2011 00:20:34 +0000 (UTC) Received: (qmail 95676 invoked by uid 500); 13 Dec 2011 00:20:34 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 95644 invoked by uid 500); 13 Dec 2011 00:20:34 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 95636 invoked by uid 99); 13 Dec 2011 00:20:34 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Dec 2011 00:20:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Dec 2011 00:20:24 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 78E7C2388860; Tue, 13 Dec 2011 00:20:02 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1213511 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ hadoop-yarn/hadoop-yarn-server... Date: Tue, 13 Dec 2011 00:20:02 -0000 To: mapreduce-commits@hadoop.apache.org From: mahadev@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111213002002.78E7C2388860@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mahadev Date: Tue Dec 13 00:20:01 2011 New Revision: 1213511 URL: http://svn.apache.org/viewvc?rev=1213511&view=rev Log: MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue -showacls (Jonathan Eagles via mahadev) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1213511&r1=1213510&r2=1213511&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Dec 13 00:20:01 2011 @@ -270,6 +270,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3328. mapred queue -list output inconsistent and missing child queues. (Ravi Prakash via mahadev) + MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue + -showacls (Jonathan Eagles via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1213511&r1=1213510&r2=1213511&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Tue Dec 13 00:20:01 2011 @@ -198,19 +198,33 @@ public class CapacitySchedulerConfigurat private static String getAclKey(QueueACL acl) { return "acl_" + acl.toString().toLowerCase(); } - + + public AccessControlList getAcl(String queue, QueueACL acl) { + String queuePrefix = getQueuePrefix(queue); + String aclString = get(queuePrefix + getAclKey(acl), DEFAULT_ACL); + return new AccessControlList(aclString); + } + + public void setAcl(String queue, QueueACL acl, String aclString) { + String queuePrefix = getQueuePrefix(queue); + set(queuePrefix + getAclKey(acl), aclString); + } + public Map getAcls(String queue) { - Map acls = + Map acls = new HashMap(); - String queuePrefix = getQueuePrefix(queue); for (QueueACL acl : QueueACL.values()) { - acls.put(acl, - new AccessControlList(get(queuePrefix + getAclKey(acl), - DEFAULT_ACL))); + acls.put(acl, getAcl(queue, acl)); } return acls; } + public void setAcls(String queue, Map acls) { + for (Map.Entry e : acls.entrySet()) { + setAcl(queue, e.getKey(), e.getValue().getAclString()); + } + } + public String[] getQueues(String queue) { LOG.info("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue)); String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1213511&r1=1213510&r2=1213511&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Tue Dec 13 00:20:01 2011 @@ -492,11 +492,8 @@ public class LeafQueue implements CSQueu QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(QueueUserACLInfo.class); List operations = new ArrayList(); - for (Map.Entry e : acls.entrySet()) { - QueueACL operation = e.getKey(); - AccessControlList acl = e.getValue(); - - if (acl.isUserAllowed(user)) { + for (QueueACL operation : QueueACL.values()) { + if (hasAccess(operation, user)) { operations.add(operation); } } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1213511&r1=1213510&r2=1213511&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Dec 13 00:20:01 2011 @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re import static org.junit.Assert.*; import static org.mockito.Mockito.*; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -30,11 +31,14 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -102,20 +106,29 @@ public class TestLeafQueue { private static final String A = "a"; private static final String B = "b"; + private static final String C = "c"; private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B}); + conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C}); conf.setCapacity(CapacityScheduler.ROOT, 100); conf.setMaximumCapacity(CapacityScheduler.ROOT, 100); + conf.setAcl(CapacityScheduler.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); final String Q_A = CapacityScheduler.ROOT + "." + A; - conf.setCapacity(Q_A, 10); + conf.setCapacity(Q_A, 9); conf.setMaximumCapacity(Q_A, 20); + conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*"); final String Q_B = CapacityScheduler.ROOT + "." + B; conf.setCapacity(Q_B, 90); conf.setMaximumCapacity(Q_B, 99); + conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*"); + + final String Q_C = CapacityScheduler.ROOT + "." + C; + conf.setCapacity(Q_C, 1); + conf.setMaximumCapacity(Q_C, 10); + conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " "); LOG.info("Setup top-level queues a and b"); } @@ -167,8 +180,8 @@ public class TestLeafQueue { //can add more sturdy test with 3-layer queues //once MAPREDUCE:3410 is resolved LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); - assertEquals(0.1, a.getCapacity(), epsilon); - assertEquals(0.1, a.getAbsoluteCapacity(), epsilon); + assertEquals(0.09, a.getCapacity(), epsilon); + assertEquals(0.09, a.getAbsoluteCapacity(), epsilon); assertEquals(0.2, a.getMaximumCapacity(), epsilon); assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon); @@ -177,6 +190,12 @@ public class TestLeafQueue { assertEquals(0.9, b.getAbsoluteCapacity(), epsilon); assertEquals(0.99, b.getMaximumCapacity(), epsilon); assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon); + + LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C)); + assertEquals(0.01, c.getCapacity(), epsilon); + assertEquals(0.01, c.getAbsoluteCapacity(), epsilon); + assertEquals(0.1, c.getMaximumCapacity(), epsilon); + assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon); } @Test @@ -1082,6 +1101,37 @@ public class TestLeafQueue { assertEquals(0, app_0.getTotalRequiredResources(priority)); } + + public boolean hasQueueACL(List aclInfos, QueueACL acl) { + for (QueueUserACLInfo aclInfo : aclInfos) { + if (aclInfo.getUserAcls().contains(acl)) { + return true; + } + } + return false; + } + + @Test + public void testInheritedQueueAcls() throws IOException { + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + + LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); + LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B)); + LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C)); + + assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertTrue(a.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertTrue(b.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + + assertTrue(hasQueueACL( + a.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); + assertTrue(hasQueueACL( + b.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); + assertFalse(hasQueueACL( + c.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); + + } @After public void tearDown() throws Exception {