Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E9AEA9278 for ; Sun, 4 Mar 2012 17:16:44 +0000 (UTC) Received: (qmail 57838 invoked by uid 500); 4 Mar 2012 17:16:44 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 57785 invoked by uid 500); 4 Mar 2012 17:16:44 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 57777 invoked by uid 99); 4 Mar 2012 17:16:44 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 04 Mar 2012 17:16:44 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 04 Mar 2012 17:16:41 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E27BB23888D2; Sun, 4 Mar 2012 17:16:19 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1296836 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ hadoop-yarn/had... Date: Sun, 04 Mar 2012 17:16:19 -0000 To: mapreduce-commits@hadoop.apache.org From: acmurthy@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120304171619.E27BB23888D2@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: acmurthy Date: Sun Mar 4 17:16:19 2012 New Revision: 1296836 URL: http://svn.apache.org/viewvc?rev=1296836&view=rev Log: Merge -c 1296835 from trunk to branch-0.23 to fix MAPREDUCE-3929. Fixed output of 'bin/mapred queue -showacl' command to clarify ACLs for users. Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1296836&r1=1296835&r2=1296836&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Sun Mar 4 17:16:19 2012 @@ -167,6 +167,9 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3792. Fix "bin/mapred job -list" to display all jobs instead of only the jobs owned by the user. (Jason Lowe via vinodkv) + MAPREDUCE-3929. Fixed output of 'bin/mapred queue -showacl' command to + clarify ACLs for users. (John George via acmurthy) + Release 0.23.1 - 2012-02-17 NEW FEATURES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1296836&r1=1296835&r2=1296836&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Sun Mar 4 17:16:19 2012 @@ -317,15 +317,12 @@ public class ParentQueue implements CSQu QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(QueueUserACLInfo.class); List operations = new ArrayList(); - for (Map.Entry e : acls.entrySet()) { - QueueACL operation = e.getKey(); - AccessControlList acl = e.getValue(); - - if (acl.isUserAllowed(user)) { + for (QueueACL operation : QueueACL.values()) { + if (hasAccess(operation, user)) { operations.add(operation); - } + } } - + userAclInfo.setQueueName(getQueueName()); userAclInfo.setUserAcls(operations); return userAclInfo; @@ -343,6 +340,7 @@ public class ParentQueue implements CSQu for (CSQueue child : childQueues) { userAcls.addAll(child.getQueueUserAclInfo(user)); } + return userAcls; } Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1296836&r1=1296835&r2=1296836&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Sun Mar 4 17:16:19 2012 @@ -24,6 +24,8 @@ import static org.mockito.Mockito.*; import java.util.HashMap; import java.util.Map; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.Resource; @@ -32,6 +34,9 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.junit.After; import org.junit.Before; @@ -255,6 +260,11 @@ public class TestParentQueue { } private static final String C = "c"; + private static final String C1 = "c1"; + private static final String C11 = "c11"; + private static final String C111 = "c111"; + private static final String C1111 = "c1111"; + private static final String D = "d"; private static final String A1 = "a1"; private static final String A2 = "a2"; @@ -265,7 +275,7 @@ public class TestParentQueue { private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; @@ -289,9 +299,24 @@ public class TestParentQueue { conf.setCapacity(Q_B + "." + B1, 10); conf.setCapacity(Q_B + "." + B2, 20); conf.setCapacity(Q_B + "." + B3, 70); - } + conf.setQueues(Q_C, new String[] {C1}); + final String Q_C1= Q_C + "." + C1; + conf.setCapacity(Q_C1, 100); + conf.setQueues(Q_C1, new String[] {C11}); + + final String Q_C11= Q_C1 + "." + C11; + conf.setCapacity(Q_C11, 100); + conf.setQueues(Q_C11, new String[] {C111}); + + final String Q_C111= Q_C11 + "." + C111; + conf.setCapacity(Q_C111, 100); + //Leaf Queue + conf.setQueues(Q_C111, new String[] {C1111}); + final String Q_C1111= Q_C111 + "." + C1111; + conf.setCapacity(Q_C1111, 100); + } @Test public void testMultiLevelQueues() throws Exception { @@ -470,6 +495,79 @@ public class TestParentQueue { } + + public boolean hasQueueACL(List aclInfos, QueueACL acl, String qName) { + for (QueueUserACLInfo aclInfo : aclInfos) { + if (aclInfo.getQueueName().equals(qName)) { + if (aclInfo.getUserAcls().contains(acl)) { + return true; + } + } + } + return false; + } + + @Test + public void testQueueAcl() throws Exception { + + setupMultiLevelQueues(csConf); + csConf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); + csConf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.ADMINISTER_QUEUE, " "); + + final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; + csConf.setAcl(Q_C, QueueACL.ADMINISTER_QUEUE, "*"); + final String Q_C11= Q_C + "." + C1 + "." + C11; + csConf.setAcl(Q_C11, QueueACL.SUBMIT_APPLICATIONS, "*"); + + Map queues = new HashMap(); + CSQueue root = + CapacityScheduler.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, queues, queues, + CapacityScheduler.queueComparator, + CapacityScheduler.applicationComparator, + TestUtils.spyHook); + + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + // Setup queue configs + ParentQueue c = (ParentQueue)queues.get(C); + ParentQueue c1 = (ParentQueue)queues.get(C1); + ParentQueue c11 = (ParentQueue)queues.get(C11); + ParentQueue c111 = (ParentQueue)queues.get(C111); + + assertFalse(root.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); + List aclInfos = root.getQueueUserAclInfo(user); + assertFalse(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "root")); + + assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertFalse(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "root")); + + // c has no SA, but QA + assertTrue(c.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); + assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "c")); + assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertFalse(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "c")); + + //Queue c1 has QA, no SA (gotten perm from parent) + assertTrue(c1.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); + assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "c1")); + assertFalse(c1.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertFalse(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "c1")); + + //Queue c11 has permissions from parent queue and SA + assertTrue(c11.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); + assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "c11")); + assertTrue(c11.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertTrue(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "c11")); + + //Queue c111 has SA and AQ, both from parent + assertTrue(c111.hasAccess(QueueACL.ADMINISTER_QUEUE, user)); + assertTrue(hasQueueACL(aclInfos, QueueACL.ADMINISTER_QUEUE, "c111")); + assertTrue(c111.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertTrue(hasQueueACL(aclInfos, QueueACL.SUBMIT_APPLICATIONS, "c111")); + + reset(c); + } + @After public void tearDown() throws Exception { }