Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9147218349 for ; Sat, 2 Jan 2016 23:08:16 +0000 (UTC) Received: (qmail 55061 invoked by uid 500); 2 Jan 2016 23:08:15 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 54923 invoked by uid 500); 2 Jan 2016 23:08:15 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 54890 invoked by uid 99); 2 Jan 2016 23:08:15 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 02 Jan 2016 23:08:15 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7A2BAE0492; Sat, 2 Jan 2016 23:08:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: asuresh@apache.org To: common-commits@hadoop.apache.org Date: Sat, 02 Jan 2016 23:08:17 -0000 Message-Id: In-Reply-To: <579fd7567b29486aa27a1ec18b9f087e@git.apache.org> References: <579fd7567b29486aa27a1ec18b9f087e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/5] hadoop git commit: YARN-4522. Queue acl can be checked at app submission. (Jian He via wangda) YARN-4522. Queue acl can be checked at app submission. (Jian He via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8310b2e9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8310b2e9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8310b2e9 Branch: refs/heads/yarn-2877 Commit: 8310b2e9ff3d6804bad703c4c15458b0dfeeb4af Parents: ab725cf Author: Wangda Tan Authored: Wed Dec 30 15:30:12 2015 -0800 Committer: Wangda Tan Committed: Wed Dec 30 15:30:12 2015 -0800 ---------------------------------------------------------------------- .../hadoop/yarn/sls/appmaster/AMSimulator.java | 2 +- hadoop-yarn-project/CHANGES.txt | 2 ++ .../hadoop/yarn/client/ProtocolHATestBase.java | 2 +- .../server/resourcemanager/ClientRMService.java | 2 +- .../server/resourcemanager/RMAppManager.java | 28 +++++++++++++++++--- .../scheduler/capacity/LeafQueue.java | 8 ------ .../server/resourcemanager/TestAppManager.java | 3 ++- .../resourcemanager/TestClientRMService.java | 2 +- .../TestRMWebServicesAppsModification.java | 10 +++++++ 9 files changed, 43 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java index 2272e3e..d61bf02 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java @@ -261,7 +261,7 @@ public abstract class AMSimulator extends TaskRunner.Task { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); ugi.doAs(new PrivilegedExceptionAction() { @Override - public Object run() throws YarnException { + public Object run() throws YarnException, IOException { rm.getClientRMService().submitApplication(subAppRequest); return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b9789b8..4c8eecb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -82,6 +82,8 @@ Release 2.9.0 - UNRELEASED YARN-3480. Remove attempts that are beyond max-attempt limit from state store. (Jun Gong via jianhe) + YARN-4522. Queue acl can be checked at app submission. (Jian He via wangda) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 45629b2..cf7fcc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -551,7 +551,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes { @Override public SubmitApplicationResponse submitApplication( - SubmitApplicationRequest request) throws YarnException { + SubmitApplicationRequest request) throws YarnException, IOException { resetStartFailoverFlag(true); // make sure failover has been triggered http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index b290487..4722e1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -549,7 +549,7 @@ public class ClientRMService extends AbstractService implements @Override public SubmitApplicationResponse submitApplication( - SubmitApplicationRequest request) throws YarnException { + SubmitApplicationRequest request) throws YarnException, IOException { ApplicationSubmissionContext submissionContext = request .getApplicationSubmissionContext(); ApplicationId applicationId = submissionContext.getApplicationId(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 711a7a7..4344914 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataInputByteBuffer; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; @@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; @@ -55,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -78,7 +81,7 @@ public class RMAppManager implements EventHandler, private final YarnScheduler scheduler; private final ApplicationACLsManager applicationACLsManager; private Configuration conf; - + private boolean isAclEnabled = false; public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) { @@ -97,6 +100,8 @@ public class RMAppManager implements EventHandler, if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) { this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } + this.isAclEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, + YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); } /** @@ -276,7 +281,7 @@ public class RMAppManager implements EventHandler, @SuppressWarnings("unchecked") protected void submitApplication( ApplicationSubmissionContext submissionContext, long submitTime, - String user) throws YarnException { + String user) throws YarnException, AccessControlException { ApplicationId applicationId = submissionContext.getApplicationId(); RMAppImpl application = @@ -325,7 +330,8 @@ public class RMAppManager implements EventHandler, private RMAppImpl createAndPopulateNewRMApp( ApplicationSubmissionContext submissionContext, long submitTime, - String user, boolean isRecovery) throws YarnException { + String user, boolean isRecovery) + throws YarnException, AccessControlException { // Do queue mapping if (!isRecovery) { if (rmContext.getQueuePlacementManager() != null) { @@ -346,6 +352,22 @@ public class RMAppManager implements EventHandler, submissionContext.getQueue(), applicationId); submissionContext.setPriority(appPriority); + UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); + // Since FairScheduler queue mapping is done inside scheduler, + // if FairScheduler is used and the queue doesn't exist, we should not + // fail here because queue will be created inside FS. Ideally, FS queue + // mapping should be done outside scheduler too like CS. + // For now, exclude FS for the acl check. + if (!isRecovery && isAclEnabled && scheduler instanceof CapacityScheduler && + !scheduler.checkAccess(userUgi, QueueACL.SUBMIT_APPLICATIONS, + submissionContext.getQueue()) && + !scheduler.checkAccess(userUgi, QueueACL.ADMINISTER_QUEUE, + submissionContext.getQueue())) { + throw new AccessControlException( + "User " + user + " does not have permission to submit " + + applicationId + " to queue " + submissionContext.getQueue()); + } + // Create RMApp RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index be1ba89..5c3f4b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -479,14 +479,6 @@ public class LeafQueue extends AbstractCSQueue { String queue) throws AccessControlException { // Careful! Locking order is important! - // Check queue ACLs - UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(userName); - if (!hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi) - && !hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { - throw new AccessControlException("User " + userName + " cannot submit" + - " applications to queue " + getQueuePath()); - } - User user = null; synchronized (this) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index c435692..075df47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; @@ -194,7 +195,7 @@ public class TestAppManager{ } public void submitApplication( ApplicationSubmissionContext submissionContext, String user) - throws YarnException { + throws YarnException, IOException { super.submitApplication(submissionContext, System.currentTimeMillis(), user); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 54ebd25..a5b0b68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -893,7 +893,7 @@ public class TestClientRMService { public void run() { try { rmService.submitApplication(submitRequest1); - } catch (YarnException e) {} + } catch (YarnException | IOException e) {} } }; t.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index df3c2bc..61579f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -735,6 +735,16 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase { client().addFilter(new LoggingFilter(System.out)); String lrKey = "example"; String queueName = "testqueue"; + + // create the queue + String[] queues = { "default", "testqueue" }; + CapacitySchedulerConfiguration csconf = + new CapacitySchedulerConfiguration(); + csconf.setQueues("root", queues); + csconf.setCapacity("root.default", 50.0f); + csconf.setCapacity("root.testqueue", 50.0f); + rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext()); + String appName = "test"; String appType = "test-type"; String urlPath = "apps";