Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8E8E710606 for ; Tue, 4 Feb 2014 22:58:02 +0000 (UTC) Received: (qmail 44501 invoked by uid 500); 4 Feb 2014 22:58:01 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 44422 invoked by uid 500); 4 Feb 2014 22:58:01 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 44414 invoked by uid 99); 4 Feb 2014 22:58:01 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Feb 2014 22:58:01 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Feb 2014 22:57:58 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E4BC72388994; Tue, 4 Feb 2014 22:57:38 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1564552 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/... Date: Tue, 04 Feb 2014 22:57:38 -0000 To: yarn-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140204225738.E4BC72388994@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Tue Feb 4 22:57:37 2014 New Revision: 1564552 URL: http://svn.apache.org/r1564552 Log: YARN-1669. Modified RM HA handling of protocol level service-ACLS to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong. svn merge --ignore-ancestry -c 1564549 ../../trunk/ Ran into minor import related conflicts that I merged manually. Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1564552&r1=1564551&r2=1564552&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Tue Feb 4 22:57:37 2014 @@ -99,6 +99,10 @@ Release 2.4.0 - UNRELEASED to be consistent with what exists (false) in the code and documentation. (Kenji Kikushima via vinodkv) + YARN-1669. Modified RM HA handling of protocol level service-ACLS to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1564552&r1=1564551&r2=1564552&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Tue Feb 4 22:57:37 2014 @@ -41,6 +41,10 @@ public class YarnConfiguration extends C public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; @Private + public static final String HADOOP_POLICY_CONFIGURATION_FILE = + "hadoop-policy.xml"; + + @Private public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; @Private Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1564552&r1=1564551&r2=1564552&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Tue Feb 4 22:57:37 2014 @@ -45,6 +45,7 @@ import org.apache.hadoop.security.author import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.conf.ConfigurationProvider; @@ -432,9 +433,8 @@ public class AdminService extends Compos @Override public RefreshServiceAclsResponse refreshServiceAcls( - RefreshServiceAclsRequest request) throws YarnException { - Configuration conf = new Configuration(); - if (!conf.getBoolean( + RefreshServiceAclsRequest request) throws YarnException, IOException { + if (!getConfig().getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { throw RPCUtil.getRemoteException( @@ -442,27 +442,38 @@ public class AdminService extends Compos CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION + ") not enabled.")); } - + + String argName = "refreshServiceAcls"; + if (!isRMActive()) { + RMAuditLogger.logFailure(UserGroupInformation.getCurrentUser() + .getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not refresh Service ACLs."); + throwStandbyException(); + } + PolicyProvider policyProvider = new RMPolicyProvider(); - + Configuration conf = + getConfiguration(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + refreshServiceAcls(conf, policyProvider); - if (isRMActive()) { - rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); - rmContext.getApplicationMasterService().refreshServiceAcls( - conf, policyProvider); - rmContext.getResourceTrackerService().refreshServiceAcls( - conf, policyProvider); - } else { - LOG.warn("ResourceManager is not active. Not refreshing ACLs for " + - "Clients, ApplicationMasters and NodeManagers"); - } + rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); + rmContext.getApplicationMasterService().refreshServiceAcls( + conf, policyProvider); + rmContext.getResourceTrackerService().refreshServiceAcls( + conf, policyProvider); return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class); } - void refreshServiceAcls(Configuration configuration, + synchronized void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.configurationProvider instanceof LocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } @Override @@ -519,4 +530,9 @@ public class AdminService extends Compos public AccessControlList getAccessControlList() { return this.adminAcl; } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1564552&r1=1564551&r2=1564552&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Tue Feb 4 22:57:37 2014 @@ -39,6 +39,7 @@ import org.apache.hadoop.security.author import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -86,6 +87,8 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import com.google.common.annotations.VisibleForTesting; + @SuppressWarnings("unchecked") @Private public class ApplicationMasterService extends AbstractService implements @@ -102,6 +105,7 @@ public class ApplicationMasterService ex private final AllocateResponse resync = recordFactory.newRecordInstance(AllocateResponse.class); private final RMContext rmContext; + private boolean useLocalConfigurationProvider; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { super(ApplicationMasterService.class.getName()); @@ -112,6 +116,15 @@ public class ApplicationMasterService ex } @Override + protected void serviceInit(Configuration conf) throws Exception { + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); + super.serviceInit(conf); + } + + @Override protected void serviceStart() throws Exception { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); @@ -578,7 +591,12 @@ public class ApplicationMasterService ex public void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } @Override @@ -604,4 +622,9 @@ public class ApplicationMasterService ex this.response = response; } } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1564552&r1=1564551&r2=1564552&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Tue Feb 4 22:57:37 2014 @@ -43,6 +43,7 @@ import org.apache.hadoop.security.UserGr import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; @@ -102,6 +103,9 @@ import org.apache.hadoop.yarn.server.sec import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; /** * The client interface to the Resource Manager. This module handles all the rpc @@ -126,6 +130,7 @@ public class ClientRMService extends Abs private final ApplicationACLsManager applicationsACLsManager; private final QueueACLsManager queueACLsManager; + private boolean useLocalConfigurationProvider; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -143,6 +148,10 @@ public class ClientRMService extends Abs @Override protected void serviceInit(Configuration conf) throws Exception { clientBindAddress = getBindAddress(conf); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); super.serviceInit(conf); } @@ -696,7 +705,12 @@ public class ClientRMService extends Abs void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } private boolean isAllowedDelegationTokenOp() throws IOException { @@ -710,4 +724,9 @@ public class ClientRMService extends Abs return true; } } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1564552&r1=1564551&r2=1564552&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Tue Feb 4 22:57:37 2014 @@ -29,6 +29,7 @@ import org.apache.hadoop.net.Node; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.VersionUtil; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -66,6 +67,8 @@ import org.apache.hadoop.yarn.server.uti import org.apache.hadoop.yarn.util.RackResolver; import org.apache.hadoop.yarn.util.YarnVersionInfo; +import com.google.common.annotations.VisibleForTesting; + public class ResourceTrackerService extends AbstractService implements ResourceTracker { @@ -92,6 +95,7 @@ public class ResourceTrackerService exte private int minAllocMb; private int minAllocVcores; + private boolean useLocalConfigurationProvider; static { resync.setNodeAction(NodeAction.RESYNC); @@ -141,6 +145,10 @@ public class ResourceTrackerService exte YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); super.serviceInit(conf); } @@ -415,6 +423,16 @@ public class ResourceTrackerService exte void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } + } + + @VisibleForTesting + public Server getServer() { + return this.server; } } Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1564552&r1=1564551&r2=1564552&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Tue Feb 4 22:57:37 2014 @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurab import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -262,10 +263,10 @@ public class CapacityScheduler extends A public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { if (!initialized) { - this.useLocalConfigurationProvider = conf.get( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals( - "org.apache.hadoop.yarn.LocalConfigurationProvider"); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); this.conf = new CapacitySchedulerConfiguration(conf, this.useLocalConfigurationProvider); Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1564552&r1=1564551&r2=1564552&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Tue Feb 4 22:57:37 2014 @@ -24,16 +24,19 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; - import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -191,6 +194,120 @@ public class TestRMAdminService { } @Test + public void testServiceAclsRefreshWithLocalConfigurationProvider() { + configuration.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + ResourceManager resourceManager = null; + + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } finally { + if (resourceManager != null) { + resourceManager.stop(); + } + } + } + + @SuppressWarnings("resource") + @Test + public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + ResourceManager resourceManager = null; + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + resourceManager.adminService + .refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: hadoop-policy.xml")); + } + + String aclsString = "alice,bob users,wheel"; + Configuration conf = new Configuration(); + conf.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + conf.set("security.applicationclient.protocol.acl", aclsString); + String hadoopConfFile = writeConfigurationXML(conf, "hadoop-policy.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(hadoopConfFile)); + + resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + + // verify service Acls refresh for AdminService + ServiceAuthorizationManager adminServiceServiceManager = + resourceManager.adminService.getServer() + .getServiceAuthorizationManager(); + verifyServiceACLsRefresh(adminServiceServiceManager, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ClientRMService + ServiceAuthorizationManager clientRMServiceServiceManager = + resourceManager.getRMContext().getClientRMService().getServer() + .getServiceAuthorizationManager(); + verifyServiceACLsRefresh(clientRMServiceServiceManager, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ApplicationMasterService + ServiceAuthorizationManager appMasterService = + resourceManager.getRMContext().getApplicationMasterService() + .getServer().getServiceAuthorizationManager(); + verifyServiceACLsRefresh(appMasterService, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ResourceTrackerService + ServiceAuthorizationManager RTService = + resourceManager.getRMContext().getResourceTrackerService() + .getServer().getServiceAuthorizationManager(); + verifyServiceACLsRefresh(RTService, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + } finally { + if (resourceManager != null) { + resourceManager.stop(); + } + } + } + + private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, + Class protocol, String aclString) { + for (Class protocolClass : manager.getProtocolsWithAcls()) { + AccessControlList accessList = + manager.getProtocolsAcls(protocolClass); + if (protocolClass == protocol) { + Assert.assertEquals(accessList.getAclString(), + aclString); + } else { + Assert.assertEquals(accessList.getAclString(), "*"); + } + } + } + + @Test public void testRefreshSuperUserGroupsWithLocalConfigurationProvider() { rm = new MockRM(configuration);