Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D3085102F9 for ; Fri, 14 Feb 2014 18:33:40 +0000 (UTC) Received: (qmail 49554 invoked by uid 500); 14 Feb 2014 18:33:40 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 49216 invoked by uid 500); 14 Feb 2014 18:33:39 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 49094 invoked by uid 99); 14 Feb 2014 18:33:39 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 14 Feb 2014 18:33:39 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 14 Feb 2014 18:33:30 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 5C1432388C29; Fri, 14 Feb 2014 18:32:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1568437 [2/2] - in /hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop... Date: Fri, 14 Feb 2014 18:32:46 -0000 To: yarn-commits@hadoop.apache.org From: jing9@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140214183248.5C1432388C29@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Fri Feb 14 18:32:37 2014 @@ -157,6 +157,7 @@ public class RMAppAttemptImpl implements private RMAppAttemptEvent eventCausingFinalSaving; private RMAppAttemptState targetedFinalState; private RMAppAttemptState recoveredFinalState; + private RMAppAttemptState stateBeforeFinalSaving; private Object transitionTodo; private static final StateMachineFactoryContainerToken, only in secure-mode - */ - Token createContainerToken( - FiCaSchedulerApp application, Container container) { - return containerTokenSecretManager.createContainerToken( - container.getId(), container.getNodeId(), - application.getUser(), container.getResource()); - } - private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority, ResourceRequest request, NodeType type, RMContainer rmContainer) { @@ -1345,14 +1335,6 @@ public class LeafQueue implements CSQueu unreserve(application, priority, node, rmContainer); } - Token containerToken = - createContainerToken(application, container); - if (containerToken == null) { - // Something went wrong... - return Resources.none(); - } - container.setContainerToken(containerToken); - // Inform the application RMContainer allocatedContainer = application.allocate(type, node, priority, request, container); Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java Fri Feb 14 18:32:37 2014 @@ -192,7 +192,7 @@ public class FiCaSchedulerApp extends Sc return Math.min(((float)requiredResources / clusterNodes), 1.0f); } - public Resource getTotalPendingRequests() { + public synchronized Resource getTotalPendingRequests() { Resource ret = Resource.newInstance(0, 0); for (ResourceRequest rr : appSchedulingInfo.getAllResourceRequests()) { // to avoid double counting we count only "ANY" resource requests Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Fri Feb 14 18:32:37 2014 @@ -82,10 +82,12 @@ public class AppSchedulable extends Sche Resources.addTo(demand, app.getCurrentConsumption()); // Add up outstanding resource requests - for (Priority p : app.getPriorities()) { - for (ResourceRequest r : app.getResourceRequests(p).values()) { - Resource total = Resources.multiply(r.getCapability(), r.getNumContainers()); - Resources.addTo(demand, total); + synchronized (app) { + for (Priority p : app.getPriorities()) { + for (ResourceRequest r : app.getResourceRequests(p).values()) { + Resource total = Resources.multiply(r.getCapability(), r.getNumContainers()); + Resources.addTo(demand, total); + } } } } @@ -149,17 +151,11 @@ public class AppSchedulable extends Sche NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); - org.apache.hadoop.yarn.api.records.Token containerToken = - containerTokenSecretManager.createContainerToken(containerId, nodeId, - application.getUser(), capability); - if (containerToken == null) { - return null; // Try again later. - } // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() - .getHttpAddress(), capability, priority, containerToken); + .getHttpAddress(), capability, priority, null); return container; } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Feb 14 18:32:37 2014 @@ -654,20 +654,11 @@ public class FifoScheduler extends Abstr NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); - Token containerToken = null; - - containerToken = - this.rmContext.getContainerTokenSecretManager() - .createContainerToken(containerId, nodeId, application.getUser(), - capability); - if (containerToken == null) { - return i; // Try again later. - } // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() - .getHttpAddress(), capability, priority, containerToken); + .getHttpAddress(), capability, priority, null); // Allocate! Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java Fri Feb 14 18:32:37 2014 @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.security.authorize; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.security.authorize.PolicyProvider; @@ -37,6 +39,23 @@ import org.apache.hadoop.yarn.server.api @InterfaceStability.Unstable public class RMPolicyProvider extends PolicyProvider { + private static RMPolicyProvider rmPolicyProvider = null; + + private RMPolicyProvider() {} + + @Private + @Unstable + public static RMPolicyProvider getInstance() { + if (rmPolicyProvider == null) { + synchronized(RMPolicyProvider.class) { + if (rmPolicyProvider == null) { + rmPolicyProvider = new RMPolicyProvider(); + } + } + } + return rmPolicyProvider; + } + private static final Service[] resourceManagerServices = new Service[] { new Service( Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Fri Feb 14 18:32:37 2014 @@ -142,8 +142,15 @@ public class MockRM extends ResourceMana public void waitForState(MockNM nm, ContainerId containerId, RMContainerState containerState) throws Exception { RMContainer container = getResourceScheduler().getRMContainer(containerId); - Assert.assertNotNull("Container shouldn't be null", container); int timeoutSecs = 0; + while(container == null && timeoutSecs++ < 20) { + nm.nodeHeartbeat(true); + container = getResourceScheduler().getRMContainer(containerId); + System.out.println("Waiting for container " + containerId + " to be allocated."); + Thread.sleep(100); + } + Assert.assertNotNull("Container shouldn't be null", container); + timeoutSecs = 0; while (!containerState.equals(container.getState()) && timeoutSecs++ < 40) { System.out.println("Container : " + containerId + " State is : " + container.getState() + " Waiting for state : " + containerState); Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Fri Feb 14 18:32:37 2014 @@ -24,11 +24,17 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; @@ -38,6 +44,7 @@ import org.apache.hadoop.yarn.server.api import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.After; @@ -105,34 +112,34 @@ public class TestRMAdminService { throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } - // clean the remoteDirectory - cleanRemoteDirectory(); + //upload default configurations + uploadDefaultConfiguration(); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); + } CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler(); int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); - try { - rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: capacity-scheduler.xml")); - } - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); - String csConfFile = writeConfigurationXML(csConf, - "capacity-scheduler.xml"); - - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(csConfFile)); + uploadConfiguration(csConf, "capacity-scheduler.xml"); rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); @@ -159,20 +166,24 @@ public class TestRMAdminService { throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } - // clean the remoteDirectory - cleanRemoteDirectory(); + //upload default configurations + uploadDefaultConfiguration(); try { - rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: yarn-site.xml")); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); } String aclStringBefore = @@ -180,10 +191,8 @@ public class TestRMAdminService { YarnConfiguration yarnConf = new YarnConfiguration(); yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda"); - String yarnConfFile = writeConfigurationXML(yarnConf, "yarn-site.xml"); + uploadConfiguration(yarnConf, "yarn-site.xml"); - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(yarnConfFile)); rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); String aclStringAfter = @@ -214,7 +223,6 @@ public class TestRMAdminService { } } - @SuppressWarnings("resource") @Test public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { @@ -224,33 +232,33 @@ public class TestRMAdminService { "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); ResourceManager resourceManager = null; try { - resourceManager = new ResourceManager(); - resourceManager.init(configuration); - resourceManager.start(); - - // clean the remoteDirectory - cleanRemoteDirectory(); - try { - resourceManager.adminService - .refreshServiceAcls(RefreshServiceAclsRequest - .newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + fail("Should throw an exception"); } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: hadoop-policy.xml")); + // expect to get an exception here } - String aclsString = "alice,bob users,wheel"; + //upload default configurations + uploadDefaultConfiguration(); Configuration conf = new Configuration(); conf.setBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); - conf.set("security.applicationclient.protocol.acl", aclsString); - String hadoopConfFile = writeConfigurationXML(conf, "hadoop-policy.xml"); + uploadConfiguration(conf, "core-site.xml"); + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(hadoopConfFile)); + String aclsString = "alice,bob users,wheel"; + Configuration newConf = new Configuration(); + newConf.set("security.applicationclient.protocol.acl", aclsString); + uploadConfiguration(newConf, "hadoop-policy.xml"); resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest .newInstance()); @@ -328,31 +336,31 @@ public class TestRMAdminService { throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch(Exception ex) { + // Expect exception here + } - // clean the remoteDirectory - cleanRemoteDirectory(); + //upload default configurations + uploadDefaultConfiguration(); try { - rm.adminService.refreshSuperUserGroupsConfiguration( - RefreshSuperUserGroupsConfigurationRequest.newInstance()); - fail("FileSystemBasedConfigurationProvider is used." + - " Should get an exception here"); - } catch (Exception ex) { - Assert.assertTrue(ex.getMessage().contains( - "Can not find Configuration: core-site.xml")); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch(Exception ex) { + fail("Should not get any exceptions"); } Configuration coreConf = new Configuration(false); coreConf.set("hadoop.proxyuser.test.groups", "test_groups"); coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts"); - String coreConfFile = writeConfigurationXML(coreConf, - "core-site.xml"); + uploadConfiguration(coreConf, "core-site.xml"); - // upload the file into Remote File System - uploadToRemoteFileSystem(new Path(coreConfFile)); rm.adminService.refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest.newInstance()); Assert.assertTrue(ProxyUsers.getProxyGroups() @@ -366,6 +374,84 @@ public class TestRMAdminService { .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); } + @Test + public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + try { + rm.adminService + .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest + .newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void + testRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + fail("Should throw an exception"); + } catch (Exception ex) { + // Expect exception here + } + + String user = UserGroupInformation.getCurrentUser().getUserName(); + List groupWithInit = + new ArrayList(Groups.getUserToGroupsMappingService( + configuration).getGroups(user)); + + // upload default configurations + uploadDefaultConfiguration(); + Configuration conf = new Configuration(); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + MockUnixGroupsMapping.class, + GroupMappingServiceProvider.class); + uploadConfiguration(conf, "core-site.xml"); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + // Make sure RM will use the updated GroupMappingServiceProvider + List groupBefore = + new ArrayList(Groups.getUserToGroupsMappingService( + configuration).getGroups(user)); + Assert.assertTrue(groupBefore.contains("test_group_A") + && groupBefore.contains("test_group_B") + && groupBefore.contains("test_group_C") && groupBefore.size() == 3); + Assert.assertTrue(groupWithInit.size() != groupBefore.size()); + Assert.assertFalse(groupWithInit.contains("test_group_A") + || groupWithInit.contains("test_group_B") + || groupWithInit.contains("test_group_C")); + + // update the groups + MockUnixGroupsMapping.updateGroups(); + + rm.adminService + .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest + .newInstance()); + List groupAfter = + Groups.getUserToGroupsMappingService(configuration).getGroups(user); + + // should get the updated groups + Assert.assertTrue(groupAfter.contains("test_group_D") + && groupAfter.contains("test_group_E") + && groupAfter.contains("test_group_F") && groupAfter.size() == 3); + + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; @@ -393,11 +479,63 @@ public class TestRMAdminService { fs.copyFromLocalFile(filePath, workingPath); } - private void cleanRemoteDirectory() throws IOException { - if (fs.exists(workingPath)) { - for (FileStatus file : fs.listStatus(workingPath)) { - fs.delete(file.getPath(), true); - } + private void uploadConfiguration(Configuration conf, String confFileName) + throws IOException { + String csConfFile = writeConfigurationXML(conf, confFileName); + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(csConfFile)); + } + + private void uploadDefaultConfiguration() throws IOException { + Configuration conf = new Configuration(); + uploadConfiguration(conf, "core-site.xml"); + + YarnConfiguration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + uploadConfiguration(yarnConf, "yarn-site.xml"); + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + uploadConfiguration(csConf, "capacity-scheduler.xml"); + + Configuration hadoopPolicyConf = new Configuration(false); + hadoopPolicyConf + .addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml"); + } + + private static class MockUnixGroupsMapping implements + GroupMappingServiceProvider { + + @SuppressWarnings("serial") + private static List group = new ArrayList() {{ + add("test_group_A"); + add("test_group_B"); + add("test_group_C"); + }}; + + @Override + public List getGroups(String user) throws IOException { + return group; + } + + @Override + public void cacheGroupsRefresh() throws IOException { + // Do nothing + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + // Do nothing + } + + public static void updateGroups() { + group.clear(); + group.add("test_group_D"); + group.add("test_group_E"); + group.add("test_group_F"); } } + } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Fri Feb 14 18:32:37 2014 @@ -23,10 +23,7 @@ import static org.junit.Assert.assertTru import static org.mockito.Mockito.mock; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,15 +31,8 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl; @@ -54,6 +44,7 @@ import org.junit.Test; public class TestZKRMStateStore extends RMStateStoreTestBase { public static final Log LOG = LogFactory.getLog(TestZKRMStateStore.class); + private static final int ZK_TIMEOUT_MS = 1000; class TestZKRMStateStoreTester implements RMStateStoreHelper { @@ -141,6 +132,7 @@ public class TestZKRMStateStore extends conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); + conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.set(YarnConfiguration.RM_HA_ID, rmId); for (String rpcAddress : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { for (String id : HAUtil.getRMHAIds(conf)) { @@ -182,26 +174,7 @@ public class TestZKRMStateStore extends HAServiceProtocol.HAServiceState.ACTIVE, rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); - // Submitting an application to RM1 to trigger a state store operation. - // RM1 should realize that it got fenced and is not the Active RM anymore. - Map mockMap = mock(Map.class); - ApplicationSubmissionContext asc = - ApplicationSubmissionContext.newInstance( - ApplicationId.newInstance(1000, 1), - "testApplication", // app Name - "default", // queue name - Priority.newInstance(0), - ContainerLaunchContext.newInstance(mockMap, mockMap, - new ArrayList(), mockMap, mock(ByteBuffer.class), - mockMap), - false, // unmanaged AM - true, // cancelTokens - 1, // max app attempts - Resource.newInstance(1024, 1)); - ClientRMService rmService = rm1.getClientRMService(); - rmService.submitApplication(SubmitApplicationRequest.newInstance(asc)); - - for (int i = 0; i < 30; i++) { + for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) { if (HAServiceProtocol.HAServiceState.ACTIVE == rm1.getRMContext().getRMAdminService().getServiceStatus().getState()) { Thread.sleep(100); Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Fri Feb 14 18:32:37 2014 @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -695,6 +696,8 @@ public class TestRMAppAttemptTransitions // launch AM and verify attempt failed applicationAttempt.handle(new RMAppAttemptRegistrationEvent( applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl")); + assertEquals(YarnApplicationAttemptState.SUBMITTED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptSubmittedToFailedState( "Unmanaged AM must register after AM attempt reaches LAUNCHED state."); } @@ -710,6 +713,8 @@ public class TestRMAppAttemptTransitions // launch AM and verify attempt failed applicationAttempt.handle(new RMAppAttemptRegistrationEvent( applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl")); + assertEquals(YarnApplicationAttemptState.SUBMITTED, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertFalse(transferStateFromPreviousAttempt); } @@ -720,6 +725,8 @@ public class TestRMAppAttemptTransitions new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.NEW, + applicationAttempt.createApplicationAttemptState()); testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS); verifyTokenCount(applicationAttempt.getAppAttemptId(), 1); } @@ -740,6 +747,8 @@ public class TestRMAppAttemptTransitions new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.SUBMITTED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS); } @@ -750,6 +759,8 @@ public class TestRMAppAttemptTransitions new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.SCHEDULED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS); } @@ -760,6 +771,8 @@ public class TestRMAppAttemptTransitions new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.ALLOCATED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptKilledState(amContainer, EMPTY_DIAGNOSTICS); } @@ -771,6 +784,8 @@ public class TestRMAppAttemptTransitions new RMAppAttemptLaunchFailedEvent( applicationAttempt.getAppAttemptId(), diagnostics)); + assertEquals(YarnApplicationAttemptState.ALLOCATED, + applicationAttempt.createApplicationAttemptState()); testAppAttemptFailedState(amContainer, diagnostics); } @@ -784,6 +799,8 @@ public class TestRMAppAttemptTransitions ContainerState.COMPLETE, containerDiagMsg, exitCode); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), cs)); + assertEquals(YarnApplicationAttemptState.ALLOCATED, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -815,7 +832,8 @@ public class TestRMAppAttemptTransitions applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState()); - + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -850,7 +868,8 @@ public class TestRMAppAttemptTransitions applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState()); - + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.KILLED, applicationAttempt.getAppAttemptState()); @@ -871,6 +890,8 @@ public class TestRMAppAttemptTransitions launchApplicationAttempt(amContainer); applicationAttempt.handle(new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); + assertEquals(YarnApplicationAttemptState.LAUNCHED, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -890,6 +911,8 @@ public class TestRMAppAttemptTransitions runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); applicationAttempt.handle(new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -1055,6 +1078,8 @@ public class TestRMAppAttemptTransitions diagnostics)); assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState()); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); // Container_finished event comes before Attempt_Saved event. applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus( @@ -1083,6 +1108,8 @@ public class TestRMAppAttemptTransitions diagnostics)); assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState()); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); // Expire event comes before Attempt_saved event. applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt .getAppAttemptId(), RMAppAttemptEventType.EXPIRE)); @@ -1118,6 +1145,8 @@ public class TestRMAppAttemptTransitions applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt .getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.LAUNCHED, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); // after attempt is killed, can not get Client Token token = applicationAttempt.createClientToken(null); @@ -1140,6 +1169,8 @@ public class TestRMAppAttemptTransitions ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId(); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( appAttemptId, cs1)); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); @@ -1178,6 +1209,8 @@ public class TestRMAppAttemptTransitions ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId(); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( appAttemptId, cs1)); + assertEquals(YarnApplicationAttemptState.RUNNING, + applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Fri Feb 14 18:32:37 2014 @@ -40,6 +40,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -104,6 +105,7 @@ public class TestCapacityScheduler { private static float B3_CAPACITY = 20; private ResourceManager resourceManager = null; + private RMContext mockContext; @Before public void setUp() throws Exception { @@ -118,6 +120,9 @@ public class TestCapacityScheduler { resourceManager.getRMContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMNMTokenSecretManager().rollMasterKey(); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); + mockContext = mock(RMContext.class); + when(mockContext.getConfigurationProvider()).thenReturn( + new LocalConfigurationProvider()); } @After @@ -133,7 +138,7 @@ public class TestCapacityScheduler { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); try { - scheduler.reinitialize(conf, null); + scheduler.reinitialize(conf, mockContext); fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation."); } catch (YarnRuntimeException e) { @@ -147,7 +152,7 @@ public class TestCapacityScheduler { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1); try { - scheduler.reinitialize(conf, null); + scheduler.reinitialize(conf, mockContext); fail("Exception is expected because the min vcores allocation is" + " larger than the max vcores allocation."); } catch (YarnRuntimeException e) { @@ -353,7 +358,7 @@ public class TestCapacityScheduler { conf.setCapacity(A, 80f); conf.setCapacity(B, 20f); - cs.reinitialize(conf,null); + cs.reinitialize(conf, mockContext); checkQueueCapacities(cs, 80f, 20f); } @@ -503,7 +508,7 @@ public class TestCapacityScheduler { conf.setCapacity(B2, B2_CAPACITY); conf.setCapacity(B3, B3_CAPACITY); conf.setCapacity(B4, B4_CAPACITY); - cs.reinitialize(conf,null); + cs.reinitialize(conf,mockContext); checkQueueCapacities(cs, 80f, 20f); // Verify parent for B4 Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java Fri Feb 14 18:32:37 2014 @@ -18,11 +18,17 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import java.util.ArrayList; +import java.util.List; + import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -30,6 +36,9 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.junit.Test; @@ -106,4 +115,38 @@ public class TestContainerAllocation { rm.stop(); } + + // This is to test container tokens are generated when the containers are + // acquired by the AM, not when the containers are allocated + @Test + public void testContainerTokenGeneratedOnPullRequest() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm1 = new MockRM(conf); + rm1.start(); + MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); + RMApp app1 = rm1.submitApp(200); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + // request a container. + am1.allocate("127.0.0.1", 1024, 1, new ArrayList()); + ContainerId containerId2 = + ContainerId.newInstance(am1.getApplicationAttemptId(), 2); + rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED); + + RMContainer container = + rm1.getResourceScheduler().getRMContainer(containerId2); + // no container token is generated. + Assert.assertEquals(containerId2, container.getContainerId()); + Assert.assertNull(container.getContainer().getContainerToken()); + + // acquire the container. + List containers = + am1.allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers(); + Assert.assertEquals(containerId2, containers.get(0).getId()); + // container token is generated. + Assert.assertNotNull(containers.get(0).getContainerToken()); + rm1.stop(); + } } \ No newline at end of file Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm Fri Feb 14 18:32:37 2014 @@ -35,7 +35,7 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| --config confdir | Overwrites the default Configuration directory. Default is ${HADOOP_PREFIX}/conf. | +| --config confdir | Overwrites the default Configuration directory. Default is $\{HADOOP_PREFIX\}/conf. | *---------------+--------------+ | COMMAND COMMAND_OPTIONS | Various commands with their options are described in the following sections. The commands have been grouped into {{User Commands}} and {{Administration Commands}}. | *---------------+--------------+ @@ -63,11 +63,22 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| -status ApplicationId | Specify an application id | +| -list | Lists applications from the RM. Supports optional use of -appTypes +| | to filter applications based on application type, and -appStates to +| | filter applications based on application state. +*---------------+--------------+ +| -appStates States | Works with -list to filter applications based on input +| | comma-separated list of application states. The valid +| | application state can be one of the following: \ +| | ALL, NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING, +| | FINISHED, FAILED, KILLED *---------------+--------------+ -| -list | Lists all the Applications from RM | +| -appTypes Types | Works with -list to filter applications based on input +| | comma-separated list of application types. *---------------+--------------+ -| -kill ApplicationId | Specify an application id | +| -status ApplicationId | Prints the status of the application. +*---------------+--------------+ +| -kill ApplicationId | Kills the application. *---------------+--------------+ ** node @@ -81,9 +92,15 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| -status NodeId | Specify a node id | +| -list | Lists all running nodes. Supports optional use of -states to filter +| | nodes based on node state, and -all to list all nodes. +*---------------+--------------+ +| -states States | Works with -list to filter nodes based on input +| | comma-separated list of node states. *---------------+--------------+ -| -list | Lists all the Nodes | +| -all | Works with -list to list all nodes. +*---------------+--------------+ +| -status NodeId | Prints the status report of the node. *---------------+--------------+ ** logs @@ -91,19 +108,22 @@ Usage: yarn [--config confdir] COMMAND Dump the container logs ------- - Usage: yarn logs + Usage: yarn logs -applicationId ------- *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| -applicationId ApplicationId | Specify an application id | +| -applicationId \ | Specifies an application id | *---------------+--------------+ -| -appOwner AppOwner | Specify an application owner | +| -appOwner AppOwner | AppOwner (assumed to be current user if not +| | specified) *---------------+--------------+ -| -containerId ContainerId | Specify a container id | +| -containerId ContainerId | ContainerId (must be specified if node address is +| | specified) *---------------+--------------+ -| -nodeAddress NodeAddress | Specify a node address | +| -nodeAddress NodeAddress | NodeAddress in the format nodename:port (must be +| | specified if container id is specified) *---------------+--------------+ ** classpath @@ -158,7 +178,11 @@ Usage: yarn [--config confdir] COMMAND ------- Usage: yarn rmadmin [-refreshQueues] [-refreshNodes] [-refreshUserToGroupsMapping] [-refreshSuperUserGroupsConfiguration] [-refreshAdminAcls] - [-refreshServiceAcl] [-help [cmd]] + [-refreshServiceAcl] [-getGroups [username]] [-help [cmd]] + [-transitionToActive ] + [-transitionToStandby ] + [-getServiceState ] + [-checkHealth ] ------- *---------------+--------------+ @@ -176,8 +200,22 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ | -refreshServiceAcl | Reload the service-level authorization policy file ResoureceManager will reload the authorization policy file. | *---------------+--------------+ +| -getGroups [username] | Get groups the specified user belongs to. +*---------------+--------------+ | -help [cmd] | Displays help for the given command or all commands if none is specified. | *---------------+--------------+ +| -transitionToActive \ | Transitions the service into Active +| | state. +*---------------+--------------+ +| -transitionToStandby \ | Transitions the service into Standby +| | state. +*---------------+--------------+ +| -getServiceState \ | Returns the state of the service. +*---------------+--------------+ +| -checkHealth \ | Requests that the service perform a health +| | check. The RMAdmin tool will exit with a +| | non-zero exit code if the check fails. +*---------------+--------------+ ** daemonlog @@ -191,9 +229,9 @@ Usage: yarn [--config confdir] COMMAND *---------------+--------------+ || COMMAND_OPTIONS || Description | *---------------+--------------+ -| -getlevel | Prints the log level of the daemon running at . This command internally connects to http:///logLevel?log= | +| -getlevel \ \ | Prints the log level of the daemon running at \. This command internally connects to http://\/logLevel?log=\ *---------------+--------------+ -| -setlevel | Sets the log level of the daemon running at . This command internally connects to http:///logLevel?log= | +| -setlevel \ \ \ | Sets the log level of the daemon running at \. This command internally connects to http://\/logLevel?log=\ *---------------+--------------+