Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A4FC610C21 for ; Mon, 2 Dec 2013 17:42:48 +0000 (UTC) Received: (qmail 28265 invoked by uid 500); 2 Dec 2013 17:42:48 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 27848 invoked by uid 500); 2 Dec 2013 17:42:43 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 27821 invoked by uid 99); 2 Dec 2013 17:42:41 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Dec 2013 17:42:41 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Dec 2013 17:42:37 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 67E7A2388C2D; Mon, 2 Dec 2013 17:41:55 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1547122 [2/2] - in /hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/proto/server/ hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/... Date: Mon, 02 Dec 2013 17:41:53 -0000 To: yarn-commits@hadoop.apache.org From: arp@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131202174155.67E7A2388C2D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java Mon Dec 2 17:41:44 2013 @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.hadoop.security.Groups; @@ -58,7 +59,7 @@ public abstract class QueuePlacementRule */ public String assignAppToQueue(String requestedQueue, String user, Groups groups, Collection configuredQueues) throws IOException { - String queue = getQueueForApp(requestedQueue, user, groups); + String queue = getQueueForApp(requestedQueue, user, groups, configuredQueues); if (create || configuredQueues.contains(queue)) { return queue; } else { @@ -103,7 +104,7 @@ public abstract class QueuePlacementRule * continue to the next rule. */ protected abstract String getQueueForApp(String requestedQueue, String user, - Groups groups) throws IOException; + Groups groups, Collection configuredQueues) throws IOException; /** * Places apps in queues by username of the submitter @@ -111,7 +112,7 @@ public abstract class QueuePlacementRule public static class User extends QueuePlacementRule { @Override protected String getQueueForApp(String requestedQueue, - String user, Groups groups) { + String user, Groups groups, Collection configuredQueues) { return "root." + user; } @@ -127,7 +128,8 @@ public abstract class QueuePlacementRule public static class PrimaryGroup extends QueuePlacementRule { @Override protected String getQueueForApp(String requestedQueue, - String user, Groups groups) throws IOException { + String user, Groups groups, + Collection configuredQueues) throws IOException { return "root." + groups.getGroups(user).get(0); } @@ -136,6 +138,33 @@ public abstract class QueuePlacementRule return create; } } + + /** + * Places apps in queues by secondary group of the submitter + * + * Match will be made on first secondary group that exist in + * queues + */ + public static class SecondaryGroupExistingQueue extends QueuePlacementRule { + @Override + protected String getQueueForApp(String requestedQueue, + String user, Groups groups, + Collection configuredQueues) throws IOException { + List groupNames = groups.getGroups(user); + for (int i = 1; i < groupNames.size(); i++) { + if (configuredQueues.contains("root." + groupNames.get(i))) { + return "root." + groupNames.get(i); + } + } + + return ""; + } + + @Override + public boolean isTerminal() { + return create; + } + } /** * Places apps in queues by requested queue of the submitter @@ -143,7 +172,7 @@ public abstract class QueuePlacementRule public static class Specified extends QueuePlacementRule { @Override protected String getQueueForApp(String requestedQueue, - String user, Groups groups) { + String user, Groups groups, Collection configuredQueues) { if (requestedQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) { return ""; } else { @@ -166,7 +195,7 @@ public abstract class QueuePlacementRule public static class Default extends QueuePlacementRule { @Override protected String getQueueForApp(String requestedQueue, String user, - Groups groups) { + Groups groups, Collection configuredQueues) { return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; } @@ -188,7 +217,7 @@ public abstract class QueuePlacementRule @Override protected String getQueueForApp(String requestedQueue, String user, - Groups groups) { + Groups groups, Collection configuredQueues) { throw new UnsupportedOperationException(); } Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java Mon Dec 2 17:41:44 2013 @@ -39,7 +39,7 @@ public class FairSchedulerLeafQueueInfo public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) { super(queue, scheduler); - Collection apps = queue.getAppSchedulables(); + Collection apps = queue.getRunnableAppSchedulables(); for (AppSchedulable app : apps) { if (app.getApp().isPending()) { numPendingApps++; @@ -47,6 +47,7 @@ public class FairSchedulerLeafQueueInfo numActiveApps++; } } + numPendingApps += queue.getNonRunnableAppSchedulables().size(); } public int getNumActiveApplications() { Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java Mon Dec 2 17:41:44 2013 @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMDTSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -106,6 +107,8 @@ public class RMStateStoreTestBase extend interface RMStateStoreHelper { RMStateStore getRMStateStore() throws Exception; boolean isFinalStateValid() throws Exception; + void writeVersion(RMStateVersion version) throws Exception; + RMStateVersion getCurrentVersion() throws Exception; } void waitNotify(TestDispatcher dispatcher) { @@ -379,4 +382,37 @@ public class RMStateStoreTestBase extend appToken.setService(new Text("appToken service")); return appToken; } + + public void testCheckVersion(RMStateStoreHelper stateStoreHelper) + throws Exception { + RMStateStore store = stateStoreHelper.getRMStateStore(); + store.setRMDispatcher(new TestDispatcher()); + + // default version + RMStateVersion defaultVersion = stateStoreHelper.getCurrentVersion(); + store.checkVersion(); + Assert.assertEquals(defaultVersion, store.loadVersion()); + + // compatible version + RMStateVersion compatibleVersion = + RMStateVersion.newInstance(defaultVersion.getMajorVersion(), + defaultVersion.getMinorVersion() + 2); + stateStoreHelper.writeVersion(compatibleVersion); + Assert.assertEquals(compatibleVersion, store.loadVersion()); + store.checkVersion(); + // overwrite the compatible version + Assert.assertEquals(defaultVersion, store.loadVersion()); + + // incompatible version + RMStateVersion incompatibleVersion = + RMStateVersion.newInstance(defaultVersion.getMajorVersion() + 2, + defaultVersion.getMinorVersion()); + stateStoreHelper.writeVersion(incompatibleVersion); + try { + store.checkVersion(); + Assert.fail("Invalid version, should fail."); + } catch (Throwable t) { + Assert.assertTrue(t instanceof RMStateVersionIncompatibleException); + } + } } Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java Mon Dec 2 17:41:44 2013 @@ -32,6 +32,8 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.Test; @@ -42,7 +44,7 @@ public class TestFSRMStateStore extends class TestFSRMStateStoreTester implements RMStateStoreHelper { Path workingDirPathURI; - FileSystemRMStateStore store; + TestFileSystemRMStore store; MiniDFSCluster cluster; class TestFileSystemRMStore extends FileSystemRMStateStore { @@ -54,6 +56,14 @@ public class TestFSRMStateStore extends start(); Assert.assertNotNull(fs); } + + public Path getVersionNode() { + return new Path(new Path(workingDirPathURI, ROOT_DIR_NAME), VERSION_NODE); + } + + public RMStateVersion getCurrentVersion() { + return CURRENT_VERSION_INFO; + } } public TestFSRMStateStoreTester(MiniDFSCluster cluster) throws Exception { @@ -81,6 +91,17 @@ public class TestFSRMStateStore extends FileStatus[] files = fs.listStatus(workingDirPathURI); return files.length == 1; } + + @Override + public void writeVersion(RMStateVersion version) throws Exception { + store.updateFile(store.getVersionNode(), ((RMStateVersionPBImpl) version) + .getProto().toByteArray()); + } + + @Override + public RMStateVersion getCurrentVersion() throws Exception { + return store.getCurrentVersion(); + } } @Test @@ -113,6 +134,7 @@ public class TestFSRMStateStore extends Assert.assertFalse(fileSystemRMStateStore.fsWorkingPath .getFileSystem(conf).exists(tempAppAttemptFile)); testRMDTSecretManagerStateStore(fsTester); + testCheckVersion(fsTester); } finally { cluster.shutdown(); } Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Mon Dec 2 17:41:44 2013 @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.conf.HAUti import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl; import org.apache.zookeeper.ZooKeeper; import org.junit.Test; @@ -54,7 +56,7 @@ public class TestZKRMStateStore extends class TestZKRMStateStoreTester implements RMStateStoreHelper { ZooKeeper client; - ZKRMStateStore store; + TestZKRMStateStoreInternal store; class TestZKRMStateStoreInternal extends ZKRMStateStore { @@ -69,6 +71,14 @@ public class TestZKRMStateStore extends public ZooKeeper getNewZooKeeper() throws IOException { return client; } + + public String getVersionNode() { + return znodeWorkingPath + "/" + ROOT_ZNODE_NAME + "/" + VERSION_NODE; + } + + public RMStateVersion getCurrentVersion() { + return CURRENT_VERSION_INFO; + } } public RMStateStore getRMStateStore() throws Exception { @@ -86,6 +96,17 @@ public class TestZKRMStateStore extends List nodes = client.getChildren(store.znodeWorkingPath, false); return nodes.size() == 1; } + + @Override + public void writeVersion(RMStateVersion version) throws Exception { + client.setData(store.getVersionNode(), ((RMStateVersionPBImpl) version) + .getProto().toByteArray(), -1); + } + + @Override + public RMStateVersion getCurrentVersion() throws Exception { + return store.getCurrentVersion(); + } } @Test @@ -93,6 +114,7 @@ public class TestZKRMStateStore extends TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester(); testRMAppStateStore(zkTester); testRMDTSecretManagerStateStore(zkTester); + testCheckVersion(zkTester); } private Configuration createHARMConf( Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Mon Dec 2 17:41:44 2013 @@ -18,19 +18,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; -import static org.junit.Assert.assertEquals; -import static org.junit.Assume.assumeTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.List; import junit.framework.Assert; @@ -41,13 +36,10 @@ import org.apache.hadoop.security.Securi import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.yarn.MockApps; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -64,11 +56,8 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; @@ -76,7 +65,6 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -290,7 +278,6 @@ public class TestRMAppTransitions { // test to make sure times are set when app finishes private void assertTimesAtFinish(RMApp application) { - sendAppUpdateSavedEvent(application); assertStartTimeSet(application); Assert.assertTrue("application finish time is not greater then 0", (application.getFinishTime() > 0)); @@ -303,7 +290,6 @@ public class TestRMAppTransitions { } private void assertKilled(RMApp application) { - sendAppUpdateSavedEvent(application); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); assertFinalAppStatus(FinalApplicationStatus.KILLED, application); @@ -314,6 +300,7 @@ public class TestRMAppTransitions { private void assertAppAndAttemptKilled(RMApp application) throws InterruptedException { + sendAppUpdateSavedEvent(application); assertKilled(application); // send attempt final state saved event. application.getCurrentAppAttempt().handle( @@ -325,7 +312,6 @@ public class TestRMAppTransitions { } private void assertFailed(RMApp application, String regex) { - sendAppUpdateSavedEvent(application); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); assertFinalAppStatus(FinalApplicationStatus.FAILED, application); @@ -475,6 +461,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); RMAppAttempt appAttempt = application.getCurrentAppAttempt(); Assert.assertEquals(1, appAttempt.getAppAttemptId().getAttemptId()); + sendAppUpdateSavedEvent(application); assertFailed(application, ".*Unmanaged application.*Failing the application.*"); } @@ -504,6 +491,7 @@ public class TestRMAppTransitions { new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertKilled(application); } @@ -518,6 +506,7 @@ public class TestRMAppTransitions { new RMAppRejectedEvent(application.getApplicationId(), rejectedText); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); } @@ -531,6 +520,7 @@ public class TestRMAppTransitions { new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertKilled(application); } @@ -545,6 +535,7 @@ public class TestRMAppTransitions { new RMAppRejectedEvent(application.getApplicationId(), rejectedText); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); } @@ -559,6 +550,7 @@ public class TestRMAppTransitions { new RMAppRejectedEvent(application.getApplicationId(), rejectedText); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); } @@ -603,6 +595,7 @@ public class TestRMAppTransitions { RMAppEventType.ATTEMPT_FAILED, message); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertFailed(application, ".*" + message + ".*Failing the application.*"); } @@ -635,7 +628,7 @@ public class TestRMAppTransitions { new RMAppFinishedAttemptEvent(application.getApplicationId(), null); application.handle(finishEvent); assertAppState(RMAppState.FINAL_SAVING, application); - + sendAppUpdateSavedEvent(application); assertKilled(application); } @@ -681,6 +674,7 @@ public class TestRMAppTransitions { RMAppEventType.ATTEMPT_FAILED, ""); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertFailed(application, ".*Failing the application.*"); // FAILED => FAILED event RMAppEventType.KILL @@ -758,6 +752,7 @@ public class TestRMAppTransitions { new RMAppRejectedEvent(application.getApplicationId(), ""); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); @@ -769,10 +764,6 @@ public class TestRMAppTransitions { assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); - // FAILED => FAILED event RMAppEventType.APP_SAVED - event = new RMAppNewSavedEvent(application.getApplicationId(), null); - application.handle(event); - rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); } @@ -788,6 +779,7 @@ public class TestRMAppTransitions { new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL); application.handle(event); rmDispatcher.await(); + sendAppUpdateSavedEvent(application); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); @@ -824,10 +816,6 @@ public class TestRMAppTransitions { assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); - // KILLED => KILLED event RMAppEventType.APP_SAVED - event = new RMAppNewSavedEvent(application.getApplicationId(), null); - application.handle(event); - rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); } @@ -841,70 +829,4 @@ public class TestRMAppTransitions { report = app.createAndGetApplicationReport("clientuser", true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); } - - @Test - public void testClientTokens() throws Exception { - assumeTrue(isSecurityEnabled); - - RMApp app = createNewTestApp(null); - assertAppState(RMAppState.NEW, app); - ApplicationReport report = app.createAndGetApplicationReport(null, true); - Assert.assertNull(report.getClientToAMToken()); - report = app.createAndGetApplicationReport("clientuser", true); - Assert.assertNull(report.getClientToAMToken()); - - app = testCreateAppRunning(null); - rmDispatcher.await(); - assertAppState(RMAppState.RUNNING, app); - - report = app.createAndGetApplicationReport("clientuser", true); - Assert.assertNull(report.getClientToAMToken()); - - // this method is to make AMLaunchedTransition invoked inside which - // ClientTokenMasterKey is registered in ClientTokenSecretManager - moveCurrentAttemptToLaunchedState(app.getCurrentAppAttempt()); - - report = app.createAndGetApplicationReport(null, true); - Assert.assertNull(report.getClientToAMToken()); - report = app.createAndGetApplicationReport("clientuser", true); - Assert.assertNotNull(report.getClientToAMToken()); - - // kill the app attempt and verify client token is unavailable - app.handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL)); - rmDispatcher.await(); - assertAppAndAttemptKilled(app); - report = app.createAndGetApplicationReport(null, true); - Assert.assertNull(report.getClientToAMToken()); - report = app.createAndGetApplicationReport("clientuser", true); - Assert.assertNull(report.getClientToAMToken()); - } - - @SuppressWarnings("unchecked") - private void moveCurrentAttemptToLaunchedState(RMAppAttempt attempt) { - attempt.handle(new RMAppAttemptEvent(attempt.getAppAttemptId(), - RMAppAttemptEventType.APP_ACCEPTED)); - // Mock the allocation of AM container - Container container = mock(Container.class); - Resource resource = BuilderUtils.newResource(2048, 1); - when(container.getId()).thenReturn( - BuilderUtils.newContainerId(attempt.getAppAttemptId(), 1)); - when(container.getResource()).thenReturn(resource); - Allocation allocation = mock(Allocation.class); - when(allocation.getContainers()).thenReturn( - Collections.singletonList(container)); - when(allocation.getContainers()). - thenReturn(Collections.singletonList(container)); - when( - scheduler.allocate(any(ApplicationAttemptId.class), any(List.class), - any(List.class), any(List.class), any(List.class))).thenReturn( - allocation); - attempt.handle(new RMAppAttemptContainerAllocatedEvent(attempt - .getAppAttemptId(), container)); - attempt - .handle(new RMAppAttemptNewSavedEvent(attempt.getAppAttemptId(), null)); - attempt.handle(new RMAppAttemptEvent(attempt.getAppAttemptId(), - RMAppAttemptEventType.LAUNCHED)); - - assertEquals(RMAppAttemptState.LAUNCHED, attempt.getAppAttemptState()); - } } Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Mon Dec 2 17:41:44 2013 @@ -23,6 +23,7 @@ import static org.junit.Assert.assertFal import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -44,6 +45,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -57,6 +59,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.InlineDispatcher; +import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -74,9 +77,9 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRejectedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; @@ -1068,6 +1071,38 @@ public class TestRMAppAttemptTransitions diagnostics, 0, false); } + // this is to test user can get client tokens only after the client token + // master key is saved in the state store and also registered in + // ClientTokenSecretManager + @Test + public void testGetClientToken() throws Exception { + assumeTrue(isSecurityEnabled); + Container amContainer = allocateApplicationAttempt(); + + // before attempt is launched, can not get ClientToken + Token token = + applicationAttempt.createClientToken(null); + Assert.assertNull(token); + token = applicationAttempt.createClientToken("clientuser"); + Assert.assertNull(token); + + launchApplicationAttempt(amContainer); + // after attempt is launched , can get ClientToken + token = applicationAttempt.createClientToken(null); + Assert.assertNull(token); + token = applicationAttempt.createClientToken("clientuser"); + Assert.assertNotNull(token); + + applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt + .getAppAttemptId(), RMAppAttemptEventType.KILL)); + sendAttemptUpdateSavedEvent(applicationAttempt); + // after attempt is killed, can not get Client Token + token = applicationAttempt.createClientToken(null); + Assert.assertNull(token); + token = applicationAttempt.createClientToken("clientuser"); + Assert.assertNull(token); + } + private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) { verify(amRMTokenManager, times(count)).applicationMasterFinished(appAttemptId); if (UserGroupInformation.isSecurityEnabled()) { Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Mon Dec 2 17:41:44 2013 @@ -596,23 +596,24 @@ public class TestCapacityScheduler { public void testConcurrentAccessOnApplications() throws Exception { CapacityScheduler cs = new CapacityScheduler(); verifyConcurrentAccessOnApplications( - cs.applications, FiCaSchedulerApp.class); + cs.applications, FiCaSchedulerApp.class, Queue.class); } - public static + public static void verifyConcurrentAccessOnApplications( - final Map applications, Class clazz) + final Map applications, Class appClazz, + final Class queueClazz) throws Exception { final int size = 10000; final ApplicationId appId = ApplicationId.newInstance(0, 0); - final Constructor ctor = clazz.getDeclaredConstructor( - ApplicationAttemptId.class, String.class, Queue.class, + final Constructor ctor = appClazz.getDeclaredConstructor( + ApplicationAttemptId.class, String.class, queueClazz, ActiveUsersManager.class, RMContext.class); ApplicationAttemptId appAttemptId0 = ApplicationAttemptId.newInstance(appId, 0); applications.put(appAttemptId0, ctor.newInstance( - appAttemptId0, null, mock(Queue.class), null, null)); + appAttemptId0, null, mock(queueClazz), null, null)); assertNotNull(applications.get(appAttemptId0)); // Imitating the thread of scheduler that will add and remove apps @@ -627,7 +628,7 @@ public class TestCapacityScheduler { = ApplicationAttemptId.newInstance(appId, i); try { applications.put(appAttemptId, ctor.newInstance( - appAttemptId, null, mock(Queue.class), null, null)); + appAttemptId, null, mock(queueClazz), null, null)); } catch (Exception e) { failed.set(true); finished.set(true); Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java Mon Dec 2 17:41:44 2013 @@ -28,7 +28,7 @@ public class SimpleGroupsMapping impleme @Override public List getGroups(String user) { - return Arrays.asList(user + "group"); + return Arrays.asList(user + "group", user + "subgroup1", user + "subgroup2"); } @Override Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java Mon Dec 2 17:41:44 2013 @@ -24,7 +24,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.util.Clock; import org.junit.Test; import org.mockito.Mockito; @@ -53,7 +52,7 @@ public class TestFSSchedulerApp { @Test public void testDelayScheduling() { - Queue queue = Mockito.mock(Queue.class); + FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); Priority prio = Mockito.mock(Priority.class); Mockito.when(prio.getPriority()).thenReturn(1); double nodeLocalityThreshold = .5; @@ -110,7 +109,7 @@ public class TestFSSchedulerApp { @Test public void testDelaySchedulingForContinuousScheduling() throws InterruptedException { - Queue queue = Mockito.mock(Queue.class); + FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); Priority prio = Mockito.mock(Priority.class); Mockito.when(prio.getPriority()).thenReturn(1); @@ -170,7 +169,7 @@ public class TestFSSchedulerApp { * no tin use), the least restrictive locality level is returned. */ public void testLocalityLevelWithoutDelays() { - Queue queue = Mockito.mock(Queue.class); + FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); Priority prio = Mockito.mock(Priority.class); Mockito.when(prio.getPriority()).thenReturn(1); Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Mon Dec 2 17:41:44 2013 @@ -100,7 +100,7 @@ import com.google.common.collect.Sets; public class TestFairScheduler { - private class MockClock implements Clock { + static class MockClock implements Clock { private long time = 0; @Override public long getTime() { @@ -613,9 +613,9 @@ public class TestFairScheduler { appAttemptId, "default", "user1"); scheduler.handle(appAddedEvent); assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) - .getAppSchedulables().size()); + .getRunnableAppSchedulables().size()); assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) - .getAppSchedulables().size()); + .getRunnableAppSchedulables().size()); assertEquals("root.user1", rmApp.getQueue()); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); @@ -625,11 +625,11 @@ public class TestFairScheduler { createAppAttemptId(2, 1), "default", "user2"); scheduler.handle(appAddedEvent2); assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) - .getAppSchedulables().size()); + .getRunnableAppSchedulables().size()); assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true) - .getAppSchedulables().size()); + .getRunnableAppSchedulables().size()); assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true) - .getAppSchedulables().size()); + .getRunnableAppSchedulables().size()); } @Test @@ -682,8 +682,10 @@ public class TestFairScheduler { rules.add(new QueuePlacementRule.Specified().initialize(true, null)); rules.add(new QueuePlacementRule.User().initialize(false, null)); rules.add(new QueuePlacementRule.PrimaryGroup().initialize(false, null)); + rules.add(new QueuePlacementRule.SecondaryGroupExistingQueue().initialize(false, null)); rules.add(new QueuePlacementRule.Default().initialize(true, null)); - Set queues = Sets.newHashSet("root.user1", "root.user3group"); + Set queues = Sets.newHashSet("root.user1", "root.user3group", + "root.user4subgroup1", "root.user4subgroup2" , "root.user5subgroup2"); scheduler.getQueueManager().placementPolicy = new QueuePlacementPolicy( rules, queues, conf); appId = createSchedulingRequest(1024, "somequeue", "user1"); @@ -692,6 +694,10 @@ public class TestFairScheduler { assertEquals("root.user1", apps.get(appId).getQueueName()); appId = createSchedulingRequest(1024, "default", "user3"); assertEquals("root.user3group", apps.get(appId).getQueueName()); + appId = createSchedulingRequest(1024, "default", "user4"); + assertEquals("root.user4subgroup1", apps.get(appId).getQueueName()); + appId = createSchedulingRequest(1024, "default", "user5"); + assertEquals("root.user5subgroup2", apps.get(appId).getQueueName()); appId = createSchedulingRequest(1024, "default", "otheruser"); assertEquals("root.default", apps.get(appId).getQueueName()); @@ -815,7 +821,7 @@ public class TestFairScheduler { // That queue should have one app assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) - .getAppSchedulables().size()); + .getRunnableAppSchedulables().size()); AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( createAppAttemptId(1, 1), RMAppAttemptState.FINISHED); @@ -825,7 +831,7 @@ public class TestFairScheduler { // Queue should have no apps assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) - .getAppSchedulables().size()); + .getRunnableAppSchedulables().size()); } @Test @@ -2394,7 +2400,158 @@ public class TestFairScheduler { public void testConcurrentAccessOnApplications() throws Exception { FairScheduler fs = new FairScheduler(); TestCapacityScheduler.verifyConcurrentAccessOnApplications( - fs.applications, FSSchedulerApp.class); + fs.applications, FSSchedulerApp.class, FSLeafQueue.class); + } + + + private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) { + FSSchedulerApp app = scheduler.applications.get(attId); + FSLeafQueue queue = app.getQueue(); + Collection runnableApps = + queue.getRunnableAppSchedulables(); + Collection nonRunnableApps = + queue.getNonRunnableAppSchedulables(); + assertEquals(runnable, runnableApps.contains(app.getAppSchedulable())); + assertEquals(!runnable, nonRunnableApps.contains(app.getAppSchedulable())); + } + + private void verifyQueueNumRunnable(String queueName, int numRunnableInQueue, + int numNonRunnableInQueue) { + FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue(queueName, false); + assertEquals(numRunnableInQueue, + queue.getRunnableAppSchedulables().size()); + assertEquals(numNonRunnableInQueue, + queue.getNonRunnableAppSchedulables().size()); + } + + @Test + public void testUserAndQueueMaxRunningApps() throws Exception { + Configuration conf = createConfiguration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("2"); + out.println(""); + out.println(""); + out.println("1"); + out.println(""); + out.println(""); + out.close(); + + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + // exceeds no limits + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1"); + verifyAppRunnable(attId1, true); + verifyQueueNumRunnable("queue1", 1, 0); + // exceeds user limit + ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue2", "user1"); + verifyAppRunnable(attId2, false); + verifyQueueNumRunnable("queue2", 0, 1); + // exceeds no limits + ApplicationAttemptId attId3 = createSchedulingRequest(1024, "queue1", "user2"); + verifyAppRunnable(attId3, true); + verifyQueueNumRunnable("queue1", 2, 0); + // exceeds queue limit + ApplicationAttemptId attId4 = createSchedulingRequest(1024, "queue1", "user2"); + verifyAppRunnable(attId4, false); + verifyQueueNumRunnable("queue1", 2, 1); + + // Remove app 1 and both app 2 and app 4 should becomes runnable in its place + AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( + attId1, RMAppAttemptState.FINISHED); + scheduler.handle(appRemovedEvent1); + verifyAppRunnable(attId2, true); + verifyQueueNumRunnable("queue2", 1, 0); + verifyAppRunnable(attId4, true); + verifyQueueNumRunnable("queue1", 2, 0); + + // A new app to queue1 should not be runnable + ApplicationAttemptId attId5 = createSchedulingRequest(1024, "queue1", "user2"); + verifyAppRunnable(attId5, false); + verifyQueueNumRunnable("queue1", 2, 1); + } + + @Test + public void testMaxRunningAppsHierarchicalQueues() throws Exception { + Configuration conf = createConfiguration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + MockClock clock = new MockClock(); + scheduler.setClock(clock); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" 3"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); + + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + // exceeds no limits + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1"); + verifyAppRunnable(attId1, true); + verifyQueueNumRunnable("queue1.sub1", 1, 0); + clock.tick(10); + // exceeds no limits + ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1.sub3", "user1"); + verifyAppRunnable(attId2, true); + verifyQueueNumRunnable("queue1.sub3", 1, 0); + clock.tick(10); + // exceeds no limits + ApplicationAttemptId attId3 = createSchedulingRequest(1024, "queue1.sub2", "user1"); + verifyAppRunnable(attId3, true); + verifyQueueNumRunnable("queue1.sub2", 1, 0); + clock.tick(10); + // exceeds queue1 limit + ApplicationAttemptId attId4 = createSchedulingRequest(1024, "queue1.sub2", "user1"); + verifyAppRunnable(attId4, false); + verifyQueueNumRunnable("queue1.sub2", 1, 1); + clock.tick(10); + // exceeds sub3 limit + ApplicationAttemptId attId5 = createSchedulingRequest(1024, "queue1.sub3", "user1"); + verifyAppRunnable(attId5, false); + verifyQueueNumRunnable("queue1.sub3", 1, 1); + clock.tick(10); + + // Even though the app was removed from sub3, the app from sub2 gets to go + // because it came in first + AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( + attId2, RMAppAttemptState.FINISHED); + scheduler.handle(appRemovedEvent1); + verifyAppRunnable(attId4, true); + verifyQueueNumRunnable("queue1.sub2", 2, 0); + verifyAppRunnable(attId5, false); + verifyQueueNumRunnable("queue1.sub3", 0, 1); + + // Now test removal of a non-runnable app + AppRemovedSchedulerEvent appRemovedEvent2 = new AppRemovedSchedulerEvent( + attId5, RMAppAttemptState.KILLED); + scheduler.handle(appRemovedEvent2); + assertEquals(0, scheduler.maxRunningEnforcer.usersNonRunnableApps + .get("user1").size()); + // verify app gone in queue accounting + verifyQueueNumRunnable("queue1.sub3", 0, 0); + // verify it doesn't become runnable when there would be space for it + AppRemovedSchedulerEvent appRemovedEvent3 = new AppRemovedSchedulerEvent( + attId4, RMAppAttemptState.FINISHED); + scheduler.handle(appRemovedEvent3); + verifyQueueNumRunnable("queue1.sub2", 1, 0); + verifyQueueNumRunnable("queue1.sub3", 0, 0); } @Test (timeout = 10000) @@ -2493,23 +2650,23 @@ public class TestFairScheduler { // Should get put into jerry createSchedulingRequest(1024, "jerry", "someuser"); - assertEquals(1, jerryQueue.getAppSchedulables().size()); + assertEquals(1, jerryQueue.getRunnableAppSchedulables().size()); // Should get forced into default createSchedulingRequest(1024, "newqueue", "someuser"); - assertEquals(1, jerryQueue.getAppSchedulables().size()); - assertEquals(1, defaultQueue.getAppSchedulables().size()); + assertEquals(1, jerryQueue.getRunnableAppSchedulables().size()); + assertEquals(1, defaultQueue.getRunnableAppSchedulables().size()); // Would get put into someuser because of user-as-default-queue, but should // be forced into default createSchedulingRequest(1024, "default", "someuser"); - assertEquals(1, jerryQueue.getAppSchedulables().size()); - assertEquals(2, defaultQueue.getAppSchedulables().size()); + assertEquals(1, jerryQueue.getRunnableAppSchedulables().size()); + assertEquals(2, defaultQueue.getRunnableAppSchedulables().size()); // Should get put into jerry because of user-as-default-queue createSchedulingRequest(1024, "default", "jerry"); - assertEquals(2, jerryQueue.getAppSchedulables().size()); - assertEquals(2, defaultQueue.getAppSchedulables().size()); + assertEquals(2, jerryQueue.getRunnableAppSchedulables().size()); + assertEquals(2, defaultQueue.getRunnableAppSchedulables().size()); } @SuppressWarnings("resource") Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Mon Dec 2 17:41:44 2013 @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; @@ -518,7 +519,7 @@ public class TestFifoScheduler { public void testConcurrentAccessOnApplications() throws Exception { FifoScheduler fs = new FifoScheduler(); TestCapacityScheduler.verifyConcurrentAccessOnApplications( - fs.applications, FiCaSchedulerApp.class); + fs.applications, FiCaSchedulerApp.class, Queue.class); } @SuppressWarnings("resource") Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java Mon Dec 2 17:41:44 2013 @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.webproxy; import java.io.IOException; +import java.net.URI; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,7 +31,6 @@ import org.apache.hadoop.util.StringUtil import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; - import org.apache.hadoop.fs.CommonConfigurationKeys; public class WebAppProxy extends AbstractService { @@ -89,7 +89,8 @@ public class WebAppProxy extends Abstrac protected void serviceStart() throws Exception { try { proxyServer = new HttpServer.Builder().setName("proxy") - .setBindAddress(bindAddress).setPort(port).setFindPort(port == 0) + .addEndpoint(URI.create("http://" + bindAddress + ":" + port)) + .setFindPort(port == 0) .setConf(getConfig()).setACL(acl).build(); proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME, ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java Mon Dec 2 17:41:44 2013 @@ -29,6 +29,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.HttpCookie; import java.net.HttpURLConnection; +import java.net.URI; import java.net.URL; import java.util.List; import java.util.Map; @@ -126,7 +127,7 @@ public class TestWebAppProxyServlet { proxy.init(configuration); proxy.start(); - int proxyPort = proxy.proxy.proxyServer.getPort(); + int proxyPort = proxy.proxy.proxyServer.getConnectorAddress(0).getPort(); AppReportFetcherForTest appReportFetcher = proxy.proxy.appReportFetcher; // wrong url @@ -285,8 +286,7 @@ public class TestWebAppProxyServlet { YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); proxyServer = new HttpServer.Builder() .setName("proxy") - .setBindAddress(bindAddress) - .setPort(0) + .addEndpoint(URI.create("http://" + bindAddress + ":0")) .setFindPort(true) .setConf(conf) .setACL(acl) @@ -306,7 +306,7 @@ public class TestWebAppProxyServlet { proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); proxyServer.start(); System.out.println("Proxy server is started at port " + - proxyServer.getPort()); + proxyServer.getConnectorAddress(0).getPort()); } catch (Exception e) { LOG.fatal("Could not start proxy web server", e); throw new YarnRuntimeException("Could not start proxy web server", e); Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1547122&r1=1547121&r2=1547122&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Mon Dec 2 17:41:44 2013 @@ -287,6 +287,10 @@ Allocation file format * primaryGroup: the app is placed into a queue with the name of the primary group of the user who submitted it. + * secondaryGroupExistingQueue: the app is placed into a queue with a name + that matches a secondary group of the user who submitted it. The first + secondary group that matches a configured queue will be selected. + * default: the app is placed into the queue named "default". * reject: the app is rejected.