Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E3F991088C for ; Sun, 22 Feb 2015 03:19:27 +0000 (UTC) Received: (qmail 34826 invoked by uid 500); 22 Feb 2015 03:19:22 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 34748 invoked by uid 500); 22 Feb 2015 03:19:22 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 34738 invoked by uid 99); 22 Feb 2015 03:19:22 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 22 Feb 2015 03:19:22 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 84BDFE0531; Sun, 22 Feb 2015 03:19:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xgong@apache.org To: common-commits@hadoop.apache.org Message-Id: <544221eb96654d34bc978bba5b68536b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-2797. TestWorkPreservingRMRestart should use ParametrizedSchedulerTestBase. Contributed by Karthik Kambatla Date: Sun, 22 Feb 2015 03:19:22 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2 f8435c5d3 -> 0ca974d0b YARN-2797. TestWorkPreservingRMRestart should use ParametrizedSchedulerTestBase. Contributed by Karthik Kambatla (cherry picked from commit fe7a302473251b7310105a936edf220e401c613f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ca974d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ca974d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ca974d0 Branch: refs/heads/branch-2 Commit: 0ca974d0b0d6d80ad68e11c28cf121bae9c930bc Parents: f8435c5 Author: Xuan Authored: Sat Feb 21 19:17:29 2015 -0800 Committer: Xuan Committed: Sat Feb 21 19:19:08 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../ParameterizedSchedulerTestBase.java | 12 +++ .../TestWorkPreservingRMRestart.java | 79 +++----------------- 3 files changed, 25 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ca974d0/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9b6800f..5a557cd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -282,6 +282,9 @@ Release 2.7.0 - UNRELEASED YARN-3236. Cleanup RMAuthenticationFilter#AUTH_HANDLER_PROPERTY. (zhihai xu via xgong) + YARN-2797. TestWorkPreservingRMRestart should use ParametrizedSchedulerTestBase + (Karthik Kambatla via xgong) + OPTIMIZATIONS YARN-2990. FairScheduler's delay-scheduling always waits for node-local and http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ca974d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java index cfd1600..b099836 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java @@ -83,10 +83,22 @@ public abstract class ParameterizedSchedulerTestBase { out.println(""); out.println(""); out.println("-1.0"); + out.println("fair"); + out.println(""); + out.println(" drf"); + out.println(" 1.0"); + out.println(" 100"); + out.println(" 120"); + out.println(" .5"); + out.println(""); out.println(""); out.close(); conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); } + + public SchedulerType getSchedulerType() { + return schedulerType; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ca974d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index a9caf77..3033496 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -97,23 +97,24 @@ import com.google.common.base.Supplier; @SuppressWarnings({"rawtypes", "unchecked"}) @RunWith(value = Parameterized.class) -public class TestWorkPreservingRMRestart { +public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase { private YarnConfiguration conf; - private Class schedulerClass; MockRM rm1 = null; MockRM rm2 = null; + public TestWorkPreservingRMRestart(SchedulerType type) { + super(type); + } + @Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); - conf = new YarnConfiguration(); + conf = getConf(); UserGroupInformation.setConfiguration(conf); conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); - conf.setClass(YarnConfiguration.RM_SCHEDULER, schedulerClass, - ResourceScheduler.class); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true); conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0); DefaultMetricsSystem.setMiniClusterMode(true); @@ -129,16 +130,6 @@ public class TestWorkPreservingRMRestart { } } - @Parameterized.Parameters - public static Collection getTestParameters() { - return Arrays.asList(new Object[][] { { CapacityScheduler.class }, - { FifoScheduler.class }, {FairScheduler.class } }); - } - - public TestWorkPreservingRMRestart(Class schedulerClass) { - this.schedulerClass = schedulerClass; - } - // Test common scheduler state including SchedulerAttempt, SchedulerNode, // AppSchedulingInfo can be reconstructed via the container recovery reports // on NM re-registration. @@ -159,9 +150,6 @@ public class TestWorkPreservingRMRestart { MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); rm1 = new MockRM(conf, memStore); - if (schedulerClass.equals(FairScheduler.class)) { - initFairScheduler(rm1); - } rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); @@ -174,9 +162,6 @@ public class TestWorkPreservingRMRestart { // Re-start RM rm2 = new MockRM(conf, memStore); - if (schedulerClass.equals(FairScheduler.class)) { - initFairScheduler(rm2); - } rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); // recover app @@ -249,11 +234,9 @@ public class TestWorkPreservingRMRestart { SchedulerApplication schedulerApp = schedulerApps.get(recoveredApp1.getApplicationId()); - if (schedulerClass.equals(CapacityScheduler.class)) { + if (getSchedulerType() == SchedulerType.CAPACITY) { checkCSQueue(rm2, schedulerApp, nmResource, nmResource, usedResources, 2); - } else if (schedulerClass.equals(FifoScheduler.class)) { - checkFifoQueue(rm2, schedulerApp, usedResources, availableResources); - } else if (schedulerClass.equals(FairScheduler.class)) { + } else { checkFSQueue(rm2, schedulerApp, usedResources, availableResources); } @@ -324,25 +307,6 @@ public class TestWorkPreservingRMRestart { .getUsed()); } - private void checkFifoQueue(ResourceManager rm, - SchedulerApplication schedulerApp, Resource usedResources, - Resource availableResources) throws Exception { - FifoScheduler scheduler = (FifoScheduler) rm.getResourceScheduler(); - // ************ check cluster used Resources ******** - assertEquals(usedResources, scheduler.getUsedResource()); - - // ************ check app headroom **************** - SchedulerApplicationAttempt schedulerAttempt = - schedulerApp.getCurrentAppAttempt(); - assertEquals(availableResources, schedulerAttempt.getHeadroom()); - - // ************ check queue metrics **************** - QueueMetrics queueMetrics = scheduler.getRootQueueMetrics(); - assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(), - availableResources.getVirtualCores(), usedResources.getMemory(), - usedResources.getVirtualCores()); - } - private void checkFSQueue(ResourceManager rm, SchedulerApplication schedulerApp, Resource usedResources, Resource availableResources) throws Exception { @@ -379,29 +343,6 @@ public class TestWorkPreservingRMRestart { usedResources.getVirtualCores()); } - private void initFairScheduler(ResourceManager rm) throws IOException { - FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); - String testDir = - new File( - System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); - String allocFile = new File(testDir, "test-queues").getAbsolutePath(); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); - - PrintWriter out = new PrintWriter(new FileWriter(allocFile)); - out.println(""); - out.println(""); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" 1.0"); - out.println(" 100"); - out.println(" 120"); - out.println(" .5"); - out.println(""); - out.println(""); - out.close(); - } - // create 3 container reports for AM public static List createNMContainerStatusForApp(MockAM am) { @@ -468,7 +409,7 @@ public class TestWorkPreservingRMRestart { // 10. Assert each user's consumption inside the queue. @Test (timeout = 30000) public void testCapacitySchedulerRecovery() throws Exception { - if (!schedulerClass.equals(CapacityScheduler.class)) { + if (getSchedulerType() != SchedulerType.CAPACITY) { return; } conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true); @@ -587,7 +528,7 @@ public class TestWorkPreservingRMRestart { //3. Verify that the expected exception was thrown @Test (timeout = 30000, expected = QueueNotFoundException.class) public void testCapacitySchedulerQueueRemovedRecovery() throws Exception { - if (!schedulerClass.equals(CapacityScheduler.class)) { + if (getSchedulerType() != SchedulerType.CAPACITY) { throw new QueueNotFoundException("Dummy"); } conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);