Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3AD8918DB0 for ; Tue, 12 May 2015 20:49:24 +0000 (UTC) Received: (qmail 31653 invoked by uid 500); 12 May 2015 20:49:19 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 31392 invoked by uid 500); 12 May 2015 20:49:19 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 31046 invoked by uid 99); 12 May 2015 20:49:19 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 May 2015 20:49:19 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D2BFDDFF07; Tue, 12 May 2015 20:49:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zjshen@apache.org To: common-commits@hadoop.apache.org Date: Tue, 12 May 2015 20:49:26 -0000 Message-Id: <8d30cefff63e41cfa1bc8065e8b932a9@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [09/36] hadoop git commit: YARN-1287. Consolidate MockClocks. (Sebastian Wong and Anubhav Dhoot via kasha) YARN-1287. Consolidate MockClocks. (Sebastian Wong and Anubhav Dhoot via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca59e771 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca59e771 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca59e771 Branch: refs/heads/YARN-2928 Commit: ca59e771062da7e857fd4a5da8cd14982221b36e Parents: 33ae623 Author: Karthik Kambatla Authored: Sat May 9 14:34:54 2015 -0700 Committer: Zhijie Shen Committed: Tue May 12 13:24:11 2015 -0700 ---------------------------------------------------------------------- .../mapreduce/v2/app/TestRuntimeEstimators.java | 26 ++-------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/util/ControlledClock.java | 15 ++++++ .../util/TestCgroupsLCEResourcesHandler.java | 18 ++----- .../scheduler/fair/FairSchedulerTestBase.java | 12 ----- .../fair/TestAllocationFileLoaderService.java | 19 ++----- .../fair/TestContinuousScheduling.java | 15 ++---- .../scheduler/fair/TestFSAppAttempt.java | 26 +++------- .../scheduler/fair/TestFairScheduler.java | 53 ++++++++++---------- .../fair/TestFairSchedulerPreemption.java | 12 +++-- .../fair/TestMaxRunningAppsEnforcer.java | 7 +-- 11 files changed, 82 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index fe0f341..69f2709 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -69,13 +69,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.hadoop.yarn.util.Clock; +import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.SystemClock; import org.junit.Assert; import org.junit.Test; @@ -90,7 +90,7 @@ public class TestRuntimeEstimators { private static int MAP_TASKS = 200; private static int REDUCE_TASKS = 150; - MockClock clock; + ControlledClock clock; Job myJob; @@ -120,7 +120,7 @@ public class TestRuntimeEstimators { private void coreTestEstimator (TaskRuntimeEstimator testedEstimator, int expectedSpeculations) { estimator = testedEstimator; - clock = new MockClock(); + clock = new ControlledClock(); dispatcher = new AsyncDispatcher(); myJob = null; slotsInUse.set(0); @@ -129,7 +129,7 @@ public class TestRuntimeEstimators { successfulSpeculations.set(0); taskTimeSavedBySpeculation.set(0); - clock.advanceTime(1000); + clock.tickMsec(1000); Configuration conf = new Configuration(); @@ -230,7 +230,7 @@ public class TestRuntimeEstimators { } } - clock.advanceTime(1000L); + clock.tickMsec(1000L); if (clock.getTime() % 10000L == 0L) { speculator.scanForSpeculations(); @@ -777,22 +777,6 @@ public class TestRuntimeEstimators { } } - static class MockClock implements Clock { - private long currentTime = 0; - - public long getTime() { - return currentTime; - } - - void setMeasuredTime(long newTime) { - currentTime = newTime; - } - - void advanceTime(long increment) { - currentTime += increment; - } - } - class MyAppMaster extends CompositeService { final Clock clock; public MyAppMaster(Clock clock) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2be7604..f7c56f1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -291,6 +291,9 @@ Release 2.8.0 - UNRELEASED YARN-1912. ResourceLocalizer started without any jvm memory control. (Masatake Iwasaki via xgong) + YARN-1287. Consolidate MockClocks. + (Sebastian Wong and Anubhav Dhoot via kasha) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/ControlledClock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/ControlledClock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/ControlledClock.java index 16bd785..6dcb1dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/ControlledClock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/ControlledClock.java @@ -21,6 +21,11 @@ package org.apache.hadoop.yarn.util; public class ControlledClock implements Clock { private long time = -1; private final Clock actualClock; + // Convenience for getting a controlled clock with overridden time + public ControlledClock() { + this(new SystemClock()); + setTime(0); + } public ControlledClock(Clock actualClock) { this.actualClock = actualClock; } @@ -30,6 +35,16 @@ public class ControlledClock implements Clock { public synchronized void reset() { time = -1; } + public synchronized void tickSec(int seconds) { + tickMsec(seconds * 1000L); + } + public synchronized void tickMsec(long millisec) { + if (time == -1) { + throw new IllegalStateException("ControlledClock setTime should be " + + "called before incrementing time"); + } + time = time + millisec; + } @Override public synchronized long getTime() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java index 8e9d787..440f9ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java @@ -22,10 +22,11 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.TestCGroupsHandlerImpl; + +import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.junit.Assert; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.util.Clock; import org.junit.Test; import org.junit.After; import org.junit.Before; @@ -39,14 +40,6 @@ import java.util.concurrent.CountDownLatch; public class TestCgroupsLCEResourcesHandler { static File cgroupDir = null; - static class MockClock implements Clock { - long time; - @Override - public long getTime() { - return time; - } - } - @Before public void setUp() throws Exception { cgroupDir = @@ -93,8 +86,7 @@ public class TestCgroupsLCEResourcesHandler { // Verify DeleteCgroup times out if "tasks" file contains data @Test public void testDeleteCgroup() throws Exception { - final MockClock clock = new MockClock(); - clock.time = System.currentTimeMillis(); + final ControlledClock clock = new ControlledClock(); CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler(); handler.setConf(new YarnConfiguration()); handler.initConfig(); @@ -118,8 +110,8 @@ public class TestCgroupsLCEResourcesHandler { } catch (InterruptedException ex) { //NOP } - clock.time += YarnConfiguration. - DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT; + clock.tickMsec(YarnConfiguration. + DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT); } }.start(); latch.await(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index 0a9c389..23b708a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -49,18 +49,6 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; public class FairSchedulerTestBase { - protected static class MockClock implements Clock { - private long time = 0; - @Override - public long getTime() { - return time; - } - - public void tick(int seconds) { - time = time + seconds * 1000; - } - } - public final static String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index b09573c..202eb09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSche import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; -import org.apache.hadoop.yarn.util.Clock; +import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Test; @@ -43,18 +43,6 @@ public class TestAllocationFileLoaderService { final static String ALLOC_FILE = new File(TEST_DIR, "test-queues").getAbsolutePath(); - private class MockClock implements Clock { - private long time = 0; - @Override - public long getTime() { - return time; - } - - public void tick(long ms) { - time += ms; - } - } - @Test public void testGetAllocationFileFromClasspath() { Configuration conf = new Configuration(); @@ -81,7 +69,8 @@ public class TestAllocationFileLoaderService { out.println(""); out.close(); - MockClock clock = new MockClock(); + ControlledClock clock = new ControlledClock(); + clock.setTime(0); Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); @@ -126,7 +115,7 @@ public class TestAllocationFileLoaderService { out.println(""); out.close(); - clock.tick(System.currentTimeMillis() + clock.tickMsec(System.currentTimeMillis() + AllocationFileLoaderService.ALLOC_RELOAD_WAIT_MS + 10000); allocLoader.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java index a72e393..53382de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java @@ -19,20 +19,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.metrics2.lib.MutableRate; -import org.apache.hadoop.metrics2.util.SampleStat; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event - .NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import static org.junit.Assert.assertEquals; @@ -41,11 +37,10 @@ import org.junit.Before; import org.junit.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.List; public class TestContinuousScheduling extends FairSchedulerTestBase { - private MockClock mockClock; + private ControlledClock mockClock; @Override public Configuration createConfiguration() { @@ -59,7 +54,7 @@ public class TestContinuousScheduling extends FairSchedulerTestBase { @Before public void setup() { - mockClock = new MockClock(); + mockClock = new ControlledClock(); conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); @@ -108,7 +103,7 @@ public class TestContinuousScheduling extends FairSchedulerTestBase { FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId); // Advance time and let continuous scheduling kick in - mockClock.tick(1); + mockClock.tickSec(1); while (1024 != app.getCurrentConsumption().getMemory()) { Thread.sleep(100); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java index 43fe186..7aa62a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java @@ -31,7 +31,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; -import org.apache.hadoop.yarn.util.Clock; + +import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Before; import org.junit.Test; @@ -39,19 +40,6 @@ import org.mockito.Mockito; public class TestFSAppAttempt extends FairSchedulerTestBase { - private class MockClock implements Clock { - private long time = 0; - @Override - public long getTime() { - return time; - } - - public void tick(int seconds) { - time = time + seconds * 1000; - } - - } - @Before public void setup() { Configuration conf = createConfiguration(); @@ -125,7 +113,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase { Priority prio = Mockito.mock(Priority.class); Mockito.when(prio.getPriority()).thenReturn(1); - MockClock clock = new MockClock(); + ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); long nodeLocalityDelayMs = 5 * 1000L; // 5 seconds @@ -143,13 +131,13 @@ public class TestFSAppAttempt extends FairSchedulerTestBase { nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); // after 4 seconds should remain node local - clock.tick(4); + clock.tickSec(4); assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); // after 6 seconds should switch to rack local - clock.tick(2); + clock.tickSec(2); assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); @@ -162,12 +150,12 @@ public class TestFSAppAttempt extends FairSchedulerTestBase { nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); // Now escalate again to rack-local, then to off-switch - clock.tick(6); + clock.tickSec(6); assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); - clock.tick(7); + clock.tickSec(7); assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index fc69de7..a26209b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -72,7 +72,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -100,6 +99,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlaceme import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Assert; @@ -1489,7 +1490,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE + ".allocation.file", ALLOC_FILE); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); - MockClock clock = new MockClock(); + ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -1587,7 +1588,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.getSchedulerApp(app2).getPreemptionContainers())); // Pretend 15 seconds have passed - clock.tick(15); + clock.tickSec(15); // Trigger a kill by insisting we want containers back scheduler.preemptResources(Resources.createResource(2 * 1024)); @@ -1617,7 +1618,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.preemptResources(Resources.createResource(2 * 1024)); // Pretend 15 seconds have passed - clock.tick(15); + clock.tickSec(15); // We should be able to claw back another container from A and B each. // For queueA (fifo), continue preempting from app2. @@ -1649,7 +1650,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); - MockClock clock = new MockClock(); + ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -1702,7 +1703,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.update(); // Let 11 sec pass - clock.tick(11); + clock.tickSec(11); scheduler.update(); Resource toPreempt = scheduler.resToPreempt(scheduler.getQueueManager() @@ -1722,7 +1723,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { */ public void testPreemptionDecision() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - MockClock clock = new MockClock(); + ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -1833,7 +1834,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { Resources.none(), scheduler.resToPreempt(schedD, clock.getTime()))); // After minSharePreemptionTime has passed, they should want to preempt min // share. - clock.tick(6); + clock.tickSec(6); assertEquals( 1024, scheduler.resToPreempt(schedC, clock.getTime()).getMemory()); assertEquals( @@ -1842,7 +1843,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { // After fairSharePreemptionTime has passed, they should want to preempt // fair share. scheduler.update(); - clock.tick(6); + clock.tickSec(6); assertEquals( 1536 , scheduler.resToPreempt(schedC, clock.getTime()).getMemory()); assertEquals( @@ -1855,7 +1856,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { */ public void testPreemptionDecisionWithVariousTimeout() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - MockClock clock = new MockClock(); + ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -1971,7 +1972,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { // After 5 seconds, queueB1 wants to preempt min share scheduler.update(); - clock.tick(6); + clock.tickSec(6); assertEquals( 1024, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory()); assertEquals( @@ -1981,7 +1982,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { // After 10 seconds, queueB2 wants to preempt min share scheduler.update(); - clock.tick(5); + clock.tickSec(5); assertEquals( 1024, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory()); assertEquals( @@ -1991,7 +1992,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { // After 15 seconds, queueC wants to preempt min share scheduler.update(); - clock.tick(5); + clock.tickSec(5); assertEquals( 1024, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory()); assertEquals( @@ -2001,7 +2002,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { // After 20 seconds, queueB2 should want to preempt fair share scheduler.update(); - clock.tick(5); + clock.tickSec(5); assertEquals( 1024, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory()); assertEquals( @@ -2011,7 +2012,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { // After 25 seconds, queueB1 should want to preempt fair share scheduler.update(); - clock.tick(5); + clock.tickSec(5); assertEquals( 1536, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory()); assertEquals( @@ -2021,7 +2022,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { // After 30 seconds, queueC should want to preempt fair share scheduler.update(); - clock.tick(5); + clock.tickSec(5); assertEquals( 1536, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory()); assertEquals( @@ -3703,7 +3704,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { @Test public void testMaxRunningAppsHierarchicalQueues() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - MockClock clock = new MockClock(); + ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -3728,28 +3729,28 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1"); verifyAppRunnable(attId1, true); verifyQueueNumRunnable("queue1.sub1", 1, 0); - clock.tick(10); + clock.tickSec(10); // exceeds no limits ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1.sub3", "user1"); verifyAppRunnable(attId2, true); verifyQueueNumRunnable("queue1.sub3", 1, 0); - clock.tick(10); + clock.tickSec(10); // exceeds no limits ApplicationAttemptId attId3 = createSchedulingRequest(1024, "queue1.sub2", "user1"); verifyAppRunnable(attId3, true); verifyQueueNumRunnable("queue1.sub2", 1, 0); - clock.tick(10); + clock.tickSec(10); // exceeds queue1 limit ApplicationAttemptId attId4 = createSchedulingRequest(1024, "queue1.sub2", "user1"); verifyAppRunnable(attId4, false); verifyQueueNumRunnable("queue1.sub2", 1, 1); - clock.tick(10); + clock.tickSec(10); // exceeds sub3 limit ApplicationAttemptId attId5 = createSchedulingRequest(1024, "queue1.sub3", "user1"); verifyAppRunnable(attId5, false); verifyQueueNumRunnable("queue1.sub3", 1, 1); - clock.tick(10); - + clock.tickSec(10); + // Even though the app was removed from sub3, the app from sub2 gets to go // because it came in first AppAttemptRemovedSchedulerEvent appRemovedEvent1 = @@ -3923,7 +3924,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { public void testRecoverRequestAfterPreemption() throws Exception { conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10); - MockClock clock = new MockClock(); + ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); scheduler.init(conf); scheduler.start(); @@ -3974,8 +3975,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.warnOrKillContainer(rmContainer); // Wait for few clock ticks - clock.tick(5); - + clock.tickSec(5); + // preempt now scheduler.warnOrKillContainer(rmContainer); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java index 458b06d..6f759ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java @@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; + +import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -43,7 +45,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase { private final static String ALLOC_FILE = new File(TEST_DIR, TestFairSchedulerPreemption.class.getName() + ".xml").getAbsolutePath(); - private MockClock clock; + private ControlledClock clock; private static class StubbedFairScheduler extends FairScheduler { public int lastPreemptMemory = -1; @@ -70,7 +72,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase { @Before public void setup() throws IOException { conf = createConfiguration(); - clock = new MockClock(); + clock = new ControlledClock(); } @After @@ -148,7 +150,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase { // Verify submitting another request triggers preemption createSchedulingRequest(1024, "queueB", "user1", 1, 1); scheduler.update(); - clock.tick(6); + clock.tickSec(6); ((StubbedFairScheduler) scheduler).resetLastPreemptResources(); scheduler.preemptTasksIfNecessary(); @@ -164,7 +166,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase { // Verify submitting another request doesn't trigger preemption createSchedulingRequest(1024, "queueB", "user1", 1, 1); scheduler.update(); - clock.tick(6); + clock.tickSec(6); ((StubbedFairScheduler) scheduler).resetLastPreemptResources(); scheduler.preemptTasksIfNecessary(); @@ -180,7 +182,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase { // Verify submitting another request triggers preemption createSchedulingRequest(1024, "queueB", "user1", 1, 1); scheduler.update(); - clock.tick(6); + clock.tickSec(6); ((StubbedFairScheduler) scheduler).resetLastPreemptResources(); scheduler.preemptTasksIfNecessary(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca59e771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index ac5748f..6cca19a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.ControlledClock; import org.junit.Before; import org.junit.Test; @@ -40,14 +41,14 @@ public class TestMaxRunningAppsEnforcer { private Map userMaxApps; private MaxRunningAppsEnforcer maxAppsEnforcer; private int appNum; - private TestFairScheduler.MockClock clock; + private ControlledClock clock; private RMContext rmContext; private FairScheduler scheduler; @Before public void setup() throws Exception { Configuration conf = new Configuration(); - clock = new TestFairScheduler.MockClock(); + clock = new ControlledClock(); scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); @@ -151,7 +152,7 @@ public class TestMaxRunningAppsEnforcer { FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); - clock.tick(20); + clock.tickSec(20); addApp(leaf1, "user"); assertEquals(1, leaf1.getNumRunnableApps()); assertEquals(1, leaf2.getNumRunnableApps());