aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wfar...@apache.org
Subject git commit: Initiate a teardown of scheduler lifecycle upon application exit.
Date Fri, 14 Mar 2014 22:01:21 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 440768235 -> 5d4e041fd


Initiate a teardown of scheduler lifecycle upon application exit.

Reviewed at https://reviews.apache.org/r/18713/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/5d4e041f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/5d4e041f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/5d4e041f

Branch: refs/heads/master
Commit: 5d4e041fd2f62f10d751125be3bbf2beaa5c52cf
Parents: 4407682
Author: Bill Farner <wfarner@apache.org>
Authored: Fri Mar 14 14:59:34 2014 -0700
Committer: Bill Farner <wfarner@apache.org>
Committed: Fri Mar 14 14:59:34 2014 -0700

----------------------------------------------------------------------
 .../aurora/scheduler/SchedulerLifecycle.java    |  19 ++-
 .../scheduler/SchedulerLifecycleTest.java       | 138 ++++++++++++++-----
 2 files changed, 118 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/5d4e041f/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java b/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java
index 1cdbff6..e7b1058 100644
--- a/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java
+++ b/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java
@@ -32,8 +32,10 @@ import com.google.common.eventbus.Subscribe;
 import com.google.common.util.concurrent.Atomics;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.twitter.common.application.Lifecycle;
+import com.twitter.common.application.ShutdownRegistry;
 import com.twitter.common.base.Closure;
 import com.twitter.common.base.Closures;
+import com.twitter.common.base.Command;
 import com.twitter.common.quantity.Amount;
 import com.twitter.common.quantity.Time;
 import com.twitter.common.stats.StatImpl;
@@ -129,7 +131,8 @@ public class SchedulerLifecycle implements EventSubscriber {
       LeadingOptions leadingOptions,
       ScheduledExecutorService executorService,
       Clock clock,
-      EventSink eventSink) {
+      EventSink eventSink,
+      ShutdownRegistry shutdownRegistry) {
 
     this(
         driverFactory,
@@ -138,7 +141,8 @@ public class SchedulerLifecycle implements EventSubscriber {
         driver,
         new DefaultDelayedActions(leadingOptions, executorService),
         clock,
-        eventSink);
+        eventSink,
+        shutdownRegistry);
   }
 
   private static final class DefaultDelayedActions implements DelayedActions {
@@ -198,7 +202,8 @@ public class SchedulerLifecycle implements EventSubscriber {
       final SettableDriver driver,
       final DelayedActions delayedActions,
       final Clock clock,
-      final EventSink eventSink) {
+      final EventSink eventSink,
+      final ShutdownRegistry shutdownRegistry) {
 
     checkNotNull(driverFactory);
     checkNotNull(storage);
@@ -207,6 +212,7 @@ public class SchedulerLifecycle implements EventSubscriber {
     checkNotNull(delayedActions);
     checkNotNull(clock);
     checkNotNull(eventSink);
+    checkNotNull(shutdownRegistry);
 
     Stats.export(new StatImpl<Integer>("framework_registered") {
       @Override
@@ -223,6 +229,13 @@ public class SchedulerLifecycle implements EventSubscriber {
       });
     }
 
+    shutdownRegistry.addAction(new Command() {
+      @Override
+      public void execute() {
+        stateMachine.transition(State.DEAD);
+      }
+    });
+
     final Closure<Transition<State>> prepareStorage = new Closure<Transition<State>>()
{
       @Override
       public void execute(Transition<State> transition) {

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/5d4e041f/src/test/java/org/apache/aurora/scheduler/SchedulerLifecycleTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/SchedulerLifecycleTest.java b/src/test/java/org/apache/aurora/scheduler/SchedulerLifecycleTest.java
index 1b1e948..e33b0d1 100644
--- a/src/test/java/org/apache/aurora/scheduler/SchedulerLifecycleTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/SchedulerLifecycleTest.java
@@ -18,6 +18,7 @@ package org.apache.aurora.scheduler;
 import java.lang.Thread.UncaughtExceptionHandler;
 
 import com.twitter.common.application.Lifecycle;
+import com.twitter.common.application.ShutdownRegistry;
 import com.twitter.common.base.Command;
 import com.twitter.common.testing.easymock.EasyMockTest;
 import com.twitter.common.util.Clock;
@@ -50,7 +51,7 @@ public class SchedulerLifecycleTest extends EasyMockTest {
 
   private DriverFactory driverFactory;
   private StorageTestUtil storageUtil;
-  private Command shutdownRegistry;
+  private ShutdownSystem shutdownRegistry;
   private SettableDriver driver;
   private LeaderControl leaderControl;
   private SchedulerDriver schedulerDriver;
@@ -63,12 +64,28 @@ public class SchedulerLifecycleTest extends EasyMockTest {
   public void setUp() {
     driverFactory = createMock(DriverFactory.class);
     storageUtil = new StorageTestUtil(this);
-    shutdownRegistry = createMock(Command.class);
+    shutdownRegistry = createMock(ShutdownSystem.class);
     driver = createMock(SettableDriver.class);
     leaderControl = createMock(LeaderControl.class);
     schedulerDriver = createMock(SchedulerDriver.class);
     delayedActions = createMock(DelayedActions.class);
     eventSink = createMock(EventSink.class);
+  }
+
+  /**
+   * Composite interface to mimick a ShutdownRegistry implementation that can be triggered.
+   */
+  private interface ShutdownSystem extends ShutdownRegistry, Command {
+  }
+
+  private Capture<Command> replayAndCreateLifecycle() {
+    Capture<Command> shutdownCommand = createCapture();
+    shutdownRegistry.addAction(capture(shutdownCommand));
+
+    Clock clock = createMock(Clock.class);
+
+    control.replay();
+
     schedulerLifecycle = new SchedulerLifecycle(
         driverFactory,
         storageUtil.storage,
@@ -80,65 +97,97 @@ public class SchedulerLifecycleTest extends EasyMockTest {
         }),
         driver,
         delayedActions,
-        createMock(Clock.class),
-        eventSink);
+        clock,
+        eventSink,
+        shutdownRegistry);
+    return shutdownCommand;
   }
 
-  @Test
-  public void testAutoFailover() throws Exception {
-    // Test that when timed failover is initiated, cleanup is done in a way that should allow
the
-    // application to tear down cleanly.  Specifically, neglecting to call leaderControl.leave()
-    // can result in a lame duck scheduler process.
-
-    storageUtil.storage.prepare();
-
+  private void expectLoadStorage() {
     storageUtil.storage.start(EasyMock.<Quiet>anyObject());
     storageUtil.expectOperations();
     expect(storageUtil.schedulerStore.fetchFrameworkId()).andReturn(FRAMEWORK_ID);
-    expect(driverFactory.apply(FRAMEWORK_ID)).andReturn(schedulerDriver);
-    Capture<Runnable> triggerFailoverCapture = createCapture();
-    delayedActions.onAutoFailover(capture(triggerFailoverCapture));
-    delayedActions.onRegistrationTimeout(EasyMock.<Runnable>anyObject());
+  }
+
+  private void expectInitializeDriver() {
     driver.initialize(schedulerDriver);
     expect(schedulerDriver.start()).andReturn(Status.DRIVER_RUNNING);
     delayedActions.blockingDriverJoin(EasyMock.<Runnable>anyObject());
+  }
 
-    Capture<Runnable> handleRegisteredCapture = createCapture();
-    delayedActions.onRegistered(capture(handleRegisteredCapture));
+  private Capture<Runnable> expectFullStartup() throws Exception {
+    Capture<Runnable> handleRegistered = createCapture();
+    delayedActions.onRegistered(capture(handleRegistered));
     leaderControl.advertise();
     eventSink.post(new SchedulerActive());
+    return handleRegistered;
+  }
+
+  private void expectShutdown() throws Exception {
     leaderControl.leave();
     driver.stop();
     storageUtil.storage.stop();
     shutdownRegistry.execute();
+  }
 
-    control.replay();
+  @Test
+  public void testAutoFailover() throws Exception {
+    // Test that when timed failover is initiated, cleanup is done in a way that should allow
the
+    // application to tear down cleanly.  Specifically, neglecting to call leaderControl.leave()
+    // can result in a lame duck scheduler process.
+
+    storageUtil.storage.prepare();
+    expectLoadStorage();
+    expect(driverFactory.apply(FRAMEWORK_ID)).andReturn(schedulerDriver);
+    Capture<Runnable> triggerFailover = createCapture();
+    delayedActions.onAutoFailover(capture(triggerFailover));
+    delayedActions.onRegistrationTimeout(EasyMock.<Runnable>anyObject());
+    expectInitializeDriver();
+
+    Capture<Runnable> handleRegistered = expectFullStartup();
+    expectShutdown();
+
+    replayAndCreateLifecycle();
 
     LeadershipListener leaderListener = schedulerLifecycle.prepare();
     leaderListener.onLeading(leaderControl);
     schedulerLifecycle.registered(new DriverRegistered());
-    handleRegisteredCapture.getValue().run();
-    triggerFailoverCapture.getValue().run();
+    handleRegistered.getValue().run();
+    triggerFailover.getValue().run();
+  }
+
+  @Test
+  public void testRegistrationTimeout() throws Exception {
+    storageUtil.storage.prepare();
+    expectLoadStorage();
+    expect(driverFactory.apply(FRAMEWORK_ID)).andReturn(schedulerDriver);
+    delayedActions.onAutoFailover(EasyMock.<Runnable>anyObject());
+    Capture<Runnable> registrationTimeout = createCapture();
+    delayedActions.onRegistrationTimeout(capture(registrationTimeout));
+    expect(schedulerDriver.start()).andReturn(Status.DRIVER_RUNNING);
+
+    expectShutdown();
+
+    replayAndCreateLifecycle();
+
+    LeadershipListener leaderListener = schedulerLifecycle.prepare();
+    leaderListener.onLeading(leaderControl);
+    registrationTimeout.getValue().run();
   }
 
   @Test
   public void testDefeatedBeforeRegistered() throws Exception {
     storageUtil.storage.prepare();
-    storageUtil.storage.start(EasyMock.<Quiet>anyObject());
-    storageUtil.expectOperations();
-    expect(storageUtil.schedulerStore.fetchFrameworkId()).andReturn(FRAMEWORK_ID);
+    expectLoadStorage();
     expect(driverFactory.apply(FRAMEWORK_ID)).andReturn(schedulerDriver);
     delayedActions.onAutoFailover(EasyMock.<Runnable>anyObject());
     delayedActions.onRegistrationTimeout(EasyMock.<Runnable>anyObject());
     expect(schedulerDriver.start()).andReturn(Status.DRIVER_RUNNING);
 
     // Important piece here is what's absent - leader presence is not advertised.
-    leaderControl.leave();
-    driver.stop();
-    storageUtil.storage.stop();
-    shutdownRegistry.execute();
+    expectShutdown();
 
-    control.replay();
+    replayAndCreateLifecycle();
 
     LeadershipListener leaderListener = schedulerLifecycle.prepare();
     leaderListener.onLeading(leaderControl);
@@ -148,17 +197,12 @@ public class SchedulerLifecycleTest extends EasyMockTest {
   @Test
   public void testStorageStartFails() throws Exception {
     storageUtil.storage.prepare();
-
     storageUtil.expectOperations();
     storageUtil.storage.start(EasyMock.<Quiet>anyObject());
     expectLastCall().andThrow(new StorageException("Recovery failed."));
+    expectShutdown();
 
-    leaderControl.leave();
-    driver.stop();
-    storageUtil.storage.stop();
-    shutdownRegistry.execute();
-
-    control.replay();
+    replayAndCreateLifecycle();
 
     LeadershipListener leaderListener = schedulerLifecycle.prepare();
 
@@ -169,4 +213,26 @@ public class SchedulerLifecycleTest extends EasyMockTest {
       // Expected.
     }
   }
+
+  @Test
+  public void testExternalShutdown() throws Exception {
+    storageUtil.storage.prepare();
+    expectLoadStorage();
+    expect(driverFactory.apply(FRAMEWORK_ID)).andReturn(schedulerDriver);
+    Capture<Runnable> triggerFailover = createCapture();
+    delayedActions.onAutoFailover(capture(triggerFailover));
+    delayedActions.onRegistrationTimeout(EasyMock.<Runnable>anyObject());
+    expectInitializeDriver();
+
+    Capture<Runnable> handleRegistered = expectFullStartup();
+    expectShutdown();
+
+    Capture<Command> shutdownCommand = replayAndCreateLifecycle();
+
+    LeadershipListener leaderListener = schedulerLifecycle.prepare();
+    leaderListener.onLeading(leaderControl);
+    schedulerLifecycle.registered(new DriverRegistered());
+    handleRegistered.getValue().run();
+    shutdownCommand.getValue().execute();
+  }
 }


Mime
View raw message