aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wfar...@apache.org
Subject git commit: Save instance update events when updating a job.
Date Thu, 18 Sep 2014 22:22:59 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 99ad092df -> 53e5cd2d0


Save instance update events when updating a job.

Bugs closed: AURORA-613

Reviewed at https://reviews.apache.org/r/25791/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/53e5cd2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/53e5cd2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/53e5cd2d

Branch: refs/heads/master
Commit: 53e5cd2d03219db56758b1745149b30d4359c4c1
Parents: 99ad092
Author: Bill Farner <wfarner@apache.org>
Authored: Thu Sep 18 15:20:37 2014 -0700
Committer: Bill Farner <wfarner@apache.org>
Committed: Thu Sep 18 15:20:37 2014 -0700

----------------------------------------------------------------------
 .../aurora/scheduler/http/RequestLogger.java    |   6 +-
 .../scheduler/storage/db/DbAttributeStore.java  |   5 +-
 .../updater/JobUpdateControllerImpl.java        |  46 ++++-
 .../aurora/scheduler/updater/SideEffect.java    |  15 +-
 .../aurora/scheduler/updater/JobUpdaterIT.java  | 180 +++++++++++++++----
 5 files changed, 209 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java b/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
index 0442191..916939d 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
@@ -27,7 +27,7 @@ import org.eclipse.jetty.server.Response;
 import org.eclipse.jetty.util.DateCache;
 import org.eclipse.jetty.util.component.AbstractLifeCycle;
 
-import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
 
 /**
  * A copy of twitter RequestLogger from twitter commons, which is a port of jetty's NCSARequestLog
@@ -63,8 +63,8 @@ public class RequestLogger extends AbstractLifeCycle implements RequestLog
{
 
   @VisibleForTesting
   RequestLogger(Clock clock, LogSink sink) {
-    this.clock = checkNotNull(clock);
-    this.sink = checkNotNull(sink);
+    this.clock = requireNonNull(clock);
+    this.sink = requireNonNull(sink);
     logDateCache = new DateCache("dd/MMM/yyyy:HH:mm:ss Z", Locale.getDefault());
     logDateCache.setTimeZoneID("GMT");
   }

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java
index 34f12a9..1b18ad5 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java
@@ -13,6 +13,7 @@
  */
 package org.apache.aurora.scheduler.storage.db;
 
+import java.util.Objects;
 import java.util.Set;
 
 import com.google.common.base.Optional;
@@ -28,8 +29,6 @@ import org.apache.aurora.scheduler.storage.AttributeStore;
 import org.apache.aurora.scheduler.storage.entities.IAttribute;
 import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 /**
  * Attribute store backed by a relational database.
  */
@@ -39,7 +38,7 @@ class DbAttributeStore implements AttributeStore.Mutable {
 
   @Inject
   DbAttributeStore(AttributeMapper mapper) {
-    this.mapper = checkNotNull(mapper);
+    this.mapper = Objects.requireNonNull(mapper);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
b/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
index 5375ed2..3e8182d 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
@@ -27,10 +27,13 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.inject.Inject;
+import com.twitter.common.collections.Pair;
 import com.twitter.common.quantity.Amount;
 import com.twitter.common.quantity.Time;
 import com.twitter.common.util.Clock;
 
+import org.apache.aurora.gen.JobInstanceUpdateEvent;
+import org.apache.aurora.gen.JobUpdateAction;
 import org.apache.aurora.gen.JobUpdateEvent;
 import org.apache.aurora.gen.JobUpdateQuery;
 import org.apache.aurora.gen.JobUpdateStatus;
@@ -46,6 +49,7 @@ import org.apache.aurora.scheduler.storage.JobUpdateStore;
 import org.apache.aurora.scheduler.storage.Storage;
 import org.apache.aurora.scheduler.storage.TaskStore;
 import org.apache.aurora.scheduler.storage.entities.IInstanceKey;
+import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
 import org.apache.aurora.scheduler.storage.entities.IJobKey;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
@@ -77,6 +81,7 @@ import static org.apache.aurora.scheduler.updater.JobUpdateStateMachine.assertTr
 import static org.apache.aurora.scheduler.updater.OneWayJobUpdater.EvaluationResult;
 import static org.apache.aurora.scheduler.updater.OneWayJobUpdater.OneWayStatus;
 import static org.apache.aurora.scheduler.updater.OneWayJobUpdater.OneWayStatus.SUCCEEDED;
+import static org.apache.aurora.scheduler.updater.SideEffect.InstanceUpdateStatus;
 
 /**
  * Implementation of an updater that orchestrates the process of gradually updating the
@@ -434,6 +439,21 @@ class JobUpdateControllerImpl implements JobUpdateController {
 
     EvaluationResult<Integer> result = update.getUpdater().evaluate(changedInstance,
stateProvider);
     LOG.info(JobKeys.canonicalString(job) + " evaluation result: " + result);
+
+    for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet())
{
+      for (InstanceUpdateStatus statusChange : entry.getValue().getStatusChanges()) {
+        JobUpdateAction action = STATE_MAP.get(Pair.of(statusChange, updaterStatus));
+        requireNonNull(action);
+
+        IJobInstanceUpdateEvent event = IJobInstanceUpdateEvent.build(
+            new JobInstanceUpdateEvent()
+                .setInstanceId(entry.getKey())
+                .setTimestampMs(clock.nowMillis())
+                .setAction(action));
+        updateStore.saveJobInstanceUpdateEvent(event, summary.getUpdateId());
+      }
+    }
+
     OneWayStatus status = result.getStatus();
     if (status == SUCCEEDED || status == OneWayStatus.FAILED) {
       if (SideEffect.hasActions(result.getSideEffects().values())) {
@@ -450,7 +470,6 @@ class JobUpdateControllerImpl implements JobUpdateController {
       LOG.info("Executing side-effects for update of " + job + ": " + result.getSideEffects());
       for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet())
{
         IInstanceKey instance = InstanceKeys.from(job, entry.getKey());
-        // TODO(wfarner): Persist SideEffect.getStatusChanges as JobInstanceUpdateEvents.
 
         Optional<InstanceAction> action = entry.getValue().getAction();
         if (action.isPresent()) {
@@ -472,6 +491,31 @@ class JobUpdateControllerImpl implements JobUpdateController {
     }
   }
 
+  /**
+   * Associates an instance updater state change and the job's update status to an action.
+   */
+  private static final Map<Pair<InstanceUpdateStatus, JobUpdateStatus>, JobUpdateAction>
STATE_MAP =
+      ImmutableMap.<Pair<InstanceUpdateStatus, JobUpdateStatus>, JobUpdateAction>builder()
+          .put(
+              Pair.of(InstanceUpdateStatus.WORKING, ROLLING_FORWARD),
+              JobUpdateAction.INSTANCE_UPDATING)
+          .put(
+              Pair.of(InstanceUpdateStatus.SUCCEEDED, ROLLING_FORWARD),
+              JobUpdateAction.INSTANCE_UPDATED)
+          .put(
+              Pair.of(InstanceUpdateStatus.FAILED, ROLLING_FORWARD),
+              JobUpdateAction.INSTANCE_UPDATE_FAILED)
+          .put(
+              Pair.of(InstanceUpdateStatus.WORKING, ROLLING_BACK),
+              JobUpdateAction.INSTANCE_ROLLING_BACK)
+          .put(
+              Pair.of(InstanceUpdateStatus.SUCCEEDED, ROLLING_BACK),
+              JobUpdateAction.INSTANCE_ROLLED_BACK)
+          .put(
+              Pair.of(InstanceUpdateStatus.FAILED, ROLLING_BACK),
+              JobUpdateAction.INSTANCE_ROLLBACK_FAILED)
+          .build();
+
   private Runnable getDeferredEvaluator(final IInstanceKey instance, final String updateId)
{
     return new Runnable() {
       @Override

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java b/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
index d9e59b6..27e0654 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
@@ -34,18 +34,25 @@ public class SideEffect {
    * @param action Action to be taken on the instance, if necessary.
    * @param statusChanges Any status changes to the instance monitor.
    */
-  public SideEffect(
-      Optional<InstanceAction> action,
-      Set<InstanceUpdateStatus> statusChanges) {
-
+  public SideEffect(Optional<InstanceAction> action, Set<InstanceUpdateStatus>
statusChanges) {
     this.action = requireNonNull(action);
     this.statusChanges = requireNonNull(statusChanges);
   }
 
+  /**
+   * Gets the action that should be performed with this side-effect, if any.
+   *
+   * @return The action associated with this side-effect.
+   */
   public Optional<InstanceAction> getAction() {
     return action;
   }
 
+  /**
+   * Gets the status changes that the instance underwent while being evaluated.
+   *
+   * @return Instance updater status changes.
+   */
   public Set<InstanceUpdateStatus> getStatusChanges() {
     return statusChanges;
   }

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
index 061d7e7..d90c87c 100644
--- a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
+++ b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
@@ -16,13 +16,18 @@ package org.apache.aurora.scheduler.updater;
 import java.util.Map;
 import java.util.concurrent.ScheduledExecutorService;
 
+import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Ordering;
 import com.google.common.eventbus.EventBus;
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
@@ -40,6 +45,7 @@ import org.apache.aurora.gen.ExecutorConfig;
 import org.apache.aurora.gen.Identity;
 import org.apache.aurora.gen.InstanceTaskConfig;
 import org.apache.aurora.gen.JobUpdate;
+import org.apache.aurora.gen.JobUpdateAction;
 import org.apache.aurora.gen.JobUpdateEvent;
 import org.apache.aurora.gen.JobUpdateInstructions;
 import org.apache.aurora.gen.JobUpdateSettings;
@@ -72,8 +78,10 @@ import org.apache.aurora.scheduler.storage.Storage.StoreProvider;
 import org.apache.aurora.scheduler.storage.Storage.Work;
 import org.apache.aurora.scheduler.storage.db.DbModule;
 import org.apache.aurora.scheduler.storage.entities.IInstanceTaskConfig;
+import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
 import org.apache.aurora.scheduler.storage.entities.IJobKey;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary;
 import org.apache.aurora.scheduler.storage.entities.ILock;
@@ -88,6 +96,12 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLBACK_FAILED;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLED_BACK;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLING_BACK;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATED;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATE_FAILED;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATING;
 import static org.apache.aurora.gen.JobUpdateStatus.ABORTED;
 import static org.apache.aurora.gen.JobUpdateStatus.ERROR;
 import static org.apache.aurora.gen.JobUpdateStatus.ROLLED_BACK;
@@ -103,7 +117,6 @@ import static org.apache.aurora.gen.ScheduleStatus.RUNNING;
 import static org.apache.aurora.gen.ScheduleStatus.STARTING;
 import static org.apache.aurora.scheduler.events.PubsubEvent.SchedulerActive;
 import static org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
-import static org.apache.aurora.scheduler.updater.JobUpdateControllerImpl.queryByUpdateId;
 import static org.apache.aurora.scheduler.updater.UpdateFactory.UpdateFactoryImpl.expandInstanceIds;
 import static org.easymock.EasyMock.expectLastCall;
 import static org.junit.Assert.assertEquals;
@@ -220,16 +233,46 @@ public class JobUpdaterIT extends EasyMockTest {
     }
   }
 
-  private void assertUpdateStatus(JobUpdateStatus expected) {
-    JobUpdateStatus status = storage.consistentRead(new Work.Quiet<JobUpdateStatus>()
{
+  private static final Ordering<IJobInstanceUpdateEvent> EVENT_ORDER = Ordering.natural()
+      .onResultOf(new Function<IJobInstanceUpdateEvent, Long>() {
+        @Override
+        public Long apply(IJobInstanceUpdateEvent event) {
+          return event.getTimestampMs();
+        }
+      });
+  private static final Function<IJobInstanceUpdateEvent, Integer> EVENT_TO_INSTANCE
=
+      new Function<IJobInstanceUpdateEvent, Integer>() {
+        @Override
+        public Integer apply(IJobInstanceUpdateEvent event) {
+          return event.getInstanceId();
+        }
+      };
+  private static final Function<IJobInstanceUpdateEvent, JobUpdateAction> EVENT_TO_ACTION
=
+      new Function<IJobInstanceUpdateEvent, JobUpdateAction>() {
+        @Override
+        public JobUpdateAction apply(IJobInstanceUpdateEvent event) {
+          return event.getAction();
+        }
+      };
+
+  private void assertState(
+      JobUpdateStatus expected,
+      Multimap<Integer, JobUpdateAction> expectedActions) {
+
+    IJobUpdateDetails details = storage.consistentRead(new Work.Quiet<IJobUpdateDetails>()
{
       @Override
-      public JobUpdateStatus apply(StoreProvider storeProvider) {
-        IJobUpdateSummary summary = Iterables.getOnlyElement(
-            storeProvider.getJobUpdateStore().fetchJobUpdateSummaries(queryByUpdateId(UPDATE_ID)));
-        return summary.getState().getStatus();
+      public IJobUpdateDetails apply(StoreProvider storeProvider) {
+        return storeProvider.getJobUpdateStore().fetchJobUpdateDetails(UPDATE_ID).get();
       }
     });
-    assertEquals(expected, status);
+    Iterable<IJobInstanceUpdateEvent> orderedEvents =
+        EVENT_ORDER.sortedCopy(details.getInstanceEvents());
+    Multimap<Integer, IJobInstanceUpdateEvent> eventsByInstance =
+        Multimaps.index(orderedEvents, EVENT_TO_INSTANCE);
+    Multimap<Integer, JobUpdateAction> actionsByInstance =
+        Multimaps.transformValues(eventsByInstance, EVENT_TO_ACTION);
+    assertEquals(expectedActions, actionsByInstance);
+    assertEquals(expected, details.getUpdate().getSummary().getState().getStatus());
   }
 
   private IExpectationSetters<String> expectTaskKilled() {
@@ -271,22 +314,31 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 2, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
     // Instance 1 is added
     updater.start(update, USER);
-    assertUpdateStatus(ROLLING_FORWARD);
+    actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING);
+    assertState(ROLLING_FORWARD, actions.build());
     changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
 
     // Updates may be paused for arbitrarily-long amounts of time, and the updater should
not
     // take action while paused.
     updater.pause(JOB);
     updater.pause(JOB);  // Pausing again is a no-op.
-    assertUpdateStatus(ROLL_FORWARD_PAUSED);
+    assertState(ROLL_FORWARD_PAUSED, actions.build());
     clock.advance(ONE_DAY);
     changeState(JOB, 1, FAILED, ASSIGNED, STARTING, RUNNING);
     changeState(JOB, 2, FAILED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
     updater.resume(JOB);
-    assertUpdateStatus(ROLLING_FORWARD);
+    // TODO(wfarner): Since the updater does not skip no-op work upfront, we end up with
redundant
+    // states.
+    actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATED)
+        .put(2, INSTANCE_UPDATING);
+    assertState(ROLLING_FORWARD, actions.build());
 
     // A task outside the scope of the update should be ignored by the updater.
     stateManager.insertPendingTasks(NEW_CONFIG, ImmutableSet.of(100));
@@ -295,7 +347,8 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 2, KILLED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
-    assertUpdateStatus(ROLLED_FORWARD);
+    actions.put(2, INSTANCE_UPDATED);
+    assertState(ROLLED_FORWARD, actions.build());
     assertJobState(
         JOB,
         ImmutableMap.of(0, NEW_CONFIG, 1, NEW_CONFIG, 2, NEW_CONFIG, 100, NEW_CONFIG));
@@ -323,7 +376,10 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
-    assertUpdateStatus(ROLLED_FORWARD);
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+    assertState(
+        ROLLED_FORWARD,
+        actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED).build());
     assertJobState(
         JOB,
         ImmutableMap.of(0, NEW_CONFIG, 1, OLD_CONFIG));
@@ -345,42 +401,58 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 3, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
     // Instance 0 is updated.
     updater.start(update, USER);
-    assertUpdateStatus(ROLLING_FORWARD);
+    actions.putAll(0, INSTANCE_UPDATING);
+    assertState(ROLLING_FORWARD, actions.build());
     changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
     // Instance 1 is added.
     changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
+    actions.putAll(0, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATED);
     clock.advance(WATCH_TIMEOUT);
 
     // Instance 2 is updated, but fails.
     changeState(JOB, 2, KILLED, ASSIGNED, STARTING, RUNNING);
+    // TODO(wfarner): Since the updater does not skip no-ops upfront, we don't touch instance
3,
+    // but record state transitions anyway.
+    actions.putAll(2, INSTANCE_UPDATING, INSTANCE_UPDATE_FAILED, INSTANCE_ROLLING_BACK)
+        .putAll(3, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK);
     clock.advance(FLAPPING_THRESHOLD);
     changeState(JOB, 2, FAILED);
 
     // Instance 2 is rolled back.
-    assertUpdateStatus(ROLLING_BACK);
+    assertState(ROLLING_BACK, actions.build());
     changeState(JOB, 2, ASSIGNED, STARTING, RUNNING);
+    actions.putAll(1, INSTANCE_ROLLING_BACK)
+        .putAll(2, INSTANCE_ROLLED_BACK);
     clock.advance(WATCH_TIMEOUT);
 
     // A rollback may be paused.
     updater.pause(JOB);
-    assertUpdateStatus(ROLL_BACK_PAUSED);
+    assertState(ROLL_BACK_PAUSED, actions.build());
     clock.advance(ONE_DAY);
     updater.resume(JOB);
-    assertUpdateStatus(ROLLING_BACK);
+    actions.putAll(1, INSTANCE_ROLLING_BACK)
+        .putAll(2, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK)
+        .putAll(3, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK);
+    assertState(ROLLING_BACK, actions.build());
 
     // Instance 1 is removed.
     changeState(JOB, 1, KILLED);
+    actions.putAll(1, INSTANCE_ROLLED_BACK);
     clock.advance(WATCH_TIMEOUT);
 
     // Instance 0 is rolled back.
     changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
+    actions.putAll(0, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK);
     clock.advance(WATCH_TIMEOUT);
 
-    assertUpdateStatus(ROLLED_BACK);
+    assertState(ROLLED_BACK, actions.build());
     assertJobState(JOB, ImmutableMap.of(0, OLD_CONFIG, 2, OLD_CONFIG, 3, OLD_CONFIG));
   }
 
@@ -402,8 +474,12 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+    actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING);
+
     updater.abort(JOB);
-    assertUpdateStatus(ABORTED);
+    assertState(ABORTED, actions.build());
     clock.advance(WATCH_TIMEOUT);
     assertJobState(JOB, ImmutableMap.of(0, NEW_CONFIG, 1, NEW_CONFIG, 2, OLD_CONFIG));
   }
@@ -422,9 +498,12 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
     // Instance 0 is updated.
     updater.start(update, USER);
-    assertUpdateStatus(ROLLING_FORWARD);
+    actions.putAll(0, INSTANCE_UPDATING);
+    assertState(ROLLING_FORWARD, actions.build());
     changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
@@ -434,12 +513,16 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 1, FAILED);
 
     // Instance 1 is rolled back, but fails.
-    assertUpdateStatus(ROLLING_BACK);
+    actions.putAll(0, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATE_FAILED, INSTANCE_ROLLING_BACK)
+        .putAll(2, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK);
+    assertState(ROLLING_BACK, actions.build());
     changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
     clock.advance(FLAPPING_THRESHOLD);
     changeState(JOB, 1, FAILED);
 
-    assertUpdateStatus(JobUpdateStatus.FAILED);
+    actions.putAll(1, INSTANCE_ROLLBACK_FAILED);
+    assertState(JobUpdateStatus.FAILED, actions.build());
     clock.advance(WATCH_TIMEOUT);
     assertJobState(JOB, ImmutableMap.of(0, NEW_CONFIG, 1, OLD_CONFIG));
   }
@@ -464,7 +547,9 @@ public class JobUpdaterIT extends EasyMockTest {
       lockManager.releaseLock(lock);
     }
     clock.advance(RUNNING_TIMEOUT);
-    assertUpdateStatus(ERROR);
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+    actions.putAll(0, INSTANCE_UPDATING);
+    assertState(ERROR, actions.build());
   }
 
   private void expectInvalid(JobUpdate update)
@@ -525,9 +610,12 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
     // Instance 0 is updated
     updater.start(update, USER);
-    assertUpdateStatus(ROLLING_FORWARD);
+    actions.putAll(0, INSTANCE_UPDATING);
+    assertState(ROLLING_FORWARD, actions.build());
 
     storage.write(new NoResult.Quiet() {
       @Override
@@ -549,9 +637,13 @@ public class JobUpdaterIT extends EasyMockTest {
 
     // Instance 1 is updated, but fails.
     changeState(JOB, 1, KILLED, ASSIGNED, STARTING, RUNNING, FAILED);
+    // Actions is reset here since we wiped the updates tables earlier in the test case.
+    actions = ImmutableMultimap.builder();
+    actions.putAll(0, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATE_FAILED);
     clock.advance(WATCH_TIMEOUT);
 
-    assertUpdateStatus(ERROR);
+    assertState(ERROR, actions.build());
   }
 
   private ILock saveJobUpdate(
@@ -607,7 +699,11 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 1, KILLED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
-    assertUpdateStatus(ROLLED_FORWARD);
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+    actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATED);
+
+    assertState(ROLLED_FORWARD, actions.build());
   }
 
   @Test
@@ -626,7 +722,7 @@ public class JobUpdaterIT extends EasyMockTest {
     });
 
     eventBus.post(new SchedulerActive());
-    assertUpdateStatus(ERROR);
+    assertState(ERROR, ImmutableMultimap.<Integer, JobUpdateAction>of());
   }
 
   @Test
@@ -640,7 +736,11 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 2, ASSIGNED, STARTING, RUNNING);
     clock.advance(ONE_DAY);
     updater.start(update, USER);
-    assertUpdateStatus(ROLLED_FORWARD);
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+    actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATED)
+        .putAll(2, INSTANCE_UPDATING, INSTANCE_UPDATED);
+    assertState(ROLLED_FORWARD, actions.build());
   }
 
   @Test
@@ -652,7 +752,16 @@ public class JobUpdaterIT extends EasyMockTest {
     insertInitialTasks(update);
     clock.advance(ONE_DAY);
     updater.start(update, USER);
-    assertUpdateStatus(JobUpdateStatus.FAILED);
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+    Iterable<JobUpdateAction> states = ImmutableList.of(
+        INSTANCE_UPDATING,
+        INSTANCE_UPDATE_FAILED,
+        INSTANCE_ROLLING_BACK,
+        INSTANCE_ROLLBACK_FAILED);
+    actions.putAll(0, states)
+        .putAll(1, states)
+        .putAll(2, states);
+    assertState(JobUpdateStatus.FAILED, actions.build());
   }
 
   @Test
@@ -666,16 +775,21 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 0, ASSIGNED, STARTING, RUNNING);
     changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
 
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
     // Instance 0 is updated.
     updater.start(update, USER);
-    assertUpdateStatus(ROLLING_FORWARD);
+    actions.putAll(0, INSTANCE_UPDATING);
+    assertState(ROLLING_FORWARD, actions.build());
     changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
     // Instance 1 is stuck in PENDING.
     changeState(JOB, 1, KILLED);
     clock.advance(RUNNING_TIMEOUT);
-    assertUpdateStatus(ROLLING_BACK);
+    actions.putAll(0, INSTANCE_UPDATED)
+        .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATE_FAILED, INSTANCE_ROLLING_BACK);
+    assertState(ROLLING_BACK, actions.build());
 
     // Instance 1 is reverted.
     changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
@@ -685,7 +799,9 @@ public class JobUpdaterIT extends EasyMockTest {
     changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
     clock.advance(WATCH_TIMEOUT);
 
-    assertUpdateStatus(ROLLED_BACK);
+    actions.putAll(0, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK)
+        .putAll(1, INSTANCE_ROLLED_BACK);
+    assertState(ROLLED_BACK, actions.build());
     assertJobState(JOB, ImmutableMap.of(0, OLD_CONFIG, 1, OLD_CONFIG));
   }
 


Mime
View raw message