aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject incubator-aurora git commit: Adding more logging into MaintenanceController.
Date Sat, 14 Mar 2015 01:28:05 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 128e5540e -> 7d68df391


Adding more logging into MaintenanceController.

Bugs closed: AURORA-1170

Reviewed at https://reviews.apache.org/r/32014/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/7d68df39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/7d68df39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/7d68df39

Branch: refs/heads/master
Commit: 7d68df39114d3fd9cea1df00ae45baae005474e3
Parents: 128e554
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Fri Mar 13 18:18:28 2015 -0700
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Fri Mar 13 18:18:28 2015 -0700

----------------------------------------------------------------------
 config/legacy_untested_classes.txt              |  2 -
 .../scheduler/state/MaintenanceController.java  |  7 ++-
 .../apache/aurora/admin/host_maintenance.py     |  6 ++-
 .../state/MaintenanceControllerImplTest.java    | 45 +++++++++++++-------
 .../aurora/admin/test_host_maintenance.py       |  9 +++-
 5 files changed, 47 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/config/legacy_untested_classes.txt
----------------------------------------------------------------------
diff --git a/config/legacy_untested_classes.txt b/config/legacy_untested_classes.txt
index 06e130f..4d090a8 100644
--- a/config/legacy_untested_classes.txt
+++ b/config/legacy_untested_classes.txt
@@ -55,8 +55,6 @@ org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule$4
 org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule$5
 org/apache/aurora/scheduler/mesos/DriverFactoryImpl
 org/apache/aurora/scheduler/mesos/LibMesosLoadingModule
-org/apache/aurora/scheduler/state/MaintenanceController$MaintenanceControllerImpl$4
-org/apache/aurora/scheduler/state/MaintenanceController$MaintenanceControllerImpl$8
 org/apache/aurora/scheduler/stats/AsyncStatsModule$OfferAdapter$1
 org/apache/aurora/scheduler/storage/log/LogStorage$RecoveryFailedException
 org/apache/aurora/scheduler/storage/mem/Util

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java b/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
index b6f642e..09be4dc 100644
--- a/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
+++ b/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
@@ -166,8 +166,9 @@ public interface MaintenanceController {
                 LOG.info(String.format("Moving host %s into DRAINED", host));
                 setMaintenanceMode(store, ImmutableSet.of(host), DRAINED);
               } else {
-                LOG.info(
-                    String.format("Host %s is DRAINING with active tasks: %s", host, activeTasks));
+                LOG.info(String.format("Host %s is DRAINING with active tasks: %s",
+                    host,
+                    Tasks.ids(activeTasks)));
               }
             }
           }
@@ -268,9 +269,11 @@ public interface MaintenanceController {
       AttributeStore.Mutable store = storeProvider.getAttributeStore();
       ImmutableSet.Builder<HostStatus> statuses = ImmutableSet.builder();
       for (String host : hosts) {
+        LOG.info(String.format("Setting maintenance mode to %s for host %s", mode, host));
         Optional<IHostAttributes> toSave = AttributeStore.Util.mergeMode(store, host,
mode);
         if (toSave.isPresent()) {
           store.saveHostAttributes(toSave.get());
+          LOG.info("Updated host attributes: " + toSave.get());
           HostStatus status = new HostStatus().setHost(host).setMode(mode);
           statuses.add(status);
         }

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/src/main/python/apache/aurora/admin/host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/admin/host_maintenance.py b/src/main/python/apache/aurora/admin/host_maintenance.py
index 8fa5182..c31e2a6 100644
--- a/src/main/python/apache/aurora/admin/host_maintenance.py
+++ b/src/main/python/apache/aurora/admin/host_maintenance.py
@@ -71,11 +71,13 @@ class HostMaintenance(object):
       self._wait_event.wait(self.STATUS_POLL_INTERVAL.as_(Time.SECONDS))
 
       statuses = self.check_status(list(not_drained_hostnames))
-      not_drained_hostnames = set([h[0] for h in statuses if h[1] != 'DRAINED'])
+      not_drained_hostnames = set(h[0] for h in statuses if h[1] != 'DRAINED')
 
       total_wait += self.STATUS_POLL_INTERVAL
       if not_drained_hostnames and total_wait > self.MAX_STATUS_WAIT:
-        log.warning('Failed to move all hosts into DRAINED within %s' % self.MAX_STATUS_WAIT)
+        log.warning('Failed to move all hosts into DRAINED within %s:\n%s' %
+            (self.MAX_STATUS_WAIT,
+            '\n'.join("\tHost:%s\tStatus:%s" % h for h in sorted(statuses) if h[1] != 'DRAINED')))
         break
 
     return not_drained_hostnames

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
b/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
index bd031a5..7b101bc 100644
--- a/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
@@ -46,7 +46,7 @@ import static org.apache.aurora.gen.MaintenanceMode.DRAINED;
 import static org.apache.aurora.gen.MaintenanceMode.DRAINING;
 import static org.apache.aurora.gen.MaintenanceMode.NONE;
 import static org.apache.aurora.gen.MaintenanceMode.SCHEDULED;
-import static org.apache.aurora.gen.ScheduleStatus.FINISHED;
+import static org.apache.aurora.gen.ScheduleStatus.KILLED;
 import static org.apache.aurora.gen.ScheduleStatus.RUNNING;
 import static org.apache.aurora.scheduler.state.MaintenanceController.MaintenanceControllerImpl;
 import static org.easymock.EasyMock.expect;
@@ -97,21 +97,23 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
 
   @Test
   public void testMaintenanceCycle() {
-    ScheduledTask task = makeTask(HOST_A, "taskA");
+    ScheduledTask task1 = makeTask(HOST_A, "taskA");
+    ScheduledTask task2 = makeTask(HOST_A, "taskB");
 
     expectMaintenanceModeChange(HOST_A, SCHEDULED);
-    expectFetchTasksByHost(HOST_A, ImmutableSet.of(task));
-    expect(stateManager.changeState(
-        storageUtil.mutableStoreProvider,
-        Tasks.id(task),
-        Optional.<ScheduleStatus>absent(),
-        ScheduleStatus.DRAINING,
-        MaintenanceControllerImpl.DRAINING_MESSAGE))
-        .andReturn(true);
+    expectFetchTasksByHost(HOST_A, ImmutableSet.of(task1, task2));
+    expectTaskDraining(task1);
+    expectTaskDraining(task2);
     expectMaintenanceModeChange(HOST_A, DRAINING);
-    expect(storageUtil.attributeStore.getHostAttributes(HOST_A)).andReturn(Optional.of(
-        IHostAttributes.build(new HostAttributes().setHost(HOST_A).setMode(DRAINING))));
-    // TaskA is FINISHED and therefore no longer active
+    IHostAttributes attributes =
+        IHostAttributes.build(new HostAttributes().setHost(HOST_A).setMode(DRAINING));
+
+    expect(storageUtil.attributeStore.getHostAttributes(HOST_A))
+        .andReturn(Optional.of(attributes)).times(2);
+
+    expect(storageUtil.attributeStore.getHostAttributes()).andReturn(ImmutableSet.of(attributes));
+    expectFetchTasksByHost(HOST_A, ImmutableSet.of(task2));
+    // TaskA is KILLED and therefore no longer active
     expectFetchTasksByHost(HOST_A, ImmutableSet.<ScheduledTask>of());
     expectMaintenanceModeChange(HOST_A, DRAINED);
     expectMaintenanceModeChange(HOST_A, NONE);
@@ -120,8 +122,11 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
 
     assertStatus(HOST_A, SCHEDULED, maintenance.startMaintenance(A));
     assertStatus(HOST_A, DRAINING, maintenance.drain(A));
+    assertStatus(HOST_A, DRAINING, maintenance.getStatus(A));
     eventSink.post(
-        TaskStateChange.transition(IScheduledTask.build(task.setStatus(FINISHED)), RUNNING));
+        TaskStateChange.transition(IScheduledTask.build(task1.setStatus(KILLED)), RUNNING));
+    eventSink.post(
+        TaskStateChange.transition(IScheduledTask.build(task2.setStatus(KILLED)), RUNNING));
     assertStatus(HOST_A, NONE, maintenance.endMaintenance(A));
   }
 
@@ -165,7 +170,7 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
     // Make sure a later transition on the host does not cause any ill effects that could
surface
     // from stale internal state.
     eventSink.post(TaskStateChange.transition(
-        IScheduledTask.build(makeTask(HOST_A, "taskA").setStatus(FINISHED)), RUNNING));
+        IScheduledTask.build(makeTask(HOST_A, "taskA").setStatus(KILLED)), RUNNING));
   }
 
   @Test
@@ -181,6 +186,16 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
     assertEquals(NONE, maintenance.getMode("unknown"));
   }
 
+  private void expectTaskDraining(ScheduledTask task) {
+    expect(stateManager.changeState(
+        storageUtil.mutableStoreProvider,
+        Tasks.id(task),
+        Optional.<ScheduleStatus>absent(),
+        ScheduleStatus.DRAINING,
+        MaintenanceControllerImpl.DRAINING_MESSAGE))
+        .andReturn(true);
+  }
+
   private void expectFetchTasksByHost(String hostName, ImmutableSet<ScheduledTask>
tasks) {
     expect(storageUtil.taskStore.fetchTasks(Query.slaveScoped(hostName).active()))
         .andReturn(IScheduledTask.setFromBuilders(tasks));

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/src/test/python/apache/aurora/admin/test_host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/admin/test_host_maintenance.py b/src/test/python/apache/aurora/admin/test_host_maintenance.py
index b7c13b7..6264d0c 100644
--- a/src/test/python/apache/aurora/admin/test_host_maintenance.py
+++ b/src/test/python/apache/aurora/admin/test_host_maintenance.py
@@ -13,6 +13,7 @@
 #
 
 import copy
+import textwrap
 import unittest
 from contextlib import contextmanager
 
@@ -103,12 +104,13 @@ class TestHostMaintenance(unittest.TestCase):
         (Hosts(set(TEST_HOSTNAMES))),
         (Hosts(set([TEST_HOSTNAMES[0]])))]
 
+  @mock.patch("twitter.common.log.warning", spec=log.warning)
   @mock.patch("apache.aurora.client.api.AuroraClientAPI.maintenance_status",
               spec=AuroraClientAPI.maintenance_status)
   @mock.patch("apache.aurora.client.api.AuroraClientAPI.drain_hosts",
               spec=AuroraClientAPI.drain_hosts)
   @mock.patch("threading._Event.wait")
-  def test_drain_hosts_timed_out_wait(self, _, mock_drain_hosts, mock_maintenance_status):
+  def test_drain_hosts_timed_out_wait(self, _, mock_drain_hosts, mock_maintenance_status,
mock_log):
     fake_maintenance_status_response = Response(
         responseCode=ResponseCode.OK,
         result=Result(maintenanceStatusResult=MaintenanceStatusResult(set([
@@ -128,6 +130,11 @@ class TestHostMaintenance(unittest.TestCase):
     assert mock_maintenance_status.call_count == 1
     mock_drain_hosts.assert_called_once_with(test_hosts)
     mock_maintenance_status.assert_called_once_with((Hosts(set(TEST_HOSTNAMES))))
+    assert mock_log.mock_calls == [mock.call(textwrap.dedent("""\
+        Failed to move all hosts into DRAINED within 1 ms:
+        \tHost:us-west-001.example.com\tStatus:SCHEDULED
+        \tHost:us-west-002.example.com\tStatus:SCHEDULED
+        \tHost:us-west-003.example.com\tStatus:SCHEDULED"""))]
 
   @mock.patch("twitter.common.log.warning", spec=log.warning)
   @mock.patch("apache.aurora.client.api.AuroraClientAPI.maintenance_status",


Mime
View raw message