aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject git commit: Filtering out unknown hosts when starting maintenance.
Date Wed, 23 Jul 2014 17:14:34 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master e1c0ade29 -> 58c0de890


Filtering out unknown hosts when starting maintenance.

Bugs closed: AURORA-443

Reviewed at https://reviews.apache.org/r/23827/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/58c0de89
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/58c0de89
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/58c0de89

Branch: refs/heads/master
Commit: 58c0de890badba6d4b512416fc79532d31a9c6c0
Parents: e1c0ade
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Wed Jul 23 10:13:59 2014 -0700
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Wed Jul 23 10:13:59 2014 -0700

----------------------------------------------------------------------
 .../apache/aurora/admin/host_maintenance.py     | 13 +++--
 .../aurora/admin/test_host_maintenance.py       |  8 ++-
 .../aurora/client/commands/test_maintenance.py  | 53 ++++++++++++++++++--
 3 files changed, 64 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/58c0de89/src/main/python/apache/aurora/admin/host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/admin/host_maintenance.py b/src/main/python/apache/aurora/admin/host_maintenance.py
index af26f7c..162f9af 100644
--- a/src/main/python/apache/aurora/admin/host_maintenance.py
+++ b/src/main/python/apache/aurora/admin/host_maintenance.py
@@ -159,8 +159,15 @@ class HostMaintenance(object):
 
     :param hostnames: List of hosts to set for initial maintenance
     :type hostnames: list of strings
+    :rtype: list of hostnames with the maintenance mode set
     """
-    check_and_log_response(self._client.start_maintenance(Hosts(set(hostnames))))
+    resp = self._client.start_maintenance(Hosts(set(hostnames)))
+    check_and_log_response(resp)
+    result = [host_status.host for host_status in resp.result.startMaintenanceResult.statuses]
+    if len(result) != len(hostnames):
+      log.warning('Skipping maintenance for unknown hosts: %s' % (set(hostnames) - set(result)))
+
+    return result
 
   def perform_maintenance(self, hostnames, grouping_function=DEFAULT_GROUPING,
                           callback=None, percentage=None, duration=None, output_file=None):
@@ -170,7 +177,7 @@ class HostMaintenance(object):
     performing an action on them once drained, then removing them from maintenance mode
     so tasks can schedule.
 
-    :param hostnames: A list of hosts to operate upon
+    :param hostnames: A list of hostnames to operate upon
     :type hostnames: list of strings
     :param grouping_function: How to split up the hostname into groups
     :type grouping_function: function
@@ -183,7 +190,7 @@ class HostMaintenance(object):
     :param output_file: file to write hosts that were not drained due to failed SLA check
     :type output_file: string
     """
-    self.start_maintenance(hostnames)
+    hostnames = self.start_maintenance(hostnames)
     not_drained_hostnames = set()
 
     for hosts in self.iter_batches(hostnames, grouping_function):

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/58c0de89/src/test/python/apache/aurora/admin/test_host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/admin/test_host_maintenance.py b/src/test/python/apache/aurora/admin/test_host_maintenance.py
index 34578ae..d8aeffb 100644
--- a/src/test/python/apache/aurora/admin/test_host_maintenance.py
+++ b/src/test/python/apache/aurora/admin/test_host_maintenance.py
@@ -34,7 +34,8 @@ from gen.apache.aurora.api.ttypes import (
     MaintenanceStatusResult,
     Response,
     ResponseCode,
-    Result
+    Result,
+    StartMaintenanceResult
 )
 
 DEFAULT_CLUSTER = Cluster(
@@ -133,7 +134,8 @@ class TestHostMaintenance(unittest.TestCase):
   @mock.patch("apache.aurora.client.api.AuroraClientAPI.start_maintenance",
       spec=AuroraClientAPI.start_maintenance)
   def test_start_maintenance(self, mock_api):
-    mock_api.return_value = Response(responseCode=ResponseCode.OK)
+    mock_api.return_value = Response(responseCode=ResponseCode.OK,
+        result=Result(startMaintenanceResult=StartMaintenanceResult(statuses=set([HostStatus()]))))
     maintenance = HostMaintenance(DEFAULT_CLUSTER, 'quiet')
     maintenance.start_maintenance(TEST_HOSTNAMES)
     mock_api.assert_called_once_with(Hosts(set(TEST_HOSTNAMES)))
@@ -152,6 +154,7 @@ class TestHostMaintenance(unittest.TestCase):
       mock_drain_hosts, mock_operate_on_hosts, mock_complete_maintenance):
     mock_callback = mock.Mock()
     mock_check_sla.return_value = set()
+    mock_start_maintenance.return_value = TEST_HOSTNAMES
     maintenance = HostMaintenance(DEFAULT_CLUSTER, 'quiet')
     maintenance.perform_maintenance(TEST_HOSTNAMES, callback=mock_callback)
     mock_start_maintenance.assert_called_once_with(TEST_HOSTNAMES)
@@ -181,6 +184,7 @@ class TestHostMaintenance(unittest.TestCase):
     mock_callback = mock.Mock()
     failed_host = 'us-west-001.example.com'
     mock_check_sla.return_value = set([failed_host])
+    mock_start_maintenance.return_value = TEST_HOSTNAMES
     drained_hosts = set(TEST_HOSTNAMES) - set([failed_host])
     maintenance = HostMaintenance(DEFAULT_CLUSTER, 'quiet')
 

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/58c0de89/src/test/python/apache/aurora/client/commands/test_maintenance.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_maintenance.py b/src/test/python/apache/aurora/client/commands/test_maintenance.py
index 07dd333..a748c01 100644
--- a/src/test/python/apache/aurora/client/commands/test_maintenance.py
+++ b/src/test/python/apache/aurora/client/commands/test_maintenance.py
@@ -51,13 +51,14 @@ class TestMaintenanceCommands(AuroraClientCommandTest):
     mock_options.reason = None
     return mock_options
 
-  def create_host_statuses(self, maintenance_mode):
-    return [HostStatus(host=hostname, mode=maintenance_mode) for hostname in self.HOSTNAMES]
+  def create_host_statuses(self, maintenance_mode, skip_hosts=None):
+    return [HostStatus(host=hostname, mode=maintenance_mode) for hostname in self.HOSTNAMES
+            if not skip_hosts or hostname not in skip_hosts]
 
-  def create_start_maintenance_result(self):
-    host_statuses = self.create_host_statuses(MaintenanceMode.SCHEDULED)
+  def create_start_maintenance_result(self, skip_hosts=None):
+    host_statuses = self.create_host_statuses(MaintenanceMode.SCHEDULED, skip_hosts)
     response = self.create_simple_success_response()
-    response.result.maintenanceStatusResult = StartMaintenanceResult(statuses=set(host_statuses))
+    response.result.startMaintenanceResult = StartMaintenanceResult(statuses=set(host_statuses))
     return response
 
   def create_end_maintenance_result(self):
@@ -153,6 +154,48 @@ class TestMaintenanceCommands(AuroraClientCommandTest):
       assert mock_scheduler_proxy.drainHosts.call_count == 3
       assert mock_scheduler_proxy.endMaintenance.call_count == 3
 
+  def test_perform_maintenance_hosts_unknown_hosts_skipped(self):
+    mock_options = self.make_mock_options()
+    mock_options.post_drain_script = None
+    mock_options.grouping = 'by_host'
+
+    def host_status_results(hostnames):
+      if isinstance(hostnames, Hosts):
+        return self.create_drained_status_result(hostnames)
+      return self.create_maintenance_status_result()
+
+    mock_api, mock_scheduler_proxy = self.create_mock_api()
+    mock_scheduler_proxy.endMaintenance.return_value = self.create_end_maintenance_result()
+    mock_scheduler_proxy.maintenanceStatus.side_effect = host_status_results
+    mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result(
+        skip_hosts=['us-grf-20'])
+    mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result()
+    mock_vector = self.create_mock_probe_hosts_vector([
+      self.create_probe_hosts(self.HOSTNAMES[0], 95, True, None),
+      self.create_probe_hosts(self.HOSTNAMES[1], 95, True, None),
+      self.create_probe_hosts(self.HOSTNAMES[2], 95, True, None)
+    ])
+
+    with contextlib.nested(
+        patch('time.sleep'),
+        patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
+        patch('apache.aurora.client.api.sla.Sla.get_domain_uptime_vector',
+              return_value=mock_vector),
+        patch('apache.aurora.client.commands.maintenance.CLUSTERS', new=self.TEST_CLUSTERS),
+        patch('twitter.common.app.get_options', return_value=mock_options)) as (
+            mock_sleep,
+            mock_scheduler_proxy_class,
+            mock_vector_class,
+            mock_clusters_maintenancepatch,
+            options):
+      perform_maintenance_hosts([self.TEST_CLUSTER])
+
+      mock_scheduler_proxy.startMaintenance.assert_called_with(Hosts(set(self.HOSTNAMES)))
+      assert mock_sleep.call_count == 2
+      assert mock_scheduler_proxy.maintenanceStatus.call_count == 4
+      assert mock_scheduler_proxy.drainHosts.call_count == 2
+      assert mock_scheduler_proxy.endMaintenance.call_count == 2
+
   def test_perform_maintenance_hosts_failed_default_sla(self):
     with temporary_file() as fp:
       mock_options = self.make_mock_options()


Mime
View raw message