aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject git commit: Refactored admin client commands to take advantage of new TaskQuery fields.
Date Fri, 07 Mar 2014 19:00:53 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 2a335d188 -> 62f0aad4b


Refactored admin client commands to take advantage of new TaskQuery fields.

Bugs closed: AURORA-239

Reviewed at https://reviews.apache.org/r/18720/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/62f0aad4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/62f0aad4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/62f0aad4

Branch: refs/heads/master
Commit: 62f0aad4bb01e73ddb6e6185c82faa50e5648a2c
Parents: 2a335d1
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Fri Mar 7 11:00:26 2014 -0800
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Fri Mar 7 11:00:26 2014 -0800

----------------------------------------------------------------------
 .../python/apache/aurora/client/api/__init__.py |  4 +-
 src/main/python/apache/aurora/client/api/sla.py | 64 +++++++++++-----
 .../apache/aurora/client/commands/admin.py      | 52 +++++++++----
 .../python/apache/aurora/client/api/test_sla.py | 45 ++++++++++-
 .../aurora/client/commands/test_admin_sla.py    | 78 ++++++++++++++++++--
 5 files changed, 199 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/62f0aad4/src/main/python/apache/aurora/client/api/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/__init__.py b/src/main/python/apache/aurora/client/api/__init__.py
index 9d9ff66..ecc3742 100644
--- a/src/main/python/apache/aurora/client/api/__init__.py
+++ b/src/main/python/apache/aurora/client/api/__init__.py
@@ -199,8 +199,8 @@ class AuroraClientAPI(object):
     self._assert_valid_job_key(job_key)
     return Sla(self._scheduler_proxy).get_job_uptime_vector(job_key)
 
-  def sla_get_safe_domain_vector(self):
-    return Sla(self._scheduler_proxy).get_domain_uptime_vector(self._cluster)
+  def sla_get_safe_domain_vector(self, hosts=None):
+    return Sla(self._scheduler_proxy).get_domain_uptime_vector(self._cluster, hosts)
 
   def _assert_valid_job_key(self, job_key):
     if not isinstance(job_key, AuroraJobKey):

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/62f0aad4/src/main/python/apache/aurora/client/api/sla.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/sla.py b/src/main/python/apache/aurora/client/api/sla.py
index 71013cd..7b4d75c 100644
--- a/src/main/python/apache/aurora/client/api/sla.py
+++ b/src/main/python/apache/aurora/client/api/sla.py
@@ -36,6 +36,38 @@ from gen.apache.aurora.ttypes import (
 SLA_LIVE_STATES = LIVE_STATES | set([ScheduleStatus.STARTING])
 
 
+def job_key_from_scheduled(task, cluster):
+  """Creates AuroraJobKey from the ScheduledTask.
+
+  Arguments:
+  task -- ScheduledTask to get job key from.
+  cluster -- Cluster the task belongs to.
+  """
+  return AuroraJobKey(
+      cluster=cluster.name,
+      role=task.assignedTask.task.owner.role,
+      env=task.assignedTask.task.environment,
+      name=task.assignedTask.task.jobName
+  )
+
+
+def task_query(job_key=None, hosts=None, job_keys=None):
+  """Creates TaskQuery optionally scoped by a job(s) or hosts.
+
+  Arguments:
+  job_key -- AuroraJobKey to scope the query by.
+  hosts -- list of hostnames to scope the query by.
+  job_keys -- list of AuroraJobKeys to scope the query by.
+  """
+  return TaskQuery(
+      owner=Identity(role=job_key.role) if job_key else None,
+      environment=job_key.env if job_key else None,
+      jobName=job_key.name if job_key else None,
+      slaveHosts=set(hosts) if hosts else None,
+      jobKeys=set(k.to_thrift() for k in job_keys) if job_keys else None,
+      statuses=SLA_LIVE_STATES)
+
+
 class JobUpTimeSlaVector(object):
   """A grouping of job active tasks by:
       - instance: Map of instance ID -> instance uptime in seconds.
@@ -213,18 +245,10 @@ class DomainUpTimeSlaVector(object):
     return filtered_percentage, total_count, filtered_vector
 
   def _init_mappings(self):
-    def job_key_from_scheduled(task):
-      return AuroraJobKey(
-          cluster=self._cluster.name,
-          role=task.assignedTask.task.owner.role,
-          env=task.assignedTask.task.environment,
-          name=task.assignedTask.task.jobName
-      )
-
     jobs = defaultdict(list)
     hosts = defaultdict(list)
     for task in self._tasks:
-      job_key = job_key_from_scheduled(task)
+      job_key = job_key_from_scheduled(task, self._cluster)
       jobs[job_key].append(task)
       hosts[task.assignedTask.slaveHost].append(job_key)
 
@@ -243,20 +267,20 @@ class Sla(object):
     Arguments:
     job_key -- job to create a task uptime vector for.
     """
-    return JobUpTimeSlaVector(self._get_tasks(self._create_task_query(job_key=job_key)))
+    return JobUpTimeSlaVector(self._get_tasks(task_query(job_key=job_key)))
+
+  def get_domain_uptime_vector(self, cluster, hosts=None):
+    """Returns a DomainUpTimeSlaVector object with all available job uptimes.
 
-  def get_domain_uptime_vector(self, cluster):
-    """Returns a DomainUpTimeSlaVector object with all available job uptimes."""
-    return DomainUpTimeSlaVector(cluster, self._get_tasks(self._create_task_query()))
+    Arguments:
+    cluster -- Cluster to get vector for.
+    hosts -- optional list of hostnames to query by.
+    """
+    tasks = self._get_tasks(task_query(hosts=hosts)) if hosts else None
+    job_keys = set(job_key_from_scheduled(t, cluster) for t in tasks) if tasks else None
+    return DomainUpTimeSlaVector(cluster, self._get_tasks(task_query(job_keys=job_keys)))
 
   def _get_tasks(self, task_query):
     resp = self._scheduler.getTasksStatus(task_query)
     check_and_log_response(resp)
     return resp.result.scheduleStatusResult.tasks
-
-  def _create_task_query(self, job_key=None):
-    return TaskQuery(
-        owner=Identity(role=job_key.role) if job_key else None,
-        environment=job_key.env if job_key else None,
-        jobName=job_key.name if job_key else None,
-        statuses=SLA_LIVE_STATES)

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/62f0aad4/src/main/python/apache/aurora/client/commands/admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/admin.py b/src/main/python/apache/aurora/client/commands/admin.py
index 40588e2..d6f571b 100644
--- a/src/main/python/apache/aurora/client/commands/admin.py
+++ b/src/main/python/apache/aurora/client/commands/admin.py
@@ -61,13 +61,31 @@ def parse_host_file(filename):
   return hosts
 
 
+def parse_host_list(host_list):
+  hosts = [hostname.strip() for hostname in host_list.split(",")]
+  if not hosts:
+    die('No valid hosts found.')
+  return hosts
+
+
+def parse_hosts_optional(list_option, file_option):
+  if bool(list_option) and bool(file_option):
+    die('Cannot specify both filename and list for the same option.')
+  hosts = None
+  if file_option:
+    hosts = parse_host_file(file_option)
+  elif list_option:
+    hosts = parse_host_list(list_option)
+  return hosts
+
+
 def parse_hosts(options):
   if bool(options.filename) == bool(options.hosts):
     die('Please specify either --filename or --hosts')
   if options.filename:
     hosts = parse_host_file(options.filename)
   elif options.hosts:
-    hosts = [hostname.strip() for hostname in options.hosts.split(",")]
+    hosts = parse_host_list(options.hosts)
   if not hosts:
     die('No valid hosts found.')
   return hosts
@@ -433,17 +451,23 @@ def scheduler_snapshot(cluster):
 
 
 @app.command
-@app.command_option('-i', '--include_hosts', dest='include_filename', default=None,
-    help='Inclusion filter. An optional text file listing hosts (one per line)'
-         'to include into the result set if found. Example: cl1-aau-dev2.test.com')
-@app.command_option('-x', '--exclude_hosts', dest='exclude_filename', default=None,
-    help='Exclusion filter. An optional text file listing hosts (one per line)'
-         'to exclude from the result set if found. Example: cl1-aau-dev1.test.com')
+@app.command_option('-I', '--include_file', dest='include_filename', default=None,
+    help='Inclusion filter. An optional text file listing host names (one per line)'
+         'to include into the result set if found.')
+@app.command_option('-i', '--include_hosts', dest='include_hosts', default=None,
+    help='Inclusion filter. An optional comma-separated list of host names'
+         'to include into the result set if found.')
+@app.command_option('-X', '--exclude_file', dest='exclude_filename', default=None,
+    help='Exclusion filter. An optional text file listing host names (one per line)'
+         'to exclude from the result set if found.')
+@app.command_option('-x', '--exclude_hosts', dest='exclude_hosts', default=None,
+    help='Exclusion filter. An optional comma-separated list of host names'
+         'to exclude from the result set if found.')
 @app.command_option('-l', '--list_jobs', dest='list_jobs', default=False, action='store_true',
     help='Lists all affected job keys with projected new SLAs if their tasks get killed'
          'in the following column format:\n'
          'HOST  JOB  PREDICTED_SLA  DURATION_SECONDS')
-@app.command_option('-o', '--override_jobs', dest='override_filename', default=None,
+@app.command_option('-o', '--override_file', dest='override_filename', default=None,
     help='An optional text file to load job specific SLAs that will override'
          'cluster-wide command line percentage and duration values.'
          'The file can have multiple lines in the following format:'
@@ -492,16 +516,18 @@ def sla_list_safe_domain(cluster, percentage, duration):
   sla_percentage = parse_sla_percentage(percentage)
   sla_duration = parse_time(duration)
 
-  exclude_hosts = parse_host_file(options.exclude_filename) if options.exclude_filename else
[]
-  include_hosts = parse_host_file(options.include_filename) if options.include_filename else
[]
+  exclude_hosts = parse_hosts_optional(options.exclude_hosts, options.exclude_filename)
+  include_hosts = parse_hosts_optional(options.include_hosts, options.include_filename)
   override_jobs = parse_jobs_file(options.override_filename) if options.override_filename
else {}
 
-  vector = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).sla_get_safe_domain_vector()
+  vector = AuroraClientAPI(
+      CLUSTERS[cluster],
+      options.verbosity).sla_get_safe_domain_vector(include_hosts)
   hosts = vector.get_safe_hosts(sla_percentage, sla_duration.as_(Time.SECONDS), override_jobs)
 
   results = []
   for host in sorted(hosts.keys()):
-    if include_hosts and host not in include_hosts or exclude_hosts and host in exclude_hosts:
+    if exclude_hosts and host in exclude_hosts:
       continue
 
     if options.list_jobs:
@@ -546,7 +572,7 @@ def sla_probe_hosts(cluster, percentage, duration):
   sla_duration = parse_time(duration)
   hosts = parse_hosts(options)
 
-  vector = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).sla_get_safe_domain_vector()
+  vector = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).sla_get_safe_domain_vector(hosts)
   probed_hosts = vector.probe_hosts(sla_percentage, sla_duration.as_(Time.SECONDS), hosts)
 
   results = []

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/62f0aad4/src/test/python/apache/aurora/client/api/test_sla.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/api/test_sla.py b/src/test/python/apache/aurora/client/api/test_sla.py
index c5ef100..fba85c9 100644
--- a/src/test/python/apache/aurora/client/api/test_sla.py
+++ b/src/test/python/apache/aurora/client/api/test_sla.py
@@ -14,14 +14,15 @@
 # limitations under the License.
 #
 
-import unittest
 import time
+import unittest
 
 from apache.aurora.client.api.sla import (
     DomainUpTimeSlaVector,
     JobUpTimeSlaVector,
     Sla,
-    SLA_LIVE_STATES
+    SLA_LIVE_STATES,
+    task_query
 )
 from apache.aurora.common.aurora_job_key import AuroraJobKey
 from apache.aurora.common.cluster import Cluster
@@ -41,7 +42,7 @@ from gen.apache.aurora.ttypes import (
     TaskQuery
 )
 
-from mock import Mock
+from mock import call, Mock, patch
 
 
 class SlaTest(unittest.TestCase):
@@ -305,3 +306,41 @@ class SlaTest(unittest.TestCase):
         self.create_task(400, 4, 'h4', self._name),
     ])
     self.assert_probe_hosts_result('h1', 80, 300, 50.0, False, None)
+
+
+  def test_get_domain_uptime_vector_with_hosts(self):
+    with patch('apache.aurora.client.api.sla.task_query', return_value=TaskQuery()) as (mock_query):
+      self.mock_get_tasks([
+          self.create_task(100, 1, 'h1', 'j1'),
+          self.create_task(200, 2, 'h1', 'j2'),
+          self.create_task(200, 3, 'h2', 'j1'),
+          self.create_task(200, 3, 'h2', 'j3'),
+          self.create_task(200, 4, 'h3', 'j4'),
+          self.create_task(200, 4, 'h3', 'j3'),
+      ])
+      hosts = ['h1', 'h2', 'h3']
+      jobs = set([
+          AuroraJobKey(self._cluster.name, self._role, self._env, 'j1'),
+          AuroraJobKey(self._cluster.name, self._role, self._env, 'j2'),
+          AuroraJobKey(self._cluster.name, self._role, self._env, 'j3'),
+          AuroraJobKey(self._cluster.name, self._role, self._env, 'j4')
+      ])
+
+      self._sla.get_domain_uptime_vector(self._cluster, hosts)
+      mock_query.assert_has_calls([call(hosts=hosts), call(job_keys=jobs)], any_order=False)
+
+  def test_task_query(self):
+    jobs = set([
+        AuroraJobKey(self._cluster.name, self._role, self._env, 'j1'),
+        AuroraJobKey(self._cluster.name, self._role, self._env, 'j2'),
+        AuroraJobKey(self._cluster.name, self._role, self._env, 'j3'),
+        AuroraJobKey(self._cluster.name, self._role, self._env, 'j4')
+    ])
+
+    query = task_query(job_keys=jobs)
+    assert len(jobs) == len(query.jobKeys), 'Expected length:%s, Actual:%s' % (
+        len(jobs), len(query.jobKeys)
+    )
+    assert SLA_LIVE_STATES == query.statuses, 'Expected:%s, Actual:%s' % (
+        SLA_LIVE_STATES, query.statuses
+    )

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/62f0aad4/src/test/python/apache/aurora/client/commands/test_admin_sla.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/commands/test_admin_sla.py b/src/test/python/apache/aurora/client/commands/test_admin_sla.py
index 6e56bc4..51e7482 100644
--- a/src/test/python/apache/aurora/client/commands/test_admin_sla.py
+++ b/src/test/python/apache/aurora/client/commands/test_admin_sla.py
@@ -32,10 +32,13 @@ from mock import Mock, patch
 class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
 
   @classmethod
-  def setup_mock_options(cls, exclude=None, include=None, override=None, list_jobs=False):
+  def setup_mock_options(cls, exclude=None, include=None, override=None,
+                         exclude_list=None, include_list=None, list_jobs=False):
     mock_options = Mock()
     mock_options.exclude_filename = exclude
+    mock_options.exclude_hosts = exclude_list
     mock_options.include_filename = include
+    mock_options.include_hosts = include_list
     mock_options.override_filename = override
     mock_options.list_jobs = list_jobs
     mock_options.verbosity = False
@@ -102,11 +105,34 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
         mock_vector.get_safe_hosts.assert_called_once_with(50.0, 100.0, {})
         mock_print_results.assert_called_once_with(['h0', 'h2'])
 
+  def test_safe_domain_exclude_hosts_from_list(self):
+    """Test successful execution of the sla_list_safe_domain command with exclude list option."""
+    mock_vector = self.create_mock_vector(self.create_hosts(3, 80, 100))
+    mock_options = self.setup_mock_options(exclude_list=','.join(['h0', 'h1']))
+    with contextlib.nested(
+        patch('apache.aurora.client.commands.admin.AuroraClientAPI', new=Mock(spec=AuroraClientAPI)),
+        patch('apache.aurora.client.commands.admin.print_results'),
+        patch('apache.aurora.client.commands.admin.CLUSTERS', new=self.TEST_CLUSTERS),
+        patch('twitter.common.app.get_options', return_value=mock_options)
+    ) as (
+        mock_api,
+        mock_print_results,
+        test_clusters,
+        mock_options):
+
+      mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
+
+      sla_list_safe_domain(['west', '50', '100s'])
+
+      mock_vector.get_safe_hosts.assert_called_once_with(50.0, 100.0, {})
+      mock_print_results.assert_called_once_with(['h2'])
+
   def test_safe_domain_include_hosts(self):
     """Test successful execution of the sla_list_safe_domain command with include hosts option."""
-    mock_vector = self.create_mock_vector(self.create_hosts(3, 80, 100))
+    mock_vector = self.create_mock_vector(self.create_hosts(1, 80, 100))
+    hostname = 'h0'
     with temporary_file() as fp:
-      fp.write('h1')
+      fp.write(hostname)
       fp.flush()
       mock_options = self.setup_mock_options(include=fp.name)
       with contextlib.nested(
@@ -124,8 +150,33 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
 
         sla_list_safe_domain(['west', '50', '100s'])
 
+        mock_api.return_value.sla_get_safe_domain_vector.assert_called_once_with([hostname])
         mock_vector.get_safe_hosts.assert_called_once_with(50.0, 100.0, {})
-        mock_print_results.assert_called_once_with(['h1'])
+        mock_print_results.assert_called_once_with([hostname])
+
+  def test_safe_domain_include_hosts_from_list(self):
+    """Test successful execution of the sla_list_safe_domain command with include list option."""
+    mock_vector = self.create_mock_vector(self.create_hosts(2, 80, 100))
+    hosts = ['h0', 'h1']
+    mock_options = self.setup_mock_options(include_list=','.join(hosts))
+    with contextlib.nested(
+        patch('apache.aurora.client.commands.admin.AuroraClientAPI', new=Mock(spec=AuroraClientAPI)),
+        patch('apache.aurora.client.commands.admin.print_results'),
+        patch('apache.aurora.client.commands.admin.CLUSTERS', new=self.TEST_CLUSTERS),
+        patch('twitter.common.app.get_options', return_value=mock_options)
+    ) as (
+        mock_api,
+        mock_print_results,
+        test_clusters,
+        mock_options):
+
+      mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
+
+      sla_list_safe_domain(['west', '50', '100s'])
+
+      mock_api.return_value.sla_get_safe_domain_vector.assert_called_once_with(hosts)
+      mock_vector.get_safe_hosts.assert_called_once_with(50.0, 100.0, {})
+      mock_print_results.assert_called_once_with(hosts)
 
   def test_safe_domain_override_jobs(self):
     """Test successful execution of the sla_list_safe_domain command with override_jobs option."""
@@ -205,6 +256,18 @@ class TestAdminSlaListSafeDomainCommand(AuroraClientCommandTest):
         else:
           assert 'Expected error is not raised.'
 
+  def test_safe_domain_hosts_error(self):
+    """Tests execution of the sla_list_safe_domain command with both include file and list"""
+    mock_options = self.setup_mock_options(include='file', include_list='list')
+    with patch('twitter.common.app.get_options', return_value=mock_options) as (mock_options):
+
+      try:
+        sla_list_safe_domain(['west', '50', '100s'])
+      except SystemExit:
+        pass
+      else:
+        assert 'Expected error is not raised.'
+
 
 class TestAdminSlaProbeHostsCommand(AuroraClientCommandTest):
 
@@ -233,7 +296,8 @@ class TestAdminSlaProbeHostsCommand(AuroraClientCommandTest):
 
   def test_probe_hosts_with_list(self):
     """Tests successful execution of the sla_probe_hosts command with host list."""
-    mock_options = self.setup_mock_options(hosts='h0,h1')
+    hosts = ['h0', 'h1']
+    mock_options = self.setup_mock_options(hosts=','.join(hosts))
     mock_vector = self.create_mock_vector(self.create_probe_hosts(2, 80, True, 0))
     with contextlib.nested(
         patch('apache.aurora.client.commands.admin.AuroraClientAPI', new=Mock(spec=AuroraClientAPI)),
@@ -249,7 +313,8 @@ class TestAdminSlaProbeHostsCommand(AuroraClientCommandTest):
       mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
       sla_probe_hosts(['west', '90', '200s'])
 
-      mock_vector.probe_hosts.assert_called_once_with(90.0, 200.0, ['h0', 'h1'])
+      mock_api.return_value.sla_get_safe_domain_vector.assert_called_once_with(hosts)
+      mock_vector.probe_hosts.assert_called_once_with(90.0, 200.0, hosts)
       mock_print_results.assert_called_once_with([
           'h0\twest/role/env/job0\t80.00\tTrue\t0',
           'h1\twest/role/env/job1\t80.00\tTrue\t0'
@@ -276,6 +341,7 @@ class TestAdminSlaProbeHostsCommand(AuroraClientCommandTest):
         mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
         sla_probe_hosts(['west', '90', '200s'])
 
+        mock_api.return_value.sla_get_safe_domain_vector.assert_called_once_with(['h0'])
         mock_vector.probe_hosts.assert_called_once_with(90.0, 200.0, ['h0'])
         mock_print_results.assert_called_once_with([
             'h0\twest/role/env/job0\t80.00\tFalse\tn/a'


Mime
View raw message