aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mchucarr...@apache.org
Subject git commit: Fix logging in the command-runner.
Date Thu, 12 Jun 2014 01:36:08 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master ea68ade10 -> f85907287


Fix logging in the command-runner.

Bugs closed: aurora-503

Reviewed at https://reviews.apache.org/r/22233/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/f8590728
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/f8590728
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/f8590728

Branch: refs/heads/master
Commit: f8590728743a18745b746d472bdc154c814ab901
Parents: ea68ade
Author: Mark Chu-Carroll <mchucarroll@twopensource.com>
Authored: Wed Jun 11 21:27:35 2014 -0400
Committer: Mark Chu-Carroll <mchucarroll@twitter.com>
Committed: Wed Jun 11 21:27:35 2014 -0400

----------------------------------------------------------------------
 .../python/apache/aurora/client/api/__init__.py |  2 ++
 .../apache/aurora/client/api/command_runner.py  | 31 ++++++++++++--------
 .../python/apache/aurora/client/cli/client.py   |  1 -
 .../python/apache/aurora/client/cli/task.py     | 16 +++++++---
 .../apache/aurora/client/cli/test_task_run.py   | 10 +++++++
 5 files changed, 42 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/f8590728/src/main/python/apache/aurora/client/api/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/__init__.py b/src/main/python/apache/aurora/client/api/__init__.py
index 99444d6..19fc678 100644
--- a/src/main/python/apache/aurora/client/api/__init__.py
+++ b/src/main/python/apache/aurora/client/api/__init__.py
@@ -12,6 +12,8 @@
 # limitations under the License.
 #
 
+from __future__ import print_function
+
 from twitter.common import log
 
 from apache.aurora.common.aurora_job_key import AuroraJobKey

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/f8590728/src/main/python/apache/aurora/client/api/command_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/command_runner.py b/src/main/python/apache/aurora/client/api/command_runner.py
index a291fd9..58c2984 100644
--- a/src/main/python/apache/aurora/client/api/command_runner.py
+++ b/src/main/python/apache/aurora/client/api/command_runner.py
@@ -14,6 +14,7 @@
 
 from __future__ import print_function
 
+import logging
 import posixpath
 import subprocess
 from multiprocessing.pool import ThreadPool
@@ -36,13 +37,6 @@ class CommandRunnerTrait(Cluster.Trait):
 
 
 class DistributedCommandRunner(object):
-  @staticmethod
-  def execute(args):
-    hostname, role, command = args
-    ssh_command = ['ssh', '-n', '-q', '%s@%s' % (role, hostname), command]
-    po = subprocess.Popen(ssh_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    output = po.communicate()
-    return '\n'.join('%s:  %s' % (hostname, line) for line in output[0].splitlines())
 
   @classmethod
   def make_executor_path(cls, cluster, executor_name):
@@ -103,19 +97,29 @@ class DistributedCommandRunner(object):
   def query_from(cls, role, env, job):
     return TaskQuery(statuses=LIVE_STATES, owner=Identity(role), jobName=job, environment=env)
 
-  def __init__(self, cluster, role, env, jobs, ssh_user=None):
+  def __init__(self, cluster, role, env, jobs, ssh_user=None,
+      log_fn=log.error):
     self._cluster = cluster
     self._api = AuroraClientAPI(cluster=cluster)
     self._role = role
     self._env = env
     self._jobs = jobs
     self._ssh_user = ssh_user if ssh_user else self._role
+    self._log = log_fn
+
+  def execute(self, args):
+    hostname, role, command = args
+    ssh_command = ['ssh', '-n', '-q', '%s@%s' % (role, hostname), command]
+    self._log(logging.DEBUG, "Running command: %s" % ssh_command)
+    po = subprocess.Popen(ssh_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    output = po.communicate()
+    return '\n'.join('%s:  %s' % (hostname, line) for line in output[0].splitlines())
 
   def resolve(self):
     for job in self._jobs:
       resp = self._api.query(self.query_from(self._role, self._env, job))
       if resp.responseCode != ResponseCode.OK:
-        log.error('Failed to query job: %s' % job)
+        self._log(logging.ERROR, 'Failed to query job: %s' % job)
         continue
       for task in resp.result.scheduleStatusResult.tasks:
         yield task
@@ -143,8 +147,9 @@ class InstanceDistributedCommandRunner(DistributedCommandRunner):
         environment=env,
         instanceIds=instances)
 
-  def __init__(self, cluster, role, env, job, ssh_user=None, instances=None):
-    super(InstanceDistributedCommandRunner, self).__init__(cluster, role, env, [job], ssh_user)
+  def __init__(self, cluster, role, env, job, ssh_user=None, instances=None, log_fn=logging.log):
+    super(InstanceDistributedCommandRunner, self).__init__(cluster, role, env, [job], ssh_user,
+        log_fn)
     self._job = job
     self._ssh_user = ssh_user if ssh_user else self._role
     self.instances = instances
@@ -155,6 +160,6 @@ class InstanceDistributedCommandRunner(DistributedCommandRunner):
       for task in resp.result.scheduleStatusResult.tasks:
         yield task
     else:
-      # This should be fixed by AURORA-503.
-      log.error('could not retrieve task information for run command: %s' % resp.messageDEPRECATED)
+      self._log(logging.ERROR,
+          "Error: could not retrieve task information for run command: %s" % resp.messageDEPRECATED)
       raise ValueError("Could not retrieve task information: %s" % resp.messageDEPRECATED)

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/f8590728/src/main/python/apache/aurora/client/cli/client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/client.py b/src/main/python/apache/aurora/client/cli/client.py
index 0cb6944..2a06cdd 100644
--- a/src/main/python/apache/aurora/client/cli/client.py
+++ b/src/main/python/apache/aurora/client/cli/client.py
@@ -18,7 +18,6 @@ from apache.aurora.client.cli.bridge import Bridge, CommandProcessor
 from apache.aurora.client.cli.logsetup import setup_default_log_handlers
 from apache.aurora.client.cli.standalone_client import AuroraCommandLine
 
-
 # TODO(mchucarroll): the entire bridged executable mechanism here is
 # intended to be deprecated once clientv2 has proven to be stable
 # and adopted by most users. Once we reach that point, this should

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/f8590728/src/main/python/apache/aurora/client/cli/task.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/task.py b/src/main/python/apache/aurora/client/cli/task.py
index c21159c..91175fa 100644
--- a/src/main/python/apache/aurora/client/cli/task.py
+++ b/src/main/python/apache/aurora/client/cli/task.py
@@ -23,7 +23,16 @@ from apache.aurora.client.api.command_runner import (
     DistributedCommandRunner,
     InstanceDistributedCommandRunner
 )
-from apache.aurora.client.cli import EXIT_INVALID_PARAMETER, Noun, Verb
+from apache.aurora.client.api.updater_util import UpdaterConfig
+from apache.aurora.client.cli import (
+    EXIT_COMMAND_FAILURE,
+    EXIT_INVALID_CONFIGURATION,
+    EXIT_INVALID_PARAMETER,
+    EXIT_OK,
+    Noun,
+    print_aurora_log,
+    Verb
+)
 from apache.aurora.client.cli.context import AuroraCommandContext
 from apache.aurora.client.cli.options import (
     CommandOption,
@@ -65,11 +74,10 @@ class RunCommand(Verb):
     ]
 
   def execute(self, context):
-    # TODO(mchucarroll): add options to specify which instances to run on (AURORA-198)
     (cluster_name, role, env, name), instances = context.options.instance_spec
     cluster = CLUSTERS[cluster_name]
-    dcr = InstanceDistributedCommandRunner(cluster, role, env, name, context.options.ssh_user,
-        instances)
+    dcr = InstanceDistributedCommandRunner(cluster, role, env, name,
+        context.options.ssh_user, instances, print_aurora_log)
     dcr.run(context.options.cmd, parallelism=context.options.num_threads,
         executor_sandbox=context.options.executor_sandbox)
 

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/f8590728/src/test/python/apache/aurora/client/cli/test_task_run.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_task_run.py b/src/test/python/apache/aurora/client/cli/test_task_run.py
index 167f490..8d9ef05 100644
--- a/src/test/python/apache/aurora/client/cli/test_task_run.py
+++ b/src/test/python/apache/aurora/client/cli/test_task_run.py
@@ -31,6 +31,12 @@ from gen.apache.aurora.api.ttypes import (
     TaskQuery
 )
 
+MOCK_LOG_CONTENTS = []
+
+
+def mock_log(level, msg):
+  MOCK_LOG_CONTENTS.append((level, msg))
+
 
 class TestRunCommand(AuroraClientCommandTest):
 
@@ -103,6 +109,7 @@ class TestRunCommand(AuroraClientCommandTest):
     mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_response()
     sandbox_args = {'slave_root': '/slaveroot', 'slave_run_directory': 'slaverun'}
     with contextlib.nested(
+        patch('apache.aurora.client.cli.task.print_aurora_log', side_effect=mock_log),
         patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
         patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
         patch('apache.aurora.client.cli.task.CLUSTERS', new=self.TEST_CLUSTERS),
@@ -110,6 +117,7 @@ class TestRunCommand(AuroraClientCommandTest):
               'InstanceDistributedCommandRunner.sandbox_args',
             return_value=sandbox_args),
         patch('subprocess.Popen', return_value=self.create_mock_process())) as (
+            _,
             mock_scheduler_proxy_class,
             mock_clusters,
             mock_clusters_cli,
@@ -132,6 +140,8 @@ class TestRunCommand(AuroraClientCommandTest):
           'cd /slaveroot/slaves/*/frameworks/*/executors/thermos-1287391823/runs/'
           'slaverun/sandbox;ls'],
           stderr=-2, stdout=-1)
+      # Check that logging worked properly:
+      assert any("Running command" in entry[1] for entry in MOCK_LOG_CONTENTS)
 
 
 class TestSshCommand(AuroraClientCommandTest):


Mime
View raw message