incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brond...@apache.org
Subject [4/49] git commit: [#5127] ticket:227 retry taskd status log read several times
Date Fri, 07 Dec 2012 22:26:35 GMT
[#5127] ticket:227 retry taskd status log read several times


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/16d2c6ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/16d2c6ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/16d2c6ca

Branch: refs/heads/db/4803
Commit: 16d2c6cad5971c9eb46ec4218b19e60931018d43
Parents: 80ee1bc
Author: Igor Bondarenko <jetmind2@gmail.com>
Authored: Tue Dec 4 13:52:18 2012 +0000
Committer: Dave Brondsema <dbrondsema@geek.net>
Committed: Thu Dec 6 22:46:33 2012 +0000

----------------------------------------------------------------------
 Allura/allura/command/taskd_cleanup.py |   35 ++++++++++++++++++--------
 Allura/allura/tests/test_commands.py   |   33 +++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/16d2c6ca/Allura/allura/command/taskd_cleanup.py
----------------------------------------------------------------------
diff --git a/Allura/allura/command/taskd_cleanup.py b/Allura/allura/command/taskd_cleanup.py
index 3af29a7..c2bfee5 100644
--- a/Allura/allura/command/taskd_cleanup.py
+++ b/Allura/allura/command/taskd_cleanup.py
@@ -1,4 +1,5 @@
 import os
+import time
 import signal
 import socket
 import subprocess
@@ -13,6 +14,9 @@ class TaskdCleanupCommand(base.Command):
     parser.add_option('-k', '--kill-stuck-taskd',
             dest='kill', action='store_true',
             help='automatically kill stuck taskd processes')
+    parser.add_option('-n', '--num-retry-status-check',
+            dest='num_retry', type='int', default=5,
+            help='number of retries to read taskd status log after sending USR1 signal (5
by default)')
     usage = '<ini file> [-k] <taskd status log file>'
     min_args = 2
     max_args = 2
@@ -109,8 +113,9 @@ class TaskdCleanupCommand(base.Command):
             tasks = [pid for pid in stdout.split('\n') if pid != ''][:-1]
         return tasks
 
-    def _taskd_status(self, pid):
-        os.kill(int(pid), signal.SIGUSR1)
+    def _taskd_status(self, pid, retry=False):
+        if not retry:
+            os.kill(int(pid), signal.SIGUSR1)
         p = subprocess.Popen(['tail', '-n1', self.taskd_status_log],
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE)
@@ -121,17 +126,25 @@ class TaskdCleanupCommand(base.Command):
         return stdout
 
     def _check_taskd_status(self, pid):
-        status = self._taskd_status(pid)
-        if ('taskd pid %s' % pid) not in status:
-            return 'STUCK'
-        return 'OK'
+        for i in range(self.options.num_retry):
+            retry = False if i == 0 else True
+            status = self._taskd_status(pid, retry)
+            if ('taskd pid %s' % pid) in status:
+                return 'OK'
+            base.log.info('retrying after one second')
+            time.sleep(1)
+        return 'STUCK'
 
     def _check_task(self, taskd_pid, task):
-        status = self._taskd_status(taskd_pid)
-        line = 'taskd pid %s is currently handling task %s' % (taskd_pid, task)
-        if line not in status:
-            return 'FAIL'
-        return 'OK'
+        for i in range(self.options.num_retry):
+            retry = False if i == 0 else True
+            status = self._taskd_status(taskd_pid, retry)
+            line = 'taskd pid %s is currently handling task %s' % (taskd_pid, task)
+            if line in status:
+                return 'OK'
+            base.log.info('retrying after one second')
+            time.sleep(1)
+        return 'FAIL'
 
     def _kill_stuck_taskd(self, pid):
         os.kill(int(pid), signal.SIGKILL)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/16d2c6ca/Allura/allura/tests/test_commands.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index 316ef01..b59bf03 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -192,13 +192,28 @@ class TestTaskdCleanupCommand(object):
 
     def setUp(self):
         self.cmd_class = taskd_cleanup.TaskdCleanupCommand
+        self.old_check_taskd_status = self.cmd_class._check_taskd_status
         self.cmd_class._check_taskd_status = lambda x, p: 'OK'
+        self.old_check_task = self.cmd_class._check_task
         self.cmd_class._check_task = lambda x, p, t: 'OK'
+        self.old_busy_tasks = self.cmd_class._busy_tasks
         self.cmd_class._busy_tasks = lambda x: []
+        self.old_taskd_pids = self.cmd_class._taskd_pids
         self.cmd_class._taskd_pids = lambda x: ['1111']
+        self.old_kill_stuck_taskd = self.cmd_class._kill_stuck_taskd
         self.cmd_class._kill_stuck_taskd = Mock()
+        self.old_complete_suspicious_tasks = self.cmd_class._complete_suspicious_tasks
         self.cmd_class._complete_suspicious_tasks = lambda x: []
 
+    def tearDown(self):
+        # need to clean up setUp mocking for unit tests below to work properly
+        self.cmd_class._check_taskd_status = self.old_check_taskd_status
+        self.cmd_class._check_task = self.old_check_task
+        self.cmd_class._busy_tasks = self.old_busy_tasks
+        self.cmd_class._taskd_pids = self.old_taskd_pids
+        self.cmd_class._kill_stuck_taskd = self.old_kill_stuck_taskd
+        self.cmd_class._complete_suspicious_tasks = self.old_complete_suspicious_tasks
+
     def test_forsaken_tasks(self):
         # forsaken task
         task = Mock(state='busy', process='host pid 1111', result='')
@@ -266,3 +281,21 @@ class TestTaskdCleanupCommand(object):
         assert cmd.suspicious_tasks == [task1], cmd.suspicious_tasks
         assert cmd.error_tasks == [], cmd.error_tasks
         assert task1.state == 'complete'
+
+
+# taskd_cleanup unit tests
+def test_status_log_retries():
+    cmd = taskd_cleanup.TaskdCleanupCommand('taskd_command')
+    cmd._taskd_status = Mock()
+    cmd._taskd_status.return_value = ''
+    cmd.options = Mock(num_retry=10)
+    cmd._check_taskd_status(123)
+    expected_calls = [call(123, False if i == 0 else True) for i in range(10)]
+    assert cmd._taskd_status.mock_calls == expected_calls
+
+    cmd._taskd_status = Mock()
+    cmd._taskd_status.return_value = ''
+    cmd.options = Mock(num_retry=3)
+    cmd._check_task(123, Mock())
+    expected_calls = [call(123, False if i == 0 else True) for i in range(3)]
+    assert cmd._taskd_status.mock_calls == expected_calls


Mime
View raw message