aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wick...@apache.org
Subject incubator-aurora git commit: Add basic test coverage to the TaskGarbageCollector.
Date Thu, 26 Feb 2015 22:48:10 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 36af017d9 -> db99eb082


Add basic test coverage to the TaskGarbageCollector.

Testing Done:
./pants test src/test/python/apache/thermos/monitoring:test_garbage

Bugs closed: AURORA-1155

Reviewed at https://reviews.apache.org/r/31491/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/db99eb08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/db99eb08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/db99eb08

Branch: refs/heads/master
Commit: db99eb0825b3ca216740cdfaac6775f79b36921c
Parents: 36af017
Author: Brian Wickman <wickman@apache.org>
Authored: Thu Feb 26 14:48:04 2015 -0800
Committer: Brian Wickman <wickman@apache.org>
Committed: Thu Feb 26 14:48:04 2015 -0800

----------------------------------------------------------------------
 .../python/apache/thermos/monitoring/garbage.py | 25 +++----
 src/test/python/apache/thermos/monitoring/BUILD | 14 ++++
 .../apache/thermos/monitoring/test_garbage.py   | 76 ++++++++++++++++++++
 3 files changed, 103 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/db99eb08/src/main/python/apache/thermos/monitoring/garbage.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/garbage.py b/src/main/python/apache/thermos/monitoring/garbage.py
index 53bb034..aa5a272 100644
--- a/src/main/python/apache/thermos/monitoring/garbage.py
+++ b/src/main/python/apache/thermos/monitoring/garbage.py
@@ -17,7 +17,7 @@ import sys
 import time
 from collections import namedtuple
 
-from twitter.common.dirutil import safe_bsize, safe_delete, safe_rmtree
+from twitter.common.dirutil import safe_bsize, safe_delete, safe_mtime, safe_rmtree
 from twitter.common.quantity import Amount, Data, Time
 
 from apache.thermos.common.ckpt import CheckpointDispatcher
@@ -37,16 +37,17 @@ class TaskGarbageCollector(object):
 
     self._detector = TaskDetector(checkpoint_root)
     self._task_id = task_id
+    self._pathspec = TaskPath(root=checkpoint_root, task_id=task_id)
     self._state = CheckpointDispatcher.from_file(self._detector.get_checkpoint(task_id))
 
   def get_age(self):
-    return os.path.getmtime(self._detector.get_checkpoint(self._task_id))
+    return safe_mtime(self._detector.get_checkpoint(self._task_id))
 
   def get_metadata(self, with_size=True):
     runner_ckpt = self._detector.get_checkpoint(self._task_id)
     process_ckpts = [ckpt for ckpt in self._detector.get_process_checkpoints(self._task_id)]
-    json_spec = TaskPath(
-        root=self._root, task_id=self._task_id, state='finished').getpath('task_path')
+    # assumes task is in finished state.
+    json_spec = self._pathspec.given(state='finished').getpath('task_path')
     for path in [json_spec, runner_ckpt] + process_ckpts:
       if with_size:
         yield path, safe_bsize(path)
@@ -54,7 +55,7 @@ class TaskGarbageCollector(object):
         yield path
 
   def get_logs(self, with_size=True):
-    if self._state.header:
+    if self._state and self._state.header and self._state.header.log_dir:
       for path in self._detector.get_process_logs(self._task_id, self._state.header.log_dir):
         if with_size:
           yield path, safe_bsize(path)
@@ -62,7 +63,7 @@ class TaskGarbageCollector(object):
           yield path
 
   def get_data(self, with_size=True):
-    if self._state.header and self._state.header.sandbox:
+    if self._state and self._state.header and self._state.header.sandbox:
       for root, dirs, files in os.walk(self._state.header.sandbox):
         for file in files:
           filename = os.path.join(root, file)
@@ -79,19 +80,19 @@ class TaskGarbageCollector(object):
   def erase_metadata(self):
     for fn in self.get_metadata(with_size=False):
       safe_delete(fn)
-    safe_rmtree(TaskPath(root=self._root, task_id=self._task_id).getpath('checkpoint_path'))
+    safe_rmtree(self._pathspec.getpath('checkpoint_path'))
 
   def erase_logs(self):
     for fn in self.get_logs(with_size=False):
       safe_delete(fn)
-    if self._state.header:
-      path = TaskPath(root=self._root, task_id=self._task_id, log_dir=self._state.header.log_dir)
-      safe_rmtree(path.getpath('process_logbase'))
+    if self._state and self._state.header:
+      path = self._pathspec.given(log_dir=self._state.header.log_dir).getpath('process_logbase')
+      safe_rmtree(path)
 
   def erase_data(self):
     for fn in self.get_data(with_size=False):
-      os.unlink(fn)
-    if self._state.header and self._state.header.sandbox:
+      safe_delete(fn)
+    if self._state and self._state.header and self._state.header.sandbox:
       safe_rmtree(self._state.header.sandbox)
 
 

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/db99eb08/src/test/python/apache/thermos/monitoring/BUILD
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/monitoring/BUILD b/src/test/python/apache/thermos/monitoring/BUILD
index 9c26777..4e99da4 100644
--- a/src/test/python/apache/thermos/monitoring/BUILD
+++ b/src/test/python/apache/thermos/monitoring/BUILD
@@ -16,6 +16,7 @@ python_test_suite(name = 'all',
   dependencies = [
     ':test_disk',
     ':test_detector',
+    ':test_garbage',
     ':test_resource',
   ]
 )
@@ -35,6 +36,19 @@ python_tests(name = 'test_detector',
   ]
 )
 
+python_tests(name = 'test_garbage',
+  sources = ['test_garbage.py'],
+  dependencies = [
+    '3rdparty/python:mock',
+    '3rdparty/python:twitter.common.contextutil',
+    '3rdparty/python:twitter.common.dirutil',
+    'api/src/main/thrift/org/apache/thermos:py-thrift',
+    'src/main/python/apache/thermos/common:path',
+    'src/main/python/apache/thermos/core:helper',
+    'src/main/python/apache/thermos/monitoring:garbage',
+  ]
+)
+
 python_tests(name = 'test_resource',
   sources = ['test_resource.py'],
   dependencies = [

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/db99eb08/src/test/python/apache/thermos/monitoring/test_garbage.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/monitoring/test_garbage.py b/src/test/python/apache/thermos/monitoring/test_garbage.py
new file mode 100644
index 0000000..8232874
--- /dev/null
+++ b/src/test/python/apache/thermos/monitoring/test_garbage.py
@@ -0,0 +1,76 @@
+import os
+
+from mock import call, patch
+from twitter.common.contextutil import temporary_dir
+from twitter.common.dirutil import safe_mkdir, touch
+
+from apache.thermos.common.path import TaskPath
+from apache.thermos.core.helper import TaskRunnerHelper
+from apache.thermos.monitoring.garbage import TaskGarbageCollector
+
+from gen.apache.thermos.ttypes import RunnerCkpt, RunnerHeader
+
+
+def test_empty_garbage_collector():
+  with temporary_dir() as checkpoint_root:
+    path = TaskPath(root=checkpoint_root, task_id='does_not_exist')
+    gc = TaskGarbageCollector(checkpoint_root, 'does_not_exist')
+
+    assert gc.get_age() == 0
+
+    # assume runner, finished task
+    assert set(gc.get_metadata(with_size=False)) == set([
+        path.getpath('runner_checkpoint'),
+        path.given(state='finished').getpath('task_path'),
+    ])
+
+    assert list(gc.get_logs()) == []
+    assert list(gc.get_data()) == []
+
+
+@patch('apache.thermos.monitoring.garbage.safe_delete')
+@patch('apache.thermos.monitoring.garbage.safe_rmtree')
+def test_garbage_collector(safe_rmtree, safe_delete):
+  with temporary_dir() as sandbox, temporary_dir() as checkpoint_root, temporary_dir() as
log_dir:
+
+    path = TaskPath(root=checkpoint_root, task_id='test', log_dir=log_dir)
+
+    touch(os.path.join(sandbox, 'test_file1'))
+    touch(os.path.join(sandbox, 'test_file2'))
+    safe_mkdir(os.path.dirname(path.given(state='finished').getpath('task_path')))
+    safe_mkdir(os.path.dirname(path.getpath('runner_checkpoint')))
+    touch(path.given(state='finished').getpath('task_path'))
+
+    header = RunnerHeader(task_id='test', sandbox=sandbox, log_dir=log_dir)
+    ckpt = TaskRunnerHelper.open_checkpoint(path.getpath('runner_checkpoint'))
+    ckpt.write(RunnerCkpt(runner_header=header))
+    ckpt.close()
+
+    gc = TaskGarbageCollector(checkpoint_root, task_id='test')
+    assert gc._state.header.log_dir == log_dir
+    assert gc._state.header.sandbox == sandbox
+
+    # erase metadata
+    gc.erase_metadata()
+    safe_delete.assert_has_calls([
+        call(path.given(state='finished').getpath('task_path')),
+        call(path.getpath('runner_checkpoint'))], any_order=True)
+    safe_rmtree.assert_has_calls([call(path.getpath('checkpoint_path'))])
+
+    safe_delete.reset_mock()
+    safe_rmtree.reset_mock()
+
+    # erase logs
+    gc.erase_logs()
+    safe_rmtree.assert_has_calls([call(log_dir)])
+
+    safe_delete.reset_mock()
+    safe_rmtree.reset_mock()
+
+    # erase sandbox
+    gc.erase_data()
+
+    safe_delete.assert_has_calls([
+        call(os.path.join(sandbox, 'test_file1')),
+        call(os.path.join(sandbox, 'test_file2'))], any_order=True)
+    safe_rmtree.assert_has_calls([call(sandbox)])


Mime
View raw message