aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wick...@apache.org
Subject incubator-aurora git commit: Cache TaskDetectors for reuse.
Date Tue, 17 Mar 2015 21:37:57 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 4b39d26f2 -> a4198634b


Cache TaskDetectors for reuse.

Testing Done:
./pants test src/test/python/apache/thermos/observer:: + manual observation

Reviewed at https://reviews.apache.org/r/32013/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/a4198634
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/a4198634
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/a4198634

Branch: refs/heads/master
Commit: a4198634bc178cff67e6faf20786f7273abc050e
Parents: 4b39d26
Author: Brian Wickman <wickman@apache.org>
Authored: Tue Mar 17 14:37:53 2015 -0700
Committer: Brian Wickman <wickman@apache.org>
Committed: Tue Mar 17 14:37:53 2015 -0700

----------------------------------------------------------------------
 .../aurora/executor/common/path_detector.py     |  2 +-
 .../python/apache/thermos/observer/detector.py  | 17 +++++++++--
 .../executor/common/test_path_detector.py       | 32 +++++++++++---------
 3 files changed, 34 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/a4198634/src/main/python/apache/aurora/executor/common/path_detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/path_detector.py b/src/main/python/apache/aurora/executor/common/path_detector.py
index e4135cc..ba85d5c 100644
--- a/src/main/python/apache/aurora/executor/common/path_detector.py
+++ b/src/main/python/apache/aurora/executor/common/path_detector.py
@@ -31,4 +31,4 @@ class MesosPathDetector(PathDetector):
     def iterate():
       for scan_result in self._detector:
         yield os.path.join(ExecutorDetector.path(scan_result), self._sandbox_path)
-    return list(iterate())
+    return [path for path in iterate() if os.path.exists(path)]

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/a4198634/src/main/python/apache/thermos/observer/detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/detector.py b/src/main/python/apache/thermos/observer/detector.py
index 5347cde..79bc632 100644
--- a/src/main/python/apache/thermos/observer/detector.py
+++ b/src/main/python/apache/thermos/observer/detector.py
@@ -41,6 +41,7 @@ class ObserverTaskDetector(object):
     self._path_detector = path_detector
     self._active_tasks = set()  # (root, task_id) tuple
     self._finished_tasks = set()  # (root, task_id) tuple
+    self._detectors = {}
     self._on_active = self.maybe_callback(on_active)
     self._on_finished = self.maybe_callback(on_finished)
     self._on_removed = self.maybe_callback(on_removed)
@@ -53,13 +54,25 @@ class ObserverTaskDetector(object):
   def finished_tasks(self):
     return self._finished_tasks.copy()
 
+  def _refresh_detectors(self):
+    new_paths = set(self._path_detector.get_paths())
+    old_paths = set(self._detectors)
+
+    for path in old_paths - new_paths:
+      self._detectors.pop(path)
+
+    for path in new_paths - old_paths:
+      self._detectors[path] = TaskDetector(root=path)
+
   def iter_tasks(self):
     # returns an iterator of root, task_id, active/finished
-    for root in self._path_detector.get_paths():
-      for status, task_id in TaskDetector(root=root).get_task_ids():
+    for root, detector in self._detectors.items():
+      for status, task_id in detector.get_task_ids():
         yield (root, task_id, status)
 
   def refresh(self):
+    self._refresh_detectors()
+
     all_active, all_finished = set(), set()
 
     for root, task_id, status in self.iter_tasks():

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/a4198634/src/test/python/apache/aurora/executor/common/test_path_detector.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/test_path_detector.py b/src/test/python/apache/aurora/executor/common/test_path_detector.py
index 2d6edae..86460bb 100644
--- a/src/test/python/apache/aurora/executor/common/test_path_detector.py
+++ b/src/test/python/apache/aurora/executor/common/test_path_detector.py
@@ -40,17 +40,21 @@ def test_path_detector():
   )
 
   with mock.patch('glob.glob', return_value=(path1, path2)) as glob:
-    mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR)
-    paths = list(mpd.get_paths())
-    assert len(paths) == 2
-    assert os.path.join(path1, FAKE_CHECKPOINT_DIR) in paths
-    assert os.path.join(path2, FAKE_CHECKPOINT_DIR) in paths
-
-    expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
-      'root': FAKE_ROOT,
-      'slave_id': '*',
-      'framework_id': '*',
-      'executor_id': '*',
-      'run': '*'
-    }
-    glob.assert_called_once_with(expected_glob_pattern)
+    with mock.patch('os.path.exists', side_effect=(True, False)) as exists:
+      mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR)
+      paths = list(mpd.get_paths())
+      assert len(paths) == 1
+      assert paths == [os.path.join(path1, FAKE_CHECKPOINT_DIR)]
+
+      expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
+        'root': FAKE_ROOT,
+        'slave_id': '*',
+        'framework_id': '*',
+        'executor_id': '*',
+        'run': '*'
+      }
+      glob.assert_called_once_with(expected_glob_pattern)
+      assert exists.mock_calls == [
+          mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)),
+          mock.call(os.path.join(path2, FAKE_CHECKPOINT_DIR)),
+      ]


Mime
View raw message