aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wick...@apache.org
Subject incubator-aurora git commit: Add an interface and implementations of PathDetector.
Date Fri, 06 Feb 2015 23:20:38 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master da296a3e3 -> 53f9c78c5


Add an interface and implementations of PathDetector.

The goal here is that any place that uses a fixed checkpoint root that
potentially needs to interact with multiple checkpoint roots (e.g.  thermos
observer, thermos cli, gc executor) will instead use a PathDetector
implementation.  The default implementation is just the FixedPathDetector to
which you can pass --checkpoint_root.  However in subsequent reviews, we
will 1) plumb PathDetector into the abovementioned components and then 2)
wire up a ChainedPathDetector that has both a FixedPathDetector and
MesosPathDetector.

Testing Done:
./pants test \
  src/test/python/apache/aurora/executor/common:: \
  src/test/python/apache/thermos/monitoring::

Bugs closed: AURORA-1024

Reviewed at https://reviews.apache.org/r/30741/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/53f9c78c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/53f9c78c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/53f9c78c

Branch: refs/heads/master
Commit: 53f9c78c52e5f354dddb8bb026c9064d6111327f
Parents: da296a3
Author: Brian Wickman <wickman@apache.org>
Authored: Fri Feb 6 15:20:22 2015 -0800
Committer: Brian Wickman <wickman@apache.org>
Committed: Fri Feb 6 15:20:22 2015 -0800

----------------------------------------------------------------------
 src/main/python/apache/aurora/executor/BUILD    |  9 +-
 .../python/apache/aurora/executor/common/BUILD  | 17 ++++
 .../aurora/executor/common/executor_detector.py | 91 ++++++++++++++++++++
 .../aurora/executor/common/path_detector.py     | 34 ++++++++
 .../apache/aurora/executor/executor_detector.py | 91 --------------------
 .../apache/aurora/executor/gc_executor.py       |  2 +-
 src/main/python/apache/thermos/monitoring/BUILD |  2 +
 .../apache/thermos/monitoring/detector.py       | 36 ++++++++
 src/test/python/apache/aurora/executor/BUILD    | 10 ---
 .../python/apache/aurora/executor/common/BUILD  | 20 +++++
 .../executor/common/test_executor_detector.py   | 81 +++++++++++++++++
 .../executor/common/test_path_detector.py       | 56 ++++++++++++
 .../aurora/executor/test_executor_detector.py   | 81 -----------------
 src/test/python/apache/thermos/monitoring/BUILD |  9 ++
 .../apache/thermos/monitoring/test_detector.py  | 37 ++++++++
 15 files changed, 385 insertions(+), 191 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/main/python/apache/aurora/executor/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/BUILD b/src/main/python/apache/aurora/executor/BUILD
index 79037bc..edd83cf 100644
--- a/src/main/python/apache/aurora/executor/BUILD
+++ b/src/main/python/apache/aurora/executor/BUILD
@@ -33,13 +33,6 @@ python_library(
   ]
 )
 
-python_library(
-  name = 'executor_detector',
-  sources = ['executor_detector.py'],
-  dependencies = [
-    '3rdparty/python:twitter.common.string',
-  ]
-)
 
 python_library(
   name = 'executor_vars',
@@ -98,7 +91,6 @@ python_library(
   name = 'gc_executor',
   sources = ['gc_executor.py'],
   dependencies = [
-    ':executor_detector',
     ':executor_base',
     '3rdparty/python:mesos.interface',
     '3rdparty/python:psutil',
@@ -114,6 +106,7 @@ python_library(
     'src/main/python/apache/thermos/monitoring:detector',
     'src/main/python/apache/thermos/monitoring:garbage',
     'src/main/python/apache/aurora/config:schema',
+    'src/main/python/apache/aurora/executor/common:executor_detector',
     'src/main/python/apache/aurora/executor/common:sandbox',
     'api/src/main/thrift/org/apache/aurora/gen:py-thrift',
     'api/src/main/thrift/org/apache/thermos:py-thrift',

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/main/python/apache/aurora/executor/common/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/BUILD b/src/main/python/apache/aurora/executor/common/BUILD
index e64362e..93f6645 100644
--- a/src/main/python/apache/aurora/executor/common/BUILD
+++ b/src/main/python/apache/aurora/executor/common/BUILD
@@ -13,6 +13,23 @@
 #
 
 python_library(
+  name = 'executor_detector',
+  sources = ['executor_detector.py'],
+  dependencies = [
+    '3rdparty/python:twitter.common.string',
+  ]
+)
+
+python_library(
+  name = 'path_detector',
+  sources = ['path_detector.py'],
+  dependencies = [
+    'src/main/python/apache/thermos/monitoring:detector',
+    ':executor_detector',
+  ]
+)
+
+python_library(
   name = 'status_checker',
   sources = ['status_checker.py'],
   dependencies = [

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/main/python/apache/aurora/executor/common/executor_detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/executor_detector.py b/src/main/python/apache/aurora/executor/common/executor_detector.py
new file mode 100644
index 0000000..a07bfc3
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/executor_detector.py
@@ -0,0 +1,91 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import glob
+import os
+
+from twitter.common.string import ScanfParser
+
+
+# TODO(wickman) MESOS-2805  This makes an assumption about the directory
+# layout Mesos provides.  Ideally we never need to do this and we should
+# work with the Mesos core team to make it unnecessary.
+class ExecutorDetector(object):
+  class Error(Exception): pass
+  class CannotFindRoot(Error): pass
+
+  LOG_PATH = 'executor_logs'
+  RESOURCE_PATH = 'resource_usage.recordio'
+  VARS_PATH = 'executor_vars.json'
+  PATTERN = [
+      '%(root)s',
+      'slaves',
+      '%(slave_id)s',
+      'frameworks',
+      '%(framework_id)s',
+      'executors',
+      '%(executor_id)s',
+      'runs',
+      '%(run)s']
+  EXTRACTOR = ScanfParser(os.path.join(*PATTERN))
+
+  @classmethod
+  def find_root(cls, path):
+    """Does this path appear to match the executor directory pattern?"""
+
+    def root_from_path(path):
+      path = os.path.normpath(path)
+      path_vector = path.split(os.path.sep)
+      pattern_vector = cls.PATTERN
+      if len(path_vector) < len(pattern_vector):
+        return None
+      for pattern, path_component in zip(reversed(pattern_vector), reversed(path_vector)):
+        if pattern.startswith('%'):
+          continue
+        if path_component != pattern:
+          return None
+      matched_path = os.path.join(*path_vector[-len(pattern_vector) + 1:])
+      return os.path.normpath(path[:-len(matched_path)])
+
+    while path != os.path.dirname(path):
+      root = root_from_path(path)
+      if root:
+        return root
+      path = os.path.dirname(path)
+
+  @classmethod
+  def match(cls, path):
+    try:
+      return cls.EXTRACTOR.parse(path)
+    except ScanfParser.ParseError:
+      return None
+
+  @classmethod
+  def path(cls, result):
+    return os.path.join(*cls.PATTERN) % result.groups()
+
+  @classmethod
+  def find(cls, root, slave_id='*', framework_id='*', executor_id='*', run='*'):
+    mixins = dict(
+        root=root, slave_id=slave_id, framework_id=framework_id, executor_id=executor_id,
run=run)
+    return filter(None, map(cls.match, glob.glob(os.path.join(*cls.PATTERN) % mixins)))
+
+  def __init__(self, root=None):
+    self.root = root or self.find_root(os.getcwd())
+    if self.root is None:
+      raise self.CannotFindRoot('Not a valid executor root!')
+
+  def __iter__(self):
+    for extraction in self.find(root=self.root):
+      yield extraction

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/main/python/apache/aurora/executor/common/path_detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/path_detector.py b/src/main/python/apache/aurora/executor/common/path_detector.py
new file mode 100644
index 0000000..e4135cc
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/path_detector.py
@@ -0,0 +1,34 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+from apache.thermos.monitoring.detector import PathDetector
+
+from .executor_detector import ExecutorDetector
+
+
+class MesosPathDetector(PathDetector):
+  DEFAULT_MESOS_ROOT = '/var/lib/mesos'
+  DEFAULT_SANDBOX_PATH = 'checkpoints'
+
+  def __init__(self, root=DEFAULT_MESOS_ROOT, sandbox_path=DEFAULT_SANDBOX_PATH):
+    self._detector = ExecutorDetector(root)
+    self._sandbox_path = sandbox_path
+
+  def get_paths(self):
+    def iterate():
+      for scan_result in self._detector:
+        yield os.path.join(ExecutorDetector.path(scan_result), self._sandbox_path)
+    return list(iterate())

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/main/python/apache/aurora/executor/executor_detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/executor_detector.py b/src/main/python/apache/aurora/executor/executor_detector.py
deleted file mode 100644
index 7b8fc4e..0000000
--- a/src/main/python/apache/aurora/executor/executor_detector.py
+++ /dev/null
@@ -1,91 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-from glob import glob
-
-from twitter.common.string import ScanfParser
-
-
-# TODO(wickman) MESOS-2805  This makes an assumption about the directory
-# layout Mesos provides.  Ideally we never need to do this and we should
-# work with the Mesos core team to make it unnecessary.
-class ExecutorDetector(object):
-  class Error(Exception): pass
-  class CannotFindRoot(Error): pass
-
-  LOG_PATH = 'executor_logs'
-  RESOURCE_PATH = 'resource_usage.recordio'
-  VARS_PATH = 'executor_vars.json'
-  PATTERN = [
-      '%(root)s',
-      'slaves',
-      '%(slave_id)s',
-      'frameworks',
-      '%(framework_id)s',
-      'executors',
-      '%(executor_id)s',
-      'runs',
-      '%(run)s']
-  EXTRACTOR = ScanfParser(os.path.join(*PATTERN))
-
-  @classmethod
-  def find_root(cls, path):
-    """Does this path appear to match the executor directory pattern?"""
-
-    def root_from_path(path):
-      path = os.path.normpath(path)
-      path_vector = path.split(os.path.sep)
-      pattern_vector = cls.PATTERN
-      if len(path_vector) < len(pattern_vector):
-        return None
-      for pattern, path_component in zip(reversed(pattern_vector), reversed(path_vector)):
-        if pattern.startswith('%'):
-          continue
-        if path_component != pattern:
-          return None
-      matched_path = os.path.join(*path_vector[-len(pattern_vector) + 1:])
-      return os.path.normpath(path[:-len(matched_path)])
-
-    while path != os.path.dirname(path):
-      root = root_from_path(path)
-      if root:
-        return root
-      path = os.path.dirname(path)
-
-  @classmethod
-  def match(cls, path):
-    try:
-      return cls.EXTRACTOR.parse(path)
-    except ScanfParser.ParseError:
-      return None
-
-  @classmethod
-  def path(cls, result):
-    return os.path.join(*cls.PATTERN) % result.groups()
-
-  @classmethod
-  def find(cls, root, slave_id='*', framework_id='*', executor_id='*', run='*'):
-    mixins = dict(
-        root=root, slave_id=slave_id, framework_id=framework_id, executor_id=executor_id,
run=run)
-    return filter(None, map(cls.match, glob(os.path.join(*cls.PATTERN) % mixins)))
-
-  def __init__(self, root=None):
-    self.root = root or self.find_root(os.getcwd())
-    if self.root is None:
-      raise self.CannotFindRoot('Not a valid executor root!')
-
-  def __iter__(self):
-    for extraction in self.find(root=self.root):
-      yield extraction

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/main/python/apache/aurora/executor/gc_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/gc_executor.py b/src/main/python/apache/aurora/executor/gc_executor.py
index 952d77d..43b415b 100644
--- a/src/main/python/apache/aurora/executor/gc_executor.py
+++ b/src/main/python/apache/aurora/executor/gc_executor.py
@@ -40,9 +40,9 @@ from apache.thermos.core.inspector import CheckpointInspector
 from apache.thermos.monitoring.detector import TaskDetector
 from apache.thermos.monitoring.garbage import TaskGarbageCollector
 
+from .common.executor_detector import ExecutorDetector
 from .common.sandbox import DirectorySandbox
 from .executor_base import ExecutorBase
-from .executor_detector import ExecutorDetector
 
 from gen.apache.aurora.api.constants import TERMINAL_STATES
 from gen.apache.aurora.api.ttypes import ScheduleStatus

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/main/python/apache/thermos/monitoring/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/BUILD b/src/main/python/apache/thermos/monitoring/BUILD
index 33259c8..a7864d4 100644
--- a/src/main/python/apache/thermos/monitoring/BUILD
+++ b/src/main/python/apache/thermos/monitoring/BUILD
@@ -18,6 +18,8 @@ python_library(
   name = 'detector',
   sources = ['detector.py'],
   dependencies = [
+    '3rdparty/python:twitter.common.lang',
+    'src/main/python/apache/thermos/common:constants',
     'src/main/python/apache/thermos/common:path',
   ]
 )

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/main/python/apache/thermos/monitoring/detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/detector.py b/src/main/python/apache/thermos/monitoring/detector.py
index 117aef5..e0922e6 100644
--- a/src/main/python/apache/thermos/monitoring/detector.py
+++ b/src/main/python/apache/thermos/monitoring/detector.py
@@ -21,10 +21,46 @@ This module contains the TaskDetector, used to detect Thermos tasks within
a giv
 import glob
 import os
 import re
+from abc import abstractmethod
 
+from twitter.common.lang import Compatibility, Interface
+
+from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
 from apache.thermos.common.path import TaskPath
 
 
+class PathDetector(Interface):
+  @abstractmethod
+  def get_paths(self):
+    """Get a list of valid checkpoint roots."""
+
+
+class FixedPathDetector(PathDetector):
+  def __init__(self, path=DEFAULT_CHECKPOINT_ROOT):
+    if not isinstance(path, Compatibility.string):
+      raise TypeError('FixedPathDetector path should be a string, got %s' % type(path))
+    self._paths = [path]
+
+  def get_paths(self):
+    return self._paths[:]
+
+
+class ChainedPathDetector(PathDetector):
+  def __init__(self, *detectors):
+    for detector in detectors:
+      if not isinstance(detector, PathDetector):
+        raise TypeError('Expected detector %r to be a PathDetector, got %s' % (
+            detector, type(detector)))
+    self._detectors = detectors
+
+  def get_paths(self):
+    def iterate():
+      for detector in self._detectors:
+        for path in detector.get_paths():
+          yield path
+    return list(set(iterate()))
+
+
 class TaskDetector(object):
   """
     Helper class in front of TaskPath to detect active/finished/running tasks. Performs no

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/test/python/apache/aurora/executor/BUILD
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/BUILD b/src/test/python/apache/aurora/executor/BUILD
index aa3bc3b..ec73285 100644
--- a/src/test/python/apache/aurora/executor/BUILD
+++ b/src/test/python/apache/aurora/executor/BUILD
@@ -24,7 +24,6 @@ python_test_suite(
   name = 'executor-small',
   dependencies = [
     ':executor_base',
-    ':executor_detector',
     ':executor_vars',
     ':status_manager',
     ':thermos_task_runner',
@@ -59,15 +58,6 @@ python_tests(name = 'gc_executor',
   ],
 )
 
-python_tests(name = 'executor_detector',
-  sources = [ 'test_executor_detector.py' ],
-  dependencies = [
-    '3rdparty/python:twitter.common.contextutil',
-    '3rdparty/python:twitter.common.dirutil',
-    'src/main/python/apache/aurora/executor:executor_detector'
-  ]
-)
-
 python_tests(name = 'thermos_executor',
   sources = ['test_thermos_executor.py'],
 #  timeout = Amount(5, Time.MINUTES),

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/test/python/apache/aurora/executor/common/BUILD
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/BUILD b/src/test/python/apache/aurora/executor/common/BUILD
index 9e3a657..7b73f69 100644
--- a/src/test/python/apache/aurora/executor/common/BUILD
+++ b/src/test/python/apache/aurora/executor/common/BUILD
@@ -17,14 +17,34 @@ python_test_suite(
   dependencies = [
     ':announcer',
     ':directory_sandbox',
+    ':executor_detector',
     ':executor_timeout',
     ':health_checker',
     ':kill_manager',
+    ':path_detector',
     ':status_checker',
     ':task_info',
   ]
 )
 
+python_tests(name = 'executor_detector',
+  sources = [ 'test_executor_detector.py' ],
+  dependencies = [
+    '3rdparty/python:twitter.common.contextutil',
+    '3rdparty/python:twitter.common.dirutil',
+    'src/main/python/apache/aurora/executor/common:executor_detector'
+  ]
+)
+
+python_tests(name = 'path_detector',
+  sources = [ 'test_path_detector.py' ],
+  dependencies = [
+    '3rdparty/python:mock',
+    'src/main/python/apache/aurora/executor/common:executor_detector',
+    'src/main/python/apache/aurora/executor/common:path_detector',
+  ]
+)
+
 python_library(
   name = 'fixtures',
   sources = ['fixtures.py'],

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/test/python/apache/aurora/executor/common/test_executor_detector.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/test_executor_detector.py b/src/test/python/apache/aurora/executor/common/test_executor_detector.py
new file mode 100644
index 0000000..d2a948f
--- /dev/null
+++ b/src/test/python/apache/aurora/executor/common/test_executor_detector.py
@@ -0,0 +1,81 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import itertools
+import os
+
+from twitter.common.contextutil import temporary_dir
+from twitter.common.dirutil import safe_mkdir
+
+from apache.aurora.executor.common.executor_detector import ExecutorDetector
+
+
+class Match(object):
+  def __init__(self, root, slave, framework, executor, run):
+    self._groups = dict(
+        root=root, slave_id=slave, framework_id=framework, executor_id=executor, run=run)
+
+  def groups(self):
+    return self._groups
+
+
+DEFAULT_MATCH = Match('abcd', 'slave', 'framework', 'executor', 'run')
+
+
+def test_find_root():
+  BAD_PATHS = (
+    os.path.sep,
+    '.',
+    os.path.sep * 10,
+    '/root/slaves',
+    '/root/slaves/S/frameworks/F/executors//runs/R',
+    'root/slaves/S/frameworks/F/executors//runs/R',
+  )
+
+  GOOD_PATHS = (
+    ExecutorDetector.path(DEFAULT_MATCH),
+    os.path.join(ExecutorDetector.path(DEFAULT_MATCH), 'some', 'other', 'path')
+  )
+
+  for cwd in BAD_PATHS:
+    assert ExecutorDetector.find_root(cwd) is None
+
+  for cwd in GOOD_PATHS:
+    assert ExecutorDetector.find_root(cwd) == 'abcd'
+
+
+def test_match_inverse():
+  assert ExecutorDetector.match(ExecutorDetector.path(DEFAULT_MATCH)).groups() == (
+      DEFAULT_MATCH.groups())
+
+
+def test_bad_match():
+  assert ExecutorDetector.match('herpderp') is None
+
+
+def test_integration():
+  SLAVES = ('slave001', 'slave123')
+  FRAMEWORKS = ('framework1', 'framework2')
+  EXECUTORS = ('executor_a', 'executor_b')
+  RUNS = ('001', '002', 'latest')
+
+  with temporary_dir() as td:
+    all_groups = set()
+    for slave, framework, executor, run in itertools.product(SLAVES, FRAMEWORKS, EXECUTORS,
RUNS):
+      match = Match(td, slave, framework, executor, run)
+      safe_mkdir(ExecutorDetector.path(match))
+      all_groups.add(tuple(sorted(match.groups().items())))
+
+    for match in ExecutorDetector(td):
+      assert tuple(sorted(match.groups().items())) in all_groups

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/test/python/apache/aurora/executor/common/test_path_detector.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/test_path_detector.py b/src/test/python/apache/aurora/executor/common/test_path_detector.py
new file mode 100644
index 0000000..2d6edae
--- /dev/null
+++ b/src/test/python/apache/aurora/executor/common/test_path_detector.py
@@ -0,0 +1,56 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+import mock
+
+from apache.aurora.executor.common.executor_detector import ExecutorDetector
+from apache.aurora.executor.common.path_detector import MesosPathDetector
+
+
+class Match(object):
+  def __init__(self, root, slave, framework, executor, run):
+    self._groups = dict(
+        root=root, slave_id=slave, framework_id=framework, executor_id=executor, run=run)
+
+  def groups(self):
+    return self._groups
+
+
+def test_path_detector():
+  ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves')
+  FAKE_ROOT = '/var/blah/blah'
+  FAKE_CHECKPOINT_DIR = 'ckpt'
+
+  path1, path2 = (
+      ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')),
+      ExecutorDetector.path(Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')),
+  )
+
+  with mock.patch('glob.glob', return_value=(path1, path2)) as glob:
+    mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR)
+    paths = list(mpd.get_paths())
+    assert len(paths) == 2
+    assert os.path.join(path1, FAKE_CHECKPOINT_DIR) in paths
+    assert os.path.join(path2, FAKE_CHECKPOINT_DIR) in paths
+
+    expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
+      'root': FAKE_ROOT,
+      'slave_id': '*',
+      'framework_id': '*',
+      'executor_id': '*',
+      'run': '*'
+    }
+    glob.assert_called_once_with(expected_glob_pattern)

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/test/python/apache/aurora/executor/test_executor_detector.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/test_executor_detector.py b/src/test/python/apache/aurora/executor/test_executor_detector.py
deleted file mode 100644
index 85d5418..0000000
--- a/src/test/python/apache/aurora/executor/test_executor_detector.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import itertools
-import os
-
-from twitter.common.contextutil import temporary_dir
-from twitter.common.dirutil import safe_mkdir
-
-from apache.aurora.executor.executor_detector import ExecutorDetector
-
-
-class Match(object):
-  def __init__(self, root, slave, framework, executor, run):
-    self._groups = dict(
-        root=root, slave_id=slave, framework_id=framework, executor_id=executor, run=run)
-
-  def groups(self):
-    return self._groups
-
-
-DEFAULT_MATCH = Match('abcd', 'slave', 'framework', 'executor', 'run')
-
-
-def test_find_root():
-  BAD_PATHS = (
-    os.path.sep,
-    '.',
-    os.path.sep * 10,
-    '/root/slaves',
-    '/root/slaves/S/frameworks/F/executors//runs/R',
-    'root/slaves/S/frameworks/F/executors//runs/R',
-  )
-
-  GOOD_PATHS = (
-    ExecutorDetector.path(DEFAULT_MATCH),
-    os.path.join(ExecutorDetector.path(DEFAULT_MATCH), 'some', 'other', 'path')
-  )
-
-  for cwd in BAD_PATHS:
-    assert ExecutorDetector.find_root(cwd) is None
-
-  for cwd in GOOD_PATHS:
-    assert ExecutorDetector.find_root(cwd) == 'abcd'
-
-
-def test_match_inverse():
-  assert ExecutorDetector.match(ExecutorDetector.path(DEFAULT_MATCH)).groups() == (
-      DEFAULT_MATCH.groups())
-
-
-def test_bad_match():
-  assert ExecutorDetector.match('herpderp') is None
-
-
-def test_integration():
-  SLAVES = ('slave001', 'slave123')
-  FRAMEWORKS = ('framework1', 'framework2')
-  EXECUTORS = ('executor_a', 'executor_b')
-  RUNS = ('001', '002', 'latest')
-
-  with temporary_dir() as td:
-    all_groups = set()
-    for slave, framework, executor, run in itertools.product(SLAVES, FRAMEWORKS, EXECUTORS,
RUNS):
-      match = Match(td, slave, framework, executor, run)
-      safe_mkdir(ExecutorDetector.path(match))
-      all_groups.add(tuple(sorted(match.groups().items())))
-
-    for match in ExecutorDetector(td):
-      assert tuple(sorted(match.groups().items())) in all_groups

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/test/python/apache/thermos/monitoring/BUILD
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/monitoring/BUILD b/src/test/python/apache/thermos/monitoring/BUILD
index 33d6bba..1b82915 100644
--- a/src/test/python/apache/thermos/monitoring/BUILD
+++ b/src/test/python/apache/thermos/monitoring/BUILD
@@ -15,6 +15,7 @@
 python_test_suite(name = 'all',
   dependencies = [
     ':test_disk',
+    ':test_detector',
   ]
 )
 
@@ -25,3 +26,11 @@ python_tests(name = 'test_disk',
   ]
 )
 
+python_tests(name = 'test_detector',
+  sources = ['test_detector.py'],
+  dependencies = [
+    'src/main/python/apache/thermos/common:constants',
+    'src/main/python/apache/thermos/monitoring:detector',
+  ]
+)
+

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53f9c78c/src/test/python/apache/thermos/monitoring/test_detector.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/monitoring/test_detector.py b/src/test/python/apache/thermos/monitoring/test_detector.py
new file mode 100644
index 0000000..e9005c4
--- /dev/null
+++ b/src/test/python/apache/thermos/monitoring/test_detector.py
@@ -0,0 +1,37 @@
+import pytest
+
+from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
+from apache.thermos.monitoring.detector import ChainedPathDetector, FixedPathDetector
+
+
+def test_fixed_path_detector():
+  # Default is TaskPath default
+  fpd = FixedPathDetector()
+  assert fpd.get_paths() == [DEFAULT_CHECKPOINT_ROOT]
+
+  # Non-default
+  root = '/var/lib/derp'
+  fpd = FixedPathDetector(path=root)
+  assert fpd.get_paths() == [root]
+
+
+def test_fixed_path_detector_constructor():
+  with pytest.raises(TypeError):
+    FixedPathDetector(path=234)
+
+
+def test_chained_path_detector():
+  root1 = '/var/lib/derp1'
+  root2 = '/var/lib/derp2'
+  fpd1 = FixedPathDetector(path=root1)
+  fpd2 = FixedPathDetector(path=root2)
+  cpd = ChainedPathDetector(fpd1, fpd2)
+  assert set(cpd.get_paths()) == set([root1, root2])
+
+
+def test_chained_path_detector_constructor():
+  with pytest.raises(TypeError):
+    ChainedPathDetector(1, 2, 3)
+
+  with pytest.raises(TypeError):
+    ChainedPathDetector(FixedPathDetector(), 'hello')


Mime
View raw message