aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject aurora git commit: Moving checkpoint root into the executor sandbox.
Date Tue, 02 Jun 2015 17:59:34 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 73ceeb22a -> 54d561eee


Moving checkpoint root into the executor sandbox.

Bugs closed: AURORA-1337

Reviewed at https://reviews.apache.org/r/34827/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/54d561ee
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/54d561ee
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/54d561ee

Branch: refs/heads/master
Commit: 54d561eee029a7fac51dab098f4db76fecde238d
Parents: 73ceeb2
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Tue Jun 2 10:47:34 2015 -0700
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Tue Jun 2 10:47:34 2015 -0700

----------------------------------------------------------------------
 examples/vagrant/aurorabuild.sh                 |  2 +-
 .../python/apache/aurora/executor/bin/BUILD     |  1 +
 .../executor/bin/thermos_executor_main.py       | 93 ++++++++++----------
 .../aurora/executor/thermos_task_runner.py      |  7 +-
 .../python/apache/aurora/executor/bin/BUILD     |  1 +
 .../bin/test_thermos_executor_entry_point.py    | 33 ++++++-
 6 files changed, 86 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/examples/vagrant/aurorabuild.sh
----------------------------------------------------------------------
diff --git a/examples/vagrant/aurorabuild.sh b/examples/vagrant/aurorabuild.sh
index 1a76e4f..fd2ce55 100755
--- a/examples/vagrant/aurorabuild.sh
+++ b/examples/vagrant/aurorabuild.sh
@@ -75,7 +75,7 @@ EOF
 }
 
 function build_observer {
-  ./pants binary src/main/python/apache/thermos/observer/bin:thermos_observer
+  ./pants binary src/main/python/apache/aurora/tools:thermos_observer
   upstart_update aurora-thermos-observer
 }
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/main/python/apache/aurora/executor/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/BUILD b/src/main/python/apache/aurora/executor/bin/BUILD
index 6ef669d..0fbb0f8 100644
--- a/src/main/python/apache/aurora/executor/bin/BUILD
+++ b/src/main/python/apache/aurora/executor/bin/BUILD
@@ -26,6 +26,7 @@ python_library(
     'src/main/python/apache/aurora/executor/common:executor_detector',
     'src/main/python/apache/aurora/executor/common:executor_timeout',
     'src/main/python/apache/aurora/executor/common:health_checker',
+    'src/main/python/apache/aurora/executor/common:path_detector',
     'src/main/python/apache/aurora/executor/common:resource_manager',
     'src/main/python/apache/aurora/executor/common:sandbox',
     'src/main/python/apache/aurora/executor:executor_vars',

http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
index 3174e1f..b3e8bf1 100644
--- a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
+++ b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
@@ -28,13 +28,13 @@ from apache.aurora.executor.aurora_executor import AuroraExecutor
 from apache.aurora.executor.common.announcer import DefaultAnnouncerCheckerProvider
 from apache.aurora.executor.common.executor_timeout import ExecutorTimeout
 from apache.aurora.executor.common.health_checker import HealthCheckerProvider
+from apache.aurora.executor.common.path_detector import MesosPathDetector
 from apache.aurora.executor.common.resource_manager import ResourceManagerProvider
 from apache.aurora.executor.common.sandbox import DefaultSandboxProvider
 from apache.aurora.executor.thermos_task_runner import (
     DefaultThermosTaskRunnerProvider,
     UserOverrideThermosTaskRunnerProvider
 )
-from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
 
 try:
   from mesos.native import MesosExecutorDriver
@@ -84,14 +84,6 @@ app.add_option(
 
 
 app.add_option(
-    '--checkpoint-root',
-    dest='checkpoint_root',
-    metavar='PATH',
-    default=DEFAULT_CHECKPOINT_ROOT,
-    help='The checkpoint root where Thermos task checkpoints are stored.')
-
-
-app.add_option(
     '--nosetuid',
     dest='nosetuid',
     action='store_true',
@@ -122,47 +114,58 @@ class UserOverrideDirectorySandboxProvider(DefaultSandboxProvider):
     return self._user_override
 
 
+def initialize(options):
+  cwd_path = os.path.abspath(CWD)
+  checkpoint_root = os.path.join(cwd_path, MesosPathDetector.DEFAULT_SANDBOX_PATH)
+
+  # status providers:
+  status_providers = [
+      HealthCheckerProvider(),
+      ResourceManagerProvider(checkpoint_root=checkpoint_root)
+  ]
+
+  if options.announcer_enable:
+    if options.announcer_ensemble is None:
+      app.error('Must specify --announcer-ensemble if the announcer is enabled.')
+    status_providers.append(DefaultAnnouncerCheckerProvider(
+      options.announcer_ensemble, options.announcer_serverset_path))
+
+  # Create executor stub
+  if options.execute_as_user or options.nosetuid:
+    # If nosetuid is set, execute_as_user is also None
+    thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
+      dump_runner_pex(),
+      checkpoint_root,
+      artifact_dir=cwd_path
+    )
+    thermos_runner_provider.set_role(None)
+
+    thermos_executor = AuroraExecutor(
+      runner_provider=thermos_runner_provider,
+      status_providers=status_providers,
+      sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user)
+    )
+  else:
+    thermos_runner_provider = DefaultThermosTaskRunnerProvider(
+      dump_runner_pex(),
+      checkpoint_root,
+      artifact_dir=cwd_path
+    )
+
+    thermos_executor = AuroraExecutor(
+      runner_provider=thermos_runner_provider,
+      status_providers=status_providers
+    )
+
+  return thermos_executor
+
+
 def proxy_main():
   def main(args, options):
     if MesosExecutorDriver is None:
       app.error('Could not load MesosExecutorDriver!')
 
-    # status providers:
-    status_providers = [
-        HealthCheckerProvider(),
-        ResourceManagerProvider(checkpoint_root=options.checkpoint_root)
-    ]
-
-    if options.announcer_enable:
-      if options.announcer_ensemble is None:
-        app.error('Must specify --announcer-ensemble if the announcer is enabled.')
-      status_providers.append(DefaultAnnouncerCheckerProvider(
-        options.announcer_ensemble, options.announcer_serverset_path))
-
-    # Create executor stub
-    if options.execute_as_user or options.nosetuid:
-      # If nosetuid is set, execute_as_user is also None
-      thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
-        dump_runner_pex(),
-        artifact_dir=os.path.abspath(CWD)
-      )
-      thermos_runner_provider.set_role(None)
-
-      thermos_executor = AuroraExecutor(
-        runner_provider=thermos_runner_provider,
-        status_providers=status_providers,
-        sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user)
-      )
-    else:
-      thermos_runner_provider = DefaultThermosTaskRunnerProvider(
-        dump_runner_pex(),
-        artifact_dir=os.path.abspath(CWD)
-      )
-
-      thermos_executor = AuroraExecutor(
-        runner_provider=thermos_runner_provider,
-        status_providers=status_providers
-      )
+    thermos_executor = initialize(options)
 
     # Create driver stub
     driver = MesosExecutorDriver(thermos_executor)

http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/main/python/apache/aurora/executor/thermos_task_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/thermos_task_runner.py b/src/main/python/apache/aurora/executor/thermos_task_runner.py
index 8ce9168..837ad5c 100644
--- a/src/main/python/apache/aurora/executor/thermos_task_runner.py
+++ b/src/main/python/apache/aurora/executor/thermos_task_runner.py
@@ -30,7 +30,6 @@ from twitter.common.log.options import LogOptions
 from twitter.common.quantity import Amount, Time
 
 from apache.aurora.common.http_signaler import HttpSignaler
-from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
 from apache.thermos.common.statuses import (
     INTERNAL_ERROR,
     INVALID_TASK,
@@ -70,7 +69,7 @@ class ThermosTaskRunner(TaskRunner):
                role,
                portmap,
                sandbox,
-               checkpoint_root=DEFAULT_CHECKPOINT_ROOT,
+               checkpoint_root,
                artifact_dir=None,
                clock=time,
                hostname=None):
@@ -353,7 +352,7 @@ class ThermosTaskRunner(TaskRunner):
 class DefaultThermosTaskRunnerProvider(TaskRunnerProvider):
   def __init__(self,
                pex_location,
-               checkpoint_root=DEFAULT_CHECKPOINT_ROOT,
+               checkpoint_root,
                artifact_dir=None,
                task_runner_class=ThermosTaskRunner,
                max_wait=Amount(1, Time.MINUTES),
@@ -393,7 +392,7 @@ class DefaultThermosTaskRunnerProvider(TaskRunnerProvider):
         role,
         mesos_ports,
         sandbox,
-        checkpoint_root=self._checkpoint_root,
+        self._checkpoint_root,
         artifact_dir=self._artifact_dir,
         clock=self._clock,
         hostname=assigned_task.slaveHost)

http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/test/python/apache/aurora/executor/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/bin/BUILD b/src/test/python/apache/aurora/executor/bin/BUILD
index b1bcfac..2caab2a 100644
--- a/src/test/python/apache/aurora/executor/bin/BUILD
+++ b/src/test/python/apache/aurora/executor/bin/BUILD
@@ -33,6 +33,7 @@ python_tests(
   name = 'thermos_executor_entry_point',
   sources = ['test_thermos_executor_entry_point.py'],
   dependencies = [
+    '3rdparty/python:mock',
     'src/main/python/apache/aurora/executor/bin:thermos_executor_source',
   ],
 )

http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py
b/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py
index e89a01e..ec5b3f9 100644
--- a/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py
+++ b/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py
@@ -11,8 +11,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import os
+import unittest
+
+from mock import create_autospec, Mock, patch
+
+from apache.aurora.executor.bin.thermos_executor_main import dump_runner_pex, initialize,
proxy_main
+from apache.aurora.executor.common.path_detector import MesosPathDetector
+from apache.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider
 
 
 def test_thermos_executor_valid_import_dependencies():
-  from apache.aurora.executor.bin.thermos_executor_main import proxy_main
   assert proxy_main is not None
+
+
+class ThermosExecutorMainTest(unittest.TestCase):
+  def test_checkpoint_path(self):
+    mock_runner_provider = create_autospec(spec=DefaultThermosTaskRunnerProvider)
+    mock_dump_runner_pex = create_autospec(spec=dump_runner_pex)
+    mock_dump_runner_pex.return_value = Mock()
+    mock_options = Mock()
+    mock_options.execute_as_user = False
+    mock_options.nosetuid = False
+    with patch(
+        'apache.aurora.executor.bin.thermos_executor_main.dump_runner_pex',
+        return_value=mock_dump_runner_pex):
+      with patch(
+          'apache.aurora.executor.bin.thermos_executor_main.DefaultThermosTaskRunnerProvider',
+          return_value=mock_runner_provider) as mock_provider:
+
+        expected_path = os.path.join(os.path.abspath('.'), MesosPathDetector.DEFAULT_SANDBOX_PATH)
+        thermos_executor = initialize(mock_options)
+
+        assert thermos_executor is not None
+        assert len(mock_provider.mock_calls) == 1
+        args = mock_provider.mock_calls[0][1]
+        assert len(args) == 2 and expected_path == args[1]


Mime
View raw message