Return-Path: X-Original-To: apmail-aurora-commits-archive@minotaur.apache.org Delivered-To: apmail-aurora-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 59A9C18CE7 for ; Tue, 2 Jun 2015 17:59:34 +0000 (UTC) Received: (qmail 48606 invoked by uid 500); 2 Jun 2015 17:59:34 -0000 Delivered-To: apmail-aurora-commits-archive@aurora.apache.org Received: (qmail 48569 invoked by uid 500); 2 Jun 2015 17:59:34 -0000 Mailing-List: contact commits-help@aurora.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@aurora.apache.org Delivered-To: mailing list commits@aurora.apache.org Received: (qmail 48559 invoked by uid 99); 2 Jun 2015 17:59:34 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Jun 2015 17:59:34 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1E29DDFBDC; Tue, 2 Jun 2015 17:59:34 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: maxim@apache.org To: commits@aurora.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: aurora git commit: Moving checkpoint root into the executor sandbox. Date: Tue, 2 Jun 2015 17:59:34 +0000 (UTC) Repository: aurora Updated Branches: refs/heads/master 73ceeb22a -> 54d561eee Moving checkpoint root into the executor sandbox. Bugs closed: AURORA-1337 Reviewed at https://reviews.apache.org/r/34827/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/54d561ee Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/54d561ee Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/54d561ee Branch: refs/heads/master Commit: 54d561eee029a7fac51dab098f4db76fecde238d Parents: 73ceeb2 Author: Maxim Khutornenko Authored: Tue Jun 2 10:47:34 2015 -0700 Committer: Maxim Khutornenko Committed: Tue Jun 2 10:47:34 2015 -0700 ---------------------------------------------------------------------- examples/vagrant/aurorabuild.sh | 2 +- .../python/apache/aurora/executor/bin/BUILD | 1 + .../executor/bin/thermos_executor_main.py | 93 ++++++++++---------- .../aurora/executor/thermos_task_runner.py | 7 +- .../python/apache/aurora/executor/bin/BUILD | 1 + .../bin/test_thermos_executor_entry_point.py | 33 ++++++- 6 files changed, 86 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/examples/vagrant/aurorabuild.sh ---------------------------------------------------------------------- diff --git a/examples/vagrant/aurorabuild.sh b/examples/vagrant/aurorabuild.sh index 1a76e4f..fd2ce55 100755 --- a/examples/vagrant/aurorabuild.sh +++ b/examples/vagrant/aurorabuild.sh @@ -75,7 +75,7 @@ EOF } function build_observer { - ./pants binary src/main/python/apache/thermos/observer/bin:thermos_observer + ./pants binary src/main/python/apache/aurora/tools:thermos_observer upstart_update aurora-thermos-observer } http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/main/python/apache/aurora/executor/bin/BUILD ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/bin/BUILD b/src/main/python/apache/aurora/executor/bin/BUILD index 6ef669d..0fbb0f8 100644 --- a/src/main/python/apache/aurora/executor/bin/BUILD +++ b/src/main/python/apache/aurora/executor/bin/BUILD @@ -26,6 +26,7 @@ python_library( 'src/main/python/apache/aurora/executor/common:executor_detector', 'src/main/python/apache/aurora/executor/common:executor_timeout', 'src/main/python/apache/aurora/executor/common:health_checker', + 'src/main/python/apache/aurora/executor/common:path_detector', 'src/main/python/apache/aurora/executor/common:resource_manager', 'src/main/python/apache/aurora/executor/common:sandbox', 'src/main/python/apache/aurora/executor:executor_vars', http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py index 3174e1f..b3e8bf1 100644 --- a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py +++ b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py @@ -28,13 +28,13 @@ from apache.aurora.executor.aurora_executor import AuroraExecutor from apache.aurora.executor.common.announcer import DefaultAnnouncerCheckerProvider from apache.aurora.executor.common.executor_timeout import ExecutorTimeout from apache.aurora.executor.common.health_checker import HealthCheckerProvider +from apache.aurora.executor.common.path_detector import MesosPathDetector from apache.aurora.executor.common.resource_manager import ResourceManagerProvider from apache.aurora.executor.common.sandbox import DefaultSandboxProvider from apache.aurora.executor.thermos_task_runner import ( DefaultThermosTaskRunnerProvider, UserOverrideThermosTaskRunnerProvider ) -from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT try: from mesos.native import MesosExecutorDriver @@ -84,14 +84,6 @@ app.add_option( app.add_option( - '--checkpoint-root', - dest='checkpoint_root', - metavar='PATH', - default=DEFAULT_CHECKPOINT_ROOT, - help='The checkpoint root where Thermos task checkpoints are stored.') - - -app.add_option( '--nosetuid', dest='nosetuid', action='store_true', @@ -122,47 +114,58 @@ class UserOverrideDirectorySandboxProvider(DefaultSandboxProvider): return self._user_override +def initialize(options): + cwd_path = os.path.abspath(CWD) + checkpoint_root = os.path.join(cwd_path, MesosPathDetector.DEFAULT_SANDBOX_PATH) + + # status providers: + status_providers = [ + HealthCheckerProvider(), + ResourceManagerProvider(checkpoint_root=checkpoint_root) + ] + + if options.announcer_enable: + if options.announcer_ensemble is None: + app.error('Must specify --announcer-ensemble if the announcer is enabled.') + status_providers.append(DefaultAnnouncerCheckerProvider( + options.announcer_ensemble, options.announcer_serverset_path)) + + # Create executor stub + if options.execute_as_user or options.nosetuid: + # If nosetuid is set, execute_as_user is also None + thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( + dump_runner_pex(), + checkpoint_root, + artifact_dir=cwd_path + ) + thermos_runner_provider.set_role(None) + + thermos_executor = AuroraExecutor( + runner_provider=thermos_runner_provider, + status_providers=status_providers, + sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user) + ) + else: + thermos_runner_provider = DefaultThermosTaskRunnerProvider( + dump_runner_pex(), + checkpoint_root, + artifact_dir=cwd_path + ) + + thermos_executor = AuroraExecutor( + runner_provider=thermos_runner_provider, + status_providers=status_providers + ) + + return thermos_executor + + def proxy_main(): def main(args, options): if MesosExecutorDriver is None: app.error('Could not load MesosExecutorDriver!') - # status providers: - status_providers = [ - HealthCheckerProvider(), - ResourceManagerProvider(checkpoint_root=options.checkpoint_root) - ] - - if options.announcer_enable: - if options.announcer_ensemble is None: - app.error('Must specify --announcer-ensemble if the announcer is enabled.') - status_providers.append(DefaultAnnouncerCheckerProvider( - options.announcer_ensemble, options.announcer_serverset_path)) - - # Create executor stub - if options.execute_as_user or options.nosetuid: - # If nosetuid is set, execute_as_user is also None - thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( - dump_runner_pex(), - artifact_dir=os.path.abspath(CWD) - ) - thermos_runner_provider.set_role(None) - - thermos_executor = AuroraExecutor( - runner_provider=thermos_runner_provider, - status_providers=status_providers, - sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user) - ) - else: - thermos_runner_provider = DefaultThermosTaskRunnerProvider( - dump_runner_pex(), - artifact_dir=os.path.abspath(CWD) - ) - - thermos_executor = AuroraExecutor( - runner_provider=thermos_runner_provider, - status_providers=status_providers - ) + thermos_executor = initialize(options) # Create driver stub driver = MesosExecutorDriver(thermos_executor) http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/main/python/apache/aurora/executor/thermos_task_runner.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/thermos_task_runner.py b/src/main/python/apache/aurora/executor/thermos_task_runner.py index 8ce9168..837ad5c 100644 --- a/src/main/python/apache/aurora/executor/thermos_task_runner.py +++ b/src/main/python/apache/aurora/executor/thermos_task_runner.py @@ -30,7 +30,6 @@ from twitter.common.log.options import LogOptions from twitter.common.quantity import Amount, Time from apache.aurora.common.http_signaler import HttpSignaler -from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT from apache.thermos.common.statuses import ( INTERNAL_ERROR, INVALID_TASK, @@ -70,7 +69,7 @@ class ThermosTaskRunner(TaskRunner): role, portmap, sandbox, - checkpoint_root=DEFAULT_CHECKPOINT_ROOT, + checkpoint_root, artifact_dir=None, clock=time, hostname=None): @@ -353,7 +352,7 @@ class ThermosTaskRunner(TaskRunner): class DefaultThermosTaskRunnerProvider(TaskRunnerProvider): def __init__(self, pex_location, - checkpoint_root=DEFAULT_CHECKPOINT_ROOT, + checkpoint_root, artifact_dir=None, task_runner_class=ThermosTaskRunner, max_wait=Amount(1, Time.MINUTES), @@ -393,7 +392,7 @@ class DefaultThermosTaskRunnerProvider(TaskRunnerProvider): role, mesos_ports, sandbox, - checkpoint_root=self._checkpoint_root, + self._checkpoint_root, artifact_dir=self._artifact_dir, clock=self._clock, hostname=assigned_task.slaveHost) http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/test/python/apache/aurora/executor/bin/BUILD ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/executor/bin/BUILD b/src/test/python/apache/aurora/executor/bin/BUILD index b1bcfac..2caab2a 100644 --- a/src/test/python/apache/aurora/executor/bin/BUILD +++ b/src/test/python/apache/aurora/executor/bin/BUILD @@ -33,6 +33,7 @@ python_tests( name = 'thermos_executor_entry_point', sources = ['test_thermos_executor_entry_point.py'], dependencies = [ + '3rdparty/python:mock', 'src/main/python/apache/aurora/executor/bin:thermos_executor_source', ], ) http://git-wip-us.apache.org/repos/asf/aurora/blob/54d561ee/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py b/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py index e89a01e..ec5b3f9 100644 --- a/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py +++ b/src/test/python/apache/aurora/executor/bin/test_thermos_executor_entry_point.py @@ -11,8 +11,39 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os +import unittest + +from mock import create_autospec, Mock, patch + +from apache.aurora.executor.bin.thermos_executor_main import dump_runner_pex, initialize, proxy_main +from apache.aurora.executor.common.path_detector import MesosPathDetector +from apache.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider def test_thermos_executor_valid_import_dependencies(): - from apache.aurora.executor.bin.thermos_executor_main import proxy_main assert proxy_main is not None + + +class ThermosExecutorMainTest(unittest.TestCase): + def test_checkpoint_path(self): + mock_runner_provider = create_autospec(spec=DefaultThermosTaskRunnerProvider) + mock_dump_runner_pex = create_autospec(spec=dump_runner_pex) + mock_dump_runner_pex.return_value = Mock() + mock_options = Mock() + mock_options.execute_as_user = False + mock_options.nosetuid = False + with patch( + 'apache.aurora.executor.bin.thermos_executor_main.dump_runner_pex', + return_value=mock_dump_runner_pex): + with patch( + 'apache.aurora.executor.bin.thermos_executor_main.DefaultThermosTaskRunnerProvider', + return_value=mock_runner_provider) as mock_provider: + + expected_path = os.path.join(os.path.abspath('.'), MesosPathDetector.DEFAULT_SANDBOX_PATH) + thermos_executor = initialize(mock_options) + + assert thermos_executor is not None + assert len(mock_provider.mock_calls) == 1 + args = mock_provider.mock_calls[0][1] + assert len(args) == 2 and expected_path == args[1]