aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From san...@apache.org
Subject aurora git commit: Use --launch_info when invoking MesosContainerizer.
Date Tue, 14 Mar 2017 02:25:31 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 2f08d9110 -> e1504a7e8


Use --launch_info when invoking MesosContainerizer.

MesosContainerizer has updated the command line parameters in 1.2.0 and
consolidated the individual arguments into a single ContainerLaunchInfo
proto buf message. Update ThermosExecutor to use the new `--launch_info`
parameter to be compatible with MesosContainerizer also check the
containerizer binary interface to determine to be backward-compatible.

Bugs closed: AURORA-1882

Reviewed at https://reviews.apache.org/r/55951/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/e1504a7e
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/e1504a7e
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/e1504a7e

Branch: refs/heads/master
Commit: e1504a7e878b1c2b51b3b747b54a634a485399a0
Parents: 2f08d91
Author: Santhosh Kumar Shanmugham <santhoshkumar.s@gmail.com>
Authored: Mon Mar 13 19:20:33 2017 -0700
Committer: Santhosh Kumar <sshanmugham@twitter.com>
Committed: Mon Mar 13 19:22:02 2017 -0700

----------------------------------------------------------------------
 RELEASE-NOTES.md                                |  2 +
 .../apache/thermos/common/process_util.py       | 51 ++++++++++++---
 .../executor/common/test_health_checker.py      | 68 +++++++++++++-------
 .../python/apache/thermos/core/test_process.py  | 63 ++++++++++++++++--
 4 files changed, 146 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/e1504a7e/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 1142f75..d58d2bd 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -13,6 +13,8 @@
   driver. Users that want to use the HTTP API should use `V1_MESOS`.
   Performance sensitive users should stick with the `SCHEDULER_DRIVER` or
   `V0_MESOS` drivers.
+- Add support for new MesosContainerizer rolled out in Mesos 1.2.0.
+* Please upgrade Aurora to 0.18 before upgrading Mesos to 1.2.0.
 
 0.17.0
 ======

http://git-wip-us.apache.org/repos/asf/aurora/blob/e1504a7e/src/main/python/apache/thermos/common/process_util.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/process_util.py b/src/main/python/apache/thermos/common/process_util.py
index 54e716b..3c04024 100644
--- a/src/main/python/apache/thermos/common/process_util.py
+++ b/src/main/python/apache/thermos/common/process_util.py
@@ -15,14 +15,28 @@
 import ctypes
 import json
 import os
+import subprocess
 
 from twitter.common import log
 
 from gen.apache.aurora.api.constants import TASK_FILESYSTEM_MOUNT_POINT
 
 
+def is_launch_info_supported(mesos_containerizer_path):
+  process = subprocess.Popen(
+          [mesos_containerizer_path, 'launch'],
+          stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+
+  _, stderr = process.communicate()
+  if "launch_info" in stderr:
+    return True
+
+  return False
+
+
 def wrap_with_mesos_containerizer(cmdline, user, cwd, mesos_containerizer_path):
-  command = json.dumps({
+  command_info = {
     'shell': False,
     'value': '/bin/bash',  # the binary to executed
     'arguments': [
@@ -30,14 +44,33 @@ def wrap_with_mesos_containerizer(cmdline, user, cwd, mesos_containerizer_path):
       '-c',
       cmdline
     ]
-  })
-  return [mesos_containerizer_path,
-          'launch',
-          '--unshare_namespace_mnt',
-          '--working_directory=%s' % cwd,
-          '--rootfs=%s' % os.path.join(os.environ['MESOS_DIRECTORY'], TASK_FILESYSTEM_MOUNT_POINT),
-          '--user=%s' % user,
-          '--command=%s' % command]
+  }
+
+  rootfs = os.path.join(os.environ['MESOS_DIRECTORY'], TASK_FILESYSTEM_MOUNT_POINT)
+
+  launch_info = {
+    'rootfs': rootfs,
+    'user': str(user),
+    'working_directory': cwd,
+    'command': command_info
+  }
+
+  # TODO(santhk) it is unfortunate that we will need to check the
+  # interface of mesos-containerizer like this. Clean this up when we have
+  # migrated to Mesos-1.2.0.
+  if is_launch_info_supported(mesos_containerizer_path):
+    return [mesos_containerizer_path,
+      'launch',
+      '--unshare_namespace_mnt',
+      '--launch_info=%s' % json.dumps(launch_info)]
+  else:
+    return [mesos_containerizer_path,
+      'launch',
+      '--unshare_namespace_mnt',
+      '--working_directory=%s' % cwd,
+      '--rootfs=%s' % rootfs,
+      '--user=%s' % user,
+      '--command=%s' % json.dumps(command_info)]
 
 
 def setup_child_subreaping():

http://git-wip-us.apache.org/repos/asf/aurora/blob/e1504a7e/src/test/python/apache/aurora/executor/common/test_health_checker.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/test_health_checker.py b/src/test/python/apache/aurora/executor/common/test_health_checker.py
index a3b5a22..adf3ac0 100644
--- a/src/test/python/apache/aurora/executor/common/test_health_checker.py
+++ b/src/test/python/apache/aurora/executor/common/test_health_checker.py
@@ -21,6 +21,8 @@ import unittest
 import mock
 import pytest
 from mesos.interface.mesos_pb2 import TaskState
+from twitter.common.contextutil import temporary_dir
+from twitter.common.dirutil import chmod_plus_x
 from twitter.common.exceptions import ExceptionalThread
 from twitter.common.testing.clock import ThreadedClock
 
@@ -594,29 +596,49 @@ class TestHealthCheckerProvider(unittest.TestCase):
     assert execconfig_data[
              'health_check_config']['health_checker']['shell']['shell_command'] == 'failed
command'
 
-    mock_sandbox = mock.Mock(spec_set=SandboxInterface)
-    type(mock_sandbox).root = mock.PropertyMock(return_value='/some/path')
-    type(mock_sandbox).is_filesystem_image = mock.PropertyMock(return_value=True)
-
-    with mock.patch('apache.aurora.executor.common.health_checker.ShellHealthCheck') as mock_shell:
-      HealthCheckerProvider(
-          nosetuid_health_checks=False,
-          mesos_containerizer_path='/some/path/mesos-containerizer').from_assigned_task(
-              assigned_task,
-              mock_sandbox)
-
-      class NotNone(object):
-        def __eq__(self, other):
-          return other is not None
-
-      assert mock_shell.mock_calls == [
-          mock.call(
-              raw_cmd='failed command',
-              wrapped_cmd=NotNone(),
-              preexec_fn=None,
-              timeout_secs=5.0
-          )
-      ]
+    with temporary_dir() as td:
+      test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
+      with open(test_isolator_path, 'w') as fd:
+        # We use a fake version of the mesos-containerizer binary that just echoes out its
args so
+        # we can assert on them in the process's output. Also imitates a failure when there
are not
+        # enough arguments, this is used to find the version of the binary (by checking the
failure
+        # message)
+        fd.write('\n'.join([
+          '#!/bin/sh',
+          'if [[ $# == 1 ]]; then',
+          '  { echo "command_info" >&2; };',
+          'else',
+          '  echo "$@";',
+          'fi'
+        ]))
+
+        fd.close()
+
+        chmod_plus_x(test_isolator_path)
+
+        mock_sandbox = mock.Mock(spec_set=SandboxInterface)
+        type(mock_sandbox).root = mock.PropertyMock(return_value=td)
+        type(mock_sandbox).is_filesystem_image = mock.PropertyMock(return_value=True)
+
+        with mock.patch('apache.aurora.executor.common.health_checker.ShellHealthCheck')
as shell:
+          HealthCheckerProvider(
+              nosetuid_health_checks=False,
+              mesos_containerizer_path=test_isolator_path).from_assigned_task(
+                  assigned_task,
+                  mock_sandbox)
+
+          class NotNone(object):
+            def __eq__(self, other):
+              return other is not None
+
+          assert shell.mock_calls == [
+              mock.call(
+                  raw_cmd='failed command',
+                  wrapped_cmd=NotNone(),
+                  preexec_fn=None,
+                  timeout_secs=5.0
+              )
+          ]
 
   def test_interpolate_cmd(self):
     """Making sure thermos.ports[foo] gets correctly substituted with assignedPorts info."""

http://git-wip-us.apache.org/repos/asf/aurora/blob/e1504a7e/src/test/python/apache/thermos/core/test_process.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/thermos/core/test_process.py b/src/test/python/apache/thermos/core/test_process.py
index 5203902..6cb9176 100644
--- a/src/test/python/apache/thermos/core/test_process.py
+++ b/src/test/python/apache/thermos/core/test_process.py
@@ -101,7 +101,7 @@ def test_simple_process():
 
 
 @mock.patch.dict(os.environ, {'MESOS_DIRECTORY': '/some/path'})
-def test_simple_process_filesystem_isolator():
+def test_simple_process_filesystem_isolator_command_info():
   with temporary_dir() as td:
     taskpath = make_taskpath(td)
     sandbox = setup_sandbox(td, taskpath)
@@ -109,10 +109,61 @@ def test_simple_process_filesystem_isolator():
     test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
     with open(test_isolator_path, 'w') as fd:
       # We use a fake version of the mesos-containerizer binary that just echoes out its
args so
-      # we can assert on them in the process's output.
+      # we can assert on them in the process's output. Also imitates a failure when there
are not
+      # enough arguments, this is used to find the version of the binary (by checking the
failure
+      # message)
       fd.write('\n'.join([
         '#!/bin/sh',
-        'echo "$@"'
+        'if [ "$#" -eq 1 ]; then',
+        '  { echo "command_info" >&2; };',
+        'else',
+        '  echo "$@";',
+        'fi'
+      ]))
+
+      fd.close()
+
+      chmod_plus_x(test_isolator_path)
+
+      p = TestProcess(
+        'process',
+        'echo hello world',
+        0,
+        taskpath,
+        sandbox,
+        mesos_containerizer_path=test_isolator_path,
+        container_sandbox=sandbox)
+      p.start()
+
+    rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
+    assert rc == 0
+    assert_log_content(
+      taskpath,
+      'stdout',
+      'launch --unshare_namespace_mnt --working_directory=%s --rootfs=/some/path/taskfs '
+      '--user=None --command={"shell": false, "arguments": ["/bin/bash", "-c", '
+      '"echo hello world"], "value": "/bin/bash"}\n' % (sandbox))
+
+
+@mock.patch.dict(os.environ, {'MESOS_DIRECTORY': '/some/path'})
+def test_simple_process_filesystem_isolator_launch_info():
+  with temporary_dir() as td:
+    taskpath = make_taskpath(td)
+    sandbox = setup_sandbox(td, taskpath)
+
+    test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
+    with open(test_isolator_path, 'w') as fd:
+      # We use a fake version of the mesos-containerizer binary that just echoes out its
args so
+      # we can assert on them in the process's output. Also imitates a failure when there
are not
+      # enough arguments, this is used to find the version of the binary (by checking the
failure
+      # message)
+      fd.write('\n'.join([
+        '#!/bin/sh',
+        'if [ "$#" -eq 1 ]; then',
+        '  { echo "launch_info" >&2; };',
+        'else',
+        '  echo "$@";',
+        'fi'
       ]))
 
       fd.close()
@@ -134,9 +185,9 @@ def test_simple_process_filesystem_isolator():
     assert_log_content(
         taskpath,
         'stdout',
-        'launch --unshare_namespace_mnt --working_directory=%s --rootfs=/some/path/taskfs
'
-        '--user=None --command={"shell": false, "arguments": ["/bin/bash", "-c", '
-        '"echo hello world"], "value": "/bin/bash"}\n' % (sandbox))
+        'launch --unshare_namespace_mnt --launch_info={"command": {"shell": false, "arguments":
'
+        '["/bin/bash", "-c", "echo hello world"], "value": "/bin/bash"}, '
+        '"working_directory": "%s", "user": "None", "rootfs": "/some/path/taskfs"}\n' % (sandbox))
 
 
 @mock.patch('os.chown')


Mime
View raw message