aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zma...@apache.org
Subject aurora git commit: Add DSL and E2E changes for per task volume mounts.
Date Wed, 15 Feb 2017 01:10:26 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 0e9c0864e -> 9ea897978


Add DSL and E2E changes for per task volume mounts.

Enables the client DSL to set per task volume mounts. This also adds a E2E test
that tests per task volume mounting.

Testing Done:
sh ./src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh

Bugs closed: AURORA-1107

Reviewed at https://reviews.apache.org/r/53333/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/9ea89797
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/9ea89797
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/9ea89797

Branch: refs/heads/master
Commit: 9ea897978a41d71121033fe5f2115465ec76581c
Parents: 0e9c086
Author: Zameer Manji <zmanji@apache.org>
Authored: Tue Feb 14 17:09:37 2017 -0800
Committer: Zameer Manji <zmanji@apache.org>
Committed: Tue Feb 14 17:09:37 2017 -0800

----------------------------------------------------------------------
 RELEASE-NOTES.md                                |  1 +
 docs/reference/configuration.md                 |  9 +++++++
 examples/vagrant/upstart/aurora-scheduler.conf  |  1 +
 .../python/apache/aurora/config/schema/base.py  |  7 ++++++
 src/main/python/apache/aurora/config/thrift.py  | 23 ++++++++++++++++--
 .../python/apache/aurora/config/test_thrift.py  | 25 +++++++++++++++++++-
 .../sh/org/apache/aurora/e2e/Dockerfile.netcat  |  1 +
 src/test/sh/org/apache/aurora/e2e/check-fs.sh   |  6 +++++
 .../apache/aurora/e2e/http/http_example.aurora  | 12 ++++++++--
 src/test/sh/org/apache/aurora/e2e/run-server.sh |  1 -
 10 files changed, 80 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index ff382ff..2391d32 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -5,6 +5,7 @@
 
 - Add message parameter to `killTasks` RPC.
 - Add prune_tasks endpoint to aurora_admin. See aurora_admin prune_tasks -h for usage information.
+- Add support for per-task volume mounts for Mesos containers to the Aurora config DSL.
 
 0.17.0
 ======

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/docs/reference/configuration.md
----------------------------------------------------------------------
diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
index 6c71142..0040de1 100644
--- a/docs/reference/configuration.md
+++ b/docs/reference/configuration.md
@@ -468,6 +468,15 @@ unified-container, the container can be omitted from your job config.
   param            | type                           | description
   -----            | :----:                         | -----------
   ```image```      | Choice(AppcImage, DockerImage) | An optional filesystem image to use
within this container.
+  ```volumes```    | List(Volume)                   | An optional list of volume mounts for
this container.
+
+### Volume Object
+
+  param                  | type     | description
+  -----                  | :----:   | -----------
+  ```container_path```   | String   | Path on the host to mount.
+  ```volume_path```      | String   | Mount point in the container.
+  ```mode```             | Enum     | Mode of the mount, can be 'RW' or 'RO'.
 
 ### AppcImage
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/examples/vagrant/upstart/aurora-scheduler.conf
----------------------------------------------------------------------
diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf
index 91f0f0c..49fdcbd 100644
--- a/examples/vagrant/upstart/aurora-scheduler.conf
+++ b/examples/vagrant/upstart/aurora-scheduler.conf
@@ -53,4 +53,5 @@ exec bin/aurora-scheduler \
   -receive_revocable_resources=true \
   -enable_revocable_ram=true \
   -allow_gpu_resource=true \
+  -allow_container_volumes=true \
   -offer_filter_duration=0secs

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/src/main/python/apache/aurora/config/schema/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py
index b15b939..b2692a6 100644
--- a/src/main/python/apache/aurora/config/schema/base.py
+++ b/src/main/python/apache/aurora/config/schema/base.py
@@ -132,9 +132,16 @@ class DockerImage(Struct):
   name = Required(String)
   tag = Required(String)
 
+Mode = Enum('RO', 'RW')
+
+class Volume(Struct):
+  container_path = Required(String)
+  host_path = Required(String)
+  mode = Required(Mode)
 
 class Mesos(Struct):
   image = Choice([AppcImage, DockerImage])
+  volumes = Default(List(Volume), [])
 
 
 class Container(Struct):

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/src/main/python/apache/aurora/config/thrift.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py
index 3539469..2158c3a 100644
--- a/src/main/python/apache/aurora/config/thrift.py
+++ b/src/main/python/apache/aurora/config/thrift.py
@@ -47,10 +47,12 @@ from gen.apache.aurora.api.ttypes import (
     LimitConstraint,
     MesosContainer,
     Metadata,
+    Mode,
     Resource,
     TaskConfig,
     TaskConstraint,
-    ValueConstraint
+    ValueConstraint,
+    Volume
 )
 
 __all__ = (
@@ -163,11 +165,28 @@ def create_container_config(container):
          else unwrapped)))
 
   if isinstance(unwrapped, Mesos):
-    return Container(MesosContainer(image_to_thrift(unwrapped.image())), None)
+    image = image_to_thrift(unwrapped.image())
+    volumes = volumes_to_thrift(unwrapped.volumes())
+
+    return Container(MesosContainer(image, volumes), None)
 
   raise InvalidConfig('If a container is specified it must set one type.')
 
 
+def volumes_to_thrift(volumes):
+  thrift_volumes = []
+  for v in volumes:
+    mode = parse_enum(Mode, v.mode())
+    thrift_volumes.append(
+      Volume(
+          containerPath=fully_interpolated(v.container_path()),
+          hostPath=fully_interpolated(v.host_path()),
+          mode=mode
+      )
+    )
+  return thrift_volumes
+
+
 def image_to_thrift(image):
   if image is Empty:
     return None

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/src/test/python/apache/aurora/config/test_thrift.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/config/test_thrift.py b/src/test/python/apache/aurora/config/test_thrift.py
index e213184..2a6a5d0 100644
--- a/src/test/python/apache/aurora/config/test_thrift.py
+++ b/src/test/python/apache/aurora/config/test_thrift.py
@@ -26,14 +26,17 @@ from apache.aurora.config.schema.base import (
     HealthCheckConfig,
     Job,
     Mesos,
+    Mode,
     Parameter,
-    SimpleTask
+    SimpleTask,
+    Volume
 )
 from apache.aurora.config.thrift import convert as convert_pystachio_to_thrift
 from apache.aurora.config.thrift import InvalidConfig, task_instance_from_job
 from apache.thermos.config.schema import Process, Resources, Task
 
 from gen.apache.aurora.api.constants import GOOD_IDENTIFIER_PATTERN_PYTHON
+from gen.apache.aurora.api.ttypes import Mode as ThriftMode
 from gen.apache.aurora.api.ttypes import CronCollisionPolicy, Identity, JobKey, Resource
 from gen.apache.aurora.test.constants import INVALID_IDENTIFIERS, VALID_IDENTIFIERS
 
@@ -108,6 +111,26 @@ def test_config_with_appc_image():
   assert job.taskConfig.container.mesos.image.appc.imageId == image_id
 
 
+def test_config_with_volumes():
+  image_name = 'some-image'
+  image_tag = 'some-tag'
+  host_path = '/etc/secrets/role/'
+  container_path = '/etc/secrets/'
+
+  volume = Volume(host_path=host_path, container_path=container_path, mode=Mode('RO'))
+
+  container = Mesos(image=DockerImage(name=image_name, tag=image_tag), volumes=[volume])
+
+  job = convert_pystachio_to_thrift(HELLO_WORLD(container=container))
+
+  assert len(job.taskConfig.container.mesos.volumes) == 1
+  thrift_volume = job.taskConfig.container.mesos.volumes[0]
+
+  assert thrift_volume.hostPath == host_path
+  assert thrift_volume.containerPath == container_path
+  assert thrift_volume.mode == ThriftMode.RO
+
+
 def test_docker_with_parameters():
   helloworld = HELLO_WORLD(
     container=Container(

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat b/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat
index c8b2f46..17a028e 100644
--- a/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat
+++ b/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat
@@ -16,3 +16,4 @@ FROM buildpack-deps:jessie
 
 RUN apt-get update && apt-get install -y netcat-openbsd
 COPY run-server.sh /usr/local/bin
+COPY check-fs.sh /usr/local/bin

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/src/test/sh/org/apache/aurora/e2e/check-fs.sh
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/check-fs.sh b/src/test/sh/org/apache/aurora/e2e/check-fs.sh
new file mode 100755
index 0000000..03221c0
--- /dev/null
+++ b/src/test/sh/org/apache/aurora/e2e/check-fs.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+echo "Checking volume mounts..."
+if [ ! -f "/etc/rsyslog.d.container/50-default.conf" ]; then
+    echo "Mounted file was not found";
+    exit 1
+fi

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora
index b2b977b..de81792 100644
--- a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora
+++ b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora
@@ -70,6 +70,11 @@ if __name__ == "__main__":
 """
 )
 
+verify_file_system = Process(
+  name = 'check_fs',
+  cmdline = 'check-fs.sh'
+)
+
 test_task = SequentialTask(
   name = 'http_example',
   resources = Resources(cpu=0.5, ram=32*MB, disk=64*MB, gpu='{{profile.gpu}}'),
@@ -92,6 +97,7 @@ no_python_task = SequentialTask(
       echo_ports,
       verify_command_escaping,
       verify_file_mount,
+      verify_file_system,
       Process(name='run_server', cmdline='run-server.sh {{thermos.ports[http]}}'),
   ]
 )
@@ -121,6 +127,8 @@ job = Service(
   ),
 )
 
+volumes = [Volume(container_path="etc/rsyslog.d.container", host_path="/etc/rsyslog.d", mode=Mode("RO"))]
+
 jobs = [
   job(
     name = 'http_example'
@@ -142,13 +150,13 @@ jobs = [
   ).bind(profile=GpuProfile),
   job(
     name = 'http_example_unified_appc',
-    container = Mesos(image=AppcImage(name='http_example_netcat', image_id='{{appc_image_id}}')),
+    container = Mesos(image=AppcImage(name='http_example_netcat', image_id='{{appc_image_id}}'),
volumes=volumes),
     task = no_python_task,
     health_check_config=shell_health_check_config
   ).bind(profile=DefaultProfile()),
   job(
     name = 'http_example_unified_docker',
-    container = Mesos(image=DockerImage(name='http_example_netcat', tag='latest')),
+    container = Mesos(image=DockerImage(name='http_example_netcat', tag='latest'), volumes=volumes),
     task = no_python_task,
     health_check_config=shell_health_check_config
   ).bind(profile=DefaultProfile())

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ea89797/src/test/sh/org/apache/aurora/e2e/run-server.sh
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/run-server.sh b/src/test/sh/org/apache/aurora/e2e/run-server.sh
index 1fe0909..834f723 100755
--- a/src/test/sh/org/apache/aurora/e2e/run-server.sh
+++ b/src/test/sh/org/apache/aurora/e2e/run-server.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 echo "Starting up server..."
 while true
 do


Mime
View raw message