Return-Path: X-Original-To: apmail-libcloud-notifications-archive@www.apache.org Delivered-To: apmail-libcloud-notifications-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0DDE518C43 for ; Wed, 20 Jan 2016 03:43:51 +0000 (UTC) Received: (qmail 90026 invoked by uid 500); 20 Jan 2016 03:43:51 -0000 Delivered-To: apmail-libcloud-notifications-archive@libcloud.apache.org Received: (qmail 89974 invoked by uid 500); 20 Jan 2016 03:43:50 -0000 Mailing-List: contact notifications-help@libcloud.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@libcloud.apache.org Delivered-To: mailing list notifications@libcloud.apache.org Received: (qmail 89001 invoked by uid 500); 20 Jan 2016 03:43:50 -0000 Delivered-To: apmail-libcloud-commits@libcloud.apache.org Received: (qmail 88988 invoked by uid 99); 20 Jan 2016 03:43:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 20 Jan 2016 03:43:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 99680DFEFE; Wed, 20 Jan 2016 03:43:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: anthonyshaw@apache.org To: commits@libcloud.apache.org Date: Wed, 20 Jan 2016 03:43:56 -0000 Message-Id: <737ae4e7c14d4c37b00750f85795c28e@git.apache.org> In-Reply-To: <1166e22aeba746c0bb4aa4b53509ba8a@git.apache.org> References: <1166e22aeba746c0bb4aa4b53509ba8a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [08/50] libcloud git commit: An additional driver to demonstrate the cluster capabilities. driver is WIP. The Amazon ECS driver An additional driver to demonstrate the cluster capabilities. driver is WIP. The Amazon ECS driver Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/f1674deb Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/f1674deb Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/f1674deb Branch: refs/heads/trunk Commit: f1674debdea969a3ef037c50200fe6451d23ad29 Parents: 6a4a772 Author: anthony-shaw Authored: Mon Dec 28 20:57:09 2015 +1100 Committer: anthony-shaw Committed: Mon Dec 28 20:57:09 2015 +1100 ---------------------------------------------------------------------- libcloud/container/base.py | 15 +- libcloud/container/drivers/docker.py | 2 +- libcloud/container/drivers/ecs.py | 291 +++++++++++++++++++ .../container/fixtures/ecs/createcluster.json | 11 + .../container/fixtures/ecs/deletecluster.json | 11 + .../fixtures/ecs/describeclusters.json | 14 + .../container/fixtures/ecs/describetasks.json | 46 +++ libcloud/test/container/test_ecs.py | 88 ++++++ libcloud/test/secrets.py-dist | 1 + 9 files changed, 471 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/container/base.py ---------------------------------------------------------------------- diff --git a/libcloud/container/base.py b/libcloud/container/base.py index ec28308..b70f1bd 100644 --- a/libcloud/container/base.py +++ b/libcloud/container/base.py @@ -341,17 +341,17 @@ class ContainerDriver(BaseDriver): raise NotImplementedError( 'restart_container not implemented for this driver') - def delete_container(self, container): + def destroy_container(self, container): """ - Delete a deployed container + Destroy a deployed container - :param container: The container to delete + :param container: The container to destroy :type container: :class:`Container` :rtype: :class:`Container` """ raise NotImplementedError( - 'delete_container not implemented for this driver') + 'destroy_container not implemented for this driver') def list_locations(self): """ @@ -377,14 +377,15 @@ class ContainerDriver(BaseDriver): raise NotImplementedError( 'create_cluster not implemented for this driver') - def delete_cluster(self, cluster): + def destroy_cluster(self, cluster): """ Delete a cluster - :rtype: ``list`` of :class:`ClusterLocation` + :return: ``True`` if the destroy was successful, otherwise ``False``. + :rtype: ``bool`` """ raise NotImplementedError( - 'delete_cluster not implemented for this driver') + 'destroy_cluster not implemented for this driver') def list_clusters(self, location=None): """ http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/container/drivers/docker.py ---------------------------------------------------------------------- diff --git a/libcloud/container/drivers/docker.py b/libcloud/container/drivers/docker.py index 9d60d20..981411a 100644 --- a/libcloud/container/drivers/docker.py +++ b/libcloud/container/drivers/docker.py @@ -451,7 +451,7 @@ class DockerContainerDriver(ContainerDriver): raise DockerException(result.status, 'failed to restart container') - def delete_container(self, container): + def destroy_container(self, container): """ Remove a container http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/container/drivers/ecs.py ---------------------------------------------------------------------- diff --git a/libcloud/container/drivers/ecs.py b/libcloud/container/drivers/ecs.py new file mode 100644 index 0000000..c4ea82a --- /dev/null +++ b/libcloud/container/drivers/ecs.py @@ -0,0 +1,291 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = [ + 'ElasticContainerDriver' +] + + +from libcloud.container.base import (ContainerDriver, Container, + ContainerCluster, ContainerImage) +from libcloud.container.types import ContainerState +from libcloud.common.base import JsonResponse +from libcloud.common.aws import SignedAWSConnection + + +VERSION = '2014-11-13' +HOST = 'ecs.%s.amazonaws.com' +ROOT = '/%s/' % (VERSION) +TARGET_BASE = 'AmazonEC2ContainerServiceV%s' % (VERSION.replace('-', '')) + + +class ECSResponse(JsonResponse): + """ + Amazon ECS response class. + ECS API uses JSON unlike the s3, elb drivers + """ + + +class ECSConnection(SignedAWSConnection): + version = VERSION + host = HOST + responseCls = ECSResponse + service_name = 'ecs' + + +class ElasticContainerDriver(ContainerDriver): + name = 'Amazon Elastic Container Service' + website = 'https://aws.amazon.com/ecs/details/' + connectionCls = ECSConnection + supports_clusters = False + status_map = { + 'RUNNING': ContainerState.RUNNING + } + + def __init__(self, access_id, secret, region): + super(ElasticContainerDriver, self).__init__(access_id, secret) + self.region = region + self.connection.host = HOST % (region) + + def list_clusters(self): + """ + Get a list of potential locations to deploy clusters into + + :param location: The location to search in + :type location: :class:`ClusterLocation` + + :rtype: ``list`` of :class:`ContainerCluster` + """ + params = {'Action': 'DescribeClusters'} + data = self.connection.request( + ROOT, + params=params, + headers=self._get_headers(params['Action']) + ).object + return self._to_clusters(data) + + def create_cluster(self, name, location=None): + """ + Create a container cluster + + :param name: The name of the cluster + :type name: ``str`` + + :param location: The location to create the cluster in + :type location: :class:`ClusterLocation` + + :rtype: :class:`ContainerCluster` + """ + params = {'Action': 'CreateCluster'} + request = {'clusterName': name} + response = self.connection.request( + ROOT, + params=params, + data=request, + headers=self._get_headers(params['Action']) + ).object + return self._to_cluster(response['cluster']) + + def destroy_cluster(self, cluster): + """ + Delete a cluster + + :return: ``True`` if the destroy was successful, otherwise ``False``. + :rtype: ``bool`` + """ + params = {'Action': 'DeleteCluster'} + request = {'cluster': cluster.id} + data = self.connection.request( + ROOT, + params=params, + data=request, + headers=self._get_headers(params['Action']) + ).object + return data['cluster']['status'] == 'INACTIVE' + + def install_image(self, path): + """ + Install a container image from a remote path. + + :param path: Path to the container image + :type path: ``str`` + + :rtype: :class:`ContainerImage` + """ + raise NotImplementedError( + 'install_image not implemented for this driver') + + def list_images(self): + """ + List the installed container images + + :rtype: ``list`` of :class:`ContainerImage` + """ + raise NotImplementedError( + 'list_images not implemented for this driver') + + def list_containers(self, image=None, cluster=None): + """ + List the deployed container images + + :param image: Filter to containers with a certain image + :type image: :class:`ContainerImage` + + :param cluster: Filter to containers in a cluster + :type cluster: :class:`ContainerCluster` + + :rtype: ``list`` of :class:`Container` + """ + params = {'Action': 'DescribeTasks'} + request = None + if cluster is not None: + request = {'cluster': cluster.id} + response = self.connection.request( + ROOT, + params=params, + data=request, + headers=self._get_headers(params['Action']) + ).object + containers = [] + for task in response['tasks']: + containers.extend(self._to_containers(task)) + return containers + + def deploy_container(self, name, image, cluster=None, + parameters=None, start=True): + """ + Deploy an installed container image + + :param name: The name of the new container + :type name: ``str`` + + :param image: The container image to deploy + :type image: :class:`ContainerImage` + + :param cluster: The cluster to deploy to, None is default + :type cluster: :class:`ContainerCluster` + + :param parameters: Container Image parameters + :type parameters: ``str`` + + :param start: Start the container on deployment + :type start: ``bool`` + + :rtype: :class:`Container` + """ + raise NotImplementedError( + 'deploy_container not implemented for this driver') + + def get_container(self, id): + """ + Get a container by ID + + :param id: The ID of the container to get + :type id: ``str`` + + :rtype: :class:`Container` + """ + raise NotImplementedError( + 'get_container not implemented for this driver') + + def start_container(self, container): + """ + Start a deployed container + + :param container: The container to start + :type container: :class:`Container` + + :rtype: :class:`Container` + """ + raise NotImplementedError( + 'start_container not implemented for this driver') + + def stop_container(self, container): + """ + Stop a deployed container + + :param container: The container to stop + :type container: :class:`Container` + + :rtype: :class:`Container` + """ + raise NotImplementedError( + 'stop_container not implemented for this driver') + + def restart_container(self, container): + """ + Restart a deployed container + + :param container: The container to restart + :type container: :class:`Container` + + :rtype: :class:`Container` + """ + raise NotImplementedError( + 'restart_container not implemented for this driver') + + def destroy_container(self, container): + """ + Destroy a deployed container + + :param container: The container to destroy + :type container: :class:`Container` + + :rtype: :class:`Container` + """ + raise NotImplementedError( + 'destroy_container not implemented for this driver') + + def _get_headers(self, action): + return {'x-amz-target': '%s.%s' % + (TARGET_BASE, action)} + + def _to_clusters(self, data): + clusters = [] + for cluster in data['clusters']: + clusters.append(self._to_cluster(cluster)) + return clusters + + def _to_cluster(self, data): + return ContainerCluster( + id=data['clusterArn'], + name=data['clusterName'], + driver=self.connection.driver + ) + + def _to_containers(self, data): + clusters = [] + for cluster in data['containers']: + clusters.append(self._to_container(cluster)) + return clusters + + def _to_container(self, data): + return Container( + id=data['containerArn'], + name=data['name'], + image=ContainerImage( + id=None, + name=data['name'], + path=None, + version=None, + driver=self.connection.driver + ), + ip_addresses=None, + state=self.status_map.get(data['lastStatus'], None), + extra={ + 'taskArn': data['taskArn'] + }, + driver=self.connection.driver + ) http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/test/container/fixtures/ecs/createcluster.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/createcluster.json b/libcloud/test/container/fixtures/ecs/createcluster.json new file mode 100644 index 0000000..1fb35e6 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/createcluster.json @@ -0,0 +1,11 @@ +{ + "cluster": { + "activeServicesCount": 0, + "clusterArn": "arn:aws:ecs:us-east-1:012345678910:cluster/jim", + "clusterName": "jim", + "pendingTasksCount": 0, + "registeredContainerInstancesCount": 0, + "runningTasksCount": 0, + "status": "ACTIVE" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/test/container/fixtures/ecs/deletecluster.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/deletecluster.json b/libcloud/test/container/fixtures/ecs/deletecluster.json new file mode 100644 index 0000000..94d5aa2 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/deletecluster.json @@ -0,0 +1,11 @@ +{ + "cluster": { + "activeServicesCount": 0, + "clusterArn": "arn:aws:ecs:us-east-1:012345678910:cluster/jim", + "clusterName": "jim", + "pendingTasksCount": 0, + "registeredContainerInstancesCount": 0, + "runningTasksCount": 0, + "status": "INACTIVE" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/test/container/fixtures/ecs/describeclusters.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/describeclusters.json b/libcloud/test/container/fixtures/ecs/describeclusters.json new file mode 100644 index 0000000..1e89426 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/describeclusters.json @@ -0,0 +1,14 @@ +{ + "clusters": [ + { + "activeServicesCount": 1, + "clusterArn": "arn:aws:ecs:us-east-1:012345678910:cluster/default", + "clusterName": "default", + "pendingTasksCount": 0, + "registeredContainerInstancesCount": 0, + "runningTasksCount": 0, + "status": "ACTIVE" + } + ], + "failures": [] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/test/container/fixtures/ecs/describetasks.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/describetasks.json b/libcloud/test/container/fixtures/ecs/describetasks.json new file mode 100644 index 0000000..d8e0624 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/describetasks.json @@ -0,0 +1,46 @@ +{ + "failures": [], + "tasks": [ + { + "clusterArn": "arn:aws:ecs:us-east-1:012345678910:cluster/default", + "containerInstanceArn": "arn:aws:ecs:us-east-1:012345678910:container-instance/84818520-995f-4d94-9d70-7714bacc2953", + "containers": [ + { + "containerArn": "arn:aws:ecs:us-east-1:012345678910:container/76c980a8-2454-4a9c-acc4-9eb103117273", + "lastStatus": "RUNNING", + "name": "mysql", + "networkBindings": [], + "taskArn": "arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe" + }, + { + "containerArn": "arn:aws:ecs:us-east-1:012345678910:container/e3c69b8f-f15e-4d33-8093-282c2d2325e9", + "lastStatus": "RUNNING", + "name": "wordpress", + "networkBindings": [ + { + "bindIP": "0.0.0.0", + "containerPort": 80, + "hostPort": 80 + } + ], + "taskArn": "arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe" + } + ], + "desiredStatus": "RUNNING", + "lastStatus": "RUNNING", + "overrides": { + "containerOverrides": [ + { + "name": "mysql" + }, + { + "name": "wordpress" + } + ] + }, + "startedBy": "ecs-svc/9223370606521064774", + "taskArn": "arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe", + "taskDefinitionArn": "arn:aws:ecs:us-east-1:012345678910:task-definition/hello_world:10" + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/test/container/test_ecs.py ---------------------------------------------------------------------- diff --git a/libcloud/test/container/test_ecs.py b/libcloud/test/container/test_ecs.py new file mode 100644 index 0000000..81bdf09 --- /dev/null +++ b/libcloud/test/container/test_ecs.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from libcloud.test import unittest + +from libcloud.container.base import ContainerCluster +from libcloud.container.drivers.ecs import ElasticContainerDriver + +from libcloud.utils.py3 import httplib +from libcloud.test.secrets import CONTAINER_PARAMS_ECS +from libcloud.test.file_fixtures import ContainerFileFixtures +from libcloud.test import MockHttp + + +class ElasticContainerDriverTestCase(unittest.TestCase): + + def setUp(self): + ElasticContainerDriver.connectionCls.conn_classes = ( + ECSMockHttp, ECSMockHttp) + ECSMockHttp.type = None + ECSMockHttp.use_param = 'a' + self.driver = ElasticContainerDriver(*CONTAINER_PARAMS_ECS) + + def test_list_clusters(self): + clusters = self.driver.list_clusters() + self.assertEqual(len(clusters), 1) + self.assertEqual(clusters[0].id, 'arn:aws:ecs:us-east-1:012345678910:cluster/default') + self.assertEqual(clusters[0].name, 'default') + + def test_create_cluster(self): + cluster = self.driver.create_cluster('jim') + self.assertEqual(cluster.name, 'jim') + + def test_destroy_cluster(self): + self.assertTrue( + self.driver.destroy_cluster( + ContainerCluster( + id='arn:aws:ecs:us-east-1:012345678910:cluster/jim', + name='jim', + driver=self.driver))) + + def test_list_containers(self): + containers = self.driver.list_containers() + self.assertEqual(len(containers), 2) + + def test_list_containers_for_cluster(self): + cluster = self.driver.list_clusters()[0] + containers = self.driver.list_containers(cluster=cluster) + self.assertEqual(len(containers), 2) + + +class ECSMockHttp(MockHttp): + fixtures = ContainerFileFixtures('ecs') + fixture_map = { + 'DescribeClusters': 'describeclusters.json', + 'CreateCluster': 'createcluster.json', + 'DeleteCluster': 'deletecluster.json', + 'DescribeTasks': 'describetasks.json' + } + + def _2014_11_13( + self, method, url, body, headers): + target = headers['x-amz-target'] + if target is not None: + type = target.split('.')[-1] + if type is None or self.fixture_map.get(type) is None: + raise AssertionError('Unsupported request type %s' % (target)) + body = self.fixtures.load(self.fixture_map.get(type)) + else: + raise AssertionError('Unsupported method') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + +if __name__ == '__main__': + sys.exit(unittest.main()) http://git-wip-us.apache.org/repos/asf/libcloud/blob/f1674deb/libcloud/test/secrets.py-dist ---------------------------------------------------------------------- diff --git a/libcloud/test/secrets.py-dist b/libcloud/test/secrets.py-dist index dad7724..608fdbd 100644 --- a/libcloud/test/secrets.py-dist +++ b/libcloud/test/secrets.py-dist @@ -83,3 +83,4 @@ DNS_PARAMS_CLOUDFLARE = ('user@example.com', 'key') # Container CONTAINER_PARAMS_DOCKER = ('user', 'password') +CONTAINER_PARAMS_ECS = ('user', 'password', 'region')