From notifications-return-16597-archive-asf-public=cust-asf.ponee.io@libcloud.apache.org Tue Dec 24 19:18:28 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id B6F5918065E for ; Tue, 24 Dec 2019 20:18:27 +0100 (CET) Received: (qmail 34615 invoked by uid 500); 24 Dec 2019 19:18:27 -0000 Mailing-List: contact notifications-help@libcloud.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@libcloud.apache.org Delivered-To: mailing list notifications@libcloud.apache.org Received: (qmail 34606 invoked by uid 99); 24 Dec 2019 19:18:27 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 24 Dec 2019 19:18:27 +0000 From: GitBox To: notifications@libcloud.apache.org Subject: [GitHub] [libcloud] Kami commented on a change in pull request #1395: Add LXD driver & tests Message-ID: <157721510581.1687.7694086530278676594.gitbox@gitbox.apache.org> References: In-Reply-To: Date: Tue, 24 Dec 2019 19:18:25 -0000 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Kami commented on a change in pull request #1395: Add LXD driver & tests URL: https://github.com/apache/libcloud/pull/1395#discussion_r361215643 ########## File path: libcloud/container/drivers/lxd.py ########## @@ -0,0 +1,1269 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import re +import os + + +try: + import simplejson as json +except Exception: + import json + +from libcloud.utils.py3 import httplib +from libcloud.utils.py3 import b + +from libcloud.common.base import JsonResponse, ConnectionUserAndKey +from libcloud.common.base import KeyCertificateConnection +from libcloud.common.types import InvalidCredsError + +from libcloud.container.base import (Container, ContainerDriver, + ContainerImage) +from libcloud.common.exceptions import BaseHTTPError + +from libcloud.compute.base import StorageVolume + +from libcloud.container.providers import Provider +from libcloud.container.types import ContainerState + +# Acceptable success strings comping from LXD API +LXD_API_SUCCESS_STATUS = ['Success'] +LXD_API_STATE_ACTIONS = ['stop', 'start', 'restart', 'freeze', 'unfreeze'] +LXD_API_IMAGE_SOURCE_TYPE = ["image", "migration", "copy", "none"] + +# the wording used by LXD to indicate that an error +# occurred for a request +LXD_ERROR_STATUS_RESP = 'error' + + +# helpers +def strip_http_prefix(host): + # strip the prefix + prefixes = ['http://', 'https://'] + for prefix in prefixes: + if host.startswith(prefix): + host = host.strip(prefix) + return host + + +def check_certificates(key_file, cert_file, **kwargs): + """ + Basic checks for the provided certificates in LXDtlsConnection + """ + + # there is no point attempting to connect if either is missing + if key_file is None or cert_file is None: + raise InvalidCredsError("TLS Connection requires specification " + "of a key file and a certificate file") + + # if they are not none they may be empty strings + # or certificates that are not appropriate + if key_file == '' or cert_file == '': + raise InvalidCredsError("TLS Connection requires specification " + "of a key file and a certificate file") + + # if none of the above check the types + if 'key_files_allowed' in kwargs.keys(): + key_file_suffix = key_file.split('.') + + if key_file_suffix[-1] not in kwargs['key_files_allowed']: + raise InvalidCredsError("Valid key files are: " + + str(kwargs['key_files_allowed']) + + "you provided: " + key_file_suffix[-1]) + + # if none of the above check the types + if 'cert_files_allowed' in kwargs.keys(): + cert_file_suffix = cert_file.split('.') + + if cert_file_suffix[-1] not in kwargs['cert_files_allowed']: + raise InvalidCredsError("Valid certification files are: " + + str(kwargs['cert_files_allowed']) + + "you provided: " + cert_file_suffix[-1]) + + # if all these are good check the paths + keypath = os.path.expanduser(key_file) + is_file_path = os.path.exists(keypath) and os.path.isfile(keypath) + if not is_file_path: + raise InvalidCredsError('You need a key file to authenticate with ' + 'LXD tls. This can be found in the server.') + + certpath = os.path.expanduser(cert_file) + is_file_path = os.path.exists(certpath) and os.path.isfile(certpath) + if not is_file_path: + raise InvalidCredsError('You need a certificate file to ' + 'authenticate with LXD tls. ' + 'This can be found in the server.') + + +def assert_response(response_dict, status_code=200): + + # if the type of the response is an error + if response_dict['type'] == LXD_ERROR_STATUS_RESP: + # an error returned + raise LXDAPIException(message="response type is error") + + # anything else apart from the status_code given should be treated as error + if response_dict['status_code'] != status_code: + # we have an unknown error + msg = "Status code should be {0}\ + but is {1}".format(status_code, response_dict['status_code']) + raise LXDAPIException(message=msg) + + +class LXDAPIException(Exception): + """ + Basic exception to be thrown when LXD API + returns with some kind of error + """ + + def __init__(self, message="Unknown Error Occurred"): + self.message = message + + super(LXDAPIException, self).__init__(message) + + def __str__(self): + return self.message + + +class LXDStoragePool(object): + """ + Utility class representing an LXD storage pool + https://lxd.readthedocs.io/en/latest/storage/ + """ + def __init__(self, name, driver, used_by, config, managed): + + # the name of the storage pool + self.name = name + + # the driver (or type of storage pool). e.g. ‘zfs’ or ‘btrfs’, etc. + self.driver = driver + + # which containers (by API endpoint /1.0/containers/) + # are using this storage-pool. + self.used_by = used_by + + # a dictionary with some information about the storage-pool. + # e.g. size, source (path), volume.size, etc. + self.config = config + + # Boolean that indicates whether LXD manages the pool or not. + self.managed = managed + + +class LXDServerInfo(object): + """ + Wraps the response form /1.0 + """ + + @classmethod + def build_from_response(cls, metadata): + + server_info = LXDServerInfo() + server_info.api_extensions = metadata.get("api_extensions", None) + server_info.api_status = metadata.get("api_status", None) + server_info.api_version = metadata.get("api_version", None) + server_info.auth = metadata.get("auth", None) + server_info.config = metadata.get("config", None) + server_info.environment = metadata.get("environment", None) + server_info.public = metadata.get("public", None) + return server_info + + def __init__(self): + + # List of API extensions added after + # the API was marked stable + self.api_extensions = None + + # API implementation status + # (one of, development, stable or deprecated) + self.api_status = None + + # The API version as a string + self.api_version = None + + # Authentication state, + # one of "guest", "untrusted" or "trusted" + self.auth = None + + self.config = None + + # Various information about the host (OS, kernel, ...) + self.environment = None + + self.public = None + + def __str__(self): + return str(self.api_extensions) + str(self.api_status) + \ + str(self.api_version) + str(self.auth) + str(self.config) + \ + str(self.environment) + \ + str(self.public) + + +class LXDResponse(JsonResponse): + valid_response_codes = [httplib.OK, httplib.ACCEPTED, httplib.CREATED, + httplib.NO_CONTENT] + + def parse_body(self): + + if len(self.body) == 0 and not self.parse_zero_length_body: + return self.body + + try: + # error responses are tricky in Docker. Eg response could be + # an error, but response status could still be 200 + content_type = self.headers.get('content-type', 'application/json') + if content_type == 'application/json' or content_type == '': + if self.headers.get('transfer-encoding') == 'chunked' and \ + 'fromImage' in self.request.url: + body = [json.loads(chunk) for chunk in + self.body.strip().replace('\r', '').split('\n')] + else: + body = json.loads(self.body) + else: + body = self.body + except ValueError: + m = re.search('Error: (.+?)"', self.body) + if m: + error_msg = m.group(1) + raise Exception(error_msg) + else: + raise Exception( + 'ConnectionError: Failed to parse JSON response') + return body + + def parse_error(self): + if self.status == 401: + raise InvalidCredsError('Invalid credentials') + return self.body + + def success(self): + return self.status in self.valid_response_codes + + +class LXDConnection(ConnectionUserAndKey): + responseCls = LXDResponse + timeout = 60 + + def add_default_headers(self, headers): + """ + Add parameters that are necessary for every request + If user and password are specified, include a base http auth + header + """ + headers['Content-Type'] = 'application/json' + if self.user_id and self.key: + user_b64 = base64.b64encode(b('%s:%s' % (self.user_id, self.key))) + headers['Authorization'] = 'Basic %s' % (user_b64.decode('utf-8')) + return headers + + +class LXDtlsConnection(KeyCertificateConnection): + + responseCls = LXDResponse + + def __init__(self, key, secret, secure=True, + host='localhost', port=8443, ca_cert='', + key_file=None, cert_file=None, **kwargs): + + if 'certificate_validator' in kwargs.keys(): + certificate_validator = kwargs.pop('certificate_validator') + certificate_validator(key_file=key_file, cert_file=cert_file) + else: + check_certificates(key_file=key_file, + cert_file=cert_file, **kwargs) + + super(LXDtlsConnection, self).__init__(key_file=key_file, + cert_file=cert_file, + secure=secure, host=host, + port=port, url=None, + proxy_url=None, + timeout=None, backoff=None, + retry_delay=None) + + self.key_file = key_file + self.cert_file = cert_file + + def add_default_headers(self, headers): + headers['Content-Type'] = 'application/json' + return headers + + +class LXDContainerDriver(ContainerDriver): + """ + Driver for LXD REST API of LXC containers + https://lxd.readthedocs.io/en/stable-2.0/rest-api/ + https://github.com/lxc/lxd/blob/master/doc/rest-api.md + """ + type = Provider.LXD + name = 'LXD' + website = 'https://linuxcontainers.org/' + connectionCls = LXDConnection + + # LXD supports clustering but still the functionality + # is not implemented yet on our side + supports_clusters = False + version = '1.0' + default_time_out = 30 + + # default configuration when creating a container + default_architecture = 'x86_64' + default_profiles = 'default' + default_ephemeral = False + + def __init__(self, key='', secret='', secure=False, + host='localhost', port=8443, key_file=None, + cert_file=None, ca_cert=None, + certificate_validator=check_certificates): + + if key_file: + + if not cert_file: + # LXD tls authentication- + # We pass two files, a key_file with the + # private key and cert_file with the certificate + # libcloud will handle them through LibcloudHTTPSConnection + + raise LXDAPIException(message='Need both private key and' + ' certificate files for ' + 'tls authentication') + + self.connectionCls = LXDtlsConnection + self.key_file = key_file + self.cert_file = cert_file + self.certificate_validator = certificate_validator + secure = True + + if host.startswith('https://'): + secure = True + + host = strip_http_prefix(host=host) + + super(LXDContainerDriver, self).__init__(key=key, secret=secret, + secure=secure, host=host, + port=port, + key_file=key_file, + cert_file=cert_file) + + if ca_cert: + self.connection.connection.ca_cert = ca_cert + else: + # do not verify SSL certificate + self.connection.connection.ca_cert = False + + self.connection.secure = secure + self.connection.host = host + self.connection.port = port + self.version = self._get_api_version() + + def ex_get_api_endpoints(self): + """ + Description: List of supported APIs + Authentication: guest + Operation: sync + Return: list of supported API endpoint URLs + + """ + response = self.connection.request("/") + response_dict = response.parse_body() + assert_response(response_dict=response_dict, status_code=200) + return response_dict["metadata"] + + def ex_get_server_configuration(self): + """ + + Description: Server configuration and environment information + Authentication: guest, untrusted or trusted + Operation: sync + Return: Dict representing server state + + The returned configuration depends on whether the connection + is trusted or not + :rtype: :class: .LXDServerInfo + + """ + response = self.connection.request("/%s" % (self.version)) + response_dict = response.parse_body() + assert_response(response_dict=response_dict, status_code=200) + meta = response_dict["metadata"] + return LXDServerInfo.build_from_response(metadata=meta) + + def deploy_container(self, name, image, cluster=None, + parameters=None, start=True, + architecture=default_architecture, + profiles=[default_profiles], + ephemeral=default_ephemeral, + config=None, devices=None, + instance_type=None): + + """ + Create a new container + Authentication: trusted + Operation: async + Return: background operation or standard error + + :param name: The name of the new container. + 64 chars max, ASCII, no slash, no colon and no comma + :type name: ``str`` + + :param image: The container image to deploy. Currently not used + :type image: :class:`.ContainerImage` + + :param cluster: The cluster to deploy to, None is default + :type cluster: :class:`.ContainerCluster` + + :param parameters: Container Image parameters. + This parameter should represent the + the ``source`` dictioanry expected by the LXD API call. For more + information how this parameter should be structured see + https://github.com/lxc/lxd/blob/master/doc/rest-api.md + :type parameters: ``str`` + + :param start: Start the container on deployment. True is the default + :type start: ``bool`` + + :param architecture: string e.g. x86_64 + :type architecture: ``str`` + + :param profiles: List of profiles + :type profiles: ``list`` + + :param ephemeral: Whether to destroy the container on shutdown + :type ephemeral: ``bool`` + + :param config: Config override e.g. {"limits.cpu": "2"} + :type config: ``dict`` + + :param devices: optional list of devices the container should have + :type devices: ``dict`` + + :param instance_type: An optional instance type + to use as basis for limits e.g. "c2.micro" + :type instance_type: ``str`` + + :rtype: :class:`.Container` + """ + + c_params = {"architecture": architecture, "profiles": profiles, + "ephemeral": ephemeral, "config": config, + "devices": devices, "instance_type": instance_type} + + if parameters: + parameters = json.loads(parameters) + + container = self._deploy_container_from_image(name=name, image=image, + parameters=parameters, + cont_params=c_params) + + if start: + container.start() + + return container + + def get_container(self, id, get_ip_addr=True): + + """ + Get a container by ID + + :param id: The ID of the container to get + :type id: ``str`` + + :param get_ip_addr: Indicates whether ip addresses + should also be included. This requires an extra GET request + :type get_ip_addr: ``boolean``` + + :rtype: :class:`libcloud.container.base.Container` + """ + req = "/%s/containers/%s" % (self.version, id) + response = self.connection.request(req) + result_dict = response.parse_body() + assert_response(response_dict=result_dict, status_code=200) + + metadata = result_dict["metadata"] + + ips = [] + if get_ip_addr: + req = "/%s/containers/%s/state" % (self.version, id) + ip_response = self.connection.request(req) + + ip_result_dict = ip_response.parse_body() + assert_response(response_dict=ip_result_dict, status_code=200) + + networks = None + + if ip_result_dict["metadata"]["network"] is not None: + networks = ip_result_dict["metadata"]["network"]["eth0"] + + # the list of addresses + addresses = networks["addresses"] + + for item in addresses: + ips.append(item["address"]) + + metadata.update({"ips": ips}) + return self._to_container(metadata=metadata) + + def start_container(self, container, timeout=default_time_out): + """ + Start a container + + :param container: The container to start + :type container: :class:`libcloud.container.base.Container` + + :param timeout: Time to wait for the operation to complete + :type timeout: ``int`` + + :rtype: :class:`libcloud.container.base.Container` + """ + return self._do_container_action(container=container, action='start', + timeout=timeout, + force=True, stateful=True) + + def stop_container(self, container, timeout=default_time_out): + """ + Stop the given container + + :param container: The container to be stopped + :type container: :class:`libcloud.container.base.Container` + + :param timeout: Time to wait for the operation to complete + :type timeout: ``int`` + + :return: The container refreshed with current data + :rtype: :class:`libcloud.container.base.Container + """ + return self._do_container_action(container=container, action='stop', + timeout=timeout, + force=True, stateful=True) + + def restart_container(self, container, timeout=default_time_out): + """ + Restart a deployed container + + :param container: The container to restart + :type container: :class:`.Container` + + :param timeout: Time to wait for the operation to complete + :type timeout: ``int`` + + :rtype: :class:`.Container` + """ + return self._do_container_action(container=container, action='restart', + timeout=timeout, + force=True, stateful=True) + + def destroy_container(self, container, timeout=default_time_out): + """ + Destroy a deployed container. Raises and exception + if he container is running + + :param container: The container to destroy + :type container: :class:`.Container` + + :param timeout: Time to wait for the operation to complete + :type timeout ``int`` + + :rtype: :class:`.Container` + """ + + # if the container is running then we cannot delete + + if container.state == ContainerState.RUNNING: Review comment: I'm fine with doing that check here although we don't really have clear rules / guidelines on how to handle such scenarios in Libcloud drivers. A lot of drivers simply let the API handle that validation instead of doing it in the driver. This way it still works if user doesn't have the latest reference to the ``container`` object locally. So another option would be to simply remove that check here. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: users@infra.apache.org With regards, Apache Git Services