Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 2B254200BDA for ; Tue, 13 Dec 2016 16:04:48 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 29BEB160B23; Tue, 13 Dec 2016 15:04:48 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 4025A160B15 for ; Tue, 13 Dec 2016 16:04:46 +0100 (CET) Received: (qmail 82965 invoked by uid 500); 13 Dec 2016 15:04:45 -0000 Mailing-List: contact dev-help@ariatosca.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ariatosca.incubator.apache.org Delivered-To: mailing list dev@ariatosca.incubator.apache.org Received: (qmail 82954 invoked by uid 99); 13 Dec 2016 15:04:45 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Dec 2016 15:04:45 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id D729E1AAB75 for ; Tue, 13 Dec 2016 15:04:44 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -5.219 X-Spam-Level: X-Spam-Status: No, score=-5.219 tagged_above=-999 required=6.31 tests=[HK_RANDOM_FROM=0.999, KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id 74hlSnOgtnmm for ; Tue, 13 Dec 2016 15:04:39 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id A6D0F5F252 for ; Tue, 13 Dec 2016 15:04:37 +0000 (UTC) Received: (qmail 82762 invoked by uid 99); 13 Dec 2016 15:04:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Dec 2016 15:04:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8BEEAE3934; Tue, 13 Dec 2016 15:04:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mxmrlv@apache.org To: dev@ariatosca.incubator.apache.org Message-Id: <4019896a022d4f7f8ed68ae0455f9173@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: incubator-ariatosca git commit: Generifying ARIA models Date: Tue, 13 Dec 2016 15:04:36 +0000 (UTC) archived-at: Tue, 13 Dec 2016 15:04:48 -0000 Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-39-Genericize-storage-models [created] a5eef4be2 Generifying ARIA models Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/a5eef4be Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/a5eef4be Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/a5eef4be Branch: refs/heads/ARIA-39-Genericize-storage-models Commit: a5eef4be228daead5f7daa55929d3562cab88d9b Parents: c6c92ae Author: mxmrlv Authored: Mon Dec 12 00:50:09 2016 +0200 Committer: mxmrlv Committed: Tue Dec 13 16:56:36 2016 +0200 ---------------------------------------------------------------------- aria/storage/models.py | 573 +++------------------------- aria/storage/models_base.py | 629 +++++++++++++++++++++++++++++++ aria/storage/structures.py | 120 +++--- tests/storage/__init__.py | 5 +- tests/storage/test_model_storage.py | 17 - tests/storage/test_models.py | 20 +- 6 files changed, 747 insertions(+), 617 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/a5eef4be/aria/storage/models.py ---------------------------------------------------------------------- diff --git a/aria/storage/models.py b/aria/storage/models.py index 6302e66..695f16c 100644 --- a/aria/storage/models.py +++ b/aria/storage/models.py @@ -36,27 +36,9 @@ classes: * ProviderContext - provider context implementation model. * Plugin - plugin implementation model. """ -from collections import namedtuple -from datetime import datetime - -from sqlalchemy.ext.declarative.base import declared_attr - -from .structures import ( - SQLModelBase, - Column, - Integer, - Text, - DateTime, - Boolean, - Enum, - String, - Float, - List, - Dict, - foreign_key, - one_to_many_relationship, - relationship_to_self, - orm) + +from .structures import ModelBase, declarative_base +from .models_base import * __all__ = ( 'Blueprint', @@ -74,499 +56,56 @@ __all__ = ( ) -#pylint: disable=no-self-argument - - -class Blueprint(SQLModelBase): - """ - Blueprint model representation. - """ - __tablename__ = 'blueprints' - - name = Column(Text, index=True) - created_at = Column(DateTime, nullable=False, index=True) - main_file_name = Column(Text, nullable=False) - plan = Column(Dict, nullable=False) - updated_at = Column(DateTime) - description = Column(Text) - - -class Deployment(SQLModelBase): - """ - Deployment model representation. - """ - __tablename__ = 'deployments' - - _private_fields = ['blueprint_id'] - - blueprint_id = foreign_key(Blueprint.id) - - name = Column(Text, index=True) - created_at = Column(DateTime, nullable=False, index=True) - description = Column(Text) - inputs = Column(Dict) - groups = Column(Dict) - permalink = Column(Text) - policy_triggers = Column(Dict) - policy_types = Column(Dict) - outputs = Column(Dict) - scaling_groups = Column(Dict) - updated_at = Column(DateTime) - workflows = Column(Dict) - - @declared_attr - def blueprint(cls): - return one_to_many_relationship(cls, Blueprint, cls.blueprint_id) - - -class Execution(SQLModelBase): - """ - Execution model representation. - """ - __tablename__ = 'executions' - - TERMINATED = 'terminated' - FAILED = 'failed' - CANCELLED = 'cancelled' - PENDING = 'pending' - STARTED = 'started' - CANCELLING = 'cancelling' - FORCE_CANCELLING = 'force_cancelling' - - STATES = [TERMINATED, FAILED, CANCELLED, PENDING, STARTED, CANCELLING, FORCE_CANCELLING] - END_STATES = [TERMINATED, FAILED, CANCELLED] - ACTIVE_STATES = [state for state in STATES if state not in END_STATES] - - VALID_TRANSITIONS = { - PENDING: [STARTED, CANCELLED], - STARTED: END_STATES + [CANCELLING], - CANCELLING: END_STATES - } - - @orm.validates('status') - def validate_status(self, key, value): - """Validation function that verifies execution status transitions are OK""" - try: - current_status = getattr(self, key) - except AttributeError: - return - valid_transitions = Execution.VALID_TRANSITIONS.get(current_status, []) - if all([current_status is not None, - current_status != value, - value not in valid_transitions]): - raise ValueError('Cannot change execution status from {current} to {new}'.format( - current=current_status, - new=value)) - return value - - deployment_id = foreign_key(Deployment.id) - blueprint_id = foreign_key(Blueprint.id) - _private_fields = ['deployment_id', 'blueprint_id'] - - created_at = Column(DateTime, index=True) - started_at = Column(DateTime, nullable=True, index=True) - ended_at = Column(DateTime, nullable=True, index=True) - error = Column(Text, nullable=True) - is_system_workflow = Column(Boolean, nullable=False, default=False) - parameters = Column(Dict) - status = Column(Enum(*STATES, name='execution_status'), default=PENDING) - workflow_name = Column(Text, nullable=False) - - @declared_attr - def deployment(cls): - return one_to_many_relationship(cls, Deployment, cls.deployment_id) - - @declared_attr - def blueprint(cls): - return one_to_many_relationship(cls, Blueprint, cls.blueprint_id) - - def __str__(self): - return '<{0} id=`{1}` (status={2})>'.format( - self.__class__.__name__, - self.id, - self.status - ) - - -class DeploymentUpdate(SQLModelBase): - """ - Deployment update model representation. - """ - __tablename__ = 'deployment_updates' - - deployment_id = foreign_key(Deployment.id) - execution_id = foreign_key(Execution.id, nullable=True) - _private_fields = ['execution_id', 'deployment_id'] - - created_at = Column(DateTime, nullable=False, index=True) - deployment_plan = Column(Dict, nullable=False) - deployment_update_node_instances = Column(Dict) - deployment_update_deployment = Column(Dict) - deployment_update_nodes = Column(Dict) - modified_entity_ids = Column(Dict) - state = Column(Text) - - @declared_attr - def execution(cls): - return one_to_many_relationship(cls, Execution, cls.execution_id) - - @declared_attr - def deployment(cls): - return one_to_many_relationship(cls, Deployment, cls.deployment_id) - - def to_dict(self, suppress_error=False, **kwargs): - dep_update_dict = super(DeploymentUpdate, self).to_dict(suppress_error) - # Taking care of the fact the DeploymentSteps are objects - dep_update_dict['steps'] = [step.to_dict() for step in self.steps] - return dep_update_dict - - -class DeploymentUpdateStep(SQLModelBase): - """ - Deployment update step model representation. - """ - __tablename__ = 'deployment_update_steps' - _action_types = namedtuple('ACTION_TYPES', 'ADD, REMOVE, MODIFY') - ACTION_TYPES = _action_types(ADD='add', REMOVE='remove', MODIFY='modify') - _entity_types = namedtuple( - 'ENTITY_TYPES', - 'NODE, RELATIONSHIP, PROPERTY, OPERATION, WORKFLOW, OUTPUT, DESCRIPTION, GROUP, ' - 'POLICY_TYPE, POLICY_TRIGGER, PLUGIN') - ENTITY_TYPES = _entity_types( - NODE='node', - RELATIONSHIP='relationship', - PROPERTY='property', - OPERATION='operation', - WORKFLOW='workflow', - OUTPUT='output', - DESCRIPTION='description', - GROUP='group', - POLICY_TYPE='policy_type', - POLICY_TRIGGER='policy_trigger', - PLUGIN='plugin' - ) - - deployment_update_id = foreign_key(DeploymentUpdate.id) - _private_fields = ['deployment_update_id'] - - action = Column(Enum(*ACTION_TYPES, name='action_type'), nullable=False) - entity_id = Column(Text, nullable=False) - entity_type = Column(Enum(*ENTITY_TYPES, name='entity_type'), nullable=False) - - @declared_attr - def deployment_update(cls): - return one_to_many_relationship(cls, - DeploymentUpdate, - cls.deployment_update_id, - backreference='steps') - - def __hash__(self): - return hash((self.id, self.entity_id)) - - def __lt__(self, other): - """ - the order is 'remove' < 'modify' < 'add' - :param other: - :return: - """ - if not isinstance(other, self.__class__): - return not self >= other - - if self.action != other.action: - if self.action == 'remove': - return_value = True - elif self.action == 'add': - return_value = False - else: - return_value = other.action == 'add' - return return_value - - if self.action == 'add': - return self.entity_type == 'node' and other.entity_type == 'relationship' - if self.action == 'remove': - return self.entity_type == 'relationship' and other.entity_type == 'node' - return False - - -class DeploymentModification(SQLModelBase): - """ - Deployment modification model representation. - """ - __tablename__ = 'deployment_modifications' - - STARTED = 'started' - FINISHED = 'finished' - ROLLEDBACK = 'rolledback' - - STATES = [STARTED, FINISHED, ROLLEDBACK] - END_STATES = [FINISHED, ROLLEDBACK] - - deployment_id = foreign_key(Deployment.id) - _private_fields = ['deployment_id'] - - context = Column(Dict) - created_at = Column(DateTime, nullable=False, index=True) - ended_at = Column(DateTime, index=True) - modified_nodes = Column(Dict) - node_instances = Column(Dict) - status = Column(Enum(*STATES, name='deployment_modification_status')) - - @declared_attr - def deployment(cls): - return one_to_many_relationship(cls, - Deployment, - cls.deployment_id, - backreference='modifications') - - -class Node(SQLModelBase): - """ - Node model representation. - """ - __tablename__ = 'nodes' - - # See base class for an explanation on these properties - is_id_unique = False - - name = Column(Text, index=True) - _private_fields = ['deployment_id', 'host_id'] - deployment_id = foreign_key(Deployment.id) - host_id = foreign_key('nodes.id', nullable=True) - - @declared_attr - def deployment(cls): - return one_to_many_relationship(cls, Deployment, cls.deployment_id) - - deploy_number_of_instances = Column(Integer, nullable=False) - # TODO: This probably should be a foreign key, but there's no guarantee - # in the code, currently, that the host will be created beforehand - max_number_of_instances = Column(Integer, nullable=False) - min_number_of_instances = Column(Integer, nullable=False) - number_of_instances = Column(Integer, nullable=False) - planned_number_of_instances = Column(Integer, nullable=False) - plugins = Column(Dict) - plugins_to_install = Column(Dict) - properties = Column(Dict) - operations = Column(Dict) - type = Column(Text, nullable=False, index=True) - type_hierarchy = Column(List) - - @declared_attr - def host(cls): - return relationship_to_self(cls, cls.host_id, cls.id) - - -class Relationship(SQLModelBase): - """ - Relationship model representation. - """ - __tablename__ = 'relationships' - - _private_fields = ['source_node_id', 'target_node_id'] - - source_node_id = foreign_key(Node.id) - target_node_id = foreign_key(Node.id) - - @declared_attr - def source_node(cls): - return one_to_many_relationship(cls, - Node, - cls.source_node_id, - 'outbound_relationships') - - @declared_attr - def target_node(cls): - return one_to_many_relationship(cls, - Node, - cls.target_node_id, - 'inbound_relationships') - - source_interfaces = Column(Dict) - source_operations = Column(Dict, nullable=False) - target_interfaces = Column(Dict) - target_operations = Column(Dict, nullable=False) - type = Column(String, nullable=False) - type_hierarchy = Column(List) - properties = Column(Dict) - - -class NodeInstance(SQLModelBase): - """ - Node instance model representation. - """ - __tablename__ = 'node_instances' - - node_id = foreign_key(Node.id) - deployment_id = foreign_key(Deployment.id) - host_id = foreign_key('node_instances.id', nullable=True) - - _private_fields = ['node_id', 'host_id'] - - name = Column(Text, index=True) - runtime_properties = Column(Dict) - scaling_groups = Column(Dict) - state = Column(Text, nullable=False) - version = Column(Integer, default=1) - - @declared_attr - def deployment(cls): - return one_to_many_relationship(cls, Deployment, cls.deployment_id) - - @declared_attr - def node(cls): - return one_to_many_relationship(cls, Node, cls.node_id) - - @declared_attr - def host(cls): - return relationship_to_self(cls, cls.host_id, cls.id) - - -class RelationshipInstance(SQLModelBase): - """ - Relationship instance model representation. - """ - __tablename__ = 'relationship_instances' - - relationship_id = foreign_key(Relationship.id) - source_node_instance_id = foreign_key(NodeInstance.id) - target_node_instance_id = foreign_key(NodeInstance.id) - - _private_fields = ['relationship_storage_id', - 'source_node_instance_id', - 'target_node_instance_id'] - - @declared_attr - def source_node_instance(cls): - return one_to_many_relationship(cls, - NodeInstance, - cls.source_node_instance_id, - 'outbound_relationship_instances') - - @declared_attr - def target_node_instance(cls): - return one_to_many_relationship(cls, - NodeInstance, - cls.target_node_instance_id, - 'inbound_relationship_instances') - - @declared_attr - def relationship(cls): - return one_to_many_relationship(cls, Relationship, cls.relationship_id) - - -class ProviderContext(SQLModelBase): - """ - Provider context model representation. - """ - __tablename__ = 'provider_context' - - name = Column(Text, nullable=False) - context = Column(Dict, nullable=False) - - -class Plugin(SQLModelBase): - """ - Plugin model representation. - """ - __tablename__ = 'plugins' - - archive_name = Column(Text, nullable=False, index=True) - distribution = Column(Text) - distribution_release = Column(Text) - distribution_version = Column(Text) - excluded_wheels = Column(Dict) - package_name = Column(Text, nullable=False, index=True) - package_source = Column(Text) - package_version = Column(Text) - supported_platform = Column(Dict) - supported_py_versions = Column(Dict) - uploaded_at = Column(DateTime, nullable=False, index=True) - wheels = Column(Dict, nullable=False) - - -class Task(SQLModelBase): - """ - A Model which represents an task - """ - - __tablename__ = 'task' - node_instance_id = foreign_key(NodeInstance.id, nullable=True) - relationship_instance_id = foreign_key(RelationshipInstance.id, nullable=True) - execution_id = foreign_key(Execution.id, nullable=True) - - _private_fields = ['node_instance_id', - 'relationship_instance_id', - 'execution_id'] - - @declared_attr - def node_instance(cls): - return one_to_many_relationship(cls, NodeInstance, cls.node_instance_id) - - @declared_attr - def relationship_instance(cls): - return one_to_many_relationship(cls, - RelationshipInstance, - cls.relationship_instance_id) - - PENDING = 'pending' - RETRYING = 'retrying' - SENT = 'sent' - STARTED = 'started' - SUCCESS = 'success' - FAILED = 'failed' - STATES = ( - PENDING, - RETRYING, - SENT, - STARTED, - SUCCESS, - FAILED, - ) - - WAIT_STATES = [PENDING, RETRYING] - END_STATES = [SUCCESS, FAILED] - - @orm.validates('max_attempts') - def validate_max_attempts(self, _, value): # pylint: disable=no-self-use - """Validates that max attempts is either -1 or a positive number""" - if value < 1 and value != Task.INFINITE_RETRIES: - raise ValueError('Max attempts can be either -1 (infinite) or any positive number. ' - 'Got {value}'.format(value=value)) - return value - - INFINITE_RETRIES = -1 - - status = Column(Enum(*STATES), name='status', default=PENDING) - - due_at = Column(DateTime, default=datetime.utcnow) - started_at = Column(DateTime, default=None) - ended_at = Column(DateTime, default=None) - max_attempts = Column(Integer, default=1) - retry_count = Column(Integer, default=0) - retry_interval = Column(Float, default=0) - ignore_failure = Column(Boolean, default=False) - - # Operation specific fields - name = Column(String) - operation_mapping = Column(String) - inputs = Column(Dict) - - @declared_attr - def execution(cls): - return one_to_many_relationship(cls, Execution, cls.execution_id) - - @property - def actor(self): - """ - Return the actor of the task - :return: - """ - return self.node_instance or self.relationship_instance - - @classmethod - def as_node_instance(cls, instance_id, **kwargs): - return cls(node_instance_id=instance_id, **kwargs) - - @classmethod - def as_relationship_instance(cls, instance_id, **kwargs): - return cls(relationship_instance_id=instance_id, **kwargs) +DeclarativeBase = declarative_base(ModelBase) + + +class Blueprint(DeclarativeBase, BlueprintBase, ModelCommon): + pass + + +class Deployment(DeploymentBase, DeclarativeBase, ModelCommon): + pass + + +class Execution(DeclarativeBase, ExecutionBase, ModelCommon): + pass + + +class DeploymentUpdate(DeclarativeBase, DeploymentUpdateBase, ModelCommon): + pass + + +class DeploymentUpdateStep(DeclarativeBase, DeploymentUpdateStepBase, ModelCommon): + pass + + +class DeploymentModification(DeclarativeBase, DeploymentModificationBase, ModelCommon): + pass + + +class Node(DeclarativeBase, NodeBase, ModelCommon): + pass + + +class Relationship(DeclarativeBase, RelationshipBase, ModelCommon): + pass + + +class NodeInstance(DeclarativeBase, NodeInstanceBase, ModelCommon): + pass + + +class RelationshipInstance(DeclarativeBase, RelationshipInstanceBase, ModelCommon): + pass + + +class ProviderContext(ProviderContextBase, DeclarativeBase, ModelCommon): + pass + + +class Plugin(DeclarativeBase, PluginBase, ModelCommon): + pass + + +class Task(DeclarativeBase, TaskBase, ModelCommon): + pass http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/a5eef4be/aria/storage/models_base.py ---------------------------------------------------------------------- diff --git a/aria/storage/models_base.py b/aria/storage/models_base.py new file mode 100644 index 0000000..33eeac6 --- /dev/null +++ b/aria/storage/models_base.py @@ -0,0 +1,629 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Aria's storage.models module +Path: aria.storage.models + +models module holds aria's models. + +classes: + * Field - represents a single field. + * IterField - represents an iterable field. + * Model - abstract model implementation. + * Snapshot - snapshots implementation model. + * Deployment - deployment implementation model. + * DeploymentUpdateStep - deployment update step implementation model. + * DeploymentUpdate - deployment update implementation model. + * DeploymentModification - deployment modification implementation model. + * Execution - execution implementation model. + * Node - node implementation model. + * Relationship - relationship implementation model. + * NodeInstance - node instance implementation model. + * RelationshipInstance - relationship instance implementation model. + * ProviderContext - provider context implementation model. + * Plugin - plugin implementation model. +""" +from collections import namedtuple +from datetime import datetime + +from .structures import ( + Column, + Integer, + Text, + DateTime, + Boolean, + Enum, + String, + Float, + List, + Dict, + foreign_key, + one_to_many_relationship, + relationship_to_self, + orm, + declared_attr, +) + +__all__ = ( + 'ModelCommon', + 'BlueprintBase', + 'DeploymentBase', + 'DeploymentUpdateStepBase', + 'DeploymentUpdateBase', + 'DeploymentModificationBase', + 'ExecutionBase', + 'NodeBase', + 'RelationshipBase', + 'NodeInstanceBase', + 'RelationshipInstanceBase', + 'ProviderContextBase', + 'PluginBase', + 'TaskBase' +) + +#pylint: disable=no-self-argument + + +class ModelCommon(object): + id = Column(Integer, primary_key=True, autoincrement=True) + + +class BlueprintBase(object): + """ + Blueprint model representation. + """ + __tablename__ = 'blueprints' + + name = Column(Text, index=True) + created_at = Column(DateTime, nullable=False, index=True) + main_file_name = Column(Text, nullable=False) + plan = Column(Dict, nullable=False) + updated_at = Column(DateTime) + description = Column(Text) + + +class DeploymentBase(object): + """ + Deployment model representation. + """ + __tablename__ = 'deployments' + + _private_fields = ['blueprint_id'] + + name = Column(Text, index=True) + created_at = Column(DateTime, nullable=False, index=True) + description = Column(Text) + inputs = Column(Dict) + groups = Column(Dict) + permalink = Column(Text) + policy_triggers = Column(Dict) + policy_types = Column(Dict) + outputs = Column(Dict) + scaling_groups = Column(Dict) + updated_at = Column(DateTime) + workflows = Column(Dict) + + @declared_attr + def blueprint_id(cls): + return foreign_key('blueprints.id', nullable=False) + + @declared_attr + def blueprint(cls): + return one_to_many_relationship(cls, 'blueprint_id', 'Blueprint') + + +class ExecutionBase(object): + """ + Execution model representation. + """ + __tablename__ = 'executions' + + TERMINATED = 'terminated' + FAILED = 'failed' + CANCELLED = 'cancelled' + PENDING = 'pending' + STARTED = 'started' + CANCELLING = 'cancelling' + FORCE_CANCELLING = 'force_cancelling' + + STATES = [TERMINATED, FAILED, CANCELLED, PENDING, STARTED, CANCELLING, FORCE_CANCELLING] + END_STATES = [TERMINATED, FAILED, CANCELLED] + ACTIVE_STATES = [state for state in STATES if state not in END_STATES] + + VALID_TRANSITIONS = { + PENDING: [STARTED, CANCELLED], + STARTED: END_STATES + [CANCELLING], + CANCELLING: END_STATES + } + + @orm.validates('status') + def validate_status(self, key, value): + """Validation function that verifies execution status transitions are OK""" + try: + current_status = getattr(self, key) + except AttributeError: + return + valid_transitions = ExecutionBase.VALID_TRANSITIONS.get(current_status, []) + if all([current_status is not None, + current_status != value, + value not in valid_transitions]): + raise ValueError('Cannot change execution status from {current} to {new}'.format( + current=current_status, + new=value)) + return value + + _private_fields = ['deployment_id', 'blueprint_id'] + + created_at = Column(DateTime, index=True) + started_at = Column(DateTime, nullable=True, index=True) + ended_at = Column(DateTime, nullable=True, index=True) + error = Column(Text, nullable=True) + is_system_workflow = Column(Boolean, nullable=False, default=False) + parameters = Column(Dict) + status = Column(Enum(*STATES, name='execution_status'), default=PENDING) + workflow_name = Column(Text, nullable=False) + + @declared_attr + def deployment_id(cls): + return foreign_key('deployments.id') + + @declared_attr + def deployment(cls): + return one_to_many_relationship(cls, 'deployment_id', 'Deployment') + + @declared_attr + def blueprint_id(cls): + return foreign_key('blueprints.id') + + @declared_attr + def blueprint(cls): + return one_to_many_relationship(cls, 'blueprint_id', 'Blueprint') + + def __str__(self): + return '<{0} id=`{1}` (status={2})>'.format( + self.__class__.__name__, + self.id, + self.status + ) + + +class DeploymentUpdateBase(object): + """ + Deployment update model representation. + """ + __tablename__ = 'deployment_updates' + + _private_fields = ['execution_id', 'deployment_id'] + + created_at = Column(DateTime, nullable=False, index=True) + deployment_plan = Column(Dict, nullable=False) + deployment_update_node_instances = Column(Dict) + deployment_update_deployment = Column(Dict) + deployment_update_nodes = Column(Dict) + modified_entity_ids = Column(Dict) + state = Column(Text) + + @declared_attr + def execution_id(cls): + return foreign_key('executions.id', nullable=True) + + @declared_attr + def execution(cls): + return one_to_many_relationship(cls, 'execution_id', 'Execution') + + @declared_attr + def deployment_id(cls): + return foreign_key('deployments.id') + + @declared_attr + def deployment(cls): + return one_to_many_relationship(cls, 'deployment_id', 'Deployment') + + def to_dict(self, suppress_error=False, **kwargs): + dep_update_dict = super(DeploymentUpdateBase, self).to_dict(suppress_error) + # Taking care of the fact the DeploymentSteps are _BaseModels + dep_update_dict['steps'] = [step.to_dict() for step in self.steps] + return dep_update_dict + + +class DeploymentUpdateStepBase(object): + """ + Deployment update step model representation. + """ + __tablename__ = 'deployment_update_steps' + + _action_types = namedtuple('ACTION_TYPES', 'ADD, REMOVE, MODIFY') + ACTION_TYPES = _action_types(ADD='add', REMOVE='remove', MODIFY='modify') + _entity_types = namedtuple( + 'ENTITY_TYPES', + 'NODE, RELATIONSHIP, PROPERTY, OPERATION, WORKFLOW, OUTPUT, DESCRIPTION, GROUP, ' + 'POLICY_TYPE, POLICY_TRIGGER, PLUGIN') + ENTITY_TYPES = _entity_types( + NODE='node', + RELATIONSHIP='relationship', + PROPERTY='property', + OPERATION='operation', + WORKFLOW='workflow', + OUTPUT='output', + DESCRIPTION='description', + GROUP='group', + POLICY_TYPE='policy_type', + POLICY_TRIGGER='policy_trigger', + PLUGIN='plugin' + ) + + _private_fields = ['deployment_update_id'] + + action = Column(Enum(*ACTION_TYPES, name='action_type'), nullable=False) + entity_id = Column(Text, nullable=False) + entity_type = Column(Enum(*ENTITY_TYPES, name='entity_type'), nullable=False) + + @declared_attr + def deployment_update_id(cls): + return foreign_key('deployment_updates.id') + + @declared_attr + def deployment_update(cls): + return one_to_many_relationship(cls, + 'deployment_update_id', + 'DeploymentUpdate', + backreference='steps') + + def __hash__(self): + return hash((self.id, self.entity_id)) + + def __lt__(self, other): + """ + the order is 'remove' < 'modify' < 'add' + :param other: + :return: + """ + if not isinstance(other, self.__class__): + return not self >= other + + if self.action != other.action: + if self.action == 'remove': + return_value = True + elif self.action == 'add': + return_value = False + else: + return_value = other.action == 'add' + return return_value + + if self.action == 'add': + return self.entity_type == 'node' and other.entity_type == 'relationship' + if self.action == 'remove': + return self.entity_type == 'relationship' and other.entity_type == 'node' + return False + + +class DeploymentModificationBase(object): + """ + Deployment modification model representation. + """ + __tablename__ = 'deployment_modifications' + + STARTED = 'started' + FINISHED = 'finished' + ROLLEDBACK = 'rolledback' + + STATES = [STARTED, FINISHED, ROLLEDBACK] + END_STATES = [FINISHED, ROLLEDBACK] + + _private_fields = ['deployment_id'] + + context = Column(Dict) + created_at = Column(DateTime, nullable=False, index=True) + ended_at = Column(DateTime, index=True) + modified_nodes = Column(Dict) + node_instances = Column(Dict) + status = Column(Enum(*STATES, name='deployment_modification_status')) + + @declared_attr + def deployment_id(cls): + return foreign_key('deployments.id') + + @declared_attr + def deployment(cls): + return one_to_many_relationship(cls, + 'deployment_id', + 'Deployment', + backreference='modifications') + + +class NodeBase(object): + """ + Node model representation. + """ + __tablename__ = 'nodes' + + # See base class for an explanation on these properties + is_id_unique = False + + name = Column(Text, index=True) + _private_fields = ['deployment_id', 'host_id'] + + @declared_attr + def host_id(cls): + return foreign_key('nodes.id', nullable=True) + + @declared_attr + def host(cls): + return relationship_to_self(cls, 'host_id') + + @declared_attr + def deployment_id(cls): + return foreign_key('deployments.id') + + @declared_attr + def deployment(cls): + return one_to_many_relationship(cls, 'deployment_id', 'Deployment') + + deploy_number_of_instances = Column(Integer, nullable=False) + max_number_of_instances = Column(Integer, nullable=False) + min_number_of_instances = Column(Integer, nullable=False) + number_of_instances = Column(Integer, nullable=False) + planned_number_of_instances = Column(Integer, nullable=False) + plugins = Column(Dict) + plugins_to_install = Column(Dict) + properties = Column(Dict) + operations = Column(Dict) + type = Column(Text, nullable=False, index=True) + type_hierarchy = Column(List) + + +class RelationshipBase(object): + """ + Relationship model representation. + """ + __tablename__ = 'relationships' + + _private_fields = ['source_node_id', 'target_node_id'] + + @declared_attr + def source_node_id(cls): + return foreign_key('nodes.id') + + @declared_attr + def source_node(cls): + return one_to_many_relationship(cls, + 'source_node_id', + 'Node', + 'outbound_relationships') + + @declared_attr + def target_node_id(cls): + return foreign_key('nodes.id') + + @declared_attr + def target_node(cls): + return one_to_many_relationship(cls, + 'target_node_id', + 'Node', + 'inbound_relationships') + + source_interfaces = Column(Dict) + source_operations = Column(Dict, nullable=False) + target_interfaces = Column(Dict) + target_operations = Column(Dict, nullable=False) + type = Column(String, nullable=False) + type_hierarchy = Column(List) + properties = Column(Dict) + + +class NodeInstanceBase(object): + """ + Node instance model representation. + """ + __tablename__ = 'node_instances' + _private_fields = ['node_id', 'host_id'] + + name = Column(Text, index=True) + runtime_properties = Column(Dict) + scaling_groups = Column(Dict) + state = Column(Text, nullable=False) + version = Column(Integer, default=1) + + @declared_attr + def host_id(cls): + return foreign_key('node_instances.id', nullable=True) + + @declared_attr + def host(cls): + return relationship_to_self(cls, 'host_id') + + @declared_attr + def deployment_id(cls): + return foreign_key('deployments.id') + + @declared_attr + def deployment(cls): + return one_to_many_relationship(cls, 'deployment_id', 'Deployment') + + @declared_attr + def node_id(cls): + return foreign_key('nodes.id') + + @declared_attr + def node(cls): + return one_to_many_relationship(cls, 'node_id', 'Node') + + +class RelationshipInstanceBase(object): + """ + Relationship instance model representation. + """ + __tablename__ = 'relationship_instances' + _private_fields = ['relationship_storage_id', + 'source_node_instance_id', + 'target_node_instance_id'] + + @declared_attr + def source_node_instance_id(cls): + return foreign_key('node_instances.id') + + @declared_attr + def source_node_instance(cls): + return one_to_many_relationship(cls, + 'source_node_instance_id', + 'NodeInstance', + 'outbound_relationship_instances') + + @declared_attr + def target_node_instance_id(cls): + return foreign_key('node_instances.id') + + @declared_attr + def target_node_instance(cls): + return one_to_many_relationship(cls, + 'target_node_instance_id', + 'NodeInstance', + 'inbound_relationship_instances') + + @declared_attr + def relationship_id(cls): + return foreign_key('relationships.id') + + @declared_attr + def relationship(cls): + return one_to_many_relationship(cls, 'relationship_id', 'Relationship') + + +class ProviderContextBase(object): + """ + Provider context model representation. + """ + __tablename__ = 'provider_context' + + name = Column(Text, nullable=False) + context = Column(Dict, nullable=False) + + +class PluginBase(object): + """ + Plugin model representation. + """ + __tablename__ = 'plugins' + + archive_name = Column(Text, nullable=False, index=True) + distribution = Column(Text) + distribution_release = Column(Text) + distribution_version = Column(Text) + excluded_wheels = Column(Dict) + package_name = Column(Text, nullable=False, index=True) + package_source = Column(Text) + package_version = Column(Text) + supported_platform = Column(Dict) + supported_py_versions = Column(Dict) + uploaded_at = Column(DateTime, nullable=False, index=True) + wheels = Column(Dict, nullable=False) + + +class TaskBase(object): + """ + A Model which represents an task + """ + __tablename__ = 'tasks' + _private_fields = ['node_instance_id', + 'relationship_instance_id', + 'execution_id'] + + + @declared_attr + def node_instance_id(cls): + return foreign_key('node_instances.id', nullable=True) + + @declared_attr + def node_instance(cls): + return one_to_many_relationship(cls, 'node_instance_id', 'NodeInstance') + + + @declared_attr + def relationship_instance_id(cls): + return foreign_key('relationship_instances.id', nullable=True) + + @declared_attr + def relationship_instance(cls): + return one_to_many_relationship(cls, + 'relationship_instance_id', + 'RelationshipInstance') + + PENDING = 'pending' + RETRYING = 'retrying' + SENT = 'sent' + STARTED = 'started' + SUCCESS = 'success' + FAILED = 'failed' + STATES = ( + PENDING, + RETRYING, + SENT, + STARTED, + SUCCESS, + FAILED, + ) + + WAIT_STATES = [PENDING, RETRYING] + END_STATES = [SUCCESS, FAILED] + + @orm.validates('max_attempts') + def validate_max_attempts(self, _, value): # pylint: disable=no-self-use + """Validates that max attempts is either -1 or a positive number""" + if value < 1 and value != TaskBase.INFINITE_RETRIES: + raise ValueError('Max attempts can be either -1 (infinite) or any positive number. ' + 'Got {value}'.format(value=value)) + return value + + INFINITE_RETRIES = -1 + + status = Column(Enum(*STATES), name='status', default=PENDING) + + due_at = Column(DateTime, default=datetime.utcnow) + started_at = Column(DateTime, default=None) + ended_at = Column(DateTime, default=None) + max_attempts = Column(Integer, default=1) + retry_count = Column(Integer, default=0) + retry_interval = Column(Float, default=0) + ignore_failure = Column(Boolean, default=False) + + # Operation specific fields + name = Column(String) + operation_mapping = Column(String) + inputs = Column(Dict) + + @declared_attr + def execution_id(cls): + return foreign_key('executions.id', nullable=True) + + @declared_attr + def execution(cls): + return one_to_many_relationship(cls, 'execution_id', 'Execution') + + @property + def actor(self): + """ + Return the actor of the task + :return: + ` """ + return self.node_instance or self.relationship_instance + + @classmethod + def as_node_instance(cls, instance_id, **kwargs): + return cls(node_instance_id=instance_id, **kwargs) + + @classmethod + def as_relationship_instance(cls, instance_id, **kwargs): + return cls(relationship_instance_id=instance_id, **kwargs) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/a5eef4be/aria/storage/structures.py ---------------------------------------------------------------------- diff --git a/aria/storage/structures.py b/aria/storage/structures.py index 8dbd2a9..50f3ad3 100644 --- a/aria/storage/structures.py +++ b/aria/storage/structures.py @@ -30,9 +30,9 @@ import json from sqlalchemy.ext.mutable import Mutable from sqlalchemy.orm import relationship, backref -from sqlalchemy.ext.declarative import declarative_base # pylint: disable=unused-import from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.declarative import declared_attr, declarative_base from sqlalchemy import ( schema, VARCHAR, @@ -49,12 +49,11 @@ from sqlalchemy import ( TypeDecorator, ForeignKey, orm, + Table, ) from aria.storage import exceptions -Model = declarative_base() - def foreign_key(foreign_key_column, nullable=False): """Return a ForeignKey object with the relevant @@ -69,33 +68,77 @@ def foreign_key(foreign_key_column, nullable=False): def one_to_many_relationship(child_class, - parent_class, foreign_key_column, + parent_class, backreference=None): """Return a one-to-many SQL relationship object Meant to be used from inside the *child* object :param parent_class: Class of the parent table :param child_class: Class of the child table - :param foreign_key_column: The column of the foreign key - :param backreference: The name to give to the reference to the child + :param foreign_key_column: The column of the foreign key (from the child table) + :param backreference: The name to give to the reference to the child (on the parent table) """ - backreference = backreference or child_class.__tablename__ + primaryjoin_str = '{parent_class_name}.id == {child_class.__name__}.{foreign_key_column}'\ + .format(parent_class_name=parent_class, + child_class=child_class, + foreign_key_column=foreign_key_column + ) return relationship( parent_class, - primaryjoin=lambda: parent_class.id == foreign_key_column, + primaryjoin=primaryjoin_str, # The following line make sure that when the *parent* is # deleted, all its connected children are deleted as well - backref=backref(backreference, cascade='all') + backref=backref(backreference or child_class.__tablename__, cascade='all') ) -def relationship_to_self(self_cls, parent_key, self_key): - return relationship( - self_cls, - foreign_keys=parent_key, - remote_side=self_key - ) +def relationship_to_self(cls, local_column, remote_column='id'): + primaryjoin_str = '{cls.__name__}.{remote_column} == {cls.__name__}.{local_column}'.format( + cls=cls, + remote_column=remote_column, + local_column=local_column) + return relationship(cls.__name__, primaryjoin=primaryjoin_str) + + +class ModelBase(object): + """ + Abstract base class for all SQL models that allows [de]serialization + """ + + _private_fields = [] + + def to_dict(self, suppress_error=False): + """Return a dict representation of the model + + :param suppress_error: If set to True, sets `None` to attributes that + it's unable to retrieve (e.g., if a relationship wasn't established + yet, and so it's impossible to access a property through it) + """ + if suppress_error: + res = dict() + for field in self.fields(): + try: + field_value = getattr(self, field) + except AttributeError: + field_value = None + res[field] = field_value + else: + # Can't simply call here `self.to_response()` because inheriting + # class might override it, but we always need the same code here + res = dict((f, getattr(self, f)) for f in self.fields()) + return res + + @classmethod + def fields(cls): + """Return the list of field names for this table + + Mostly for backwards compatibility in the code (that uses `fields`) + """ + return set(cls.__table__.columns.keys()) - set(cls._private_fields) + + def __repr__(self): + return '<{0} id=`{1}`>'.format(self.__class__.__name__, self.id) class _MutableType(TypeDecorator): @@ -195,50 +238,3 @@ class _MutableList(Mutable, list): Dict = _MutableDict.as_mutable(_DictType) List = _MutableList.as_mutable(_ListType) - - -class SQLModelBase(Model): - """ - Abstract base class for all SQL models that allows [de]serialization - """ - # SQLAlchemy syntax - __abstract__ = True - - # This would be overridden once the models are created. Created for pylint. - __table__ = None - - _private_fields = [] - - id = Column(Integer, primary_key=True, autoincrement=True) - - def to_dict(self, suppress_error=False): - """Return a dict representation of the model - - :param suppress_error: If set to True, sets `None` to attributes that - it's unable to retrieve (e.g., if a relationship wasn't established - yet, and so it's impossible to access a property through it) - """ - if suppress_error: - res = dict() - for field in self.fields(): - try: - field_value = getattr(self, field) - except AttributeError: - field_value = None - res[field] = field_value - else: - # Can't simply call here `self.to_response()` because inheriting - # class might override it, but we always need the same code here - res = dict((f, getattr(self, f)) for f in self.fields()) - return res - - @classmethod - def fields(cls): - """Return the list of field names for this table - - Mostly for backwards compatibility in the code (that uses `fields`) - """ - return set(cls.__table__.columns.keys()) - set(cls._private_fields) - - def __repr__(self): - return '<{0} id=`{1}`>'.format(self.__class__.__name__, self.id) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/a5eef4be/tests/storage/__init__.py ---------------------------------------------------------------------- diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index edff982..8f9dda8 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -17,6 +17,7 @@ import platform from tempfile import mkdtemp from shutil import rmtree +from aria.storage import models from sqlalchemy import ( create_engine, orm) @@ -60,7 +61,7 @@ def get_sqlite_api_kwargs(base_dir=None, filename='db.sqlite'): session_factory = orm.sessionmaker(bind=engine) session = scoped_session(session_factory=session_factory) if base_dir else session_factory() - structures.Model.metadata.create_all(engine) + models.DeclarativeBase.metadata.create_all(bind=engine) return dict(engine=engine, session=session) @@ -77,4 +78,4 @@ def release_sqlite_storage(storage): session.rollback() session.close() for engine in set(mapi._engine for mapi in mapis): - structures.Model.metadata.drop_all(engine) + models.DeclarativeBase.metadata.drop_all(engine) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/a5eef4be/tests/storage/test_model_storage.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py index 48cd02c..e5aee5a 100644 --- a/tests/storage/test_model_storage.py +++ b/tests/storage/test_model_storage.py @@ -58,23 +58,6 @@ def test_model_storage(storage): storage.provider_context.get(pc.id) -def test_storage_driver(storage): - storage.register(models.ProviderContext) - - pc = models.ProviderContext(context={}, name='context_name') - storage.registered['provider_context'].put(entry=pc) - - assert storage.registered['provider_context'].get_by_name('context_name') == pc - - assert next(i for i in storage.registered['provider_context'].iter()) == pc - assert [i for i in storage.provider_context] == [pc] - - storage.registered['provider_context'].delete(pc) - - with pytest.raises(exceptions.StorageError): - storage.registered['provider_context'].get(pc.id) - - def test_application_storage_factory(): storage = application_model_storage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs()) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/a5eef4be/tests/storage/test_models.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_models.py b/tests/storage/test_models.py index 0ae5d1c..454a003 100644 --- a/tests/storage/test_models.py +++ b/tests/storage/test_models.py @@ -692,24 +692,6 @@ class TestRelationshipInstance(object): assert relationship_instance.target_node_instance == target_node_instance -class TestProviderContext(object): - @pytest.mark.parametrize( - 'is_valid, name, context', - [ - (False, None, {}), - (False, 'name', None), - (True, 'name', {}), - ] - ) - def test_provider_context_model_creation(self, empty_storage, is_valid, name, context): - _test_model(is_valid=is_valid, - storage=empty_storage, - model_name='provider_context', - model_cls=ProviderContext, - model_kwargs=dict(name=name, context=context) - ) - - class TestPlugin(object): @pytest.mark.parametrize( 'is_valid, archive_name, distribution, distribution_release, ' @@ -860,7 +842,7 @@ class TestTask(object): def test_inner_dict_update(empty_storage): inner_dict = {'inner_value': 1} pc = ProviderContext(name='name', context={ - 'inner_dict': {'inner_value': inner_dict}, + 'inner_dict': inner_dict, 'value': 0 }) empty_storage.provider_context.put(pc)