Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 9C059200C0F for ; Thu, 2 Feb 2017 09:47:44 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 9A8F5160B54; Thu, 2 Feb 2017 08:47:44 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 411CA160B57 for ; Thu, 2 Feb 2017 09:47:41 +0100 (CET) Received: (qmail 11918 invoked by uid 500); 2 Feb 2017 08:47:40 -0000 Mailing-List: contact dev-help@ariatosca.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ariatosca.incubator.apache.org Delivered-To: mailing list dev@ariatosca.incubator.apache.org Received: (qmail 11907 invoked by uid 99); 2 Feb 2017 08:47:40 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 02 Feb 2017 08:47:40 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 21EF21A0465 for ; Thu, 2 Feb 2017 08:47:40 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -5.22 X-Spam-Level: X-Spam-Status: No, score=-5.22 tagged_above=-999 required=6.31 tests=[HK_RANDOM_FROM=0.999, KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id bkOtJJeoSvLt for ; Thu, 2 Feb 2017 08:47:36 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id 7409D5FBB2 for ; Thu, 2 Feb 2017 08:47:33 +0000 (UTC) Received: (qmail 11775 invoked by uid 99); 2 Feb 2017 08:47:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 02 Feb 2017 08:47:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 064F0DFF6D; Thu, 2 Feb 2017 08:47:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mxmrlv@apache.org To: dev@ariatosca.incubator.apache.org Date: Thu, 02 Feb 2017 08:47:33 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/3] incubator-ariatosca git commit: wip archived-at: Thu, 02 Feb 2017 08:47:44 -0000 wip Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/99ae9ca6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/99ae9ca6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/99ae9ca6 Branch: refs/heads/ARIA-42-Generic-ctx-serialization-mechanism Commit: 99ae9ca63597ee7bb99eaa33c76ccce7f1192d4d Parents: 4cfbe34 Author: mxmrlv Authored: Wed Feb 1 16:16:01 2017 +0200 Committer: mxmrlv Committed: Thu Feb 2 10:47:17 2017 +0200 ---------------------------------------------------------------------- aria/__init__.py | 14 ++----- aria/orchestrator/context/serialization.py | 43 ++++---------------- aria/orchestrator/runner.py | 2 +- aria/storage/core.py | 8 ++-- aria/storage/sql_mapi.py | 42 +++++++++++++++++++ tests/mock/context.py | 19 ++++----- .../context/test_resource_render.py | 2 +- tests/orchestrator/context/test_serialize.py | 18 ++++---- tests/orchestrator/context/test_workflow.py | 2 +- .../orchestrator/execution_plugin/test_local.py | 2 +- tests/orchestrator/execution_plugin/test_ssh.py | 2 +- .../workflows/executor/test_process_executor.py | 2 +- tests/storage/test_model_storage.py | 2 +- tests/storage/test_models.py | 2 +- tests/utils/test_plugin.py | 2 +- 15 files changed, 82 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index 248aa1a..37cec1a 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -57,7 +57,7 @@ def install_aria_extensions(): extension.init() -def application_model_storage(api, api_kwargs=None): +def application_model_storage(api, driver_kwargs=None): """ Initiate model storage """ @@ -79,18 +79,12 @@ def application_model_storage(api, api_kwargs=None): storage.model.Task, ] # if api not in _model_storage: - return storage.ModelStorage(api, items=models, api_kwargs=api_kwargs or {}) + return storage.ModelStorage(api, items=models, driver_kwargs=driver_kwargs or {}) -def application_resource_storage(api, api_kwargs=None): +def application_resource_storage(api, driver_kwargs=None): """ Initiate resource storage """ return storage.ResourceStorage( - api, - api_kwargs=api_kwargs or {}, - items=[ - 'blueprint', - 'deployment', - 'plugin', - ]) + api, driver_kwargs=driver_kwargs or {}, items=['blueprint', 'deployment', 'plugin', ]) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/orchestrator/context/serialization.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/context/serialization.py b/aria/orchestrator/context/serialization.py index 760818f..c583a71 100644 --- a/aria/orchestrator/context/serialization.py +++ b/aria/orchestrator/context/serialization.py @@ -13,9 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sqlalchemy.orm -import sqlalchemy.pool - import aria @@ -32,7 +29,7 @@ def operation_context_to_dict(context): model = context.model context_dict['model_storage'] = { 'api_cls': model.api, - 'api_kwargs': _serialize_sql_mapi_kwargs(model) + 'driver_kwargs': model._driver_kwargs, } else: context_dict['model_storage'] = None @@ -40,7 +37,7 @@ def operation_context_to_dict(context): resource = context.resource context_dict['resource_storage'] = { 'api_cls': resource.api, - 'api_kwargs': _serialize_file_rapi_kwargs(resource) + 'driver_kwargs': resource._driver_kwargs } else: context_dict['resource_storage'] = None @@ -57,39 +54,15 @@ def operation_context_from_dict(context_dict): model_storage = context['model_storage'] if model_storage: api_cls = model_storage['api_cls'] - api_kwargs = _deserialize_sql_mapi_kwargs(model_storage.get('api_kwargs', {})) - context['model_storage'] = aria.application_model_storage(api=api_cls, - api_kwargs=api_kwargs) + driver_kwargs = model_storage['driver_kwargs'] + context['model_storage'] = aria.application_model_storage( + api_cls, driver_kwargs=driver_kwargs) resource_storage = context['resource_storage'] if resource_storage: api_cls = resource_storage['api_cls'] - api_kwargs = _deserialize_file_rapi_kwargs(resource_storage.get('api_kwargs', {})) - context['resource_storage'] = aria.application_resource_storage(api=api_cls, - api_kwargs=api_kwargs) + driver_kwargs = resource_storage['driver_kwargs'] + context['resource_storage'] = aria.application_resource_storage( + api=api_cls, driver_kwargs=driver_kwargs) return context_cls(**context) - - -def _serialize_sql_mapi_kwargs(model): - engine_url = str(model._api_kwargs['engine'].url) - assert ':memory:' not in engine_url - return {'engine_url': engine_url} - - -def _deserialize_sql_mapi_kwargs(api_kwargs): - engine_url = api_kwargs.get('engine_url') - if not engine_url: - return {} - engine = sqlalchemy.create_engine(engine_url) - session_factory = sqlalchemy.orm.sessionmaker(bind=engine) - session = sqlalchemy.orm.scoped_session(session_factory=session_factory) - return {'session': session, 'engine': engine} - - -def _serialize_file_rapi_kwargs(resource): - return {'directory': resource._api_kwargs['directory']} - - -def _deserialize_file_rapi_kwargs(api_kwargs): - return api_kwargs http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/orchestrator/runner.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/runner.py b/aria/orchestrator/runner.py index 16acc19..b571b34 100644 --- a/aria/orchestrator/runner.py +++ b/aria/orchestrator/runner.py @@ -120,7 +120,7 @@ class Runner(object): sqlite_kwargs = dict(engine=sqlite_engine, session=sqlite_session) return application_model_storage( SQLAlchemyModelAPI, - api_kwargs=sqlite_kwargs) + driver_kwargs=sqlite_kwargs) def create_fs_resource_storage(self, directory='.'): # pylint: disable=no-self-use fs_kwargs = dict(directory=directory) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/storage/core.py ---------------------------------------------------------------------- diff --git a/aria/storage/core.py b/aria/storage/core.py index 94b4fe0..d46afaa 100644 --- a/aria/storage/core.py +++ b/aria/storage/core.py @@ -52,10 +52,10 @@ class Storage(LoggerMixin): """ Represents the storage """ - def __init__(self, api_cls, api_kwargs=None, items=(), **kwargs): + def __init__(self, api_cls, driver_kwargs=None, items=(), **kwargs): super(Storage, self).__init__(**kwargs) self.api = api_cls - self._api_kwargs = api_kwargs or {} + self._driver_kwargs = driver_kwargs or {} self.registered = {} for item in items: self.register(item) @@ -90,7 +90,7 @@ class ResourceStorage(Storage): :param name: :return: """ - self.registered[name] = self.api(name=name, **self._api_kwargs) + self.registered[name] = self.api(name=name, **self._driver_kwargs) self.registered[name].create() self.logger.debug('setup {name} in storage {self!r}'.format(name=name, self=self)) @@ -112,7 +112,7 @@ class ModelStorage(Storage): return self.registered[model_name] = self.api(name=model_name, model_cls=model_cls, - **self._api_kwargs) + **self._driver_kwargs) self.registered[model_name].create() self.logger.debug('setup {name} in storage {self!r}'.format(name=model_name, self=self)) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/storage/sql_mapi.py ---------------------------------------------------------------------- diff --git a/aria/storage/sql_mapi.py b/aria/storage/sql_mapi.py index 809f677..3982eaa 100644 --- a/aria/storage/sql_mapi.py +++ b/aria/storage/sql_mapi.py @@ -15,7 +15,13 @@ """ SQLAlchemy based MAPI """ +import os +import platform +from functools import partial +from sqlalchemy import create_engine +from sqlalchemy import orm +from sqlalchemy import pool from sqlalchemy.exc import SQLAlchemyError from aria.utils.collections import OrderedDict @@ -25,11 +31,47 @@ from aria.storage import ( ) +def init_storage(init_func=None): + if init_func is None: + return partial(init_storage, func=init_storage) + + track = { + 'engine': None, + 'session': None + } + + def _wrapper(self, base_dir=None, filename='db.sqlite', *args, **kwargs): + if not (track['engine'] and track['session']): + if base_dir is not None: + uri = 'sqlite:///{platform_char}{path}'.format( + # Handles the windows behavior where there is not root, but drivers. + # Thus behaving as relative path. + platform_char='' if 'Windows' in platform.system() else '/', + + path=os.path.join(base_dir, filename)) + engine_kwargs = {} + else: + uri = 'sqlite:///:memory:' + engine_kwargs = dict(connect_args={'check_same_thread': False}, + poolclass=pool.StaticPool) + + track['engine'] = create_engine(uri, **engine_kwargs) + session_factory = orm.sessionmaker(bind=track['engine']) + track['session'] = orm.scoped_session(session_factory=session_factory) if base_dir else \ + session_factory() + + return init_func( + self=self, engine=track['engine'], session=track['session'], *args, **kwargs) + + return _wrapper + + class SQLAlchemyModelAPI(api.ModelAPI): """ SQL based MAPI. """ + @init_storage def __init__(self, engine, session, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/mock/context.py ---------------------------------------------------------------------- diff --git a/tests/mock/context.py b/tests/mock/context.py index ec4bfb8..4fbae14 100644 --- a/tests/mock/context.py +++ b/tests/mock/context.py @@ -22,19 +22,14 @@ from . import models from .topology import create_simple_topology_two_nodes -def simple(mapi_kwargs, resources_dir=None, **kwargs): - model_storage = aria.application_model_storage(SQLAlchemyModelAPI, api_kwargs=mapi_kwargs) +def simple(model_driver_kwargs=None, resources_driver_kwargs=None, context_kwargs=None): + model_storage = aria.application_model_storage( + SQLAlchemyModelAPI, driver_kwargs=model_driver_kwargs or {}) - deployment_id = create_simple_topology_two_nodes(model_storage) + resource_storage = aria.application_resource_storage( + FileSystemResourceAPI, driver_kwargs=resources_driver_kwargs or {}) - # pytest tmpdir - if resources_dir: - resource_storage = aria.application_resource_storage( - FileSystemResourceAPI, - api_kwargs={'directory': resources_dir} - ) - else: - resource_storage = None + deployment_id = create_simple_topology_two_nodes(model_storage) final_kwargs = dict( name='simple_context', @@ -45,5 +40,5 @@ def simple(mapi_kwargs, resources_dir=None, **kwargs): task_max_attempts=models.TASK_MAX_ATTEMPTS, task_retry_interval=models.TASK_RETRY_INTERVAL ) - final_kwargs.update(kwargs) + final_kwargs.update(context_kwargs or {}) return context.workflow.WorkflowContext(**final_kwargs) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/context/test_resource_render.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_resource_render.py b/tests/orchestrator/context/test_resource_render.py index ca2ef42..a5e4fa8 100644 --- a/tests/orchestrator/context/test_resource_render.py +++ b/tests/orchestrator/context/test_resource_render.py @@ -54,7 +54,7 @@ def test_download_resource_and_render_provided_variables(tmpdir, ctx): @pytest.fixture def ctx(tmpdir): context = mock.context.simple(storage.get_sqlite_api_kwargs(), - resources_dir=str(tmpdir.join('resources'))) + resources_driver_kwargs=str(tmpdir.join('resources'))) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/context/test_serialize.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_serialize.py b/tests/orchestrator/context/test_serialize.py index 76930b1..8d4cb1f 100644 --- a/tests/orchestrator/context/test_serialize.py +++ b/tests/orchestrator/context/test_serialize.py @@ -16,13 +16,12 @@ import pytest import aria +from aria.storage import model from aria.storage.sql_mapi import SQLAlchemyModelAPI from aria.orchestrator.workflows import api from aria.orchestrator.workflows.core import engine from aria.orchestrator.workflows.executor import process from aria.orchestrator import workflow, operation -from aria.orchestrator.context import serialization - import tests from tests import mock from tests import storage @@ -42,10 +41,6 @@ def test_serialize_operation_context(context, executor, tmpdir): eng.execute() -def test_illegal_serialize_of_memory_model_storage(memory_model_storage): - with pytest.raises(AssertionError): - serialization._serialize_sql_mapi_kwargs(memory_model_storage) - @workflow def _mock_workflow(ctx, graph): @@ -93,9 +88,12 @@ def executor(): @pytest.fixture def context(tmpdir): - result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)), - resources_dir=str(tmpdir.join('resources')), - workdir=str(tmpdir.join('workdir'))) + result = mock.context.simple( + model_driver_kwargs=dict(base_dir=str(tmpdir)), + resources_driver_kwargs=dict(directory=str(tmpdir.join('resources'))), + context_kwargs=dict(workdir=str(tmpdir.join('workdir'))) + ) + yield result storage.release_sqlite_storage(result.model) @@ -103,6 +101,6 @@ def context(tmpdir): @pytest.fixture def memory_model_storage(): result = aria.application_model_storage( - SQLAlchemyModelAPI, api_kwargs=storage.get_sqlite_api_kwargs()) + SQLAlchemyModelAPI, driver_kwargs=storage.get_sqlite_api_kwargs()) yield result storage.release_sqlite_storage(result) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/context/test_workflow.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py index 496c1ff..13e7639 100644 --- a/tests/orchestrator/context/test_workflow.py +++ b/tests/orchestrator/context/test_workflow.py @@ -61,7 +61,7 @@ class TestWorkflowContext(object): @pytest.fixture(scope='function') def storage(): api_kwargs = test_storage.get_sqlite_api_kwargs() - workflow_storage = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) + workflow_storage = application_model_storage(SQLAlchemyModelAPI, driver_kwargs=api_kwargs) workflow_storage.blueprint.put(models.get_blueprint()) blueprint = workflow_storage.blueprint.get_by_name(models.BLUEPRINT_NAME) workflow_storage.deployment.put(models.get_deployment(blueprint)) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/execution_plugin/test_local.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/execution_plugin/test_local.py b/tests/orchestrator/execution_plugin/test_local.py index 497da48..30447e2 100644 --- a/tests/orchestrator/execution_plugin/test_local.py +++ b/tests/orchestrator/execution_plugin/test_local.py @@ -505,7 +505,7 @@ if __name__ == '__main__': def workflow_context(self, tmpdir): workflow_context = mock.context.simple( storage.get_sqlite_api_kwargs(str(tmpdir)), - resources_dir=str(tmpdir.join('resources'))) + resources_driver_kwargs=str(tmpdir.join('resources'))) workflow_context.states = [] workflow_context.exception = None yield workflow_context http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/execution_plugin/test_ssh.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/execution_plugin/test_ssh.py b/tests/orchestrator/execution_plugin/test_ssh.py index 6b5c783..5d21ca4 100644 --- a/tests/orchestrator/execution_plugin/test_ssh.py +++ b/tests/orchestrator/execution_plugin/test_ssh.py @@ -267,7 +267,7 @@ class TestWithActualSSHServer(object): def workflow_context(self, tmpdir): workflow_context = mock.context.simple( storage.get_sqlite_api_kwargs(str(tmpdir)), - resources_dir=str(tmpdir.join('resources'))) + resources_driver_kwargs=str(tmpdir.join('resources'))) workflow_context.states = [] workflow_context.exception = None yield workflow_context http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/workflows/executor/test_process_executor.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/executor/test_process_executor.py b/tests/orchestrator/workflows/executor/test_process_executor.py index 687e245..bf910db 100644 --- a/tests/orchestrator/workflows/executor/test_process_executor.py +++ b/tests/orchestrator/workflows/executor/test_process_executor.py @@ -75,7 +75,7 @@ class TestProcessExecutor(object): @pytest.fixture def model(tmpdir): api_kwargs = tests.storage.get_sqlite_api_kwargs(str(tmpdir)) - result = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) + result = application_model_storage(SQLAlchemyModelAPI, driver_kwargs=api_kwargs) yield result tests.storage.release_sqlite_storage(result) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/storage/test_model_storage.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py index d1596e3..09d59c4 100644 --- a/tests/storage/test_model_storage.py +++ b/tests/storage/test_model_storage.py @@ -61,7 +61,7 @@ def test_model_storage(storage): def test_application_storage_factory(): storage = application_model_storage(sql_mapi.SQLAlchemyModelAPI, - api_kwargs=get_sqlite_api_kwargs()) + driver_kwargs=get_sqlite_api_kwargs()) assert storage.node assert storage.node_instance assert storage.plugin http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/storage/test_models.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_models.py b/tests/storage/test_models.py index 2088676..56516f1 100644 --- a/tests/storage/test_models.py +++ b/tests/storage/test_models.py @@ -55,7 +55,7 @@ def sql_storage(storage_func): def _empty_storage(): return application_model_storage(sql_mapi.SQLAlchemyModelAPI, - api_kwargs=get_sqlite_api_kwargs()) + driver_kwargs=get_sqlite_api_kwargs()) def _blueprint_storage(): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/utils/test_plugin.py ---------------------------------------------------------------------- diff --git a/tests/utils/test_plugin.py b/tests/utils/test_plugin.py index 6f2dd92..e80cb87 100644 --- a/tests/utils/test_plugin.py +++ b/tests/utils/test_plugin.py @@ -50,7 +50,7 @@ class TestPluginManager(object): @pytest.fixture def model(): api_kwargs = storage.get_sqlite_api_kwargs() - model = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) + model = application_model_storage(SQLAlchemyModelAPI, driver_kwargs=api_kwargs) yield model storage.release_sqlite_storage(model)