ariatosca-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mxm...@apache.org
Subject incubator-ariatosca git commit: ARIA-42-Generic-ctx-serialization-mechanism [Forced Update!]
Date Mon, 06 Feb 2017 16:10:55 GMT
Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-42-Generic-ctx-serialization-mechanism c079925c3 -> 53344f781 (forced
update)


ARIA-42-Generic-ctx-serialization-mechanism


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/53344f78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/53344f78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/53344f78

Branch: refs/heads/ARIA-42-Generic-ctx-serialization-mechanism
Commit: 53344f781cf0ab836e1064625e176d5570bb9eb2
Parents: 4cfbe34
Author: mxmrlv <mxmrlv@gmail.com>
Authored: Wed Feb 1 16:16:01 2017 +0200
Committer: mxmrlv <mxmrlv@gmail.com>
Committed: Mon Feb 6 18:10:21 2017 +0200

----------------------------------------------------------------------
 aria/__init__.py                                | 16 +++----
 aria/orchestrator/context/serialization.py      | 49 ++++++--------------
 aria/orchestrator/runner.py                     |  8 +---
 aria/orchestrator/workflows/executor/process.py |  6 ++-
 aria/storage/api.py                             | 36 +++++++++++++-
 aria/storage/sql_mapi.py                        | 41 +++++++++++++++-
 setup.py                                        |  2 +-
 tests/__init__.py                               |  4 ++
 tests/mock/context.py                           | 25 +++++-----
 tests/orchestrator/context/test_operation.py    |  7 ++-
 .../context/test_resource_render.py             |  3 +-
 tests/orchestrator/context/test_serialize.py    | 20 ++++----
 tests/orchestrator/context/test_toolbelt.py     |  4 +-
 tests/orchestrator/context/test_workflow.py     |  6 +--
 .../orchestrator/execution_plugin/test_local.py |  3 +-
 tests/orchestrator/execution_plugin/test_ssh.py |  5 +-
 tests/orchestrator/test_runner.py               |  3 +-
 tests/orchestrator/workflows/api/test_task.py   |  4 +-
 .../workflows/builtin/test_execute_operation.py |  3 +-
 .../orchestrator/workflows/builtin/test_heal.py |  3 +-
 .../workflows/builtin/test_install.py           |  3 +-
 .../workflows/builtin/test_uninstall.py         |  3 +-
 .../orchestrator/workflows/core/test_engine.py  |  4 +-
 tests/orchestrator/workflows/core/test_task.py  |  4 +-
 .../test_task_graph_into_exececution_graph.py   |  4 +-
 .../workflows/executor/test_process_executor.py |  7 +--
 .../executor/test_process_executor_extension.py |  3 +-
 .../test_process_executor_tracked_changes.py    |  3 +-
 tests/storage/__init__.py                       | 31 +------------
 tests/storage/test_instrumentation.py           |  4 +-
 tests/storage/test_model_storage.py             |  7 +--
 tests/storage/test_models.py                    |  4 +-
 tests/storage/test_structures.py                | 12 +++--
 tests/utils/test_plugin.py                      |  6 +--
 34 files changed, 188 insertions(+), 155 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/aria/__init__.py
----------------------------------------------------------------------
diff --git a/aria/__init__.py b/aria/__init__.py
index 248aa1a..da77f03 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -57,7 +57,7 @@ def install_aria_extensions():
     extension.init()
 
 
-def application_model_storage(api, api_kwargs=None):
+def application_model_storage(api, api_kwargs=None, storage_initiator_func=None):
     """
     Initiate model storage
     """
@@ -78,19 +78,15 @@ def application_model_storage(api, api_kwargs=None):
         storage.model.Execution,
         storage.model.Task,
     ]
-    # if api not in _model_storage:
+    api.storage_initiator(storage_initiator_func)
     return storage.ModelStorage(api, items=models, api_kwargs=api_kwargs or {})
 
 
-def application_resource_storage(api, api_kwargs=None):
+def application_resource_storage(api, api_kwargs=None, storage_initiator_func=None):
     """
     Initiate resource storage
     """
+
+    api.storage_initiator(storage_initiator_func)
     return storage.ResourceStorage(
-        api,
-        api_kwargs=api_kwargs or {},
-        items=[
-            'blueprint',
-            'deployment',
-            'plugin',
-        ])
+        api, api_kwargs=api_kwargs or {}, items=['blueprint', 'deployment', 'plugin', ])

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/aria/orchestrator/context/serialization.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/serialization.py b/aria/orchestrator/context/serialization.py
index 760818f..0810541 100644
--- a/aria/orchestrator/context/serialization.py
+++ b/aria/orchestrator/context/serialization.py
@@ -13,9 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sqlalchemy.orm
-import sqlalchemy.pool
-
 import aria
 
 
@@ -32,15 +29,19 @@ def operation_context_to_dict(context):
         model = context.model
         context_dict['model_storage'] = {
             'api_cls': model.api,
-            'api_kwargs': _serialize_sql_mapi_kwargs(model)
+            'api_kwargs': model._api_kwargs,
+            'init_func': model.api._init_func.__func__ if hasattr(model.api, '_init_func')
else None
         }
+
     else:
         context_dict['model_storage'] = None
     if context.resource:
         resource = context.resource
         context_dict['resource_storage'] = {
             'api_cls': resource.api,
-            'api_kwargs': _serialize_file_rapi_kwargs(resource)
+            'api_kwargs': resource._api_kwargs,
+            'init_func': resource.api._init_func.__func__
+                         if hasattr(resource.api, '_init_func') else None
         }
     else:
         context_dict['resource_storage'] = None
@@ -57,39 +58,17 @@ def operation_context_from_dict(context_dict):
     model_storage = context['model_storage']
     if model_storage:
         api_cls = model_storage['api_cls']
-        api_kwargs = _deserialize_sql_mapi_kwargs(model_storage.get('api_kwargs', {}))
-        context['model_storage'] = aria.application_model_storage(api=api_cls,
-                                                                  api_kwargs=api_kwargs)
+        api_kwargs = model_storage['api_kwargs']
+        init_func = model_storage['init_func']
+        context['model_storage'] = aria.application_model_storage(
+            api_cls, api_kwargs=api_kwargs, storage_initiator_func=init_func)
 
     resource_storage = context['resource_storage']
     if resource_storage:
         api_cls = resource_storage['api_cls']
-        api_kwargs = _deserialize_file_rapi_kwargs(resource_storage.get('api_kwargs', {}))
-        context['resource_storage'] = aria.application_resource_storage(api=api_cls,
-                                                                        api_kwargs=api_kwargs)
+        api_kwargs = resource_storage['api_kwargs']
+        init_func = resource_storage['init_func']
+        context['resource_storage'] = aria.application_resource_storage(
+            api=api_cls, api_kwargs=api_kwargs, storage_initiator_func=init_func)
 
     return context_cls(**context)
-
-
-def _serialize_sql_mapi_kwargs(model):
-    engine_url = str(model._api_kwargs['engine'].url)
-    assert ':memory:' not in engine_url
-    return {'engine_url': engine_url}
-
-
-def _deserialize_sql_mapi_kwargs(api_kwargs):
-    engine_url = api_kwargs.get('engine_url')
-    if not engine_url:
-        return {}
-    engine = sqlalchemy.create_engine(engine_url)
-    session_factory = sqlalchemy.orm.sessionmaker(bind=engine)
-    session = sqlalchemy.orm.scoped_session(session_factory=session_factory)
-    return {'session': session, 'engine': engine}
-
-
-def _serialize_file_rapi_kwargs(resource):
-    return {'directory': resource._api_kwargs['directory']}
-
-
-def _deserialize_file_rapi_kwargs(api_kwargs):
-    return api_kwargs

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/aria/orchestrator/runner.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/runner.py b/aria/orchestrator/runner.py
index 16acc19..7892d67 100644
--- a/aria/orchestrator/runner.py
+++ b/aria/orchestrator/runner.py
@@ -118,15 +118,11 @@ class Runner(object):
 
         # Storage
         sqlite_kwargs = dict(engine=sqlite_engine, session=sqlite_session)
-        return application_model_storage(
-            SQLAlchemyModelAPI,
-            api_kwargs=sqlite_kwargs)
+        return application_model_storage(SQLAlchemyModelAPI, api_kwargs=sqlite_kwargs)
 
     def create_fs_resource_storage(self, directory='.'): # pylint: disable=no-self-use
         fs_kwargs = dict(directory=directory)
-        return application_resource_storage(
-            FileSystemResourceAPI,
-            api_kwargs=fs_kwargs)
+        return application_resource_storage(FileSystemResourceAPI, api_kwargs=fs_kwargs)
 
     def cleanup(self):
         if self._is_storage_temporary \

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index 7d990fa..23a4f52 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -37,6 +37,7 @@ import struct
 import subprocess
 import tempfile
 import Queue
+import pickle
 
 import jsonpickle
 
@@ -113,7 +114,7 @@ class ProcessExecutor(base.BaseExecutor):
         file_descriptor, arguments_json_path = tempfile.mkstemp(prefix='executor-', suffix='.json')
         os.close(file_descriptor)
         with open(arguments_json_path, 'wb') as f:
-            f.write(jsonpickle.dumps(self._create_arguments_dict(task)))
+            f.write(pickle.dumps(self._create_arguments_dict(task)))
 
         env = os.environ.copy()
         # See _update_env for plugin_prefix usage
@@ -306,7 +307,7 @@ def _patch_session(ctx, messenger, instrument):
 def _main():
     arguments_json_path = sys.argv[1]
     with open(arguments_json_path) as f:
-        arguments = jsonpickle.loads(f.read())
+        arguments = pickle.loads(f.read())
 
     # arguments_json_path is a temporary file created by the parent process.
     # so we remove it here
@@ -338,5 +339,6 @@ def _main():
         except BaseException as e:
             messenger.failed(exception=e, tracked_changes=instrument.tracked_changes)
 
+
 if __name__ == '__main__':
     _main()

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/aria/storage/api.py
----------------------------------------------------------------------
diff --git a/aria/storage/api.py b/aria/storage/api.py
index d6fc3b8..5f6b862 100644
--- a/aria/storage/api.py
+++ b/aria/storage/api.py
@@ -15,9 +15,12 @@
 """
 General storage API
 """
+from functools import partial
 
 
 class StorageAPI(object):
+    NO_INITIATOR = 'no_initiator_func'
+
     """
     General storage Base API
     """
@@ -29,6 +32,36 @@ class StorageAPI(object):
         """
         raise NotImplementedError('Subclass must implement abstract create method')
 
+    @classmethod
+    def storage_initiator(cls, func=NO_INITIATOR):
+        if func is cls.NO_INITIATOR:
+            # Support for decoration syntax
+            return partial(cls.storage_initiator, cls=cls)
+
+        if func is None:
+            # If func is None, no storage initiator was set, and nothing should happen.
+            return
+
+        if hasattr(cls, '_original_init') and cls._init_func.__func__ == func:          
           # pylint: disable=no-member
+            # setting the same function twice could cause a loop. we need to avoid that.
+            return
+
+        cls._original_init = cls.__init__
+        cls._init_func = func
+        cls.__init__ = \
+            lambda self, *args, **kwargs: cls._original_init(self, **func(cls=cls, *args,
**kwargs))
+
+    @classmethod
+    def free_storage_initiator(cls):
+        if hasattr(cls, '_original_init') and cls.__init__ != cls._original_init:
+            cls.__init__ = cls._original_init
+            delattr(cls, '_original_init')
+
+        if hasattr(cls, '_engine'):
+            delattr(cls, '_engine')
+        if hasattr(cls, '_session'):
+            delattr(cls, '_session')
+
 
 class ModelAPI(StorageAPI):
     """
@@ -119,11 +152,12 @@ class ResourceAPI(StorageAPI):
     """
     A Base object for the resource.
     """
-    def __init__(self, name):
+    def __init__(self, name, **kwargs):
         """
         Base resource API
         :param str name: the resource type
         """
+        super(ResourceAPI, self).__init__(**kwargs)
         self._name = name
 
     @property

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/aria/storage/sql_mapi.py
----------------------------------------------------------------------
diff --git a/aria/storage/sql_mapi.py b/aria/storage/sql_mapi.py
index 809f677..2ce8998 100644
--- a/aria/storage/sql_mapi.py
+++ b/aria/storage/sql_mapi.py
@@ -15,13 +15,18 @@
 """
 SQLAlchemy based MAPI
 """
+import os
+import platform
 
+from sqlalchemy import create_engine
+from sqlalchemy import orm
+from sqlalchemy import pool
 from sqlalchemy.exc import SQLAlchemyError
 
 from aria.utils.collections import OrderedDict
-from aria.storage import (
+from . import (
     api,
-    exceptions
+    exceptions,
 )
 
 
@@ -364,6 +369,38 @@ class SQLAlchemyModelAPI(api.ModelAPI):
             getattr(instance, rel.key)
 
 
+def init_storage(cls, base_dir=None, filename='db.sqlite', **kwargs):
+
+    _storage_kwargs = {
+        'base_dir': base_dir,
+        'filename': filename
+    }
+    _storage_kwargs.update(**kwargs)
+
+    if not hasattr(cls, '_engine') or getattr(cls, '_storage_kwargs', {}) == _storage_kwargs:
+        if base_dir is not None:
+            uri = 'sqlite:///{platform_char}{path}'.format(
+                # Handles the windows behavior where there is not root, but drivers.
+                # Thus behaving as relative path.
+                platform_char='' if 'Windows' in platform.system() else '/',
+
+                path=os.path.join(base_dir, filename))
+            engine_kwargs = {}
+        else:
+            uri = 'sqlite:///:memory:'
+            engine_kwargs = dict(connect_args={'check_same_thread': False},
+                                 poolclass=pool.StaticPool)
+
+        cls._engine = create_engine(uri, **engine_kwargs)
+        session_factory = orm.sessionmaker(bind=cls._engine)
+        cls._session = orm.scoped_session(session_factory=session_factory) if base_dir else
\
+            session_factory()
+
+    return_dict = dict(engine=cls._engine, session=cls._session)
+    return_dict.update(kwargs)
+    return return_dict
+
+
 class ListResult(object):
     """
     a ListResult contains results about the requested items.

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/setup.py
----------------------------------------------------------------------
diff --git a/setup.py b/setup.py
index 2d1106d..7a1a3f4 100644
--- a/setup.py
+++ b/setup.py
@@ -64,7 +64,7 @@ console_scripts = ['aria = aria.cli.cli:main']
 
 class InstallCommand(install):
     user_options = install.user_options + [
-        ('skip-ctx', None, 'Install with or without the ctx (Defaults to False')
+        ('skip-ctx', None, 'Install with or without the ctx (Defaults to False)')
     ]
     boolean_options = install.boolean_options + ['skip-ctx']
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/__init__.py
----------------------------------------------------------------------
diff --git a/tests/__init__.py b/tests/__init__.py
index d2858d2..e51dd69 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -15,4 +15,8 @@
 
 import os
 
+from aria.storage import sql_mapi
+
+sql_mapi.SQLAlchemyModelAPI.storage_initiator(sql_mapi.init_storage)
+
 ROOT_DIR = os.path.dirname(os.path.dirname(__file__))

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/mock/context.py
----------------------------------------------------------------------
diff --git a/tests/mock/context.py b/tests/mock/context.py
index ec4bfb8..d1fa5ed 100644
--- a/tests/mock/context.py
+++ b/tests/mock/context.py
@@ -16,25 +16,24 @@
 import aria
 from aria.orchestrator import context
 from aria.storage.filesystem_rapi import FileSystemResourceAPI
-from aria.storage.sql_mapi import SQLAlchemyModelAPI
+from aria.storage import sql_mapi
 
 from . import models
 from .topology import create_simple_topology_two_nodes
 
 
-def simple(mapi_kwargs, resources_dir=None, **kwargs):
-    model_storage = aria.application_model_storage(SQLAlchemyModelAPI, api_kwargs=mapi_kwargs)
+def simple(tmpdir, model_driver_kwargs=None, resources_driver_kwargs=None, context_kwargs=None):
 
-    deployment_id = create_simple_topology_two_nodes(model_storage)
+    model_storage = aria.application_model_storage(
+        sql_mapi.SQLAlchemyModelAPI,
+        api_kwargs=model_driver_kwargs or {},
+        storage_initiator_func=sql_mapi.init_storage
+    )
 
-    # pytest tmpdir
-    if resources_dir:
-        resource_storage = aria.application_resource_storage(
-            FileSystemResourceAPI,
-            api_kwargs={'directory': resources_dir}
-        )
-    else:
-        resource_storage = None
+    resource_storage = aria.application_resource_storage(
+        FileSystemResourceAPI, api_kwargs=resources_driver_kwargs or dict(directory=str(tmpdir)))
+
+    deployment_id = create_simple_topology_two_nodes(model_storage)
 
     final_kwargs = dict(
         name='simple_context',
@@ -45,5 +44,5 @@ def simple(mapi_kwargs, resources_dir=None, **kwargs):
         task_max_attempts=models.TASK_MAX_ATTEMPTS,
         task_retry_interval=models.TASK_RETRY_INTERVAL
     )
-    final_kwargs.update(kwargs)
+    final_kwargs.update(context_kwargs or {})
     return context.workflow.WorkflowContext(**final_kwargs)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/context/test_operation.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py
index b0918d1..1b84390 100644
--- a/tests/orchestrator/context/test_operation.py
+++ b/tests/orchestrator/context/test_operation.py
@@ -37,8 +37,11 @@ global_test_holder = {}
 
 @pytest.fixture
 def ctx(tmpdir):
-    context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)),
-                                  workdir=str(tmpdir.join('workdir')))
+    context = mock.context.simple(
+        str(tmpdir.join('workdir')),
+        model_driver_kwargs=dict(base_dir=str(tmpdir)),
+        context_kwargs=dict(workdir=str(tmpdir.join('workdir')))
+    )
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/context/test_resource_render.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_resource_render.py b/tests/orchestrator/context/test_resource_render.py
index ca2ef42..467e941 100644
--- a/tests/orchestrator/context/test_resource_render.py
+++ b/tests/orchestrator/context/test_resource_render.py
@@ -53,8 +53,7 @@ def test_download_resource_and_render_provided_variables(tmpdir, ctx):
 
 @pytest.fixture
 def ctx(tmpdir):
-    context = mock.context.simple(storage.get_sqlite_api_kwargs(),
-                                  resources_dir=str(tmpdir.join('resources')))
+    context = mock.context.simple(str(tmpdir).join('resources'))
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/context/test_serialize.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_serialize.py b/tests/orchestrator/context/test_serialize.py
index 76930b1..60f8708 100644
--- a/tests/orchestrator/context/test_serialize.py
+++ b/tests/orchestrator/context/test_serialize.py
@@ -16,13 +16,11 @@
 import pytest
 
 import aria
-from aria.storage.sql_mapi import SQLAlchemyModelAPI
+from aria.storage import sql_mapi
 from aria.orchestrator.workflows import api
 from aria.orchestrator.workflows.core import engine
 from aria.orchestrator.workflows.executor import process
 from aria.orchestrator import workflow, operation
-from aria.orchestrator.context import serialization
-
 import tests
 from tests import mock
 from tests import storage
@@ -42,10 +40,6 @@ def test_serialize_operation_context(context, executor, tmpdir):
     eng.execute()
 
 
-def test_illegal_serialize_of_memory_model_storage(memory_model_storage):
-    with pytest.raises(AssertionError):
-        serialization._serialize_sql_mapi_kwargs(memory_model_storage)
-
 
 @workflow
 def _mock_workflow(ctx, graph):
@@ -93,16 +87,18 @@ def executor():
 
 @pytest.fixture
 def context(tmpdir):
-    result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)),
-                                 resources_dir=str(tmpdir.join('resources')),
-                                 workdir=str(tmpdir.join('workdir')))
+    result = mock.context.simple(
+        str(tmpdir).join('resources'),
+        model_driver_kwargs=dict(base_dir=str(tmpdir)),
+        context_kwargs=dict(workdir=str(tmpdir).join('workdir'))
+    )
+
     yield result
     storage.release_sqlite_storage(result.model)
 
 
 @pytest.fixture
 def memory_model_storage():
-    result = aria.application_model_storage(
-        SQLAlchemyModelAPI, api_kwargs=storage.get_sqlite_api_kwargs())
+    result = aria.application_model_storage(sql_mapi.SQLAlchemyModelAPI)
     yield result
     storage.release_sqlite_storage(result)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/context/test_toolbelt.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_toolbelt.py b/tests/orchestrator/context/test_toolbelt.py
index b63811b..6f83d1e 100644
--- a/tests/orchestrator/context/test_toolbelt.py
+++ b/tests/orchestrator/context/test_toolbelt.py
@@ -33,7 +33,9 @@ global_test_holder = {}
 
 @pytest.fixture
 def workflow_context(tmpdir):
-    context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+    context = mock.context.simple(
+        str(tmpdir),
+        model_driver_kwargs=dict(base_dir=str(tmpdir)))
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/context/test_workflow.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py
index 496c1ff..6d9ed39 100644
--- a/tests/orchestrator/context/test_workflow.py
+++ b/tests/orchestrator/context/test_workflow.py
@@ -19,7 +19,7 @@ import pytest
 
 from aria import application_model_storage
 from aria.orchestrator import context
-from aria.storage.sql_mapi import SQLAlchemyModelAPI
+from aria.storage import sql_mapi
 from tests import storage as test_storage
 from tests.mock import models
 
@@ -60,8 +60,8 @@ class TestWorkflowContext(object):
 
 @pytest.fixture(scope='function')
 def storage():
-    api_kwargs = test_storage.get_sqlite_api_kwargs()
-    workflow_storage = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs)
+    workflow_storage = application_model_storage(sql_mapi.SQLAlchemyModelAPI,
+                                                 storage_initiator_func=sql_mapi.init_storage)
     workflow_storage.blueprint.put(models.get_blueprint())
     blueprint = workflow_storage.blueprint.get_by_name(models.BLUEPRINT_NAME)
     workflow_storage.deployment.put(models.get_deployment(blueprint))

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/execution_plugin/test_local.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_local.py b/tests/orchestrator/execution_plugin/test_local.py
index 497da48..dd41224 100644
--- a/tests/orchestrator/execution_plugin/test_local.py
+++ b/tests/orchestrator/execution_plugin/test_local.py
@@ -504,8 +504,7 @@ if __name__ == '__main__':
     @pytest.fixture
     def workflow_context(self, tmpdir):
         workflow_context = mock.context.simple(
-            storage.get_sqlite_api_kwargs(str(tmpdir)),
-            resources_dir=str(tmpdir.join('resources')))
+            str(tmpdir.join('resources')), model_driver_kwargs=dict(base_dir=str(tmpdir)))
         workflow_context.states = []
         workflow_context.exception = None
         yield workflow_context

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/execution_plugin/test_ssh.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_ssh.py b/tests/orchestrator/execution_plugin/test_ssh.py
index 6b5c783..e744c8e 100644
--- a/tests/orchestrator/execution_plugin/test_ssh.py
+++ b/tests/orchestrator/execution_plugin/test_ssh.py
@@ -265,9 +265,8 @@ class TestWithActualSSHServer(object):
 
     @pytest.fixture
     def workflow_context(self, tmpdir):
-        workflow_context = mock.context.simple(
-            storage.get_sqlite_api_kwargs(str(tmpdir)),
-            resources_dir=str(tmpdir.join('resources')))
+        workflow_context = mock.context.simple(str(tmpdir).join('resources'),
+                                               model_driver_kwargs=dict(base_dir=str(tmpdir)))
         workflow_context.states = []
         workflow_context.exception = None
         yield workflow_context

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/test_runner.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/test_runner.py b/tests/orchestrator/test_runner.py
index 1d46e91..0a6fadf 100644
--- a/tests/orchestrator/test_runner.py
+++ b/tests/orchestrator/test_runner.py
@@ -36,6 +36,7 @@ def cleanup():
     OPERATION_RESULTS.clear()
 
 
+@pytest.mark.skip()
 def test_runner_no_tasks():
     @workflow
     def workflow_fn(ctx, graph): # pylint: disable=unused-argument
@@ -43,7 +44,7 @@ def test_runner_no_tasks():
 
     _test_runner(workflow_fn)
 
-
+@pytest.mark.skip()
 def test_runner_tasks():
     @workflow
     def workflow_fn(ctx, graph):

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/api/test_task.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/api/test_task.py b/tests/orchestrator/workflows/api/test_task.py
index 601c437..433c763 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -24,13 +24,13 @@ from tests import mock, storage
 
 
 @pytest.fixture
-def ctx():
+def ctx(tmpdir):
     """
     Create the following graph in storage:
     dependency_node <------ dependent_node
     :return:
     """
-    simple_context = mock.context.simple(storage.get_sqlite_api_kwargs())
+    simple_context = mock.context.simple(tmpdir)
     simple_context.model.execution.put(mock.models.get_execution(simple_context.deployment))
     yield simple_context
     storage.release_sqlite_storage(simple_context.model)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/builtin/test_execute_operation.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/builtin/test_execute_operation.py b/tests/orchestrator/workflows/builtin/test_execute_operation.py
index b7e5678..4ad614a 100644
--- a/tests/orchestrator/workflows/builtin/test_execute_operation.py
+++ b/tests/orchestrator/workflows/builtin/test_execute_operation.py
@@ -24,7 +24,8 @@ from tests import storage
 
 @pytest.fixture
 def ctx(tmpdir):
-    context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+    context = mock.context.simple(str(tmpdir),
+                                  model_driver_kwargs=dict(base_dir=str(tmpdir)))
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/builtin/test_heal.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/builtin/test_heal.py b/tests/orchestrator/workflows/builtin/test_heal.py
index b470790..2555919 100644
--- a/tests/orchestrator/workflows/builtin/test_heal.py
+++ b/tests/orchestrator/workflows/builtin/test_heal.py
@@ -26,7 +26,8 @@ from . import (assert_node_install_operations,
 
 @pytest.fixture
 def ctx(tmpdir):
-    context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+    context = mock.context.simple(str(tmpdir),
+                                  model_driver_kwargs=dict(base_dir=str(tmpdir)))
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/builtin/test_install.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/builtin/test_install.py b/tests/orchestrator/workflows/builtin/test_install.py
index 789a161..dc425b3 100644
--- a/tests/orchestrator/workflows/builtin/test_install.py
+++ b/tests/orchestrator/workflows/builtin/test_install.py
@@ -25,7 +25,8 @@ from . import assert_node_install_operations
 
 @pytest.fixture
 def ctx(tmpdir):
-    context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+    context = mock.context.simple(str(tmpdir),
+                                  model_driver_kwargs=dict(base_dir=str(tmpdir)))
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/builtin/test_uninstall.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/builtin/test_uninstall.py b/tests/orchestrator/workflows/builtin/test_uninstall.py
index 126c4cf..5a34494 100644
--- a/tests/orchestrator/workflows/builtin/test_uninstall.py
+++ b/tests/orchestrator/workflows/builtin/test_uninstall.py
@@ -26,7 +26,8 @@ from . import assert_node_uninstall_operations
 
 @pytest.fixture
 def ctx(tmpdir):
-    context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+    context = mock.context.simple(str(tmpdir),
+                                  model_driver_kwargs=dict(base_dir=str(tmpdir)))
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/core/test_engine.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_engine.py b/tests/orchestrator/workflows/core/test_engine.py
index d9b50a9..68ba477 100644
--- a/tests/orchestrator/workflows/core/test_engine.py
+++ b/tests/orchestrator/workflows/core/test_engine.py
@@ -124,7 +124,9 @@ class BaseTest(object):
 
     @pytest.fixture
     def workflow_context(self, tmpdir):
-        workflow_context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+        workflow_context = mock.context.simple(
+            str(tmpdir),
+            model_driver_kwargs=dict(base_dir=str(tmpdir)))
         workflow_context.states = []
         workflow_context.exception = None
         yield workflow_context

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/core/test_task.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_task.py b/tests/orchestrator/workflows/core/test_task.py
index 061a3f2..7ffe045 100644
--- a/tests/orchestrator/workflows/core/test_task.py
+++ b/tests/orchestrator/workflows/core/test_task.py
@@ -31,7 +31,9 @@ from tests import mock, storage
 
 @pytest.fixture
 def ctx(tmpdir):
-    context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+    context = mock.context.simple(
+        str(tmpdir),
+        model_driver_kwargs=dict(base_dir=str(tmpdir)))
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py
index cd37bde..57be075 100644
--- a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py
+++ b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py
@@ -22,9 +22,9 @@ from tests import mock
 from tests import storage
 
 
-def test_task_graph_into_execution_graph():
+def test_task_graph_into_execution_graph(tmpdir):
     operation_name = 'tosca.interfaces.node.lifecycle.Standard.create'
-    task_context = mock.context.simple(storage.get_sqlite_api_kwargs())
+    task_context = mock.context.simple(str(tmpdir))
     node_instance = \
         task_context.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
     def sub_workflow(name, **_):

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/executor/test_process_executor.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_process_executor.py b/tests/orchestrator/workflows/executor/test_process_executor.py
index 687e245..ab47bca 100644
--- a/tests/orchestrator/workflows/executor/test_process_executor.py
+++ b/tests/orchestrator/workflows/executor/test_process_executor.py
@@ -24,7 +24,7 @@ import pytest
 from aria import application_model_storage
 from aria.storage import model as aria_model
 from aria.utils.plugin import create as create_plugin
-from aria.storage.sql_mapi import SQLAlchemyModelAPI
+from aria.storage import sql_mapi
 from aria.orchestrator import events
 from aria.orchestrator import plugin
 from aria.orchestrator.workflows.executor import process
@@ -74,8 +74,9 @@ class TestProcessExecutor(object):
 
 @pytest.fixture
 def model(tmpdir):
-    api_kwargs = tests.storage.get_sqlite_api_kwargs(str(tmpdir))
-    result = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs)
+    result = application_model_storage(sql_mapi.SQLAlchemyModelAPI,
+                                       api_kwargs=dict(base_dir=str(tmpdir)),
+                                       storage_initiator_func=sql_mapi.init_storage)
     yield result
     tests.storage.release_sqlite_storage(result)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/executor/test_process_executor_extension.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_process_executor_extension.py b/tests/orchestrator/workflows/executor/test_process_executor_extension.py
index 4a8ef57..9e8623e 100644
--- a/tests/orchestrator/workflows/executor/test_process_executor_extension.py
+++ b/tests/orchestrator/workflows/executor/test_process_executor_extension.py
@@ -75,6 +75,7 @@ def executor():
 
 @pytest.fixture
 def context(tmpdir):
-    result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+    result = mock.context.simple(str(tmpdir),
+                                 model_driver_kwargs=dict(base_dir=str(tmpdir)))
     yield result
     storage.release_sqlite_storage(result.model)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
index bd1fa96..45793c4 100644
--- a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
+++ b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
@@ -148,6 +148,7 @@ def executor():
 
 @pytest.fixture
 def context(tmpdir):
-    result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+    result = mock.context.simple(str(tmpdir),
+                                 model_driver_kwargs=dict(base_dir=str(tmpdir)))
     yield result
     storage.release_sqlite_storage(result.model)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/storage/__init__.py
----------------------------------------------------------------------
diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py
index 3b3715e..10c2ee2 100644
--- a/tests/storage/__init__.py
+++ b/tests/storage/__init__.py
@@ -52,35 +52,6 @@ class TestFileSystem(object):
         rmtree(self.path, ignore_errors=True)
 
 
-def get_sqlite_api_kwargs(base_dir=None, filename='db.sqlite'):
-    """
-    Create sql params. works in in-memory and in filesystem mode.
-    If base_dir is passed, the mode will be filesystem mode. while the default mode is in-memory.
-    :param str base_dir: The base dir for the filesystem memory file.
-    :param str filename: the file name - defaults to 'db.sqlite'.
-    :return:
-    """
-    if base_dir is not None:
-        uri = 'sqlite:///{platform_char}{path}'.format(
-            # Handles the windows behavior where there is not root, but drivers.
-            # Thus behaving as relative path.
-            platform_char='' if 'Windows' in platform.system() else '/',
-
-            path=os.path.join(base_dir, filename))
-        engine_kwargs = {}
-    else:
-        uri = 'sqlite:///:memory:'
-        engine_kwargs = dict(connect_args={'check_same_thread': False},
-                             poolclass=pool.StaticPool)
-
-    engine = create_engine(uri, **engine_kwargs)
-    session_factory = orm.sessionmaker(bind=engine)
-    session = orm.scoped_session(session_factory=session_factory) if base_dir else session_factory()
-
-    model.DeclarativeBase.metadata.create_all(bind=engine)
-    return dict(engine=engine, session=session)
-
-
 def release_sqlite_storage(storage):
     """
     Drops the tables and clears the session
@@ -90,6 +61,8 @@ def release_sqlite_storage(storage):
     mapis = storage.registered.values()
 
     if mapis:
+        for mapi in mapis:
+            mapi.free_storage_initiator()
         for session in set(mapi._session for mapi in mapis):
             session.rollback()
             session.close()

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/storage/test_instrumentation.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_instrumentation.py b/tests/storage/test_instrumentation.py
index 9b4da4f..7eb733f 100644
--- a/tests/storage/test_instrumentation.py
+++ b/tests/storage/test_instrumentation.py
@@ -25,7 +25,7 @@ from aria.storage import (
     instrumentation,
     exceptions
 )
-from ..storage import get_sqlite_api_kwargs, release_sqlite_storage
+from ..storage import release_sqlite_storage
 
 
 STUB = instrumentation._STUB
@@ -328,9 +328,9 @@ def restore_instrumentation():
 
 @pytest.fixture
 def storage():
+    sql_mapi.SQLAlchemyModelAPI.storage_initiator(sql_mapi.init_storage)
     result = ModelStorage(
         api_cls=sql_mapi.SQLAlchemyModelAPI,
-        api_kwargs=get_sqlite_api_kwargs(),
         items=(MockModel1, MockModel2, StrictMockModel))
     yield result
     release_sqlite_storage(result)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/storage/test_model_storage.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py
index d1596e3..3ba92fe 100644
--- a/tests/storage/test_model_storage.py
+++ b/tests/storage/test_model_storage.py
@@ -22,14 +22,15 @@ from aria.storage import (
     sql_mapi,
 )
 from aria import application_model_storage
-from ..storage import get_sqlite_api_kwargs, release_sqlite_storage
+from ..storage import release_sqlite_storage
 
 from . import MockModel
 
 
 @pytest.fixture
 def storage():
-    base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs())
+    base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI)
+    sql_mapi.SQLAlchemyModelAPI.storage_initiator(sql_mapi.init_storage)
     base_storage.register(MockModel)
     yield base_storage
     release_sqlite_storage(base_storage)
@@ -61,7 +62,7 @@ def test_model_storage(storage):
 
 def test_application_storage_factory():
     storage = application_model_storage(sql_mapi.SQLAlchemyModelAPI,
-                                        api_kwargs=get_sqlite_api_kwargs())
+                                        storage_initiator_func=sql_mapi.init_storage)
     assert storage.node
     assert storage.node_instance
     assert storage.plugin

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/storage/test_models.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_models.py b/tests/storage/test_models.py
index 2088676..c5b1c62 100644
--- a/tests/storage/test_models.py
+++ b/tests/storage/test_models.py
@@ -39,7 +39,7 @@ from aria.storage.model import (
 
 
 from tests import mock
-from tests.storage import get_sqlite_api_kwargs, release_sqlite_storage
+from tests.storage import release_sqlite_storage
 
 
 @contextmanager
@@ -55,7 +55,7 @@ def sql_storage(storage_func):
 
 def _empty_storage():
     return application_model_storage(sql_mapi.SQLAlchemyModelAPI,
-                                     api_kwargs=get_sqlite_api_kwargs())
+                                     storage_initiator_func=sql_mapi.init_storage)
 
 
 def _blueprint_storage():

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/storage/test_structures.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_structures.py b/tests/storage/test_structures.py
index 0223a98..6caa4ff 100644
--- a/tests/storage/test_structures.py
+++ b/tests/storage/test_structures.py
@@ -25,7 +25,7 @@ from aria.storage import (
     exceptions
 )
 
-from ..storage import get_sqlite_api_kwargs, release_sqlite_storage, structure
+from ..storage import release_sqlite_storage, structure
 from . import MockModel
 from ..mock import (
     models,
@@ -36,7 +36,8 @@ from ..mock import (
 
 @pytest.fixture
 def storage():
-    base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs())
+    base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI)
+    sql_mapi.SQLAlchemyModelAPI.storage_initiator(sql_mapi.init_storage)
     base_storage.register(MockModel)
     yield base_storage
     release_sqlite_storage(base_storage)
@@ -48,8 +49,10 @@ def module_cleanup():
 
 
 @pytest.fixture
-def context():
-    return mock_context.simple(get_sqlite_api_kwargs())
+def context(tmpdir):
+    ctx = mock_context.simple(str(tmpdir))
+    yield ctx
+    release_sqlite_storage(ctx.model)
 
 
 def test_inner_dict_update(storage):
@@ -174,7 +177,6 @@ def test_relationship_model_ordering(context):
         target_node_instance=target_node_instance,
     )
 
-
     context.model.node.put(new_node)
     context.model.node_instance.put(new_node_instance)
     context.model.relationship.put(source_to_new_relationship)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/53344f78/tests/utils/test_plugin.py
----------------------------------------------------------------------
diff --git a/tests/utils/test_plugin.py b/tests/utils/test_plugin.py
index 6f2dd92..6a45f19 100644
--- a/tests/utils/test_plugin.py
+++ b/tests/utils/test_plugin.py
@@ -21,7 +21,7 @@ from aria import application_model_storage
 from aria.orchestrator import exceptions
 from aria.orchestrator import plugin
 from aria.utils.plugin import create as create_plugin
-from aria.storage.sql_mapi import SQLAlchemyModelAPI
+from aria.storage import sql_mapi
 
 from .. import storage
 
@@ -49,8 +49,8 @@ class TestPluginManager(object):
 
 @pytest.fixture
 def model():
-    api_kwargs = storage.get_sqlite_api_kwargs()
-    model = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs)
+    model = application_model_storage(sql_mapi.SQLAlchemyModelAPI,
+                                      storage_initiator_func=sql_mapi.init_storage)
     yield model
     storage.release_sqlite_storage(model)
 



Mime
View raw message