ariatosca-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mxm...@apache.org
Subject [2/3] incubator-ariatosca git commit: wip
Date Thu, 02 Feb 2017 08:47:33 GMT
wip


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/99ae9ca6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/99ae9ca6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/99ae9ca6

Branch: refs/heads/ARIA-42-Generic-ctx-serialization-mechanism
Commit: 99ae9ca63597ee7bb99eaa33c76ccce7f1192d4d
Parents: 4cfbe34
Author: mxmrlv <mxmrlv@gmail.com>
Authored: Wed Feb 1 16:16:01 2017 +0200
Committer: mxmrlv <mxmrlv@gmail.com>
Committed: Thu Feb 2 10:47:17 2017 +0200

----------------------------------------------------------------------
 aria/__init__.py                                | 14 ++-----
 aria/orchestrator/context/serialization.py      | 43 ++++----------------
 aria/orchestrator/runner.py                     |  2 +-
 aria/storage/core.py                            |  8 ++--
 aria/storage/sql_mapi.py                        | 42 +++++++++++++++++++
 tests/mock/context.py                           | 19 ++++-----
 .../context/test_resource_render.py             |  2 +-
 tests/orchestrator/context/test_serialize.py    | 18 ++++----
 tests/orchestrator/context/test_workflow.py     |  2 +-
 .../orchestrator/execution_plugin/test_local.py |  2 +-
 tests/orchestrator/execution_plugin/test_ssh.py |  2 +-
 .../workflows/executor/test_process_executor.py |  2 +-
 tests/storage/test_model_storage.py             |  2 +-
 tests/storage/test_models.py                    |  2 +-
 tests/utils/test_plugin.py                      |  2 +-
 15 files changed, 82 insertions(+), 80 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/__init__.py
----------------------------------------------------------------------
diff --git a/aria/__init__.py b/aria/__init__.py
index 248aa1a..37cec1a 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -57,7 +57,7 @@ def install_aria_extensions():
     extension.init()
 
 
-def application_model_storage(api, api_kwargs=None):
+def application_model_storage(api, driver_kwargs=None):
     """
     Initiate model storage
     """
@@ -79,18 +79,12 @@ def application_model_storage(api, api_kwargs=None):
         storage.model.Task,
     ]
     # if api not in _model_storage:
-    return storage.ModelStorage(api, items=models, api_kwargs=api_kwargs or {})
+    return storage.ModelStorage(api, items=models, driver_kwargs=driver_kwargs or {})
 
 
-def application_resource_storage(api, api_kwargs=None):
+def application_resource_storage(api, driver_kwargs=None):
     """
     Initiate resource storage
     """
     return storage.ResourceStorage(
-        api,
-        api_kwargs=api_kwargs or {},
-        items=[
-            'blueprint',
-            'deployment',
-            'plugin',
-        ])
+        api, driver_kwargs=driver_kwargs or {}, items=['blueprint', 'deployment', 'plugin',
])

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/orchestrator/context/serialization.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/serialization.py b/aria/orchestrator/context/serialization.py
index 760818f..c583a71 100644
--- a/aria/orchestrator/context/serialization.py
+++ b/aria/orchestrator/context/serialization.py
@@ -13,9 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sqlalchemy.orm
-import sqlalchemy.pool
-
 import aria
 
 
@@ -32,7 +29,7 @@ def operation_context_to_dict(context):
         model = context.model
         context_dict['model_storage'] = {
             'api_cls': model.api,
-            'api_kwargs': _serialize_sql_mapi_kwargs(model)
+            'driver_kwargs': model._driver_kwargs,
         }
     else:
         context_dict['model_storage'] = None
@@ -40,7 +37,7 @@ def operation_context_to_dict(context):
         resource = context.resource
         context_dict['resource_storage'] = {
             'api_cls': resource.api,
-            'api_kwargs': _serialize_file_rapi_kwargs(resource)
+            'driver_kwargs': resource._driver_kwargs
         }
     else:
         context_dict['resource_storage'] = None
@@ -57,39 +54,15 @@ def operation_context_from_dict(context_dict):
     model_storage = context['model_storage']
     if model_storage:
         api_cls = model_storage['api_cls']
-        api_kwargs = _deserialize_sql_mapi_kwargs(model_storage.get('api_kwargs', {}))
-        context['model_storage'] = aria.application_model_storage(api=api_cls,
-                                                                  api_kwargs=api_kwargs)
+        driver_kwargs = model_storage['driver_kwargs']
+        context['model_storage'] = aria.application_model_storage(
+            api_cls, driver_kwargs=driver_kwargs)
 
     resource_storage = context['resource_storage']
     if resource_storage:
         api_cls = resource_storage['api_cls']
-        api_kwargs = _deserialize_file_rapi_kwargs(resource_storage.get('api_kwargs', {}))
-        context['resource_storage'] = aria.application_resource_storage(api=api_cls,
-                                                                        api_kwargs=api_kwargs)
+        driver_kwargs = resource_storage['driver_kwargs']
+        context['resource_storage'] = aria.application_resource_storage(
+             api=api_cls, driver_kwargs=driver_kwargs)
 
     return context_cls(**context)
-
-
-def _serialize_sql_mapi_kwargs(model):
-    engine_url = str(model._api_kwargs['engine'].url)
-    assert ':memory:' not in engine_url
-    return {'engine_url': engine_url}
-
-
-def _deserialize_sql_mapi_kwargs(api_kwargs):
-    engine_url = api_kwargs.get('engine_url')
-    if not engine_url:
-        return {}
-    engine = sqlalchemy.create_engine(engine_url)
-    session_factory = sqlalchemy.orm.sessionmaker(bind=engine)
-    session = sqlalchemy.orm.scoped_session(session_factory=session_factory)
-    return {'session': session, 'engine': engine}
-
-
-def _serialize_file_rapi_kwargs(resource):
-    return {'directory': resource._api_kwargs['directory']}
-
-
-def _deserialize_file_rapi_kwargs(api_kwargs):
-    return api_kwargs

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/orchestrator/runner.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/runner.py b/aria/orchestrator/runner.py
index 16acc19..b571b34 100644
--- a/aria/orchestrator/runner.py
+++ b/aria/orchestrator/runner.py
@@ -120,7 +120,7 @@ class Runner(object):
         sqlite_kwargs = dict(engine=sqlite_engine, session=sqlite_session)
         return application_model_storage(
             SQLAlchemyModelAPI,
-            api_kwargs=sqlite_kwargs)
+            driver_kwargs=sqlite_kwargs)
 
     def create_fs_resource_storage(self, directory='.'): # pylint: disable=no-self-use
         fs_kwargs = dict(directory=directory)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/storage/core.py
----------------------------------------------------------------------
diff --git a/aria/storage/core.py b/aria/storage/core.py
index 94b4fe0..d46afaa 100644
--- a/aria/storage/core.py
+++ b/aria/storage/core.py
@@ -52,10 +52,10 @@ class Storage(LoggerMixin):
     """
     Represents the storage
     """
-    def __init__(self, api_cls, api_kwargs=None, items=(), **kwargs):
+    def __init__(self, api_cls, driver_kwargs=None, items=(), **kwargs):
         super(Storage, self).__init__(**kwargs)
         self.api = api_cls
-        self._api_kwargs = api_kwargs or {}
+        self._driver_kwargs = driver_kwargs or {}
         self.registered = {}
         for item in items:
             self.register(item)
@@ -90,7 +90,7 @@ class ResourceStorage(Storage):
         :param name:
         :return:
         """
-        self.registered[name] = self.api(name=name, **self._api_kwargs)
+        self.registered[name] = self.api(name=name, **self._driver_kwargs)
         self.registered[name].create()
         self.logger.debug('setup {name} in storage {self!r}'.format(name=name, self=self))
 
@@ -112,7 +112,7 @@ class ModelStorage(Storage):
             return
         self.registered[model_name] = self.api(name=model_name,
                                                model_cls=model_cls,
-                                               **self._api_kwargs)
+                                               **self._driver_kwargs)
         self.registered[model_name].create()
         self.logger.debug('setup {name} in storage {self!r}'.format(name=model_name, self=self))
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/aria/storage/sql_mapi.py
----------------------------------------------------------------------
diff --git a/aria/storage/sql_mapi.py b/aria/storage/sql_mapi.py
index 809f677..3982eaa 100644
--- a/aria/storage/sql_mapi.py
+++ b/aria/storage/sql_mapi.py
@@ -15,7 +15,13 @@
 """
 SQLAlchemy based MAPI
 """
+import os
+import platform
+from functools import partial
 
+from sqlalchemy import create_engine
+from sqlalchemy import orm
+from sqlalchemy import pool
 from sqlalchemy.exc import SQLAlchemyError
 
 from aria.utils.collections import OrderedDict
@@ -25,11 +31,47 @@ from aria.storage import (
 )
 
 
+def init_storage(init_func=None):
+    if init_func is None:
+        return partial(init_storage, func=init_storage)
+
+    track = {
+        'engine': None,
+        'session': None
+    }
+
+    def _wrapper(self, base_dir=None, filename='db.sqlite', *args, **kwargs):
+        if not (track['engine'] and track['session']):
+            if base_dir is not None:
+                uri = 'sqlite:///{platform_char}{path}'.format(
+                    # Handles the windows behavior where there is not root, but drivers.
+                    # Thus behaving as relative path.
+                    platform_char='' if 'Windows' in platform.system() else '/',
+
+                    path=os.path.join(base_dir, filename))
+                engine_kwargs = {}
+            else:
+                uri = 'sqlite:///:memory:'
+                engine_kwargs = dict(connect_args={'check_same_thread': False},
+                                     poolclass=pool.StaticPool)
+
+            track['engine'] = create_engine(uri, **engine_kwargs)
+            session_factory = orm.sessionmaker(bind=track['engine'])
+            track['session'] = orm.scoped_session(session_factory=session_factory) if base_dir
else \
+                session_factory()
+
+        return init_func(
+            self=self, engine=track['engine'], session=track['session'], *args, **kwargs)
+
+    return _wrapper
+
+
 class SQLAlchemyModelAPI(api.ModelAPI):
     """
     SQL based MAPI.
     """
 
+    @init_storage
     def __init__(self,
                  engine,
                  session,

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/mock/context.py
----------------------------------------------------------------------
diff --git a/tests/mock/context.py b/tests/mock/context.py
index ec4bfb8..4fbae14 100644
--- a/tests/mock/context.py
+++ b/tests/mock/context.py
@@ -22,19 +22,14 @@ from . import models
 from .topology import create_simple_topology_two_nodes
 
 
-def simple(mapi_kwargs, resources_dir=None, **kwargs):
-    model_storage = aria.application_model_storage(SQLAlchemyModelAPI, api_kwargs=mapi_kwargs)
+def simple(model_driver_kwargs=None, resources_driver_kwargs=None, context_kwargs=None):
+    model_storage = aria.application_model_storage(
+        SQLAlchemyModelAPI, driver_kwargs=model_driver_kwargs or {})
 
-    deployment_id = create_simple_topology_two_nodes(model_storage)
+    resource_storage = aria.application_resource_storage(
+        FileSystemResourceAPI, driver_kwargs=resources_driver_kwargs or {})
 
-    # pytest tmpdir
-    if resources_dir:
-        resource_storage = aria.application_resource_storage(
-            FileSystemResourceAPI,
-            api_kwargs={'directory': resources_dir}
-        )
-    else:
-        resource_storage = None
+    deployment_id = create_simple_topology_two_nodes(model_storage)
 
     final_kwargs = dict(
         name='simple_context',
@@ -45,5 +40,5 @@ def simple(mapi_kwargs, resources_dir=None, **kwargs):
         task_max_attempts=models.TASK_MAX_ATTEMPTS,
         task_retry_interval=models.TASK_RETRY_INTERVAL
     )
-    final_kwargs.update(kwargs)
+    final_kwargs.update(context_kwargs or {})
     return context.workflow.WorkflowContext(**final_kwargs)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/context/test_resource_render.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_resource_render.py b/tests/orchestrator/context/test_resource_render.py
index ca2ef42..a5e4fa8 100644
--- a/tests/orchestrator/context/test_resource_render.py
+++ b/tests/orchestrator/context/test_resource_render.py
@@ -54,7 +54,7 @@ def test_download_resource_and_render_provided_variables(tmpdir, ctx):
 @pytest.fixture
 def ctx(tmpdir):
     context = mock.context.simple(storage.get_sqlite_api_kwargs(),
-                                  resources_dir=str(tmpdir.join('resources')))
+                                  resources_driver_kwargs=str(tmpdir.join('resources')))
     yield context
     storage.release_sqlite_storage(context.model)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/context/test_serialize.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_serialize.py b/tests/orchestrator/context/test_serialize.py
index 76930b1..8d4cb1f 100644
--- a/tests/orchestrator/context/test_serialize.py
+++ b/tests/orchestrator/context/test_serialize.py
@@ -16,13 +16,12 @@
 import pytest
 
 import aria
+from aria.storage import model
 from aria.storage.sql_mapi import SQLAlchemyModelAPI
 from aria.orchestrator.workflows import api
 from aria.orchestrator.workflows.core import engine
 from aria.orchestrator.workflows.executor import process
 from aria.orchestrator import workflow, operation
-from aria.orchestrator.context import serialization
-
 import tests
 from tests import mock
 from tests import storage
@@ -42,10 +41,6 @@ def test_serialize_operation_context(context, executor, tmpdir):
     eng.execute()
 
 
-def test_illegal_serialize_of_memory_model_storage(memory_model_storage):
-    with pytest.raises(AssertionError):
-        serialization._serialize_sql_mapi_kwargs(memory_model_storage)
-
 
 @workflow
 def _mock_workflow(ctx, graph):
@@ -93,9 +88,12 @@ def executor():
 
 @pytest.fixture
 def context(tmpdir):
-    result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)),
-                                 resources_dir=str(tmpdir.join('resources')),
-                                 workdir=str(tmpdir.join('workdir')))
+    result = mock.context.simple(
+        model_driver_kwargs=dict(base_dir=str(tmpdir)),
+        resources_driver_kwargs=dict(directory=str(tmpdir.join('resources'))),
+        context_kwargs=dict(workdir=str(tmpdir.join('workdir')))
+    )
+
     yield result
     storage.release_sqlite_storage(result.model)
 
@@ -103,6 +101,6 @@ def context(tmpdir):
 @pytest.fixture
 def memory_model_storage():
     result = aria.application_model_storage(
-        SQLAlchemyModelAPI, api_kwargs=storage.get_sqlite_api_kwargs())
+        SQLAlchemyModelAPI, driver_kwargs=storage.get_sqlite_api_kwargs())
     yield result
     storage.release_sqlite_storage(result)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/context/test_workflow.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py
index 496c1ff..13e7639 100644
--- a/tests/orchestrator/context/test_workflow.py
+++ b/tests/orchestrator/context/test_workflow.py
@@ -61,7 +61,7 @@ class TestWorkflowContext(object):
 @pytest.fixture(scope='function')
 def storage():
     api_kwargs = test_storage.get_sqlite_api_kwargs()
-    workflow_storage = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs)
+    workflow_storage = application_model_storage(SQLAlchemyModelAPI, driver_kwargs=api_kwargs)
     workflow_storage.blueprint.put(models.get_blueprint())
     blueprint = workflow_storage.blueprint.get_by_name(models.BLUEPRINT_NAME)
     workflow_storage.deployment.put(models.get_deployment(blueprint))

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/execution_plugin/test_local.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_local.py b/tests/orchestrator/execution_plugin/test_local.py
index 497da48..30447e2 100644
--- a/tests/orchestrator/execution_plugin/test_local.py
+++ b/tests/orchestrator/execution_plugin/test_local.py
@@ -505,7 +505,7 @@ if __name__ == '__main__':
     def workflow_context(self, tmpdir):
         workflow_context = mock.context.simple(
             storage.get_sqlite_api_kwargs(str(tmpdir)),
-            resources_dir=str(tmpdir.join('resources')))
+            resources_driver_kwargs=str(tmpdir.join('resources')))
         workflow_context.states = []
         workflow_context.exception = None
         yield workflow_context

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/execution_plugin/test_ssh.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_ssh.py b/tests/orchestrator/execution_plugin/test_ssh.py
index 6b5c783..5d21ca4 100644
--- a/tests/orchestrator/execution_plugin/test_ssh.py
+++ b/tests/orchestrator/execution_plugin/test_ssh.py
@@ -267,7 +267,7 @@ class TestWithActualSSHServer(object):
     def workflow_context(self, tmpdir):
         workflow_context = mock.context.simple(
             storage.get_sqlite_api_kwargs(str(tmpdir)),
-            resources_dir=str(tmpdir.join('resources')))
+            resources_driver_kwargs=str(tmpdir.join('resources')))
         workflow_context.states = []
         workflow_context.exception = None
         yield workflow_context

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/orchestrator/workflows/executor/test_process_executor.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_process_executor.py b/tests/orchestrator/workflows/executor/test_process_executor.py
index 687e245..bf910db 100644
--- a/tests/orchestrator/workflows/executor/test_process_executor.py
+++ b/tests/orchestrator/workflows/executor/test_process_executor.py
@@ -75,7 +75,7 @@ class TestProcessExecutor(object):
 @pytest.fixture
 def model(tmpdir):
     api_kwargs = tests.storage.get_sqlite_api_kwargs(str(tmpdir))
-    result = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs)
+    result = application_model_storage(SQLAlchemyModelAPI, driver_kwargs=api_kwargs)
     yield result
     tests.storage.release_sqlite_storage(result)
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/storage/test_model_storage.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py
index d1596e3..09d59c4 100644
--- a/tests/storage/test_model_storage.py
+++ b/tests/storage/test_model_storage.py
@@ -61,7 +61,7 @@ def test_model_storage(storage):
 
 def test_application_storage_factory():
     storage = application_model_storage(sql_mapi.SQLAlchemyModelAPI,
-                                        api_kwargs=get_sqlite_api_kwargs())
+                                        driver_kwargs=get_sqlite_api_kwargs())
     assert storage.node
     assert storage.node_instance
     assert storage.plugin

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/storage/test_models.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_models.py b/tests/storage/test_models.py
index 2088676..56516f1 100644
--- a/tests/storage/test_models.py
+++ b/tests/storage/test_models.py
@@ -55,7 +55,7 @@ def sql_storage(storage_func):
 
 def _empty_storage():
     return application_model_storage(sql_mapi.SQLAlchemyModelAPI,
-                                     api_kwargs=get_sqlite_api_kwargs())
+                                     driver_kwargs=get_sqlite_api_kwargs())
 
 
 def _blueprint_storage():

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/99ae9ca6/tests/utils/test_plugin.py
----------------------------------------------------------------------
diff --git a/tests/utils/test_plugin.py b/tests/utils/test_plugin.py
index 6f2dd92..e80cb87 100644
--- a/tests/utils/test_plugin.py
+++ b/tests/utils/test_plugin.py
@@ -50,7 +50,7 @@ class TestPluginManager(object):
 @pytest.fixture
 def model():
     api_kwargs = storage.get_sqlite_api_kwargs()
-    model = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs)
+    model = application_model_storage(SQLAlchemyModelAPI, driver_kwargs=api_kwargs)
     yield model
     storage.release_sqlite_storage(model)
 


Mime
View raw message