ariatosca-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mxm...@apache.org
Subject incubator-ariatosca git commit: review 2
Date Wed, 22 Feb 2017 17:20:53 GMT
Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-106-Create-sqla-logging-handler c37f075b5 -> 73dab0013


review 2


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/73dab001
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/73dab001
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/73dab001

Branch: refs/heads/ARIA-106-Create-sqla-logging-handler
Commit: 73dab0013bc576ba8c6295b01d27a993d013adc3
Parents: c37f075
Author: max-orlov <maxim@gigaspaces.com>
Authored: Wed Feb 22 19:20:45 2017 +0200
Committer: max-orlov <maxim@gigaspaces.com>
Committed: Wed Feb 22 19:20:45 2017 +0200

----------------------------------------------------------------------
 aria/__init__.py                             |  2 +
 aria/logger.py                               | 22 ++++---
 aria/orchestrator/context/common.py          |  7 ++-
 aria/orchestrator/decorators.py              |  4 +-
 aria/orchestrator/workflows/executor/base.py | 17 +++--
 aria/utils/imports.py                        | 35 +++++------
 tests/.pylintrc                              |  2 +-
 tests/orchestrator/context/test_operation.py | 76 +++++++++++++----------
 8 files changed, 89 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/73dab001/aria/__init__.py
----------------------------------------------------------------------
diff --git a/aria/__init__.py b/aria/__init__.py
index 7fd0db9..6b10501 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -32,6 +32,8 @@ from . import (
 )
 
 if sys.version_info < (2, 7):
+    # pkgutil in python2.6 has a bug where it fails to import from protected modules, which
causes
+    # the entire process to fail. In order to overcome this issue we use our custom iter_modules
     from .utils.imports import iter_modules
 else:
     from pkgutil import iter_modules

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/73dab001/aria/logger.py
----------------------------------------------------------------------
diff --git a/aria/logger.py b/aria/logger.py
index 1fcdf03..8d1bb7d 100644
--- a/aria/logger.py
+++ b/aria/logger.py
@@ -17,11 +17,12 @@
 Logging related mixins and functions
 """
 
-
 import logging
 from logging import handlers as logging_handlers
 from datetime import datetime
 
+import sqlalchemy
+
 _base_logger = logging.getLogger('aria')
 
 
@@ -99,8 +100,16 @@ def create_console_log_handler(level=logging.DEBUG, formatter=None):
     return console
 
 
-def create_sqla_log_handler(session, engine, level=logging.DEBUG):
-    return _SQLAlchemyHandler(session=session, engine=engine, level=level)
+def create_sqla_log_handler(session, engine, log_cls, level=logging.DEBUG):
+
+    # This is needed since the engine and session are entirely new we need to reflect the
db
+    # schema of the logging model into the engine and session.
+    sqlalchemy.Table(log_cls.__tablename__,
+                     log_cls.__table__.metadata,
+                     autoload=True,
+                     autoload_with=engine)
+
+    return _SQLAlchemyHandler(session=session, engine=engine, log_cls=log_cls, level=level)
 
 
 class _DefaultConsoleFormat(logging.Formatter):
@@ -144,16 +153,13 @@ def create_file_log_handler(
 
 class _SQLAlchemyHandler(logging.Handler):
 
-    def __init__(self, session, engine, **kwargs):
+    def __init__(self, session, engine, log_cls, **kwargs):
         logging.Handler.__init__(self, **kwargs)
         self._session = session
         self._engine = engine
 
         # Cyclic dependency
-        from aria.storage.modeling.model import Log
-        self._cls = Log
-        # This is needed since the engine and session are entirely new
-        self._cls.__table__.create(bind=self._engine, checkfirst=True)
+        self._cls = log_cls
 
     def emit(self, record):
         created_at = datetime.strptime(logging.Formatter('%(asctime)s').formatTime(record),

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/73dab001/aria/orchestrator/context/common.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/common.py b/aria/orchestrator/context/common.py
index 4b1730a..b34cd5d 100644
--- a/aria/orchestrator/context/common.py
+++ b/aria/orchestrator/context/common.py
@@ -23,7 +23,10 @@ from uuid import uuid4
 import jinja2
 
 from aria import logger as aria_logger
-from aria.storage import exceptions
+from aria.storage import (
+    exceptions,
+    modeling
+)
 
 
 class BaseContext(object):
@@ -71,7 +74,7 @@ class BaseContext(object):
         if self._model._initiator:
             api_kwargs.update(self._model._initiator(**self._model._initiator_kwargs))
         api_kwargs.update(**self._model._api_kwargs)
-        return aria_logger.create_sqla_log_handler(**api_kwargs)
+        return aria_logger.create_sqla_log_handler(log_cls=modeling.model.Log, **api_kwargs)
 
     def __repr__(self):
         return (

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/73dab001/aria/orchestrator/decorators.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/decorators.py b/aria/orchestrator/decorators.py
index 1a8ead0..3ced61c 100644
--- a/aria/orchestrator/decorators.py
+++ b/aria/orchestrator/decorators.py
@@ -71,9 +71,7 @@ def operation(func=None, toolbelt=False, suffix_template='', logging_handlers=No
             operation_toolbelt = context.toolbelt(func_kwargs['ctx'])
             func_kwargs.setdefault('toolbelt', operation_toolbelt)
         validate_function_arguments(func, func_kwargs)
-
-        with func_kwargs['ctx'].logging_handlers(logging_handlers):
-            return func(**func_kwargs)
+        return func(**func_kwargs)
     return _wrapper
 
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/73dab001/aria/orchestrator/workflows/executor/base.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/base.py b/aria/orchestrator/workflows/executor/base.py
index cd8e7ee..4ae046d 100644
--- a/aria/orchestrator/workflows/executor/base.py
+++ b/aria/orchestrator/workflows/executor/base.py
@@ -39,15 +39,14 @@ class BaseExecutor(logger.LoggerMixin):
         """
         pass
 
-    def _task_started(self, task):
-        self._signal(events.start_task_signal, task)
-
-    def _task_failed(self, task, exception):
-        self._signal(events.on_failure_task_signal, task, exception=exception)
+    @staticmethod
+    def _task_started(task):
+        events.start_task_signal.send(task)
 
-    def _task_succeeded(self, task):
-        self._signal(events.on_success_task_signal, task)
+    @staticmethod
+    def _task_failed(task, exception):
+        events.on_failure_task_signal.send(task, exception=exception)
 
     @staticmethod
-    def _signal(signal, task, **kwargs):
-        signal.send(task, **kwargs)
+    def _task_succeeded(task):
+        events.on_success_task_signal.send(task)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/73dab001/aria/utils/imports.py
----------------------------------------------------------------------
diff --git a/aria/utils/imports.py b/aria/utils/imports.py
index 1782d16..64a48cf 100644
--- a/aria/utils/imports.py
+++ b/aria/utils/imports.py
@@ -79,23 +79,18 @@ def load_attribute(attribute_path):
         raise
 
 
-class _SafeModuleImporter(object):
-    def __init__(self):
-        self._yielded = {}
-
-    def iter_modules(self):
-        # apparently pkgutil had some issues in python 2.6. Accessing any root level directories
-        # failed. and it got the entire process of importing fail. Since we only need any
-        # aria_extension related loading, in the meantime we could try to import only those
-        # (and assume they are not located at the root level.
-        # [In python 2.7 it does actually ignore any OSError].
-        for importer in pkgutil.iter_importers():
-            try:
-                for module_name, ispkg in pkgutil.iter_importer_modules(importer):
-                    if module_name not in self._yielded:
-                        self._yielded[module_name] = True
-                        yield importer, module_name, ispkg
-            except OSError:
-                pass
-
-iter_modules = _SafeModuleImporter().iter_modules
+def iter_modules():
+    # apparently pkgutil had some issues in python 2.6. Accessing any root level directories
+    # failed. and it got the entire process of importing fail. Since we only need any
+    # aria_extension related loading, in the meantime we could try to import only those
+    # (and assume they are not located at the root level.
+    # [In python 2.7 it does actually ignore any OSError].
+    yielded = {}
+    for importer in pkgutil.iter_importers():
+        try:
+            for module_name, ispkg in pkgutil.iter_importer_modules(importer):
+                if module_name not in yielded:
+                    yielded[module_name] = True
+                    yield importer, module_name, ispkg
+        except OSError:
+            pass

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/73dab001/tests/.pylintrc
----------------------------------------------------------------------
diff --git a/tests/.pylintrc b/tests/.pylintrc
index 5de0691..06409e9 100644
--- a/tests/.pylintrc
+++ b/tests/.pylintrc
@@ -77,7 +77,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,no-self-use,missing-docstring,attribute-defined-outside-init,redefined-outer-name,import-error,redefined-variable-type,broad
 -except,protected-access,global-statement,too-many-locals,abstract-method
+disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,no-self-use,missing-docstring,attribute-defined-outside-init,redefined-outer-name,import-error,redefined-variable-type,broad
 -except,protected-access,global-statement,too-many-locals,abstract-method,no-member
 
 [REPORTS]
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/73dab001/tests/orchestrator/context/test_operation.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py
index 50e0af4..2b26bb1 100644
--- a/tests/orchestrator/context/test_operation.py
+++ b/tests/orchestrator/context/test_operation.py
@@ -15,8 +15,6 @@
 
 import os
 import time
-import logging
-import tempfile
 
 import pytest
 
@@ -50,18 +48,6 @@ def ctx(tmpdir):
     storage.release_sqlite_storage(context.model)
 
 
-tmp_file = os.path.join(tempfile.gettempdir(), 'op_testing_const_file_name')
-
-
-@pytest.fixture(autouse=True)
-def tmpfile_cleanup():
-    try:
-        yield
-    finally:
-        if os.path.isfile(tmp_file):
-            os.remove(tmp_file)
-
-
 @pytest.fixture
 def thread_executor():
     result = thread.ThreadExecutor()
@@ -227,19 +213,20 @@ def test_plugin_workdir(ctx, thread_executor, tmpdir):
 
 
 @pytest.fixture(params=[
-    (thread.ThreadExecutor()),
-    (process.ProcessExecutor(python_path=tests.ROOT_DIR))
+    (thread.ThreadExecutor, dict()),
+    (process.ProcessExecutor, dict(python_path=tests.ROOT_DIR))
 ])
 def executor(request):
-    ex = request.param
+    ex_cls, kwargs = request.param
+    ex = ex_cls(**kwargs)
     try:
         yield ex
     finally:
         ex.close()
 
 
-def test_operation_logging(ctx, executor):
-    operation_name = 'aria.interfaces.lifecycle.create'
+def test_node_operation_logging(ctx, executor):
+    operation_name = mock.operations.NODE_OPERATIONS_INSTALL[0]
 
     node = ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
     interface = mock.models.get_interface(
@@ -265,9 +252,44 @@ def test_operation_logging(ctx, executor):
         )
 
     execute(workflow_func=basic_workflow, workflow_context=ctx, executor=executor)
+    _assert_loggins(ctx.model.log.list(), inputs)
+
+
+def test_relationship_operation_logging(ctx, executor):
+    operation_name = mock.operations.RELATIONSHIP_OPERATIONS_INSTALL[0].rsplit('_', 1)[0]
 
-    logs = ctx.model.log.list()
+    relationship = ctx.model.relationship.list()[0]
+    relationship.interfaces = [mock.models.get_interface(
+        operation_name,
+        operation_kwargs=dict(implementation=op_path(logged_operation, module_path=__name__)),
+        edge='source'
+    )]
+    ctx.model.relationship.update(relationship)
+
+    inputs = {
+        'op_start': 'op_start',
+        'op_end': 'op_end',
+    }
 
+    @workflow
+    def basic_workflow(graph, **_):
+        graph.add_tasks(
+            api.task.OperationTask.relationship(
+                name=operation_name,
+                instance=relationship,
+                inputs=inputs,
+                edge='source',
+            )
+        )
+
+    execute(workflow_func=basic_workflow, workflow_context=ctx, executor=executor)
+    _assert_loggins(ctx.model.log.list(), inputs)
+
+
+def _assert_loggins(logs, inputs):
+
+    # The logs should contain the following: Workflow Start, Operation Start, custom operation
+    # log string (op_start), custom operation log string (op_end), Operation End, Workflow
End.
     assert len(logs) == 6
 
     op_start_log = [l for l in logs if inputs['op_start'] in l.msg and l.level.lower() ==
'info']
@@ -280,20 +302,8 @@ def test_operation_logging(ctx, executor):
 
     assert op_start_log.created_at < op_end_log.created_at
 
-    with open(tmp_file, 'r') as f:
-        logs = [l.strip() for l in f.readlines()]
-
-    assert inputs['op_start'] in logs
-    assert inputs['op_end'] in logs
 
-
-class MockLogHandler(logging.Handler):
-    def emit(self, record):
-        with open(tmp_file, 'a+') as f:
-            f.write(record.msg + '\n')
-
-
-@operation(logging_handlers=[MockLogHandler()])
+@operation
 def logged_operation(ctx, **_):
     ctx.logger.info(ctx.task.inputs['op_start'])
     # enables to check the relation between the created_at field properly


Mime
View raw message