Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-106-Create-sqla-logging-handler 51b17799f -> 6bff90672 (forced update)
wip Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/6bff9067 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/6bff9067 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/6bff9067 Branch: refs/heads/ARIA-106-Create-sqla-logging-handler Commit: 6bff90672b0781fb425beffcbda950dfbecc37a7 Parents: 2286b0d Author: mxmrlv <mxm...@gmail.com> Authored: Mon Feb 13 13:50:05 2017 +0200 Committer: mxmrlv <mxm...@gmail.com> Committed: Mon Feb 13 19:09:29 2017 +0200 ---------------------------------------------------------------------- aria/__init__.py | 1 - aria/logger.py | 37 +++++++++++++------- aria/orchestrator/context/common.py | 23 ++++++------ aria/orchestrator/workflows/core/engine.py | 5 +++ aria/orchestrator/workflows/core/task.py | 2 -- aria/orchestrator/workflows/executor/process.py | 1 - aria/storage/model.py | 2 +- tests/orchestrator/context/test_operation.py | 6 ++-- tests/storage/__init__.py | 13 +++---- 9 files changed, 50 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index 4e2982f..2302b06 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -77,7 +77,6 @@ def application_model_storage(api, api_kwargs=None, initiator=None, initiator_kw storage.model.Execution, storage.model.Task, - storage.model.Log ] return storage.ModelStorage(api_cls=api, http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/aria/logger.py ---------------------------------------------------------------------- diff --git a/aria/logger.py b/aria/logger.py index 5840cff..f19d286 100644 --- a/aria/logger.py +++ b/aria/logger.py @@ -17,10 +17,10 @@ Logging related mixins and functions """ -import logging +import logging +from logging import handlers as logging_handlers from datetime import datetime -from logging.handlers import RotatingFileHandler _base_logger = logging.getLogger('aria') @@ -126,7 +126,7 @@ def create_file_log_handler( """ Create a logging.handlers.RotatingFileHandler """ - rotating_file = RotatingFileHandler( + rotating_file = logging_handlers.RotatingFileHandler( filename=file_path, maxBytes=max_bytes, backupCount=backup_count, @@ -137,15 +137,6 @@ def create_file_log_handler( return rotating_file -def create_sqla_log_handler(session, engine, model_cls=None, level=logging.DEBUG): - from aria.storage.model import Log - return SQLAlchemyHandler(session, engine, model_cls or Log, level=level) - - -_default_file_formatter = logging.Formatter( - '%(asctime)s [%(name)s:%(levelname)s] %(message)s <%(pathname)s:%(lineno)d>') - - class SQLAlchemyHandler(logging.Handler): def __init__(self, session, engine, log_cls, **kwargs): self._session = session @@ -162,3 +153,25 @@ class SQLAlchemyHandler(logging.Handler): ) self._session.add(log) self._session.commit() + + +class _SQLAlchemyHandlerFactory(object): + from aria.storage.model import Log + + def __init__(self): + self._handler = None + + def __call__(self, session, engine, model_cls=Log, level=logging.DEBUG): + if self._handler is None or not self._is_eq(session, engine, model_cls): + self._handler = SQLAlchemyHandler(session, engine, model_cls, level=level) + return self._handler + + def _is_eq(self, session, engine, model_cls): + return all([self._handler._session == session, + self._handler._engine == engine, + self._handler._cls == model_cls]) + +create_sqla_log_handler = _SQLAlchemyHandlerFactory() + +_default_file_formatter = logging.Formatter( + '%(asctime)s [%(name)s:%(levelname)s] %(message)s <%(pathname)s:%(lineno)d>') http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/aria/orchestrator/context/common.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/context/common.py b/aria/orchestrator/context/common.py index 87f88b1..3f29ac0 100644 --- a/aria/orchestrator/context/common.py +++ b/aria/orchestrator/context/common.py @@ -17,9 +17,10 @@ A common context for both workflow and operation """ from uuid import uuid4 -import jinja2 import logging +import jinja2 + from aria import logger from aria.storage import exceptions @@ -44,18 +45,20 @@ class BaseContext(object): self._model = model_storage self._resource = resource_storage self._deployment_id = deployment_id - if ctx_logger is None: - sqla_logging_handler = logger.create_sqla_log_handler(**model_storage.all_api_kwargs) - ctx_logger = logging.getLogger('aria_ctx') + self._logger = self._init_logger(ctx_logger) + self._workdir = workdir - # A handler should be registered only once. - if len(ctx_logger.handlers) == 0: - ctx_logger.addHandler(sqla_logging_handler) + def _init_logger(self, ctx_logger=None): + ctx_logger = ctx_logger or logging.getLogger('aria_ctx') - ctx_logger.setLevel(logging.DEBUG) - self._logger = ctx_logger + # A handler should be registered only once. + sqla_handler = logger.create_sqla_log_handler(**self._model.all_api_kwargs) + if sqla_handler not in ctx_logger.handlers: + ctx_logger.addHandler(sqla_handler) - self._workdir = workdir + ctx_logger.setLevel(logging.DEBUG) + + return ctx_logger def __repr__(self): return ( http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/aria/orchestrator/workflows/core/engine.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/core/engine.py b/aria/orchestrator/workflows/core/engine.py index fd83614..fcd46c7 100644 --- a/aria/orchestrator/workflows/core/engine.py +++ b/aria/orchestrator/workflows/core/engine.py @@ -71,6 +71,11 @@ class Engine(logger.LoggerMixin): except BaseException as e: events.on_failure_workflow_signal.send(self._workflow_context, exception=e) raise + finally: + # Each context creates its own handlers an assign them to the logger. + # This enables easy serialization. In order the handlers would not overlap, we + # need to clear them each execution. + self._workflow_context.logger.handlers = [] def cancel_execution(self): """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/aria/orchestrator/workflows/core/task.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/core/task.py b/aria/orchestrator/workflows/core/task.py index 4d3a7f0..f44e6c5 100644 --- a/aria/orchestrator/workflows/core/task.py +++ b/aria/orchestrator/workflows/core/task.py @@ -23,8 +23,6 @@ from functools import ( wraps, ) -import logging - from aria import logger from aria.storage import model from aria.orchestrator.context import operation as operation_context http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/aria/orchestrator/workflows/executor/process.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py index c4b8ba1..0cec889 100644 --- a/aria/orchestrator/workflows/executor/process.py +++ b/aria/orchestrator/workflows/executor/process.py @@ -324,7 +324,6 @@ def _main(): # This is required for the instrumentation work properly. # See docstring of `remove_mutable_association_listener` for further details storage_type.remove_mutable_association_listener() - with instrumentation.track_changes() as instrument: try: ctx = context_dict['context_cls'].deserialize_from_dict(**context_dict['context']) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/aria/storage/model.py ---------------------------------------------------------------------- diff --git a/aria/storage/model.py b/aria/storage/model.py index 23fbb88..b3cb5d4 100644 --- a/aria/storage/model.py +++ b/aria/storage/model.py @@ -111,4 +111,4 @@ class Task(DeclarativeBase, base.TaskBase): class Log(DeclarativeBase, base.LogBase): - pass \ No newline at end of file + pass http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/tests/orchestrator/context/test_operation.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py index bab3fb9..2c256f8 100644 --- a/tests/orchestrator/context/test_operation.py +++ b/tests/orchestrator/context/test_operation.py @@ -15,7 +15,6 @@ import os -import logging import pytest from aria import ( @@ -39,8 +38,7 @@ global_test_holder = {} @pytest.fixture def ctx(tmpdir): context = mock.context.simple( - str(tmpdir.join('workdir')), - inmemory=True, + str(tmpdir), context_kwargs=dict(workdir=str(tmpdir.join('workdir'))) ) yield context @@ -262,10 +260,10 @@ def test_operation_logging(ctx, executor): op_start_log.created_at < op_end_log.created_at) + @operation def logged_operation(ctx, **_): ctx.logger.info(ctx.task.inputs['op_start']) - x = 1 + 2 ctx.logger.debug(ctx.task.inputs['op_end']) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/6bff9067/tests/storage/__init__.py ---------------------------------------------------------------------- diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 2ba6da9..2befd3a 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -22,7 +22,8 @@ from sqlalchemy import ( Column, Text, Integer, - pool + pool, + MetaData ) @@ -56,14 +57,8 @@ def release_sqlite_storage(storage): :param storage: :return: """ - mapis = storage.registered.values() - - if mapis: - for session in set(mapi._session for mapi in mapis): - session.rollback() - session.close() - for engine in set(mapi._engine for mapi in mapis): - model.DeclarativeBase.metadata.drop_all(engine) + storage.all_api_kwargs['session'].close() + MetaData(bind=storage.all_api_kwargs['engine']).drop_all() def init_inmemory_model_storage():
