Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-42-Generic-ctx-serialization-mechanism efdd85f66 -> 01f963699 (forced update)
ARIA-42-Generic-ctx-serialization-mechanism Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/01f96369 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/01f96369 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/01f96369 Branch: refs/heads/ARIA-42-Generic-ctx-serialization-mechanism Commit: 01f96369973bd9564bb19bdb44ffc9ac15166504 Parents: e282f23 Author: mxmrlv <[email protected]> Authored: Wed Feb 1 16:16:01 2017 +0200 Committer: mxmrlv <[email protected]> Committed: Mon Feb 6 23:15:30 2017 +0200 ---------------------------------------------------------------------- aria/__init__.py | 16 +++---- aria/orchestrator/context/serialization.py | 49 ++++++-------------- aria/orchestrator/runner.py | 8 +--- aria/orchestrator/workflows/executor/process.py | 6 ++- aria/storage/api.py | 36 +++++++++++++- aria/storage/sql_mapi.py | 41 +++++++++++++++- setup.py | 2 +- tests/__init__.py | 4 ++ tests/mock/context.py | 25 +++++----- tests/orchestrator/context/test_operation.py | 7 ++- .../context/test_resource_render.py | 3 +- tests/orchestrator/context/test_serialize.py | 20 ++++---- tests/orchestrator/context/test_toolbelt.py | 4 +- tests/orchestrator/context/test_workflow.py | 6 +-- .../orchestrator/execution_plugin/test_local.py | 3 +- tests/orchestrator/execution_plugin/test_ssh.py | 5 +- tests/orchestrator/test_runner.py | 3 +- tests/orchestrator/workflows/api/test_task.py | 4 +- .../workflows/builtin/test_execute_operation.py | 3 +- .../orchestrator/workflows/builtin/test_heal.py | 3 +- .../workflows/builtin/test_install.py | 3 +- .../workflows/builtin/test_uninstall.py | 3 +- .../orchestrator/workflows/core/test_engine.py | 4 +- tests/orchestrator/workflows/core/test_task.py | 4 +- .../test_task_graph_into_exececution_graph.py | 4 +- .../workflows/executor/test_process_executor.py | 7 +-- .../executor/test_process_executor_extension.py | 3 +- .../test_process_executor_tracked_changes.py | 3 +- tests/storage/__init__.py | 31 +------------ tests/storage/test_instrumentation.py | 4 +- tests/storage/test_model_storage.py | 7 +-- tests/storage/test_models.py | 4 +- tests/storage/test_structures.py | 12 +++-- tests/utils/test_plugin.py | 6 +-- 34 files changed, 188 insertions(+), 155 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index 248aa1a..da77f03 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -57,7 +57,7 @@ def install_aria_extensions(): extension.init() -def application_model_storage(api, api_kwargs=None): +def application_model_storage(api, api_kwargs=None, storage_initiator_func=None): """ Initiate model storage """ @@ -78,19 +78,15 @@ def application_model_storage(api, api_kwargs=None): storage.model.Execution, storage.model.Task, ] - # if api not in _model_storage: + api.storage_initiator(storage_initiator_func) return storage.ModelStorage(api, items=models, api_kwargs=api_kwargs or {}) -def application_resource_storage(api, api_kwargs=None): +def application_resource_storage(api, api_kwargs=None, storage_initiator_func=None): """ Initiate resource storage """ + + api.storage_initiator(storage_initiator_func) return storage.ResourceStorage( - api, - api_kwargs=api_kwargs or {}, - items=[ - 'blueprint', - 'deployment', - 'plugin', - ]) + api, api_kwargs=api_kwargs or {}, items=['blueprint', 'deployment', 'plugin', ]) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/aria/orchestrator/context/serialization.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/context/serialization.py b/aria/orchestrator/context/serialization.py index 760818f..0810541 100644 --- a/aria/orchestrator/context/serialization.py +++ b/aria/orchestrator/context/serialization.py @@ -13,9 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sqlalchemy.orm -import sqlalchemy.pool - import aria @@ -32,15 +29,19 @@ def operation_context_to_dict(context): model = context.model context_dict['model_storage'] = { 'api_cls': model.api, - 'api_kwargs': _serialize_sql_mapi_kwargs(model) + 'api_kwargs': model._api_kwargs, + 'init_func': model.api._init_func.__func__ if hasattr(model.api, '_init_func') else None } + else: context_dict['model_storage'] = None if context.resource: resource = context.resource context_dict['resource_storage'] = { 'api_cls': resource.api, - 'api_kwargs': _serialize_file_rapi_kwargs(resource) + 'api_kwargs': resource._api_kwargs, + 'init_func': resource.api._init_func.__func__ + if hasattr(resource.api, '_init_func') else None } else: context_dict['resource_storage'] = None @@ -57,39 +58,17 @@ def operation_context_from_dict(context_dict): model_storage = context['model_storage'] if model_storage: api_cls = model_storage['api_cls'] - api_kwargs = _deserialize_sql_mapi_kwargs(model_storage.get('api_kwargs', {})) - context['model_storage'] = aria.application_model_storage(api=api_cls, - api_kwargs=api_kwargs) + api_kwargs = model_storage['api_kwargs'] + init_func = model_storage['init_func'] + context['model_storage'] = aria.application_model_storage( + api_cls, api_kwargs=api_kwargs, storage_initiator_func=init_func) resource_storage = context['resource_storage'] if resource_storage: api_cls = resource_storage['api_cls'] - api_kwargs = _deserialize_file_rapi_kwargs(resource_storage.get('api_kwargs', {})) - context['resource_storage'] = aria.application_resource_storage(api=api_cls, - api_kwargs=api_kwargs) + api_kwargs = resource_storage['api_kwargs'] + init_func = resource_storage['init_func'] + context['resource_storage'] = aria.application_resource_storage( + api=api_cls, api_kwargs=api_kwargs, storage_initiator_func=init_func) return context_cls(**context) - - -def _serialize_sql_mapi_kwargs(model): - engine_url = str(model._api_kwargs['engine'].url) - assert ':memory:' not in engine_url - return {'engine_url': engine_url} - - -def _deserialize_sql_mapi_kwargs(api_kwargs): - engine_url = api_kwargs.get('engine_url') - if not engine_url: - return {} - engine = sqlalchemy.create_engine(engine_url) - session_factory = sqlalchemy.orm.sessionmaker(bind=engine) - session = sqlalchemy.orm.scoped_session(session_factory=session_factory) - return {'session': session, 'engine': engine} - - -def _serialize_file_rapi_kwargs(resource): - return {'directory': resource._api_kwargs['directory']} - - -def _deserialize_file_rapi_kwargs(api_kwargs): - return api_kwargs http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/aria/orchestrator/runner.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/runner.py b/aria/orchestrator/runner.py index 16acc19..7892d67 100644 --- a/aria/orchestrator/runner.py +++ b/aria/orchestrator/runner.py @@ -118,15 +118,11 @@ class Runner(object): # Storage sqlite_kwargs = dict(engine=sqlite_engine, session=sqlite_session) - return application_model_storage( - SQLAlchemyModelAPI, - api_kwargs=sqlite_kwargs) + return application_model_storage(SQLAlchemyModelAPI, api_kwargs=sqlite_kwargs) def create_fs_resource_storage(self, directory='.'): # pylint: disable=no-self-use fs_kwargs = dict(directory=directory) - return application_resource_storage( - FileSystemResourceAPI, - api_kwargs=fs_kwargs) + return application_resource_storage(FileSystemResourceAPI, api_kwargs=fs_kwargs) def cleanup(self): if self._is_storage_temporary \ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/aria/orchestrator/workflows/executor/process.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py index 7d990fa..23a4f52 100644 --- a/aria/orchestrator/workflows/executor/process.py +++ b/aria/orchestrator/workflows/executor/process.py @@ -37,6 +37,7 @@ import struct import subprocess import tempfile import Queue +import pickle import jsonpickle @@ -113,7 +114,7 @@ class ProcessExecutor(base.BaseExecutor): file_descriptor, arguments_json_path = tempfile.mkstemp(prefix='executor-', suffix='.json') os.close(file_descriptor) with open(arguments_json_path, 'wb') as f: - f.write(jsonpickle.dumps(self._create_arguments_dict(task))) + f.write(pickle.dumps(self._create_arguments_dict(task))) env = os.environ.copy() # See _update_env for plugin_prefix usage @@ -306,7 +307,7 @@ def _patch_session(ctx, messenger, instrument): def _main(): arguments_json_path = sys.argv[1] with open(arguments_json_path) as f: - arguments = jsonpickle.loads(f.read()) + arguments = pickle.loads(f.read()) # arguments_json_path is a temporary file created by the parent process. # so we remove it here @@ -338,5 +339,6 @@ def _main(): except BaseException as e: messenger.failed(exception=e, tracked_changes=instrument.tracked_changes) + if __name__ == '__main__': _main() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/aria/storage/api.py ---------------------------------------------------------------------- diff --git a/aria/storage/api.py b/aria/storage/api.py index d6fc3b8..5f6b862 100644 --- a/aria/storage/api.py +++ b/aria/storage/api.py @@ -15,9 +15,12 @@ """ General storage API """ +from functools import partial class StorageAPI(object): + NO_INITIATOR = 'no_initiator_func' + """ General storage Base API """ @@ -29,6 +32,36 @@ class StorageAPI(object): """ raise NotImplementedError('Subclass must implement abstract create method') + @classmethod + def storage_initiator(cls, func=NO_INITIATOR): + if func is cls.NO_INITIATOR: + # Support for decoration syntax + return partial(cls.storage_initiator, cls=cls) + + if func is None: + # If func is None, no storage initiator was set, and nothing should happen. + return + + if hasattr(cls, '_original_init') and cls._init_func.__func__ == func: # pylint: disable=no-member + # setting the same function twice could cause a loop. we need to avoid that. + return + + cls._original_init = cls.__init__ + cls._init_func = func + cls.__init__ = \ + lambda self, *args, **kwargs: cls._original_init(self, **func(cls=cls, *args, **kwargs)) + + @classmethod + def free_storage_initiator(cls): + if hasattr(cls, '_original_init') and cls.__init__ != cls._original_init: + cls.__init__ = cls._original_init + delattr(cls, '_original_init') + + if hasattr(cls, '_engine'): + delattr(cls, '_engine') + if hasattr(cls, '_session'): + delattr(cls, '_session') + class ModelAPI(StorageAPI): """ @@ -119,11 +152,12 @@ class ResourceAPI(StorageAPI): """ A Base object for the resource. """ - def __init__(self, name): + def __init__(self, name, **kwargs): """ Base resource API :param str name: the resource type """ + super(ResourceAPI, self).__init__(**kwargs) self._name = name @property http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/aria/storage/sql_mapi.py ---------------------------------------------------------------------- diff --git a/aria/storage/sql_mapi.py b/aria/storage/sql_mapi.py index 809f677..2ce8998 100644 --- a/aria/storage/sql_mapi.py +++ b/aria/storage/sql_mapi.py @@ -15,13 +15,18 @@ """ SQLAlchemy based MAPI """ +import os +import platform +from sqlalchemy import create_engine +from sqlalchemy import orm +from sqlalchemy import pool from sqlalchemy.exc import SQLAlchemyError from aria.utils.collections import OrderedDict -from aria.storage import ( +from . import ( api, - exceptions + exceptions, ) @@ -364,6 +369,38 @@ class SQLAlchemyModelAPI(api.ModelAPI): getattr(instance, rel.key) +def init_storage(cls, base_dir=None, filename='db.sqlite', **kwargs): + + _storage_kwargs = { + 'base_dir': base_dir, + 'filename': filename + } + _storage_kwargs.update(**kwargs) + + if not hasattr(cls, '_engine') or getattr(cls, '_storage_kwargs', {}) == _storage_kwargs: + if base_dir is not None: + uri = 'sqlite:///{platform_char}{path}'.format( + # Handles the windows behavior where there is not root, but drivers. + # Thus behaving as relative path. + platform_char='' if 'Windows' in platform.system() else '/', + + path=os.path.join(base_dir, filename)) + engine_kwargs = {} + else: + uri = 'sqlite:///:memory:' + engine_kwargs = dict(connect_args={'check_same_thread': False}, + poolclass=pool.StaticPool) + + cls._engine = create_engine(uri, **engine_kwargs) + session_factory = orm.sessionmaker(bind=cls._engine) + cls._session = orm.scoped_session(session_factory=session_factory) if base_dir else \ + session_factory() + + return_dict = dict(engine=cls._engine, session=cls._session) + return_dict.update(kwargs) + return return_dict + + class ListResult(object): """ a ListResult contains results about the requested items. http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 2d1106d..7a1a3f4 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ console_scripts = ['aria = aria.cli.cli:main'] class InstallCommand(install): user_options = install.user_options + [ - ('skip-ctx', None, 'Install with or without the ctx (Defaults to False') + ('skip-ctx', None, 'Install with or without the ctx (Defaults to False)') ] boolean_options = install.boolean_options + ['skip-ctx'] http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/__init__.py ---------------------------------------------------------------------- diff --git a/tests/__init__.py b/tests/__init__.py index d2858d2..e51dd69 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -15,4 +15,8 @@ import os +from aria.storage import sql_mapi + +sql_mapi.SQLAlchemyModelAPI.storage_initiator(sql_mapi.init_storage) + ROOT_DIR = os.path.dirname(os.path.dirname(__file__)) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/mock/context.py ---------------------------------------------------------------------- diff --git a/tests/mock/context.py b/tests/mock/context.py index ec4bfb8..d1fa5ed 100644 --- a/tests/mock/context.py +++ b/tests/mock/context.py @@ -16,25 +16,24 @@ import aria from aria.orchestrator import context from aria.storage.filesystem_rapi import FileSystemResourceAPI -from aria.storage.sql_mapi import SQLAlchemyModelAPI +from aria.storage import sql_mapi from . import models from .topology import create_simple_topology_two_nodes -def simple(mapi_kwargs, resources_dir=None, **kwargs): - model_storage = aria.application_model_storage(SQLAlchemyModelAPI, api_kwargs=mapi_kwargs) +def simple(tmpdir, model_driver_kwargs=None, resources_driver_kwargs=None, context_kwargs=None): - deployment_id = create_simple_topology_two_nodes(model_storage) + model_storage = aria.application_model_storage( + sql_mapi.SQLAlchemyModelAPI, + api_kwargs=model_driver_kwargs or {}, + storage_initiator_func=sql_mapi.init_storage + ) - # pytest tmpdir - if resources_dir: - resource_storage = aria.application_resource_storage( - FileSystemResourceAPI, - api_kwargs={'directory': resources_dir} - ) - else: - resource_storage = None + resource_storage = aria.application_resource_storage( + FileSystemResourceAPI, api_kwargs=resources_driver_kwargs or dict(directory=str(tmpdir))) + + deployment_id = create_simple_topology_two_nodes(model_storage) final_kwargs = dict( name='simple_context', @@ -45,5 +44,5 @@ def simple(mapi_kwargs, resources_dir=None, **kwargs): task_max_attempts=models.TASK_MAX_ATTEMPTS, task_retry_interval=models.TASK_RETRY_INTERVAL ) - final_kwargs.update(kwargs) + final_kwargs.update(context_kwargs or {}) return context.workflow.WorkflowContext(**final_kwargs) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/context/test_operation.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py index b0918d1..1b84390 100644 --- a/tests/orchestrator/context/test_operation.py +++ b/tests/orchestrator/context/test_operation.py @@ -37,8 +37,11 @@ global_test_holder = {} @pytest.fixture def ctx(tmpdir): - context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)), - workdir=str(tmpdir.join('workdir'))) + context = mock.context.simple( + str(tmpdir.join('workdir')), + model_driver_kwargs=dict(base_dir=str(tmpdir)), + context_kwargs=dict(workdir=str(tmpdir.join('workdir'))) + ) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/context/test_resource_render.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_resource_render.py b/tests/orchestrator/context/test_resource_render.py index ca2ef42..20ad694 100644 --- a/tests/orchestrator/context/test_resource_render.py +++ b/tests/orchestrator/context/test_resource_render.py @@ -53,8 +53,7 @@ def test_download_resource_and_render_provided_variables(tmpdir, ctx): @pytest.fixture def ctx(tmpdir): - context = mock.context.simple(storage.get_sqlite_api_kwargs(), - resources_dir=str(tmpdir.join('resources'))) + context = mock.context.simple(str(tmpdir.join('resources'))) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/context/test_serialize.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_serialize.py b/tests/orchestrator/context/test_serialize.py index 76930b1..ef553d2 100644 --- a/tests/orchestrator/context/test_serialize.py +++ b/tests/orchestrator/context/test_serialize.py @@ -16,13 +16,11 @@ import pytest import aria -from aria.storage.sql_mapi import SQLAlchemyModelAPI +from aria.storage import sql_mapi from aria.orchestrator.workflows import api from aria.orchestrator.workflows.core import engine from aria.orchestrator.workflows.executor import process from aria.orchestrator import workflow, operation -from aria.orchestrator.context import serialization - import tests from tests import mock from tests import storage @@ -42,10 +40,6 @@ def test_serialize_operation_context(context, executor, tmpdir): eng.execute() -def test_illegal_serialize_of_memory_model_storage(memory_model_storage): - with pytest.raises(AssertionError): - serialization._serialize_sql_mapi_kwargs(memory_model_storage) - @workflow def _mock_workflow(ctx, graph): @@ -93,16 +87,18 @@ def executor(): @pytest.fixture def context(tmpdir): - result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)), - resources_dir=str(tmpdir.join('resources')), - workdir=str(tmpdir.join('workdir'))) + result = mock.context.simple( + str(tmpdir.join('resources')), + model_driver_kwargs=dict(base_dir=str(tmpdir)), + context_kwargs=dict(workdir=str(tmpdir.join('workdir'))) + ) + yield result storage.release_sqlite_storage(result.model) @pytest.fixture def memory_model_storage(): - result = aria.application_model_storage( - SQLAlchemyModelAPI, api_kwargs=storage.get_sqlite_api_kwargs()) + result = aria.application_model_storage(sql_mapi.SQLAlchemyModelAPI) yield result storage.release_sqlite_storage(result) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/context/test_toolbelt.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_toolbelt.py b/tests/orchestrator/context/test_toolbelt.py index b63811b..6f83d1e 100644 --- a/tests/orchestrator/context/test_toolbelt.py +++ b/tests/orchestrator/context/test_toolbelt.py @@ -33,7 +33,9 @@ global_test_holder = {} @pytest.fixture def workflow_context(tmpdir): - context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + context = mock.context.simple( + str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/context/test_workflow.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py index 496c1ff..6d9ed39 100644 --- a/tests/orchestrator/context/test_workflow.py +++ b/tests/orchestrator/context/test_workflow.py @@ -19,7 +19,7 @@ import pytest from aria import application_model_storage from aria.orchestrator import context -from aria.storage.sql_mapi import SQLAlchemyModelAPI +from aria.storage import sql_mapi from tests import storage as test_storage from tests.mock import models @@ -60,8 +60,8 @@ class TestWorkflowContext(object): @pytest.fixture(scope='function') def storage(): - api_kwargs = test_storage.get_sqlite_api_kwargs() - workflow_storage = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) + workflow_storage = application_model_storage(sql_mapi.SQLAlchemyModelAPI, + storage_initiator_func=sql_mapi.init_storage) workflow_storage.blueprint.put(models.get_blueprint()) blueprint = workflow_storage.blueprint.get_by_name(models.BLUEPRINT_NAME) workflow_storage.deployment.put(models.get_deployment(blueprint)) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/execution_plugin/test_local.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/execution_plugin/test_local.py b/tests/orchestrator/execution_plugin/test_local.py index 497da48..dd41224 100644 --- a/tests/orchestrator/execution_plugin/test_local.py +++ b/tests/orchestrator/execution_plugin/test_local.py @@ -504,8 +504,7 @@ if __name__ == '__main__': @pytest.fixture def workflow_context(self, tmpdir): workflow_context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)), - resources_dir=str(tmpdir.join('resources'))) + str(tmpdir.join('resources')), model_driver_kwargs=dict(base_dir=str(tmpdir))) workflow_context.states = [] workflow_context.exception = None yield workflow_context http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/execution_plugin/test_ssh.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/execution_plugin/test_ssh.py b/tests/orchestrator/execution_plugin/test_ssh.py index 6b5c783..b023284 100644 --- a/tests/orchestrator/execution_plugin/test_ssh.py +++ b/tests/orchestrator/execution_plugin/test_ssh.py @@ -265,9 +265,8 @@ class TestWithActualSSHServer(object): @pytest.fixture def workflow_context(self, tmpdir): - workflow_context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)), - resources_dir=str(tmpdir.join('resources'))) + workflow_context = mock.context.simple(str(tmpdir.join('resources')), + model_driver_kwargs=dict(base_dir=str(tmpdir))) workflow_context.states = [] workflow_context.exception = None yield workflow_context http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/test_runner.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/test_runner.py b/tests/orchestrator/test_runner.py index 1d46e91..0a6fadf 100644 --- a/tests/orchestrator/test_runner.py +++ b/tests/orchestrator/test_runner.py @@ -36,6 +36,7 @@ def cleanup(): OPERATION_RESULTS.clear() [email protected]() def test_runner_no_tasks(): @workflow def workflow_fn(ctx, graph): # pylint: disable=unused-argument @@ -43,7 +44,7 @@ def test_runner_no_tasks(): _test_runner(workflow_fn) - [email protected]() def test_runner_tasks(): @workflow def workflow_fn(ctx, graph): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/api/test_task.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/api/test_task.py b/tests/orchestrator/workflows/api/test_task.py index 601c437..433c763 100644 --- a/tests/orchestrator/workflows/api/test_task.py +++ b/tests/orchestrator/workflows/api/test_task.py @@ -24,13 +24,13 @@ from tests import mock, storage @pytest.fixture -def ctx(): +def ctx(tmpdir): """ Create the following graph in storage: dependency_node <------ dependent_node :return: """ - simple_context = mock.context.simple(storage.get_sqlite_api_kwargs()) + simple_context = mock.context.simple(tmpdir) simple_context.model.execution.put(mock.models.get_execution(simple_context.deployment)) yield simple_context storage.release_sqlite_storage(simple_context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/builtin/test_execute_operation.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_execute_operation.py b/tests/orchestrator/workflows/builtin/test_execute_operation.py index b7e5678..4ad614a 100644 --- a/tests/orchestrator/workflows/builtin/test_execute_operation.py +++ b/tests/orchestrator/workflows/builtin/test_execute_operation.py @@ -24,7 +24,8 @@ from tests import storage @pytest.fixture def ctx(tmpdir): - context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + context = mock.context.simple(str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/builtin/test_heal.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_heal.py b/tests/orchestrator/workflows/builtin/test_heal.py index b470790..2555919 100644 --- a/tests/orchestrator/workflows/builtin/test_heal.py +++ b/tests/orchestrator/workflows/builtin/test_heal.py @@ -26,7 +26,8 @@ from . import (assert_node_install_operations, @pytest.fixture def ctx(tmpdir): - context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + context = mock.context.simple(str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/builtin/test_install.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_install.py b/tests/orchestrator/workflows/builtin/test_install.py index 789a161..dc425b3 100644 --- a/tests/orchestrator/workflows/builtin/test_install.py +++ b/tests/orchestrator/workflows/builtin/test_install.py @@ -25,7 +25,8 @@ from . import assert_node_install_operations @pytest.fixture def ctx(tmpdir): - context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + context = mock.context.simple(str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/builtin/test_uninstall.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_uninstall.py b/tests/orchestrator/workflows/builtin/test_uninstall.py index 126c4cf..5a34494 100644 --- a/tests/orchestrator/workflows/builtin/test_uninstall.py +++ b/tests/orchestrator/workflows/builtin/test_uninstall.py @@ -26,7 +26,8 @@ from . import assert_node_uninstall_operations @pytest.fixture def ctx(tmpdir): - context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + context = mock.context.simple(str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/core/test_engine.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_engine.py b/tests/orchestrator/workflows/core/test_engine.py index d9b50a9..68ba477 100644 --- a/tests/orchestrator/workflows/core/test_engine.py +++ b/tests/orchestrator/workflows/core/test_engine.py @@ -124,7 +124,9 @@ class BaseTest(object): @pytest.fixture def workflow_context(self, tmpdir): - workflow_context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + workflow_context = mock.context.simple( + str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) workflow_context.states = [] workflow_context.exception = None yield workflow_context http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/core/test_task.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_task.py b/tests/orchestrator/workflows/core/test_task.py index 061a3f2..7ffe045 100644 --- a/tests/orchestrator/workflows/core/test_task.py +++ b/tests/orchestrator/workflows/core/test_task.py @@ -31,7 +31,9 @@ from tests import mock, storage @pytest.fixture def ctx(tmpdir): - context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + context = mock.context.simple( + str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) yield context storage.release_sqlite_storage(context.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py index cd37bde..57be075 100644 --- a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py +++ b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py @@ -22,9 +22,9 @@ from tests import mock from tests import storage -def test_task_graph_into_execution_graph(): +def test_task_graph_into_execution_graph(tmpdir): operation_name = 'tosca.interfaces.node.lifecycle.Standard.create' - task_context = mock.context.simple(storage.get_sqlite_api_kwargs()) + task_context = mock.context.simple(str(tmpdir)) node_instance = \ task_context.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME) def sub_workflow(name, **_): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/executor/test_process_executor.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/executor/test_process_executor.py b/tests/orchestrator/workflows/executor/test_process_executor.py index 687e245..ab47bca 100644 --- a/tests/orchestrator/workflows/executor/test_process_executor.py +++ b/tests/orchestrator/workflows/executor/test_process_executor.py @@ -24,7 +24,7 @@ import pytest from aria import application_model_storage from aria.storage import model as aria_model from aria.utils.plugin import create as create_plugin -from aria.storage.sql_mapi import SQLAlchemyModelAPI +from aria.storage import sql_mapi from aria.orchestrator import events from aria.orchestrator import plugin from aria.orchestrator.workflows.executor import process @@ -74,8 +74,9 @@ class TestProcessExecutor(object): @pytest.fixture def model(tmpdir): - api_kwargs = tests.storage.get_sqlite_api_kwargs(str(tmpdir)) - result = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) + result = application_model_storage(sql_mapi.SQLAlchemyModelAPI, + api_kwargs=dict(base_dir=str(tmpdir)), + storage_initiator_func=sql_mapi.init_storage) yield result tests.storage.release_sqlite_storage(result) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/executor/test_process_executor_extension.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/executor/test_process_executor_extension.py b/tests/orchestrator/workflows/executor/test_process_executor_extension.py index 4a8ef57..9e8623e 100644 --- a/tests/orchestrator/workflows/executor/test_process_executor_extension.py +++ b/tests/orchestrator/workflows/executor/test_process_executor_extension.py @@ -75,6 +75,7 @@ def executor(): @pytest.fixture def context(tmpdir): - result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + result = mock.context.simple(str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) yield result storage.release_sqlite_storage(result.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py index bd1fa96..45793c4 100644 --- a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py +++ b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py @@ -148,6 +148,7 @@ def executor(): @pytest.fixture def context(tmpdir): - result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) + result = mock.context.simple(str(tmpdir), + model_driver_kwargs=dict(base_dir=str(tmpdir))) yield result storage.release_sqlite_storage(result.model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/storage/__init__.py ---------------------------------------------------------------------- diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 3b3715e..10c2ee2 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -52,35 +52,6 @@ class TestFileSystem(object): rmtree(self.path, ignore_errors=True) -def get_sqlite_api_kwargs(base_dir=None, filename='db.sqlite'): - """ - Create sql params. works in in-memory and in filesystem mode. - If base_dir is passed, the mode will be filesystem mode. while the default mode is in-memory. - :param str base_dir: The base dir for the filesystem memory file. - :param str filename: the file name - defaults to 'db.sqlite'. - :return: - """ - if base_dir is not None: - uri = 'sqlite:///{platform_char}{path}'.format( - # Handles the windows behavior where there is not root, but drivers. - # Thus behaving as relative path. - platform_char='' if 'Windows' in platform.system() else '/', - - path=os.path.join(base_dir, filename)) - engine_kwargs = {} - else: - uri = 'sqlite:///:memory:' - engine_kwargs = dict(connect_args={'check_same_thread': False}, - poolclass=pool.StaticPool) - - engine = create_engine(uri, **engine_kwargs) - session_factory = orm.sessionmaker(bind=engine) - session = orm.scoped_session(session_factory=session_factory) if base_dir else session_factory() - - model.DeclarativeBase.metadata.create_all(bind=engine) - return dict(engine=engine, session=session) - - def release_sqlite_storage(storage): """ Drops the tables and clears the session @@ -90,6 +61,8 @@ def release_sqlite_storage(storage): mapis = storage.registered.values() if mapis: + for mapi in mapis: + mapi.free_storage_initiator() for session in set(mapi._session for mapi in mapis): session.rollback() session.close() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/storage/test_instrumentation.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_instrumentation.py b/tests/storage/test_instrumentation.py index 9b4da4f..7eb733f 100644 --- a/tests/storage/test_instrumentation.py +++ b/tests/storage/test_instrumentation.py @@ -25,7 +25,7 @@ from aria.storage import ( instrumentation, exceptions ) -from ..storage import get_sqlite_api_kwargs, release_sqlite_storage +from ..storage import release_sqlite_storage STUB = instrumentation._STUB @@ -328,9 +328,9 @@ def restore_instrumentation(): @pytest.fixture def storage(): + sql_mapi.SQLAlchemyModelAPI.storage_initiator(sql_mapi.init_storage) result = ModelStorage( api_cls=sql_mapi.SQLAlchemyModelAPI, - api_kwargs=get_sqlite_api_kwargs(), items=(MockModel1, MockModel2, StrictMockModel)) yield result release_sqlite_storage(result) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/storage/test_model_storage.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py index d1596e3..3ba92fe 100644 --- a/tests/storage/test_model_storage.py +++ b/tests/storage/test_model_storage.py @@ -22,14 +22,15 @@ from aria.storage import ( sql_mapi, ) from aria import application_model_storage -from ..storage import get_sqlite_api_kwargs, release_sqlite_storage +from ..storage import release_sqlite_storage from . import MockModel @pytest.fixture def storage(): - base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs()) + base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI) + sql_mapi.SQLAlchemyModelAPI.storage_initiator(sql_mapi.init_storage) base_storage.register(MockModel) yield base_storage release_sqlite_storage(base_storage) @@ -61,7 +62,7 @@ def test_model_storage(storage): def test_application_storage_factory(): storage = application_model_storage(sql_mapi.SQLAlchemyModelAPI, - api_kwargs=get_sqlite_api_kwargs()) + storage_initiator_func=sql_mapi.init_storage) assert storage.node assert storage.node_instance assert storage.plugin http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/storage/test_models.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_models.py b/tests/storage/test_models.py index 2088676..c5b1c62 100644 --- a/tests/storage/test_models.py +++ b/tests/storage/test_models.py @@ -39,7 +39,7 @@ from aria.storage.model import ( from tests import mock -from tests.storage import get_sqlite_api_kwargs, release_sqlite_storage +from tests.storage import release_sqlite_storage @contextmanager @@ -55,7 +55,7 @@ def sql_storage(storage_func): def _empty_storage(): return application_model_storage(sql_mapi.SQLAlchemyModelAPI, - api_kwargs=get_sqlite_api_kwargs()) + storage_initiator_func=sql_mapi.init_storage) def _blueprint_storage(): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/storage/test_structures.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_structures.py b/tests/storage/test_structures.py index 0223a98..6caa4ff 100644 --- a/tests/storage/test_structures.py +++ b/tests/storage/test_structures.py @@ -25,7 +25,7 @@ from aria.storage import ( exceptions ) -from ..storage import get_sqlite_api_kwargs, release_sqlite_storage, structure +from ..storage import release_sqlite_storage, structure from . import MockModel from ..mock import ( models, @@ -36,7 +36,8 @@ from ..mock import ( @pytest.fixture def storage(): - base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs()) + base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI) + sql_mapi.SQLAlchemyModelAPI.storage_initiator(sql_mapi.init_storage) base_storage.register(MockModel) yield base_storage release_sqlite_storage(base_storage) @@ -48,8 +49,10 @@ def module_cleanup(): @pytest.fixture -def context(): - return mock_context.simple(get_sqlite_api_kwargs()) +def context(tmpdir): + ctx = mock_context.simple(str(tmpdir)) + yield ctx + release_sqlite_storage(ctx.model) def test_inner_dict_update(storage): @@ -174,7 +177,6 @@ def test_relationship_model_ordering(context): target_node_instance=target_node_instance, ) - context.model.node.put(new_node) context.model.node_instance.put(new_node_instance) context.model.relationship.put(source_to_new_relationship) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/01f96369/tests/utils/test_plugin.py ---------------------------------------------------------------------- diff --git a/tests/utils/test_plugin.py b/tests/utils/test_plugin.py index 6f2dd92..6a45f19 100644 --- a/tests/utils/test_plugin.py +++ b/tests/utils/test_plugin.py @@ -21,7 +21,7 @@ from aria import application_model_storage from aria.orchestrator import exceptions from aria.orchestrator import plugin from aria.utils.plugin import create as create_plugin -from aria.storage.sql_mapi import SQLAlchemyModelAPI +from aria.storage import sql_mapi from .. import storage @@ -49,8 +49,8 @@ class TestPluginManager(object): @pytest.fixture def model(): - api_kwargs = storage.get_sqlite_api_kwargs() - model = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) + model = application_model_storage(sql_mapi.SQLAlchemyModelAPI, + storage_initiator_func=sql_mapi.init_storage) yield model storage.release_sqlite_storage(model)
