Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-30-SQL-based-storage-implementation e5bbeaa03 -> 402aeaca4 (forced update)
major threading fix Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/402aeaca Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/402aeaca Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/402aeaca Branch: refs/heads/ARIA-30-SQL-based-storage-implementation Commit: 402aeaca4c600317a0178de102c29ca16f8893f0 Parents: 7ec1183 Author: mxmrlv <mxm...@gmail.com> Authored: Tue Dec 6 22:57:42 2016 +0200 Committer: mxmrlv <mxm...@gmail.com> Committed: Wed Dec 7 11:16:48 2016 +0200 ---------------------------------------------------------------------- aria/__init__.py | 13 ++++--- aria/orchestrator/context/operation.py | 16 +++++---- aria/orchestrator/context/toolbelt.py | 12 ------- aria/orchestrator/workflows/core/task.py | 3 +- aria/storage/structures.py | 2 +- tests/mock/context.py | 6 +--- tests/orchestrator/context/test_operation.py | 8 +++-- tests/orchestrator/context/test_toolbelt.py | 38 ++------------------ tests/orchestrator/context/test_workflow.py | 2 +- tests/orchestrator/workflows/api/test_task.py | 8 ++--- .../workflows/builtin/test_execute_operation.py | 7 ++-- .../orchestrator/workflows/builtin/test_heal.py | 8 +++-- .../workflows/builtin/test_install.py | 7 ++-- .../workflows/builtin/test_uninstall.py | 7 ++-- .../orchestrator/workflows/core/test_engine.py | 11 +++--- tests/orchestrator/workflows/core/test_task.py | 10 ++++-- .../test_task_graph_into_exececution_graph.py | 5 ++- tests/storage/__init__.py | 26 +++++++++++--- tests/storage/test_model_storage.py | 5 ++- 19 files changed, 97 insertions(+), 97 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/aria/__init__.py ---------------------------------------------------------------------- diff --git a/aria/__init__.py b/aria/__init__.py index f5151e3..b000397 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -61,18 +61,21 @@ def application_model_storage(api, api_kwargs=None): Initiate model storage for the supplied storage driver """ models = [ + storage.models.Plugin, + storage.models.ProviderContext, + storage.models.Blueprint, storage.models.Deployment, + storage.models.DeploymentUpdate, + storage.models.DeploymentUpdateStep, + storage.models.DeploymentModification, + storage.models.Node, storage.models.NodeInstance, storage.models.Relationship, storage.models.RelationshipInstance, - storage.models.Plugin, - storage.models.DeploymentUpdate, - storage.models.DeploymentUpdateStep, - storage.models.DeploymentModification, + storage.models.Execution, - storage.models.ProviderContext, storage.models.Task, ] # if api not in _model_storage: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/aria/orchestrator/context/operation.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/context/operation.py b/aria/orchestrator/context/operation.py index 02469a7..a789e24 100644 --- a/aria/orchestrator/context/operation.py +++ b/aria/orchestrator/context/operation.py @@ -26,7 +26,7 @@ class BaseOperationContext(BaseContext): Context object used during operation creation and execution """ - def __init__(self, name, workflow_context, task, **kwargs): + def __init__(self, name, workflow_context, task, actor, **kwargs): super(BaseOperationContext, self).__init__( name=name, model_storage=workflow_context.model, @@ -35,7 +35,8 @@ class BaseOperationContext(BaseContext): workflow_name=workflow_context._workflow_name, **kwargs) self._task_model = task - self._actor = self.task.actor + self._task_id = task.id + self._actor_id = actor.id def __repr__(self): details = 'operation_mapping={task.operation_mapping}; ' \ @@ -49,7 +50,7 @@ class BaseOperationContext(BaseContext): The task in the model storage :return: Task model """ - return self._task_model + return self.model.task.get(self._task_id) class NodeOperationContext(BaseOperationContext): @@ -62,7 +63,7 @@ class NodeOperationContext(BaseOperationContext): the node of the current operation :return: """ - return self._actor.node + return self.node_instance.node @property def node_instance(self): @@ -70,7 +71,7 @@ class NodeOperationContext(BaseOperationContext): The node instance of the current operation :return: """ - return self._actor + return self.model.node_instance.get(self._task_id) class RelationshipOperationContext(BaseOperationContext): @@ -115,7 +116,8 @@ class RelationshipOperationContext(BaseOperationContext): The relationship of the current operation :return: """ - return self._actor.relationship + + return self.relationship_instance.relationship @property def relationship_instance(self): @@ -123,4 +125,4 @@ class RelationshipOperationContext(BaseOperationContext): The relationship instance of the current operation :return: """ - return self._actor + return self.model.relationship_instance.get(self._actor_id) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/aria/orchestrator/context/toolbelt.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/context/toolbelt.py b/aria/orchestrator/context/toolbelt.py index 298d334..301b013 100644 --- a/aria/orchestrator/context/toolbelt.py +++ b/aria/orchestrator/context/toolbelt.py @@ -27,18 +27,6 @@ class NodeToolBelt(object): self._op_context = operation_context @property - def dependent_node_instances(self): - """ - Any node instance which has a relationship to the current node instance. - :return: - """ - assert isinstance(self._op_context, operation.NodeOperationContext) - filters = {'target_node_instance_id': self._op_context.node_instance.id} - for relationship_instance in \ - self._op_context.model.relationship_instance.iter(filters=filters): - yield relationship_instance.source_node_instance - - @property def host_ip(self): """ The host ip of the current node http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/aria/orchestrator/workflows/core/task.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/core/task.py b/aria/orchestrator/workflows/core/task.py index 16c0491..0be17fe 100644 --- a/aria/orchestrator/workflows/core/task.py +++ b/aria/orchestrator/workflows/core/task.py @@ -132,7 +132,8 @@ class OperationTask(BaseTask): self._ctx = context_class(name=api_task.name, workflow_context=self._workflow_context, - task=operation_task) + task=operation_task, + actor=operation_task.actor) self._task_id = operation_task.id self._update_fields = None http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/aria/storage/structures.py ---------------------------------------------------------------------- diff --git a/aria/storage/structures.py b/aria/storage/structures.py index a170497..8dbd2a9 100644 --- a/aria/storage/structures.py +++ b/aria/storage/structures.py @@ -193,10 +193,10 @@ class _MutableList(Mutable, list): list.__delitem__(self, key) - Dict = _MutableDict.as_mutable(_DictType) List = _MutableList.as_mutable(_ListType) + class SQLModelBase(Model): """ Abstract base class for all SQL models that allows [de]serialization http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/mock/context.py ---------------------------------------------------------------------- diff --git a/tests/mock/context.py b/tests/mock/context.py index 8231d08..7eb57ed 100644 --- a/tests/mock/context.py +++ b/tests/mock/context.py @@ -15,19 +15,15 @@ import pytest - from aria import application_model_storage from aria.orchestrator import context from aria.storage.sql_mapi import SQLAlchemyModelAPI -from tests.storage import get_sqlite_api_kwargs - from . import models @pytest.fixture -def simple(**kwargs): - api_kwargs = get_sqlite_api_kwargs() +def simple(api_kwargs, **kwargs): model_storage = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) blueprint = models.get_blueprint() model_storage.blueprint.put(blueprint) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/context/test_operation.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py index ab442a4..dff181a 100644 --- a/tests/orchestrator/context/test_operation.py +++ b/tests/orchestrator/context/test_operation.py @@ -23,7 +23,7 @@ from aria.orchestrator import context from aria.orchestrator.workflows import api from aria.orchestrator.workflows.executor import thread -from tests import mock +from tests import mock, storage from . import ( op_path, op_name, @@ -34,8 +34,10 @@ global_test_holder = {} @pytest.fixture -def ctx(): - context = mock.context.simple() +def ctx(tmpdir): + context = mock.context.simple( + storage.get_sqlite_api_kwargs(tmpdir) + ) yield context context.model.drop() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/context/test_toolbelt.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_toolbelt.py b/tests/orchestrator/context/test_toolbelt.py index 7b3f1a3..f611c9c 100644 --- a/tests/orchestrator/context/test_toolbelt.py +++ b/tests/orchestrator/context/test_toolbelt.py @@ -21,7 +21,7 @@ from aria.orchestrator.workflows import api from aria.orchestrator.workflows.executor import thread from aria.orchestrator.context.toolbelt import RelationshipToolBelt -from tests import mock +from tests import mock, storage from . import ( op_path, op_name, @@ -32,8 +32,8 @@ global_test_holder = {} @pytest.fixture -def workflow_context(): - context = mock.context.simple() +def workflow_context(tmpdir): + context = mock.context.simple(storage.get_sqlite_api_kwargs(tmpdir)) yield context context.model.drop() @@ -98,33 +98,6 @@ def test_host_ip(workflow_context, executor): dependency_node_instance.runtime_properties.get('ip') -def test_dependent_node_instances(workflow_context, executor): - operation_name = 'aria.interfaces.lifecycle.create' - dependency_node, dependency_node_instance, _, dependent_node_instance, _, _ = \ - _get_elements(workflow_context) - dependency_node.operations[operation_name] = { - 'operation': op_path(dependent_nodes, module_path=__name__) - - } - workflow_context.model.node.put(dependency_node) - inputs = {'putput': True} - - @workflow - def basic_workflow(graph, **_): - graph.add_tasks( - api.task.OperationTask.node_instance( - instance=dependency_node_instance, - name=operation_name, - inputs=inputs - ) - ) - - execute(workflow_func=basic_workflow, workflow_context=workflow_context, executor=executor) - - assert global_test_holder.get('dependent_node_instances')[0].to_dict == \ - dependent_node_instance.to_dict - - def test_relationship_tool_belt(workflow_context, executor): operation_name = 'aria.interfaces.relationship_lifecycle.postconfigure' _, _, _, _, relationship, relationship_instance = \ @@ -164,11 +137,6 @@ def host_ip(toolbelt, **_): @operation(toolbelt=True) -def dependent_nodes(toolbelt, **_): - global_test_holder['dependent_node_instances'] = list(toolbelt.dependent_node_instances) - - -@operation(toolbelt=True) def relationship_operation(ctx, toolbelt, **_): global_test_holder[ctx.name] = toolbelt http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/context/test_workflow.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py index 84fe95a..3bd305e 100644 --- a/tests/orchestrator/context/test_workflow.py +++ b/tests/orchestrator/context/test_workflow.py @@ -28,7 +28,7 @@ class TestWorkflowContext(object): def test_execution_creation_on_workflow_context_creation(self, storage): ctx = self._create_ctx(storage) - execution = storage.execution.get(ctx.execution.id) + execution = storage.execution.get(ctx.execution.id) # pylint: disable=no-member assert execution.deployment == storage.deployment.get_by_name(models.DEPLOYMENT_NAME) assert execution.workflow_name == models.WORKFLOW_NAME assert execution.blueprint == storage.blueprint.get_by_name(models.BLUEPRINT_NAME) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/workflows/api/test_task.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/api/test_task.py b/tests/orchestrator/workflows/api/test_task.py index 22dddd8..d87e2be 100644 --- a/tests/orchestrator/workflows/api/test_task.py +++ b/tests/orchestrator/workflows/api/test_task.py @@ -19,7 +19,7 @@ import pytest from aria.orchestrator import context from aria.orchestrator.workflows import api -from tests import mock +from tests import mock, storage @pytest.fixture @@ -29,11 +29,11 @@ def ctx(): dependency_node <------ dependent_node :return: """ - simple_context = mock.context.simple() + simple_context = mock.context.simple(storage.get_sqlite_api_kwargs()) simple_context.model.execution.put(mock.models.get_execution(simple_context.deployment)) - return simple_context - + yield simple_context + simple_context.model.drop() class TestOperationTask(object): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/workflows/builtin/test_execute_operation.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_execute_operation.py b/tests/orchestrator/workflows/builtin/test_execute_operation.py index c7ab524..b2e8884 100644 --- a/tests/orchestrator/workflows/builtin/test_execute_operation.py +++ b/tests/orchestrator/workflows/builtin/test_execute_operation.py @@ -19,11 +19,14 @@ from aria.orchestrator.workflows.api import task from aria.orchestrator.workflows.builtin.execute_operation import execute_operation from tests import mock +from tests import storage @pytest.fixture -def ctx(): - context = mock.context.simple() +def ctx(tmpdir): + context = mock.context.simple( + storage.get_sqlite_api_kwargs(tmpdir) + ) yield context context.model.drop() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/workflows/builtin/test_heal.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_heal.py b/tests/orchestrator/workflows/builtin/test_heal.py index e874e9e..aa35517 100644 --- a/tests/orchestrator/workflows/builtin/test_heal.py +++ b/tests/orchestrator/workflows/builtin/test_heal.py @@ -18,15 +18,17 @@ import pytest from aria.orchestrator.workflows.api import task from aria.orchestrator.workflows.builtin.heal import heal -from tests import mock +from tests import mock, storage from . import (assert_node_install_operations, assert_node_uninstall_operations) @pytest.fixture -def ctx(): - context = mock.context.simple() +def ctx(tmpdir): + context = mock.context.simple( + storage.get_sqlite_api_kwargs(tmpdir) + ) yield context context.model.drop() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/workflows/builtin/test_install.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_install.py b/tests/orchestrator/workflows/builtin/test_install.py index d6c2a97..bef29be 100644 --- a/tests/orchestrator/workflows/builtin/test_install.py +++ b/tests/orchestrator/workflows/builtin/test_install.py @@ -18,13 +18,16 @@ from aria.orchestrator.workflows.api import task from aria.orchestrator.workflows.builtin.install import install from tests import mock +from tests import storage from . import assert_node_install_operations @pytest.fixture -def ctx(): - context = mock.context.simple() +def ctx(tmpdir): + context = mock.context.simple( + storage.get_sqlite_api_kwargs(tmpdir) + ) yield context context.model.drop() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/workflows/builtin/test_uninstall.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_uninstall.py b/tests/orchestrator/workflows/builtin/test_uninstall.py index 3b1ab4f..a2acda1 100644 --- a/tests/orchestrator/workflows/builtin/test_uninstall.py +++ b/tests/orchestrator/workflows/builtin/test_uninstall.py @@ -19,13 +19,16 @@ from aria.orchestrator.workflows.api import task from aria.orchestrator.workflows.builtin.uninstall import uninstall from tests import mock +from tests import storage from . import assert_node_uninstall_operations @pytest.fixture -def ctx(): - context = mock.context.simple() +def ctx(tmpdir): + context = mock.context.simple( + storage.get_sqlite_api_kwargs(tmpdir) + ) yield context context.model.drop() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/workflows/core/test_engine.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_engine.py b/tests/orchestrator/workflows/core/test_engine.py index 5864495..e8c237f 100644 --- a/tests/orchestrator/workflows/core/test_engine.py +++ b/tests/orchestrator/workflows/core/test_engine.py @@ -31,7 +31,7 @@ from aria.orchestrator.workflows import ( from aria.orchestrator.workflows.core import engine from aria.orchestrator.workflows.executor import thread -from tests import mock +from tests import mock, storage global_test_holder = {} @@ -123,11 +123,14 @@ class BaseTest(object): result.close() @pytest.fixture(scope='function') - def workflow_context(self): - workflow_context = mock.context.simple() + def workflow_context(self, tmpdir): + workflow_context = mock.context.simple( + storage.get_sqlite_api_kwargs(tmpdir) + ) workflow_context.states = [] workflow_context.exception = None - return workflow_context + yield workflow_context + workflow_context.model.drop() class TestEngine(BaseTest): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/workflows/core/test_task.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_task.py b/tests/orchestrator/workflows/core/test_task.py index bc86576..afb50a9 100644 --- a/tests/orchestrator/workflows/core/test_task.py +++ b/tests/orchestrator/workflows/core/test_task.py @@ -26,12 +26,16 @@ from aria.orchestrator.workflows import ( exceptions, ) -from tests import mock +from tests import mock, storage @pytest.fixture -def ctx(): - return mock.context.simple() +def ctx(tmpdir): + context = mock.context.simple( + api_kwargs=storage.get_sqlite_api_kwargs(tmpdir) + ) + yield context + context.model.drop() class TestOperationTask(object): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py index e452cc6..2736a2e 100644 --- a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py +++ b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py @@ -19,11 +19,14 @@ from aria.orchestrator import context from aria.orchestrator.workflows import api, core from tests import mock +from tests import storage def test_task_graph_into_execution_graph(): operation_name = 'aria.interfaces.lifecycle.create' - task_context = mock.context.simple() + task_context = mock.context.simple( + storage.get_sqlite_api_kwargs() + ) node_instance = \ task_context.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME) def sub_workflow(name, **_): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/storage/__init__.py ---------------------------------------------------------------------- diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index ba1269d..164c0e1 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -19,6 +19,7 @@ from shutil import rmtree from sqlalchemy import ( create_engine, orm) +from sqlalchemy.orm import scoped_session from sqlalchemy.pool import StaticPool from aria.storage import structures @@ -33,10 +34,25 @@ class TestFileSystem(object): rmtree(self.path, ignore_errors=True) -def get_sqlite_api_kwargs(): - engine = create_engine('sqlite:///:memory:', - connect_args={'check_same_thread': False}, - poolclass=StaticPool) - session = orm.sessionmaker(bind=engine)() +def get_sqlite_api_kwargs(base_dir=None, filename='memory'): + """ + Create sql params. works in in-memory and in filesystem mode. + If base_dir is passed, the mode will be filesystem mode. while the default mode is in-memory. + :param str base_dir: The base dir for the filesystem memory file. + :param str filename: the file name - defaults to 'memory'. + :return: + """ + if base_dir is not None: + uri = 'sqlite:////{base_dir}/{filename}'.format(base_dir=base_dir, filename=filename) + engine_kwargs = {} + else: + uri = 'sqlite:///:memory:' + engine_kwargs = dict(connect_args={'check_same_thread': False}, + poolclass=StaticPool) + + engine = create_engine(uri, **engine_kwargs) + session_factory = orm.sessionmaker(bind=engine) + session = scoped_session(session_factory=session_factory) if base_dir else session_factory() + structures.Model.metadata.create_all(engine) return dict(engine=engine, session=session) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402aeaca/tests/storage/test_model_storage.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py index 4b93a8b..e602884 100644 --- a/tests/storage/test_model_storage.py +++ b/tests/storage/test_model_storage.py @@ -27,7 +27,10 @@ from tests.storage import get_sqlite_api_kwargs @pytest.fixture def storage(): - return ModelStorage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs()) + storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI, + api_kwargs=get_sqlite_api_kwargs()) + yield storage + storage.drop() def test_storage_base(storage):
