Repository: incubator-ariatosca Updated Branches: refs/heads/ARIA-30-SQL-based-storage-implementation e01575cbc -> 395b9b0b1 (forced update)
added cleanup func, better cleaning, and actor/instace.id test Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/395b9b0b Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/395b9b0b Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/395b9b0b Branch: refs/heads/ARIA-30-SQL-based-storage-implementation Commit: 395b9b0b1417339f7b37b6ac6841ea1fb384fa09 Parents: 0003e47 Author: mxmrlv <mxm...@gmail.com> Authored: Wed Dec 7 14:36:43 2016 +0200 Committer: mxmrlv <mxm...@gmail.com> Committed: Wed Dec 7 15:09:47 2016 +0200 ---------------------------------------------------------------------- aria/orchestrator/context/operation.py | 2 +- tests/mock/context.py | 3 - tests/orchestrator/context/test_operation.py | 44 +++++++- tests/orchestrator/context/test_toolbelt.py | 2 +- tests/orchestrator/context/test_workflow.py | 12 ++- tests/orchestrator/workflows/api/test_task.py | 4 +- .../workflows/builtin/test_execute_operation.py | 6 +- .../orchestrator/workflows/builtin/test_heal.py | 6 +- .../workflows/builtin/test_install.py | 6 +- .../workflows/builtin/test_uninstall.py | 6 +- .../orchestrator/workflows/core/test_engine.py | 8 +- tests/orchestrator/workflows/core/test_task.py | 6 +- .../test_task_graph_into_exececution_graph.py | 5 +- tests/storage/__init__.py | 25 ++++- tests/storage/test_model_storage.py | 11 +- tests/storage/test_models.py | 104 ++++++++++++++----- 16 files changed, 172 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/aria/orchestrator/context/operation.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/context/operation.py b/aria/orchestrator/context/operation.py index a789e24..a73bad1 100644 --- a/aria/orchestrator/context/operation.py +++ b/aria/orchestrator/context/operation.py @@ -71,7 +71,7 @@ class NodeOperationContext(BaseOperationContext): The node instance of the current operation :return: """ - return self.model.node_instance.get(self._task_id) + return self.model.node_instance.get(self._actor_id) class RelationshipOperationContext(BaseOperationContext): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/mock/context.py ---------------------------------------------------------------------- diff --git a/tests/mock/context.py b/tests/mock/context.py index 7eb57ed..1904140 100644 --- a/tests/mock/context.py +++ b/tests/mock/context.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from aria import application_model_storage from aria.orchestrator import context from aria.storage.sql_mapi import SQLAlchemyModelAPI @@ -22,7 +20,6 @@ from aria.storage.sql_mapi import SQLAlchemyModelAPI from . import models -@pytest.fixture def simple(api_kwargs, **kwargs): model_storage = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) blueprint = models.get_blueprint() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/context/test_operation.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py index f3af0c5..b5f52a3 100644 --- a/tests/orchestrator/context/test_operation.py +++ b/tests/orchestrator/context/test_operation.py @@ -35,11 +35,9 @@ global_test_holder = {} @pytest.fixture def ctx(tmpdir): - context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)) - ) + context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) yield context - context.model.drop() + storage.release_sqlite_storage(context.model) @pytest.fixture @@ -142,11 +140,49 @@ def test_relationship_operation_task_execution(ctx, executor): assert operation_context.source_node_instance == dependent_node_instance +def test_invalid_task_operation_id(ctx, executor): + """ + Checks that the right id is used. The task created with id == 1, thus running the task on + node_instance with id == 2. will check that indeed the node_instance uses the correct id. + :param ctx: + :param executor: + :return: + """ + operation_name = 'aria.interfaces.lifecycle.create' + other_node_instance, node_instance = ctx.model.node_instance.list() + assert other_node_instance.id == 1 + assert node_instance.id == 2 + + node = node_instance.node + node.operations[operation_name] = { + 'operation': op_path(get_node_instance_id, module_path=__name__) + + } + ctx.model.node.update(node) + + @workflow + def basic_workflow(graph, **_): + graph.add_tasks( + api.task.OperationTask.node_instance(name=operation_name, instance=node_instance) + ) + + execute(workflow_func=basic_workflow, workflow_context=ctx, executor=executor) + + op_node_instance_id = global_test_holder[op_name(node_instance, operation_name)] + assert op_node_instance_id == node_instance.id + assert op_node_instance_id != other_node_instance.id + + @operation def my_operation(ctx, **_): global_test_holder[ctx.name] = ctx +@operation +def get_node_instance_id(ctx, **_): + global_test_holder[ctx.name] = ctx.node_instance.id + + @pytest.fixture(autouse=True) def cleanup(): global_test_holder.clear() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/context/test_toolbelt.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_toolbelt.py b/tests/orchestrator/context/test_toolbelt.py index 235004e..da46696 100644 --- a/tests/orchestrator/context/test_toolbelt.py +++ b/tests/orchestrator/context/test_toolbelt.py @@ -35,7 +35,7 @@ global_test_holder = {} def workflow_context(tmpdir): context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) yield context - context.model.drop() + storage.release_sqlite_storage(context.model) @pytest.fixture http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/context/test_workflow.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py index 3bd305e..3e86de2 100644 --- a/tests/orchestrator/context/test_workflow.py +++ b/tests/orchestrator/context/test_workflow.py @@ -61,8 +61,10 @@ class TestWorkflowContext(object): @pytest.fixture(scope='function') def storage(): api_kwargs = test_storage.get_sqlite_api_kwargs() - result = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) - result.blueprint.put(models.get_blueprint()) - blueprint = result.blueprint.get_by_name(models.BLUEPRINT_NAME) - result.deployment.put(models.get_deployment(blueprint)) - return result + workflow_storage = application_model_storage(SQLAlchemyModelAPI, api_kwargs=api_kwargs) + workflow_storage.blueprint.put(models.get_blueprint()) + blueprint = workflow_storage.blueprint.get_by_name(models.BLUEPRINT_NAME) + workflow_storage.deployment.put(models.get_deployment(blueprint)) + yield workflow_storage + test_storage.release_sqlite_storage(workflow_storage) + http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/workflows/api/test_task.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/api/test_task.py b/tests/orchestrator/workflows/api/test_task.py index d87e2be..1a90338 100644 --- a/tests/orchestrator/workflows/api/test_task.py +++ b/tests/orchestrator/workflows/api/test_task.py @@ -31,9 +31,9 @@ def ctx(): """ simple_context = mock.context.simple(storage.get_sqlite_api_kwargs()) simple_context.model.execution.put(mock.models.get_execution(simple_context.deployment)) - yield simple_context - simple_context.model.drop() + storage.release_sqlite_storage(simple_context.model) + class TestOperationTask(object): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/workflows/builtin/test_execute_operation.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_execute_operation.py b/tests/orchestrator/workflows/builtin/test_execute_operation.py index 56f8ef3..b7e5678 100644 --- a/tests/orchestrator/workflows/builtin/test_execute_operation.py +++ b/tests/orchestrator/workflows/builtin/test_execute_operation.py @@ -24,11 +24,9 @@ from tests import storage @pytest.fixture def ctx(tmpdir): - context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)) - ) + context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) yield context - context.model.drop() + storage.release_sqlite_storage(context.model) def test_execute_operation(ctx): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/workflows/builtin/test_heal.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_heal.py b/tests/orchestrator/workflows/builtin/test_heal.py index 854f379..97121b9 100644 --- a/tests/orchestrator/workflows/builtin/test_heal.py +++ b/tests/orchestrator/workflows/builtin/test_heal.py @@ -26,11 +26,9 @@ from . import (assert_node_install_operations, @pytest.fixture def ctx(tmpdir): - context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)) - ) + context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) yield context - context.model.drop() + storage.release_sqlite_storage(context.model) def test_heal_dependent_node(ctx): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/workflows/builtin/test_install.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_install.py b/tests/orchestrator/workflows/builtin/test_install.py index b285ebd..789a161 100644 --- a/tests/orchestrator/workflows/builtin/test_install.py +++ b/tests/orchestrator/workflows/builtin/test_install.py @@ -25,11 +25,9 @@ from . import assert_node_install_operations @pytest.fixture def ctx(tmpdir): - context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)) - ) + context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) yield context - context.model.drop() + storage.release_sqlite_storage(context.model) def test_install(ctx): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/workflows/builtin/test_uninstall.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/builtin/test_uninstall.py b/tests/orchestrator/workflows/builtin/test_uninstall.py index d7d919a..126c4cf 100644 --- a/tests/orchestrator/workflows/builtin/test_uninstall.py +++ b/tests/orchestrator/workflows/builtin/test_uninstall.py @@ -26,11 +26,9 @@ from . import assert_node_uninstall_operations @pytest.fixture def ctx(tmpdir): - context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)) - ) + context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) yield context - context.model.drop() + storage.release_sqlite_storage(context.model) def test_uninstall(ctx): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/workflows/core/test_engine.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_engine.py b/tests/orchestrator/workflows/core/test_engine.py index 0416ab8..c3e5532 100644 --- a/tests/orchestrator/workflows/core/test_engine.py +++ b/tests/orchestrator/workflows/core/test_engine.py @@ -124,15 +124,11 @@ class BaseTest(object): @pytest.fixture(scope='function') def workflow_context(self, tmpdir): - workflow_context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)) - ) + workflow_context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) workflow_context.states = [] workflow_context.exception = None yield workflow_context - workflow_context.model.drop() - for mapi in workflow_context.model.registered.values(): - mapi._session.remove() + storage.release_sqlite_storage(workflow_context.model) class TestEngine(BaseTest): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/workflows/core/test_task.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_task.py b/tests/orchestrator/workflows/core/test_task.py index ae50c24..c572501 100644 --- a/tests/orchestrator/workflows/core/test_task.py +++ b/tests/orchestrator/workflows/core/test_task.py @@ -31,11 +31,9 @@ from tests import mock, storage @pytest.fixture def ctx(tmpdir): - context = mock.context.simple( - storage.get_sqlite_api_kwargs(str(tmpdir)) - ) + context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir))) yield context - context.model.drop() + storage.release_sqlite_storage(context.model) class TestOperationTask(object): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py ---------------------------------------------------------------------- diff --git a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py index 2736a2e..18540f4 100644 --- a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py +++ b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py @@ -24,9 +24,7 @@ from tests import storage def test_task_graph_into_execution_graph(): operation_name = 'aria.interfaces.lifecycle.create' - task_context = mock.context.simple( - storage.get_sqlite_api_kwargs() - ) + task_context = mock.context.simple(storage.get_sqlite_api_kwargs()) node_instance = \ task_context.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME) def sub_workflow(name, **_): @@ -87,6 +85,7 @@ def test_task_graph_into_execution_graph(): simple_after_task) assert isinstance(_get_task_by_name(execution_tasks[6], execution_graph), core.task.EndWorkflowTask) + storage.release_sqlite_storage(task_context.model) def _assert_execution_is_api_task(execution_task, api_task): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/storage/__init__.py ---------------------------------------------------------------------- diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 90e113c..c1d6f18 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -20,7 +20,11 @@ from shutil import rmtree from sqlalchemy import ( create_engine, orm) -from sqlalchemy.orm import scoped_session +from sqlalchemy.orm import ( + Session, + scoped_session +) +from sqlalchemy.orm.scoping import ScopedSession from sqlalchemy.pool import StaticPool from aria.storage import structures @@ -35,12 +39,12 @@ class TestFileSystem(object): rmtree(self.path, ignore_errors=True) -def get_sqlite_api_kwargs(base_dir=None, filename='memory'): +def get_sqlite_api_kwargs(base_dir=None, filename='db.sqlite'): """ Create sql params. works in in-memory and in filesystem mode. If base_dir is passed, the mode will be filesystem mode. while the default mode is in-memory. :param str base_dir: The base dir for the filesystem memory file. - :param str filename: the file name - defaults to 'memory'. + :param str filename: the file name - defaults to 'db.sqlite'. :return: """ if base_dir is not None: @@ -62,3 +66,18 @@ def get_sqlite_api_kwargs(base_dir=None, filename='memory'): structures.Model.metadata.create_all(engine) return dict(engine=engine, session=session) + + +def release_sqlite_storage(storage): + """ + Drops the tables and clears the session + :param storage: + :return: + """ + storage.drop() + for mapi in storage.registered.values(): + if isinstance(mapi._session, ScopedSession): + mapi._session.remove() + elif isinstance(mapi._session, Session): + mapi._session.close() + mapi._engine.dispose() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/storage/test_model_storage.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py index e602884..48cd02c 100644 --- a/tests/storage/test_model_storage.py +++ b/tests/storage/test_model_storage.py @@ -22,15 +22,14 @@ from aria.storage import ( sql_mapi, ) from aria import application_model_storage -from tests.storage import get_sqlite_api_kwargs +from tests.storage import get_sqlite_api_kwargs, release_sqlite_storage @pytest.fixture def storage(): - storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI, - api_kwargs=get_sqlite_api_kwargs()) - yield storage - storage.drop() + base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs()) + yield base_storage + release_sqlite_storage(base_storage) def test_storage_base(storage): @@ -89,3 +88,5 @@ def test_application_storage_factory(): assert storage.deployment_modification assert storage.execution assert storage.provider_context + + release_sqlite_storage(storage) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/395b9b0b/tests/storage/test_models.py ---------------------------------------------------------------------- diff --git a/tests/storage/test_models.py b/tests/storage/test_models.py index d63d9c5..ff056d4 100644 --- a/tests/storage/test_models.py +++ b/tests/storage/test_models.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +from contextlib import contextmanager from datetime import datetime import pytest @@ -37,34 +37,41 @@ from aria.storage.models import ( from tests import mock -from tests.storage import get_sqlite_api_kwargs +from tests.storage import get_sqlite_api_kwargs, release_sqlite_storage -@pytest.fixture -def empty_storage(): +@contextmanager +def sql_storage(storage_func): + storage = None + try: + storage = storage_func() + yield storage + finally: + if storage: + release_sqlite_storage(storage) + + +def _empty_storage(): return application_model_storage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs()) -@pytest.fixture -def blueprint_storage(): - storage = empty_storage() +def _blueprint_storage(): + storage = _empty_storage() blueprint = mock.models.get_blueprint() storage.blueprint.put(blueprint) return storage -@pytest.fixture -def deployment_storage(): - storage = blueprint_storage() +def _deployment_storage(): + storage = _blueprint_storage() deployment = mock.models.get_deployment(storage.blueprint.list()[0]) storage.deployment.put(deployment) return storage -@pytest.fixture -def deployment_update_storage(): - storage = deployment_storage() +def _deployment_update_storage(): + storage = _deployment_storage() deployment_update = DeploymentUpdate( deployment_id=storage.deployment.list()[0].id, created_at=now, @@ -74,17 +81,15 @@ def deployment_update_storage(): return storage -@pytest.fixture -def node_storage(): - storage = deployment_storage() +def _node_storage(): + storage = _deployment_storage() node = mock.models.get_dependency_node(storage.deployment.list()[0]) storage.node.put(node) return storage -@pytest.fixture -def nodes_storage(): - storage = deployment_storage() +def _nodes_storage(): + storage = _deployment_storage() dependent_node = mock.models.get_dependent_node(storage.deployment.list()[0]) dependency_node = mock.models.get_dependency_node(storage.deployment.list()[0]) storage.node.put(dependent_node) @@ -92,9 +97,8 @@ def nodes_storage(): return storage -@pytest.fixture -def node_instances_storage(): - storage = nodes_storage() +def _node_instances_storage(): + storage = _nodes_storage() dependent_node = storage.node.get_by_name(mock.models.DEPENDENT_NODE_NAME) dependency_node = storage.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) dependency_node_instance = mock.models.get_dependency_node_instance(dependency_node) @@ -104,13 +108,63 @@ def node_instances_storage(): return storage -@pytest.fixture -def execution_storage(): - storage = deployment_storage() +def _execution_storage(): + storage = _deployment_storage() execution = mock.models.get_execution(storage.deployment.list()[0]) storage.execution.put(execution) return storage + +########################### + +@pytest.fixture +def empty_storage(): + with sql_storage(_empty_storage) as storage: + yield storage + + +@pytest.fixture +def blueprint_storage(): + with sql_storage(_blueprint_storage) as storage: + yield storage + + +@pytest.fixture +def deployment_storage(): + with sql_storage(_deployment_storage) as storage: + yield storage + + +@pytest.fixture +def deployment_update_storage(): + with sql_storage(_deployment_update_storage) as storage: + yield storage + + +@pytest.fixture +def node_storage(): + with sql_storage(_node_storage) as storage: + yield storage + + +@pytest.fixture +def nodes_storage(): + with sql_storage(_nodes_storage) as storage: + yield storage + + +@pytest.fixture +def node_instances_storage(): + with sql_storage(_node_instances_storage) as storage: + yield storage + + +@pytest.fixture +def execution_storage(): + with sql_storage(_execution_storage) as storage: + yield storage + + m_cls = type('MockClass') now = datetime.utcnow()
