ARIA-153 Write end-to-end tests for ARIA Created infrastructure for end-to-end tests, plus a test for the hello-world example.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/d91696bd Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/d91696bd Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/d91696bd Branch: refs/heads/ARIA-146-Support-colorful-execution-logging Commit: d91696bdf5d208ff57496df23026341bc6cfa351 Parents: 5cd7aec Author: Ran Ziv <r...@gigaspaces.com> Authored: Wed Apr 26 20:21:19 2017 +0300 Committer: Ran Ziv <r...@gigaspaces.com> Committed: Thu Apr 27 14:28:10 2017 +0300 ---------------------------------------------------------------------- .travis.yml | 2 + tests/end2end/__init__.py | 0 tests/end2end/test_hello_world.py | 61 +++++++++++++++++++++ tests/end2end/testenv.py | 96 ++++++++++++++++++++++++++++++++++ tests/helpers.py | 31 +++++++++++ tests/parser/service_templates.py | 9 ++-- tests/parser/utils.py | 16 ------ tests/requirements.txt | 1 + tox.ini | 14 +++-- 9 files changed, 207 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 5413ff2..b11ed62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,8 @@ env: - TOX_ENV=pylint_tests - TOX_ENV=py27 - TOX_ENV=py26 +- TOX_ENV=py27e2e +- TOX_ENV=py26e2e install: - pip install --upgrade pip - pip install --upgrade setuptools http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/tests/end2end/__init__.py ---------------------------------------------------------------------- diff --git a/tests/end2end/__init__.py b/tests/end2end/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/tests/end2end/test_hello_world.py ---------------------------------------------------------------------- diff --git a/tests/end2end/test_hello_world.py b/tests/end2end/test_hello_world.py new file mode 100644 index 0000000..09e5d06 --- /dev/null +++ b/tests/end2end/test_hello_world.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +from .testenv import testenv # pylint: disable=unused-import +from .. import helpers + + +def test_hello_world(testenv): + hello_world_template_uri = helpers.get_example_uri('hello-world', 'helloworld.yaml') + service_name = testenv.install_service(hello_world_template_uri) + + try: + _verify_deployed_service_in_storage(service_name, testenv.model_storage) + _verify_webserver_up('http://localhost:9090') + finally: + # Even if some assertions failed, attempt to execute uninstall so the + # webserver process doesn't stay up once the test is finished + # TODO: remove force_service_delete=True + testenv.uninstall_service(force_service_delete=True) + + _verify_webserver_down('http://localhost:9090') + testenv.verify_clean_storage() + + +def _verify_webserver_up(http_endpoint): + server_response = requests.get(http_endpoint, timeout=10) + assert server_response.status_code == 200 + + +def _verify_webserver_down(http_endpoint): + try: + requests.get(http_endpoint, timeout=10) + assert False + except requests.exceptions.ConnectionError: + pass + + +def _verify_deployed_service_in_storage(service_name, model_storage): + service_templates = model_storage.service_template.list() + assert len(service_templates) == 1 + assert len(service_templates[0].services) == 1 + service = service_templates[0].services[0] + assert service.name == service_name + assert len(service.executions) == 1 + assert len(service.nodes) == 2 + # TODO: validate node states + assert len(service.executions[0].logs) > 0 http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/tests/end2end/testenv.py ---------------------------------------------------------------------- diff --git a/tests/end2end/testenv.py b/tests/end2end/testenv.py new file mode 100644 index 0000000..3950b20 --- /dev/null +++ b/tests/end2end/testenv.py @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import pytest +import sh + + +@pytest.fixture +def testenv(tmpdir, request, monkeypatch): + test_name = request.node.name + workdir = str(tmpdir) + + # setting the workdir environment variable for the CLI to work with + monkeypatch.setenv('ARIA_WORKDIR', workdir) + return TestEnvironment(workdir, test_name) + + +class TestEnvironment(object): + + def __init__(self, workdir, test_name): + self.workdir = workdir + self.test_name = test_name + + self.cli = self._get_cli() + env = self._get_aria_env() + self.model_storage = env.model_storage + self.resource_storage = env.resource_storage + self.plugin_manager = env.plugin_manager + + def install_service(self, service_template_path, dry=False, service_template_name=None, + service_name=None): + service_template_name = service_template_name or self.test_name + service_name = service_name or self.test_name + + self.cli.service_templates.store(service_template_path, service_template_name) + self.cli.services.create(service_name, service_template_name=service_template_name) + self.execute_workflow(service_name, 'install', dry=dry) + return service_name + + def uninstall_service(self, service_name=None, service_template_name=None, dry=False, + force_service_delete=False): + service_name = service_name or self.test_name + self.execute_workflow(service_name, 'uninstall', dry=dry) + self.cli.services.delete(service_name, force=force_service_delete) + self.cli.service_templates.delete(service_template_name or self.test_name) + + def execute_workflow(self, service_name, workflow_name, dry=False): + self.cli.executions.start(workflow_name, service_name=service_name, dry=dry) + + def verify_clean_storage(self): + assert len(self.model_storage.service_template.list()) == 0 + assert len(self.model_storage.service.list()) == 0 + assert len(self.model_storage.execution.list()) == 0 + assert len(self.model_storage.node_template.list()) == 0 + assert len(self.model_storage.node.list()) == 0 + assert len(self.model_storage.log.list()) == 0 + + def _get_cli(self): + cli = sh.aria.bake(_out=sys.stdout.write, _err=sys.stderr.write) + + # the `sh` library supports underscore-dash auto-replacement for commands and option flags + # yet not for subcommands (e.g. `aria service-templates`); The following class fixes this. + class PatchedCli(object): + def __getattr__(self, attr): + if '_' in attr: + return cli.bake(attr.replace('_', '-')) + return getattr(cli, attr) + + def __call__(self, *args, **kwargs): + # this is to support the `aria` command itself (e.g. `aria --version` calls) + return cli(*args, **kwargs) + + return PatchedCli() + + def _get_aria_env(self): + # a somewhat hackish but most simple way of acquiring environment context such as + # the model storage, resource storage etc. + # note that the `ARIA_WORKDIR` environment variable must be exported before the import + # below is used, as the import itself will initialize the `.aria` directory. + from aria.cli import env as cli_env + reload(cli_env) # reloading the module in-between tests + return cli_env.env http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/tests/helpers.py ---------------------------------------------------------------------- diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..472d696 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from . import ROOT_DIR +from .resources import DIR as RESOURCES_DIR + + +def get_example_uri(*args): + return os.path.join(ROOT_DIR, 'examples', *args) + + +def get_resource_uri(*args): + return os.path.join(RESOURCES_DIR, *args) + + +def get_service_template_uri(*args): + return os.path.join(RESOURCES_DIR, 'service-templates', *args) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/tests/parser/service_templates.py ---------------------------------------------------------------------- diff --git a/tests/parser/service_templates.py b/tests/parser/service_templates.py index a8fde14..56f75ab 100644 --- a/tests/parser/service_templates.py +++ b/tests/parser/service_templates.py @@ -17,7 +17,8 @@ import os from aria.utils.caching import cachedmethod -from .utils import (get_example_uri, get_test_uri, create_context, create_consumer) +from .utils import (create_context, create_consumer) +from ..helpers import (get_example_uri, get_service_template_uri) def consume_use_case(use_case_name, consumer_class_name='instance', cache=True): @@ -37,10 +38,10 @@ def consume_use_case(use_case_name, consumer_class_name='instance', cache=True): def consume_node_cellar(consumer_class_name='instance', cache=True): cachedmethod.ENABLED = cache - uri = get_test_uri('tosca-simple-1.0', 'node-cellar', 'node-cellar.yaml') + uri = get_service_template_uri('tosca-simple-1.0', 'node-cellar', 'node-cellar.yaml') context = create_context(uri) - context.args.append('--inputs=' + get_test_uri('tosca-simple-1.0', 'node-cellar', - 'inputs.yaml')) + context.args.append('--inputs=' + get_service_template_uri('tosca-simple-1.0', 'node-cellar', + 'inputs.yaml')) consumer, dumper = create_consumer(context, consumer_class_name) consumer.consume() context.validation.dump_issues() http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/tests/parser/utils.py ---------------------------------------------------------------------- diff --git a/tests/parser/utils.py b/tests/parser/utils.py index 8460de8..f0e890f 100644 --- a/tests/parser/utils.py +++ b/tests/parser/utils.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from aria.parser.loading import UriLocation from aria.parser.consumption import ( ConsumptionContext, @@ -28,20 +26,6 @@ from aria.parser.consumption import ( ) from aria.utils.imports import import_fullname -from tests import ROOT_DIR -from tests.resources import DIR - - -SERVICE_TEMPLATES_DIR = os.path.join(DIR, 'service-templates') - - -def get_example_uri(*args): - return os.path.join(ROOT_DIR, 'examples', *args) - - -def get_test_uri(*args): - return os.path.join(SERVICE_TEMPLATES_DIR, *args) - def create_context(uri, loader_source='aria.parser.loading.DefaultLoaderSource', http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/tests/requirements.txt ---------------------------------------------------------------------- diff --git a/tests/requirements.txt b/tests/requirements.txt index 2f0245a..71a227a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -12,6 +12,7 @@ testtools fasteners==0.13.0 +sh==1.12.13 mock==1.0.1 pylint==1.6.4 pytest==3.0.2 http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d91696bd/tox.ini ---------------------------------------------------------------------- diff --git a/tox.ini b/tox.ini index 6ad048f..4d86c6e 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ # limitations under the License. [tox] -envlist=py27,py26,pywin,pylint_code,pylint_tests +envlist=py27,py26,py27e2e,py26e2e,pywin,pylint_code,pylint_tests [testenv] passenv = @@ -27,15 +27,23 @@ deps = basepython = py26: python2.6 py27: python2.7 + py26e2e: python2.6 + py27e2e: python2.7 pywin: {env:PYTHON:}\python.exe pylint_code: python2.7 pylint_tests: python2.7 [testenv:py27] -commands=pytest tests --cov-report term-missing --cov aria +commands=pytest tests --ignore=tests/end2end --cov-report term-missing --cov aria [testenv:py26] -commands=pytest tests --cov-report term-missing --cov aria +commands=pytest tests --ignore=tests/end2end --cov-report term-missing --cov aria + +[testenv:py27e2e] +commands=pytest tests/end2end --cov-report term-missing --cov aria + +[testenv:py26e2e] +commands=pytest tests/end2end --cov-report term-missing --cov aria [testenv:pywin] commands=pytest tests --cov-report term-missing --cov aria