ARIA-149 Enhance operation configuration * Parse special "dependencies" configuration parameters as YAML and treat as Parameter models, allowing them full use of intrinsic functions, type coersions, and validations * Rename various functions that process "properties" to more generically process "parameters" (properties, inputs, attributes, arguments, etc.) * The "configuration" field in OperationTemplate and Operation models is now now a dict of Parameter models * Added "function" and "arguments" fields to Operation model to preserve user data (in "implementation" and "inputs") and to clearly demarcate orchestration data from user data * Some cleanup of parser code touched by this commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/84d8da59 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/84d8da59 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/84d8da59 Branch: refs/heads/ARIA-149-functions-in-operation-configuration Commit: 84d8da592b3534cecbc896e7ee60704d660f4a26 Parents: 3d22d36 Author: Tal Liron <tal.li...@gmail.com> Authored: Thu Apr 20 17:54:47 2017 -0500 Committer: Tal Liron <tal.li...@gmail.com> Committed: Mon May 22 18:33:11 2017 -0500 ---------------------------------------------------------------------- aria/cli/commands/services.py | 2 +- aria/core.py | 18 +- aria/modeling/constraints.py | 28 +++ aria/modeling/contraints.py | 28 --- aria/modeling/exceptions.py | 16 +- aria/modeling/service_common.py | 4 + aria/modeling/service_instance.py | 69 +++--- aria/modeling/service_template.py | 54 ++--- aria/modeling/utils.py | 105 ++++----- .../execution_plugin/instantiation.py | 133 +++++++----- aria/orchestrator/workflow_runner.py | 2 +- aria/orchestrator/workflows/api/task.py | 4 +- aria/orchestrator/workflows/core/task.py | 1 - .../profiles/aria-1.0/aria-1.0.yaml | 16 +- .../simple_v1_0/assignments.py | 4 +- .../simple_v1_0/modeling/__init__.py | 65 ++++-- .../simple_v1_0/modeling/artifacts.py | 2 +- .../simple_v1_0/modeling/capabilities.py | 24 ++- .../simple_v1_0/modeling/constraints.py | 2 +- .../simple_v1_0/modeling/data_types.py | 16 ++ .../simple_v1_0/modeling/interfaces.py | 34 ++- .../simple_v1_0/modeling/parameters.py | 211 +++++++++++++++++++ .../simple_v1_0/modeling/policies.py | 2 + .../simple_v1_0/modeling/properties.py | 202 ------------------ .../simple_v1_0/modeling/requirements.py | 20 +- .../modeling/substitution_mappings.py | 4 + .../simple_v1_0/templates.py | 13 +- .../aria_extension_tosca/simple_v1_0/types.py | 24 +-- tests/cli/test_services.py | 14 +- tests/mock/models.py | 10 +- tests/mock/topology.py | 12 +- tests/orchestrator/context/test_operation.py | 53 +++-- tests/orchestrator/context/test_serialize.py | 2 +- tests/orchestrator/context/test_toolbelt.py | 14 +- .../orchestrator/execution_plugin/test_local.py | 10 +- tests/orchestrator/execution_plugin/test_ssh.py | 12 +- tests/orchestrator/test_workflow_runner.py | 8 +- tests/orchestrator/workflows/api/test_task.py | 26 +-- .../workflows/builtin/test_execute_operation.py | 2 +- .../orchestrator/workflows/core/test_engine.py | 6 +- .../orchestrator/workflows/core/test_events.py | 3 +- tests/orchestrator/workflows/core/test_task.py | 6 +- .../test_task_graph_into_execution_graph.py | 2 +- ...process_executor_concurrent_modifications.py | 10 +- .../executor/test_process_executor_extension.py | 13 +- .../test_process_executor_tracked_changes.py | 8 +- .../node-cellar/node-cellar.yaml | 24 ++- 47 files changed, 739 insertions(+), 599 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/cli/commands/services.py ---------------------------------------------------------------------- diff --git a/aria/cli/commands/services.py b/aria/cli/commands/services.py index 24de7c5..476387c 100644 --- a/aria/cli/commands/services.py +++ b/aria/cli/commands/services.py @@ -151,7 +151,7 @@ def create(service_template_name, except storage_exceptions.StorageError as e: utils.check_overriding_storage_exceptions(e, 'service', service_name) raise - except modeling_exceptions.InputsException: + except modeling_exceptions.ParameterException: service_templates.print_service_template_inputs(model_storage, service_template_name, logger) raise http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/core.py ---------------------------------------------------------------------- diff --git a/aria/core.py b/aria/core.py index cc943ef..12a2683 100644 --- a/aria/core.py +++ b/aria/core.py @@ -2,13 +2,13 @@ # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with +# (the 'License'); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, +# distributed under the License is distributed on an 'AS IS' BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. @@ -56,7 +56,8 @@ class Core(object): service_template = self.model_storage.service_template.get(service_template_id) if service_template.services: raise exceptions.DependentServicesError( - "Can't delete service template {0} - Service template has existing services") + 'Can\'t delete service template `{0}` - service template has existing services' + .format(service_template.name)) self.model_storage.service_template.delete(service_template) self.resource_storage.service_template.delete(entry_id=str(service_template.id)) @@ -87,7 +88,8 @@ class Core(object): consumption.CoerceServiceInstanceValues )).consume() if context.validation.dump_issues(): - raise exceptions.InstantiationError('Failed to instantiate service template') + raise exceptions.InstantiationError('Failed to instantiate service template `{0}`' + .format(service_template.name)) storage_session.flush() # flushing so service.id would auto-populate service.name = service_name or '{0}_{1}'.format(service_template.name, service.id) @@ -100,15 +102,15 @@ class Core(object): active_executions = [e for e in service.executions if e.is_active()] if active_executions: raise exceptions.DependentActiveExecutionsError( - "Can't delete service {0} - there is an active execution for this service. " - "Active execution id: {1}".format(service.name, active_executions[0].id)) + 'Can\'t delete service `{0}` - there is an active execution for this service. ' + 'Active execution ID: {1}'.format(service.name, active_executions[0].id)) if not force: available_nodes = [str(n.id) for n in service.nodes.values() if n.is_available()] if available_nodes: raise exceptions.DependentAvailableNodesError( - "Can't delete service {0} - there are available nodes for this service. " - "Available node ids: {1}".format(service.name, ', '.join(available_nodes))) + 'Can\'t delete service `{0}` - there are available nodes for this service. ' + 'Available node IDs: {1}'.format(service.name, ', '.join(available_nodes))) self.model_storage.service.delete(service) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/modeling/constraints.py ---------------------------------------------------------------------- diff --git a/aria/modeling/constraints.py b/aria/modeling/constraints.py new file mode 100644 index 0000000..107b010 --- /dev/null +++ b/aria/modeling/constraints.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class NodeTemplateConstraint(object): + """ + Used to constrain requirements for node templates. + + Must be serializable. + """ + + def matches(self, source_node_template, target_node_template): + """ + Returns true is the target matches the constraint for the source. + """ + raise NotImplementedError http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/modeling/contraints.py ---------------------------------------------------------------------- diff --git a/aria/modeling/contraints.py b/aria/modeling/contraints.py deleted file mode 100644 index 107b010..0000000 --- a/aria/modeling/contraints.py +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class NodeTemplateConstraint(object): - """ - Used to constrain requirements for node templates. - - Must be serializable. - """ - - def matches(self, source_node_template, target_node_template): - """ - Returns true is the target matches the constraint for the source. - """ - raise NotImplementedError http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/modeling/exceptions.py ---------------------------------------------------------------------- diff --git a/aria/modeling/exceptions.py b/aria/modeling/exceptions.py index 19fd942..d0e3e22 100644 --- a/aria/modeling/exceptions.py +++ b/aria/modeling/exceptions.py @@ -22,9 +22,9 @@ class ModelingException(AriaException): """ -class InputsException(ModelingException): +class ParameterException(ModelingException): """ - ARIA inputs exception. + ARIA parameter exception. """ pass @@ -41,19 +41,19 @@ class CannotEvaluateFunctionException(ModelingException): """ -class MissingRequiredInputsException(InputsException): +class MissingRequiredParametersException(ParameterException): """ - ARIA modeling exception: Required inputs have been omitted. + ARIA modeling exception: Required parameters have been omitted. """ -class InputsOfWrongTypeException(InputsException): +class ParametersOfWrongTypeException(ParameterException): """ - ARIA modeling exception: Inputs of the wrong types have been provided. + ARIA modeling exception: Parameters of the wrong types have been provided. """ -class UndeclaredInputsException(InputsException): +class UndeclaredParametersException(ParameterException): """ - ARIA modeling exception: Undeclared inputs have been provided. + ARIA modeling exception: Undeclared parameters have been provided. """ http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/modeling/service_common.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_common.py b/aria/modeling/service_common.py index e9c96a4..3caa147 100644 --- a/aria/modeling/service_common.py +++ b/aria/modeling/service_common.py @@ -211,6 +211,10 @@ class ParameterBase(TemplateModelMixin, caching.HasCachedMethods): """ Wraps an arbitrary value as a parameter. The type will be guessed via introspection. + For primitive types, we will prefer their TOSCA aliases. See the `TOSCA Simple Profile v1.0 + cos01 specification <http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/cos01 + /TOSCA-Simple-Profile-YAML-v1.0-cos01.html#_Toc373867862>`__ + :param name: Parameter name :type name: basestring :param value: Parameter value http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/modeling/service_instance.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_instance.py b/aria/modeling/service_instance.py index 41a388d..d14cdc5 100644 --- a/aria/modeling/service_instance.py +++ b/aria/modeling/service_instance.py @@ -1633,20 +1633,24 @@ class OperationBase(InstanceModelMixin): :vartype operation_template: :class:`OperationTemplate` :ivar description: Human-readable description :vartype description: string - :ivar plugin: Associated plugin - :vartype plugin: :class:`Plugin` :ivar relationship_edge: When true specified that the operation is on the relationship's target edge instead of its source (only used by relationship operations) :vartype relationship_edge: bool :ivar implementation: Implementation (interpreted by the plugin) :vartype implementation: basestring - :ivar configuration: Configuration (interpreted by the plugin) - :vartype configuration: {basestring, object} :ivar dependencies: Dependency strings (interpreted by the plugin) :vartype dependencies: [basestring] :ivar inputs: Parameters that can be used by this operation :vartype inputs: {basestring: :class:`Parameter`} + :ivar plugin: Associated plugin + :vartype plugin: :class:`Plugin` + :ivar configuration: Configuration (interpreted by the plugin) + :vartype configuration: {basestring, :class:`Parameter`} + :ivar function: Name of the operation function + :vartype function: basestring + :ivar arguments: Arguments to send to the operation function + :vartype arguments: {basestring: :class:`Parameter`} :ivar executor: Name of executor to run the operation with :vartype executor: basestring :ivar max_attempts: Maximum number of attempts allowed in case of failure @@ -1728,34 +1732,41 @@ class OperationBase(InstanceModelMixin): def inputs(cls): return relationship.many_to_many(cls, 'parameter', prefix='inputs', dict_key='name') + @declared_attr + def configuration(cls): + return relationship.many_to_many(cls, 'parameter', prefix='configuration', dict_key='name') + + @declared_attr + def arguments(cls): + return relationship.many_to_many(cls, 'parameter', prefix='arguments', dict_key='name') + # endregion description = Column(Text) relationship_edge = Column(Boolean) implementation = Column(Text) - configuration = Column(modeling_types.StrictDict(key_cls=basestring)) dependencies = Column(modeling_types.StrictList(item_cls=basestring)) + function = Column(Text) executor = Column(Text) max_attempts = Column(Integer) retry_interval = Column(Integer) def configure(self): - from . import models - # Note: for workflows (operations attached directly to the service) "interface" will be None - if (self.implementation is None) or (self.interface is None): + if (self.implementation is None) and (self.function is None): return - if self.plugin is None: - arguments = execution_plugin.instantiation.configure_operation(self) + if (self.plugin is None) and (self.interface is not None): + # Default to execution plugin ("interface" is None for workflow operations) + execution_plugin.instantiation.configure_operation(self) else: # In the future plugins may be able to add their own "configure_operation" hook that - # can validate the configuration and otherwise return specially derived arguments - arguments = self.configuration + # can validate the configuration and otherwise create specially derived arguments. For + # now, we just send all configuration parameters as arguments + utils.instantiate_dict(self, self.arguments, self.configuration) - # Note: the arguments will *override* operation inputs of the same name - if arguments: - for k, v in arguments.iteritems(): - self.inputs[k] = models.Parameter.wrap(k, v) + # Send all inputs as extra arguments. Note that they will override existing arguments of the + # same names. + utils.instantiate_dict(self, self.arguments, self.inputs) @property def as_raw(self): @@ -1764,17 +1775,18 @@ class OperationBase(InstanceModelMixin): ('description', self.description), ('implementation', self.implementation), ('dependencies', self.dependencies), - ('executor', self.executor), - ('max_attempts', self.max_attempts), - ('retry_interval', self.retry_interval), ('inputs', formatting.as_raw_dict(self.inputs)))) def validate(self): - # TODO must be associated with interface or service + # TODO must be associated with either interface or service utils.validate_dict_values(self.inputs) + utils.validate_dict_values(self.configuration) + utils.validate_dict_values(self.arguments) def coerce_values(self, report_issues): utils.coerce_dict_values(self.inputs, report_issues) + utils.coerce_dict_values(self.configuration, report_issues) + utils.coerce_dict_values(self.arguments, report_issues) def dump(self): context = ConsumptionContext.get_thread_local() @@ -1782,21 +1794,14 @@ class OperationBase(InstanceModelMixin): if self.description: console.puts(context.style.meta(self.description)) with context.style.indent: - if self.plugin is not None: - console.puts('Plugin: {0}'.format( - context.style.literal(self.plugin.name))) if self.implementation is not None: console.puts('Implementation: {0}'.format( context.style.literal(self.implementation))) - if self.configuration: - with context.style.indent: - for k, v in self.configuration.iteritems(): - console.puts('{0}: {1}'.format(context.style.property(k), - context.style.literal(v))) if self.dependencies: console.puts( 'Dependencies: {0}'.format( ', '.join((str(context.style.literal(v)) for v in self.dependencies)))) + utils.dump_dict_values(self.inputs, 'Inputs') if self.executor is not None: console.puts('Executor: {0}'.format(context.style.literal(self.executor))) if self.max_attempts is not None: @@ -1804,7 +1809,13 @@ class OperationBase(InstanceModelMixin): if self.retry_interval is not None: console.puts('Retry interval: {0}'.format( context.style.literal(self.retry_interval))) - utils.dump_dict_values(self.inputs, 'Inputs') + if self.plugin is not None: + console.puts('Plugin: {0}'.format( + context.style.literal(self.plugin.name))) + utils.dump_dict_values(self.configuration, 'Configuration') + if self.function is not None: + console.puts('Function: {0}'.format(context.style.literal(self.function))) + utils.dump_dict_values(self.arguments, 'Arguments') class ArtifactBase(InstanceModelMixin): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/modeling/service_template.py ---------------------------------------------------------------------- diff --git a/aria/modeling/service_template.py b/aria/modeling/service_template.py index 12195a1..496aacf 100644 --- a/aria/modeling/service_template.py +++ b/aria/modeling/service_template.py @@ -287,7 +287,7 @@ class ServiceTemplateBase(TemplateModelMixin): service_template=self) context.modeling.instance = service - service.inputs = utils.create_inputs(inputs or {}, self.inputs) + service.inputs = utils.create_parameters(inputs or {}, self.inputs) # TODO: now that we have inputs, we should scan properties and inputs and evaluate functions for plugin_specification in self.plugin_specifications.itervalues(): @@ -1763,20 +1763,22 @@ class OperationTemplateBase(TemplateModelMixin): :vartype name: basestring :ivar description: Human-readable description :vartype description: basestring - :ivar plugin_specification: Associated plugin - :vartype plugin_specification: :class:`PluginSpecification` :ivar relationship_edge: When true specified that the operation is on the relationship's target edge instead of its source (only used by relationship operations) :vartype relationship_edge: bool :ivar implementation: Implementation (interpreted by the plugin) :vartype implementation: basestring - :ivar configuration: Configuration (interpreted by the plugin) - :vartype configuration: {basestring, object} :ivar dependencies: Dependency strings (interpreted by the plugin) :vartype dependencies: [basestring] :ivar inputs: Parameters that can be used by this operation :vartype inputs: {basestring: :class:`Parameter`} + :ivar plugin_specification: Associated plugin + :vartype plugin_specification: :class:`PluginSpecification` + :ivar configuration: Configuration (interpreted by the plugin) + :vartype configuration: {basestring, :class:`Parameter`} + :ivar function: Name of the operation function + :vartype function: basestring :ivar executor: Name of executor to run the operation with :vartype executor: basestring :ivar max_attempts: Maximum number of attempts allowed in case of failure @@ -1856,13 +1858,17 @@ class OperationTemplateBase(TemplateModelMixin): def inputs(cls): return relationship.many_to_many(cls, 'parameter', prefix='inputs', dict_key='name') + @declared_attr + def configuration(cls): + return relationship.many_to_many(cls, 'parameter', prefix='configuration', dict_key='name') + # endregion description = Column(Text) relationship_edge = Column(Boolean) implementation = Column(Text) - configuration = Column(modeling_types.StrictDict(key_cls=basestring)) dependencies = Column(modeling_types.StrictList(item_cls=basestring)) + function = Column(Text) executor = Column(Text) max_attempts = Column(Integer) retry_interval = Column(Integer) @@ -1874,9 +1880,6 @@ class OperationTemplateBase(TemplateModelMixin): ('description', self.description), ('implementation', self.implementation), ('dependencies', self.dependencies), - ('executor', self.executor), - ('max_attempts', self.max_attempts), - ('retry_interval', self.retry_interval), ('inputs', formatting.as_raw_dict(self.inputs)))) def instantiate(self, container): @@ -1884,38 +1887,41 @@ class OperationTemplateBase(TemplateModelMixin): if self.plugin_specification: if self.plugin_specification.enabled: plugin = self.plugin_specification.plugin - implementation = self.implementation if plugin is not None else None + function = self.function if plugin is not None else None # "plugin" would be none if a match was not found. In that case, a validation error # should already have been reported in ServiceTemplateBase.instantiate, so we will # continue silently here else: # If the plugin is disabled, the operation should be disabled, too plugin = None - implementation = None + function = None else: - # Using the execution plugin + # Using the default execution plugin (plugin=None) plugin = None - implementation = self.implementation + function = self.function operation = models.Operation(name=self.name, description=deepcopy_with_locators(self.description), relationship_edge=self.relationship_edge, - plugin=plugin, - implementation=implementation, - configuration=self.configuration, + implementation=self.implementation, dependencies=self.dependencies, executor=self.executor, + plugin=plugin, + function=function, max_attempts=self.max_attempts, retry_interval=self.retry_interval, operation_template=self) utils.instantiate_dict(container, operation.inputs, self.inputs) + utils.instantiate_dict(container, operation.configuration, self.configuration) return operation def validate(self): utils.validate_dict_values(self.inputs) + utils.validate_dict_values(self.configuration) def coerce_values(self, report_issues): utils.coerce_dict_values(self.inputs, report_issues) + utils.coerce_dict_values(self.configuration, report_issues) def dump(self): context = ConsumptionContext.get_thread_local() @@ -1923,20 +1929,13 @@ class OperationTemplateBase(TemplateModelMixin): if self.description: console.puts(context.style.meta(self.description)) with context.style.indent: - if self.plugin_specification is not None: - console.puts('Plugin specification: {0}'.format( - context.style.literal(self.plugin_specification.name))) if self.implementation is not None: console.puts('Implementation: {0}'.format( context.style.literal(self.implementation))) - if self.configuration: - with context.style.indent: - for k, v in self.configuration.iteritems(): - console.puts('{0}: {1}'.format(context.style.property(k), - context.style.literal(v))) if self.dependencies: console.puts('Dependencies: {0}'.format( ', '.join((str(context.style.literal(v)) for v in self.dependencies)))) + utils.dump_dict_values(self.inputs, 'Inputs') if self.executor is not None: console.puts('Executor: {0}'.format(context.style.literal(self.executor))) if self.max_attempts is not None: @@ -1944,7 +1943,12 @@ class OperationTemplateBase(TemplateModelMixin): if self.retry_interval is not None: console.puts('Retry interval: {0}'.format( context.style.literal(self.retry_interval))) - utils.dump_dict_values(self.inputs, 'Inputs') + if self.plugin_specification is not None: + console.puts('Plugin specification: {0}'.format( + context.style.literal(self.plugin_specification.name))) + utils.dump_dict_values(self.configuration, 'Configuration') + if self.function is not None: + console.puts('Function: {0}'.format(context.style.literal(self.function))) class ArtifactTemplateBase(TemplateModelMixin): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/modeling/utils.py ---------------------------------------------------------------------- diff --git a/aria/modeling/utils.py b/aria/modeling/utils.py index 0404fe4..6f4022c 100644 --- a/aria/modeling/utils.py +++ b/aria/modeling/utils.py @@ -21,6 +21,7 @@ from . import exceptions from ..parser.consumption import ConsumptionContext from ..utils.console import puts from ..utils.type import validate_value_type +from ..utils.collections import OrderedDict class ModelJSONEncoder(JSONEncoder): @@ -39,7 +40,7 @@ class ModelJSONEncoder(JSONEncoder): class NodeTemplateContainerHolder(object): """ Wrapper that allows using a :class:`aria.modeling.models.NodeTemplate` model directly as the - ``container_holder`` argument for :func:`aria.modeling.functions.evaluate`. + ``container_holder`` input for :func:`aria.modeling.functions.evaluate`. """ def __init__(self, node_template): @@ -51,74 +52,84 @@ class NodeTemplateContainerHolder(object): return self.container.service_template -def create_inputs(inputs, template_inputs): +def create_parameters(parameters, declared_parameters): """ - :param inputs: key-value dict - :param template_inputs: parameter name to parameter object dict - :return: dict of parameter name to Parameter models + Validates, merges, and wraps parameter values according to those declared by a type. + + Exceptions will be raised for validation errors: + + * :class:`aria.modeling.exceptions.UndeclaredParametersException` if a key in ``parameters`` + does not exist in ``declared_parameters`` + * :class:`aria.modeling.exceptions.MissingRequiredParametersException` if a key in + ``declared_parameters`` does not exist in ``parameters`` and also has no default value + * :class:`aria.modeling.exceptions.ParametersOfWrongTypeException` if a value in ``parameters`` + does not match its type in ``declared_parameters`` + + :param parameters: Provided parameter values + :type parameters: {basestring, object} + :param declared_parameters: Declared parameters + :type declared_parameters: {basestring, :class:`aria.modeling.models.Parameter`} + :return: The merged parameters + :rtype: {basestring, :class:`aria.modeling.models.Parameter`} """ - merged_inputs = _merge_and_validate_inputs(inputs, template_inputs) + + merged_parameters = _merge_and_validate_parameters(parameters, declared_parameters) from . import models - input_models = [] - for input_name, input_val in merged_inputs.iteritems(): + parameters_models = OrderedDict() + for parameter_name, parameter_value in merged_parameters.iteritems(): parameter = models.Parameter( # pylint: disable=unexpected-keyword-arg - name=input_name, - type_name=template_inputs[input_name].type_name, - description=template_inputs[input_name].description, - value=input_val) - input_models.append(parameter) + name=parameter_name, + type_name=declared_parameters[parameter_name].type_name, + description=declared_parameters[parameter_name].description, + value=parameter_value) + parameters_models[parameter.name] = parameter - return dict((inp.name, inp) for inp in input_models) + return parameters_models -def _merge_and_validate_inputs(inputs, template_inputs): - """ - :param inputs: key-value dict - :param template_inputs: parameter name to parameter object dict - :return: - """ - merged_inputs = inputs.copy() - - missing_inputs = [] - wrong_type_inputs = {} - for input_name, input_template in template_inputs.iteritems(): - if input_name not in inputs: - if input_template.value is not None: - merged_inputs[input_name] = input_template.value # apply default value +def _merge_and_validate_parameters(parameters, declared_parameters): + merged_parameters = OrderedDict(parameters) + + missing_parameters = [] + wrong_type_parameters = OrderedDict() + for parameter_name, declared_parameter in declared_parameters.iteritems(): + if parameter_name not in parameters: + if declared_parameter.value is not None: + merged_parameters[parameter_name] = declared_parameter.value # apply default value else: - missing_inputs.append(input_name) + missing_parameters.append(parameter_name) else: - # Validate input type + # Validate parameter type try: - validate_value_type(inputs[input_name], input_template.type_name) + validate_value_type(parameters[parameter_name], declared_parameter.type_name) except ValueError: - wrong_type_inputs[input_name] = input_template.type_name + wrong_type_parameters[parameter_name] = declared_parameter.type_name except RuntimeError: # TODO: This error shouldn't be raised (or caught), but right now we lack support # for custom data_types, which will raise this error. Skipping their validation. pass - if missing_inputs: - raise exceptions.MissingRequiredInputsException( - 'Required inputs {0} have not been specified - expected inputs: {1}' - .format(missing_inputs, template_inputs.keys())) + if missing_parameters: + raise exceptions.MissingRequiredParametersException( + 'Required parameters {0} have not been specified; Expected parameters: {1}' + .format(missing_parameters, declared_parameters.keys())) - if wrong_type_inputs: + if wrong_type_parameters: error_message = StringIO() - for param_name, param_type in wrong_type_inputs.iteritems(): - error_message.write('Input "{0}" must be of type {1}{2}' + for param_name, param_type in wrong_type_parameters.iteritems(): + error_message.write('Parameter "{0}" must be of type {1}{2}' .format(param_name, param_type, os.linesep)) - raise exceptions.InputsOfWrongTypeException(error_message.getvalue()) + raise exceptions.ParametersOfWrongTypeException(error_message.getvalue()) - undeclared_inputs = [input_name for input_name in inputs.keys() - if input_name not in template_inputs] - if undeclared_inputs: - raise exceptions.UndeclaredInputsException( - 'Undeclared inputs have been specified: {0}; Expected inputs: {1}' - .format(undeclared_inputs, template_inputs.keys())) + undeclared_parameters = [parameter_name for parameter_name in parameters.keys() + if parameter_name not in declared_parameters] + if undeclared_parameters: + raise exceptions.UndeclaredParametersException( + 'Undeclared parameters have been specified: {0}; Expected parameters: {1}' + .format(undeclared_parameters, declared_parameters.keys())) - return merged_inputs + return merged_parameters def coerce_dict_values(the_dict, report_issues=False): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/orchestrator/execution_plugin/instantiation.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/execution_plugin/instantiation.py b/aria/orchestrator/execution_plugin/instantiation.py index c09434e..26c3913 100644 --- a/aria/orchestrator/execution_plugin/instantiation.py +++ b/aria/orchestrator/execution_plugin/instantiation.py @@ -16,19 +16,13 @@ # TODO: this module will eventually be moved to a new "aria.instantiation" package from ...utils.type import full_type_name -from ...utils.collections import OrderedDict +from ...utils.formatting import safe_repr from ...parser import validation from ...parser.consumption import ConsumptionContext +from ...modeling.functions import Function def configure_operation(operation): - configuration = OrderedDict(operation.configuration) if operation.configuration else {} - - arguments = OrderedDict() - arguments['script_path'] = operation.implementation - arguments['process'] = _get_process(configuration.pop('process')) \ - if 'process' in configuration else dict() - host = None interface = operation.interface if interface.node is not None: @@ -36,87 +30,118 @@ def configure_operation(operation): elif interface.relationship is not None: if operation.relationship_edge is True: host = interface.relationship.target_node.host - else: # either False or None + else: # either False or None (None meaning that edge was not specified) host = interface.relationship.source_node.host + _configure_common(operation) if host is None: _configure_local(operation) else: - _configure_remote(operation, configuration, arguments) + _configure_remote(operation) + + # Any remaining un-handled configuration parameters will become extra arguments, available as + # kwargs in either "run_script_locally" or "run_script_with_ssh" + for key, value in operation.configuration.iteritems(): + if key not in ('process', 'ssh'): + operation.arguments[key] = value.instantiate() - # Any remaining unhandled configuration values will become extra arguments, available as kwargs - # in either "run_script_locally" or "run_script_with_ssh" - arguments.update(configuration) - return arguments +def _configure_common(operation): + """ + Local and remote operations. + """ + + from ...modeling.models import Parameter + operation.arguments['script_path'] = Parameter.wrap('script_path', operation.implementation, + 'Relative path to the executable file.') + operation.arguments['process'] = Parameter.wrap('process', _get_process(operation), + 'Sub-process configuration.') + def _configure_local(operation): """ Local operation. """ + from . import operations - operation.implementation = '{0}.{1}'.format(operations.__name__, - operations.run_script_locally.__name__) + operation.function = '{0}.{1}'.format(operations.__name__, + operations.run_script_locally.__name__) -def _configure_remote(operation, configuration, arguments): +def _configure_remote(operation): """ Remote SSH operation via Fabric. """ + + from ...modeling.models import Parameter + from . import operations + + ssh = _get_ssh(operation) + + # Defaults # TODO: find a way to configure these generally in the service template default_user = '' default_password = '' - - ssh = _get_ssh(configuration.pop('ssh')) if 'ssh' in configuration else {} if 'user' not in ssh: ssh['user'] = default_user if ('password' not in ssh) and ('key' not in ssh) and ('key_filename' not in ssh): ssh['password'] = default_password - arguments['use_sudo'] = ssh.get('use_sudo', False) - arguments['hide_output'] = ssh.get('hide_output', []) - arguments['fabric_env'] = {} + operation.arguments['use_sudo'] = Parameter.wrap('use_sudo', ssh.get('use_sudo', False), + 'Whether to execute with sudo.') + + operation.arguments['hide_output'] = Parameter.wrap('hide_output', ssh.get('hide_output', []), + 'Hide output of these Fabric groups.') + + fabric_env = {} if 'warn_only' in ssh: - arguments['fabric_env']['warn_only'] = ssh['warn_only'] - arguments['fabric_env']['user'] = ssh.get('user') - arguments['fabric_env']['password'] = ssh.get('password') - arguments['fabric_env']['key'] = ssh.get('key') - arguments['fabric_env']['key_filename'] = ssh.get('key_filename') + fabric_env['warn_only'] = ssh['warn_only'] + fabric_env['user'] = ssh.get('user') + fabric_env['password'] = ssh.get('password') + fabric_env['key'] = ssh.get('key') + fabric_env['key_filename'] = ssh.get('key_filename') if 'address' in ssh: - arguments['fabric_env']['host_string'] = ssh['address'] + fabric_env['host_string'] = ssh['address'] - if arguments['fabric_env'].get('user') is None: + # Make sure we have a user + if fabric_env.get('user') is None: context = ConsumptionContext.get_thread_local() context.validation.report('must configure "ssh.user" for "{0}"' .format(operation.implementation), level=validation.Issue.BETWEEN_TYPES) - if (arguments['fabric_env'].get('password') is None) and \ - (arguments['fabric_env'].get('key') is None) and \ - (arguments['fabric_env'].get('key_filename') is None): + + # Make sure we have an authentication value + if (fabric_env.get('password') is None) and \ + (fabric_env.get('key') is None) and \ + (fabric_env.get('key_filename') is None): context = ConsumptionContext.get_thread_local() context.validation.report('must configure "ssh.password", "ssh.key", or "ssh.key_filename" ' 'for "{0}"' .format(operation.implementation), level=validation.Issue.BETWEEN_TYPES) - from . import operations - operation.implementation = '{0}.{1}'.format(operations.__name__, - operations.run_script_with_ssh.__name__) + operation.arguments['fabric_env'] = Parameter.wrap('fabric_env', fabric_env, + 'Fabric configuration.') + operation.function = '{0}.{1}'.format(operations.__name__, + operations.run_script_with_ssh.__name__) -def _get_process(value): + +def _get_process(operation): + value = operation.configuration.get('process')._value \ + if 'process' in operation.configuration else None if value is None: - return None + return {} _validate_type(value, dict, 'process') for k, v in value.iteritems(): if k == 'eval_python': - value[k] = _str_to_bool(v, 'process.eval_python') + value[k] = _coerce_bool(v, 'process.eval_python') elif k == 'cwd': _validate_type(v, basestring, 'process.cwd') elif k == 'command_prefix': _validate_type(v, basestring, 'process.command_prefix') elif k == 'args': - value[k] = _dict_to_list(v, 'process.args') + value[k] = _dict_to_list_of_strings(v, 'process.args') elif k == 'env': _validate_type(v, dict, 'process.env') else: @@ -126,17 +151,19 @@ def _get_process(value): return value -def _get_ssh(value): +def _get_ssh(operation): + value = operation.configuration.get('ssh')._value \ + if 'process' in operation.configuration else None if value is None: return {} _validate_type(value, dict, 'ssh') for k, v in value.iteritems(): if k == 'use_sudo': - value[k] = _str_to_bool(v, 'ssh.use_sudo') + value[k] = _coerce_bool(v, 'ssh.use_sudo') elif k == 'hide_output': - value[k] = _dict_to_list(v, 'ssh.hide_output') + value[k] = _dict_to_list_of_strings(v, 'ssh.hide_output') elif k == 'warn_only': - value[k] = _str_to_bool(v, 'ssh.warn_only') + value[k] = _coerce_bool(v, 'ssh.warn_only') elif k == 'user': _validate_type(v, basestring, 'ssh.user') elif k == 'password': @@ -155,16 +182,20 @@ def _get_ssh(value): def _validate_type(value, the_type, name): + if isinstance(value, Function): + return if not isinstance(value, the_type): context = ConsumptionContext.get_thread_local() - context.validation.report('"{0}" configuration is not a {1}' - .format(name, full_type_name(the_type)), + context.validation.report('"{0}" configuration is not a {1}: {2}' + .format(name, full_type_name(the_type), safe_repr(value)), level=validation.Issue.BETWEEN_TYPES) -def _str_to_bool(value, name): +def _coerce_bool(value, name): if value is None: return None + if isinstance(value, bool): + return value _validate_type(value, basestring, name) if value == 'true': return True @@ -173,19 +204,15 @@ def _str_to_bool(value, name): else: context = ConsumptionContext.get_thread_local() context.validation.report('"{0}" configuration is not "true" or "false": {1}' - .format(name, repr(value)), + .format(name, safe_repr(value)), level=validation.Issue.BETWEEN_TYPES) -def _dict_to_list(the_dict, name): +def _dict_to_list_of_strings(the_dict, name): _validate_type(the_dict, dict, name) value = [] for k in sorted(the_dict): v = the_dict[k] - if not isinstance(v, basestring): - context = ConsumptionContext.get_thread_local() - context.validation.report('"{0}.{1}" configuration is not a string: {2}' - .format(name, k, repr(v)), - level=validation.Issue.BETWEEN_TYPES) + _validate_type(v, basestring, '{0}.{1}'.format(name, k)) value.append(v) return value http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/orchestrator/workflow_runner.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflow_runner.py b/aria/orchestrator/workflow_runner.py index 8f25cce..0c6321f 100644 --- a/aria/orchestrator/workflow_runner.py +++ b/aria/orchestrator/workflow_runner.py @@ -119,7 +119,7 @@ class WorkflowRunner(object): else: workflow_inputs = self.service.workflows[self._workflow_name].inputs - execution.inputs = modeling_utils.create_inputs(inputs, workflow_inputs) + execution.inputs = modeling_utils.create_parameters(inputs, workflow_inputs) # TODO: these two following calls should execute atomically self._validate_no_active_executions(execution) self._model_storage.execution.put(execution) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/orchestrator/workflows/api/task.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py index cb79eb3..aa6ac45 100644 --- a/aria/orchestrator/workflows/api/task.py +++ b/aria/orchestrator/workflows/api/task.py @@ -99,8 +99,8 @@ class OperationTask(BaseTask): operation = self.actor.interfaces[self.interface_name].operations[self.operation_name] self.plugin = operation.plugin - self.inputs = modeling_utils.create_inputs(inputs or {}, operation.inputs) - self.implementation = operation.implementation + self.inputs = modeling_utils.create_parameters(inputs or {}, operation.arguments) + self.implementation = operation.function def __repr__(self): return self.name http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/aria/orchestrator/workflows/core/task.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/core/task.py b/aria/orchestrator/workflows/core/task.py index b3dfb3c..0d6eb11 100644 --- a/aria/orchestrator/workflows/core/task.py +++ b/aria/orchestrator/workflows/core/task.py @@ -148,7 +148,6 @@ class OperationTask(BaseTask): plugin=api_task.plugin, implementation=api_task.implementation, inputs=api_task.inputs - ) self._workflow_context.model.task.put(task_model) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/profiles/aria-1.0/aria-1.0.yaml ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/profiles/aria-1.0/aria-1.0.yaml b/extensions/aria_extension_tosca/profiles/aria-1.0/aria-1.0.yaml index 0c5e77f..abac03b 100644 --- a/extensions/aria_extension_tosca/profiles/aria-1.0/aria-1.0.yaml +++ b/extensions/aria_extension_tosca/profiles/aria-1.0/aria-1.0.yaml @@ -52,18 +52,10 @@ policy_types: should be inherited and extended with additional properties. derived_from: tosca.policies.Root properties: - implementation: + function: description: >- - The interpretation of the implementation string depends on the orchestrator. In ARIA it is - the full path to a Python @workflow function that generates a task graph based on the - service topology. + The interpretation of the function string depends on the orchestrator. In ARIA it is the + full path to a Python @workflow function that generates a task graph based on the service + topology. type: string required: true - dependencies: - description: >- - The optional ordered list of one or more dependent or secondary implementation artifact - name which are referenced by the primary implementation artifact (e.g., a library the - script installs or a secondary script). - type: list - entry_schema: string - required: false http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/simple_v1_0/assignments.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/assignments.py b/extensions/aria_extension_tosca/simple_v1_0/assignments.py index d929ce0..79f6377 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/assignments.py +++ b/extensions/aria_extension_tosca/simple_v1_0/assignments.py @@ -23,7 +23,7 @@ from aria.parser.presentation import (AsIsPresentation, has_fields, allow_unknow from .filters import NodeFilter from .misc import Description, OperationImplementation -from .modeling.properties import get_assigned_and_defined_property_values +from .modeling.parameters import get_assigned_and_defined_parameter_values from .presentation.extensible import ExtensiblePresentation from .presentation.field_validators import (node_template_or_type_validator, relationship_template_or_type_validator, @@ -428,7 +428,7 @@ class ArtifactAssignment(ExtensiblePresentation): @cachedmethod def _get_property_values(self, context): - return FrozenDict(get_assigned_and_defined_property_values(context, self)) + return FrozenDict(get_assigned_and_defined_parameter_values(context, self, 'property')) @cachedmethod def _validate(self, context): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py index 99389e4..0b04fdc 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py @@ -26,14 +26,17 @@ import re from types import FunctionType from datetime import datetime +from ruamel import yaml + from aria.parser.validation import Issue -from aria.utils.collections import StrictDict +from aria.utils.collections import (StrictDict, OrderedDict) from aria.modeling.models import (Type, ServiceTemplate, NodeTemplate, RequirementTemplate, RelationshipTemplate, CapabilityTemplate, GroupTemplate, PolicyTemplate, SubstitutionTemplate, SubstitutionTemplateMapping, InterfaceTemplate, OperationTemplate, ArtifactTemplate, Metadata, Parameter, PluginSpecification) +from .parameters import coerce_parameter_value from .constraints import (Equal, GreaterThan, GreaterOrEqual, LessThan, LessOrEqual, InRange, ValidValues, Length, MinLength, MaxLength, Pattern) from ..data_types import coerce_value @@ -375,7 +378,7 @@ def create_operation_template_model(context, service_template, operation): implementation = operation.implementation if implementation is not None: primary = implementation.primary - parse_implementation_string(context, service_template, operation, model, primary) + set_implementation(context, service_template, operation, model, primary) relationship_edge = operation._get_extensions(context).get('relationship_edge') if relationship_edge is not None: if relationship_edge == 'source': @@ -384,18 +387,37 @@ def create_operation_template_model(context, service_template, operation): model.relationship_edge = True dependencies = implementation.dependencies + configuration = OrderedDict() if dependencies: for dependency in dependencies: key, value = split_prefix(dependency) if key is not None: - if model.configuration is None: - model.configuration = {} - set_nested(model.configuration, key.split('.'), value) + # Parse as YAML + try: + value = yaml.load(value) + except yaml.parser.MarkedYAMLError as e: + context.validation.report( + 'YAML parser {0} in operation configuration: {1}' + .format(e.problem, value), + locator=implementation._locator, + level=Issue.FIELD) + continue + + # Coerce to intrinsic functions, if there are any + value = coerce_parameter_value(context, implementation, None, value).value + + # Support dot-notation nesting + set_nested(configuration, key.split('.'), value) else: if model.dependencies is None: model.dependencies = [] model.dependencies.append(dependency) + # Convert configuration to Parameter models + for key, value in configuration.iteritems(): + model.configuration[key] = Parameter.wrap(key, value, + description='Operation configuration.') + inputs = operation.inputs if inputs: for input_name, the_input in inputs.iteritems(): @@ -490,15 +512,15 @@ def create_workflow_operation_template_model(context, service_template, policy): properties = policy._get_property_values(context) for prop_name, prop in properties.iteritems(): - if prop_name == 'implementation': - parse_implementation_string(context, service_template, policy, model, prop.value) + if prop_name == 'function': + model.function = prop.value elif prop_name == 'dependencies': model.dependencies = prop.value else: - model.inputs[prop_name] = Parameter(name=prop_name, # pylint: disable=unexpected-keyword-arg - type_name=prop.type, - value=prop.value, - description=prop.description) + model.configuration[prop_name] = Parameter(name=prop_name, # pylint: disable=unexpected-keyword-arg + type_name=prop.type, + value=prop.value, + description=prop.description) return model @@ -639,12 +661,12 @@ def create_constraint(context, node_filter, constraint_clause, property_name, ca def split_prefix(string): """ - Splits the prefix on the first unescaped ">". + Splits the prefix on the first non-escaped ">". """ - split = IMPLEMENTATION_PREFIX_REGEX.split(string, 2) + split = IMPLEMENTATION_PREFIX_REGEX.split(string, 1) if len(split) < 2: - return None, string + return None, None return split[0].strip(), split[1].lstrip() @@ -671,13 +693,18 @@ def set_nested(the_dict, keys, value): set_nested(the_dict[key], keys, value) -def parse_implementation_string(context, service_template, presentation, model, implementation): - plugin_name, model.implementation = split_prefix(implementation) - if plugin_name is not None: - model.plugin_specification = service_template.plugin_specifications.get(plugin_name) +def set_implementation(context, service_template, presentation, model, primary): + prefix, postfix = split_prefix(primary) + if prefix: + # Special ARIA prefix + model.plugin_specification = service_template.plugin_specifications.get(prefix) + model.function = postfix if model.plugin_specification is None: context.validation.report( 'no policy for plugin "{0}" specified in operation implementation: {1}' - .format(plugin_name, implementation), + .format(prefix, primary), locator=presentation._get_child_locator('properties', 'implementation'), level=Issue.BETWEEN_TYPES) + else: + # Standard TOSCA artifact + model.implementation = primary http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py index 4f61ef5..dd9eeb4 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/artifacts.py @@ -15,11 +15,11 @@ from aria.utils.collections import OrderedDict + # # NodeType, NodeTemplate # - def get_inherited_artifact_definitions(context, presentation, for_presentation=None): if hasattr(presentation, '_get_type'): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py index 6df7177..a90a9fc 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/capabilities.py @@ -16,8 +16,9 @@ from aria.utils.collections import deepcopy_with_locators, OrderedDict from aria.parser.validation import Issue -from .properties import (convert_property_definitions_to_values, merge_raw_property_definitions, - get_assigned_and_defined_property_values) +from .parameters import (convert_parameter_definitions_to_values, merge_raw_parameter_definitions, + get_assigned_and_defined_parameter_values) + # # CapabilityType @@ -38,6 +39,7 @@ def get_inherited_valid_source_types(context, presentation): return valid_source_types + # # NodeType # @@ -92,6 +94,7 @@ def get_inherited_capability_definitions(context, presentation, for_presentation return capability_definitions + # # NodeTemplate # @@ -127,8 +130,9 @@ def get_template_capabilities(context, presentation): capability_assignment = capability_assignments[capability_name] # Assign properties - values = get_assigned_and_defined_property_values(context, - our_capability_assignment) + values = get_assigned_and_defined_parameter_values(context, + our_capability_assignment, + 'property') if values: capability_assignment._raw['properties'] = values else: @@ -139,6 +143,7 @@ def get_template_capabilities(context, presentation): return capability_assignments + # # Utils # @@ -150,24 +155,25 @@ def convert_capability_from_definition_to_assignment(context, presentation, cont properties = presentation.properties if properties is not None: - raw['properties'] = convert_property_definitions_to_values(context, properties) + raw['properties'] = convert_parameter_definitions_to_values(context, properties) # TODO attributes return CapabilityAssignment(name=presentation._name, raw=raw, container=container) + def merge_capability_definition_from_type(context, presentation, capability_definition): raw_properties = OrderedDict() # Merge properties from type the_type = capability_definition._get_type(context) type_property_defintions = the_type._get_properties(context) - merge_raw_property_definitions(context, presentation, raw_properties, type_property_defintions, - 'properties') + merge_raw_parameter_definitions(context, presentation, raw_properties, type_property_defintions, + 'properties') # Merge our properties - merge_raw_property_definitions(context, presentation, raw_properties, - capability_definition.properties, 'properties') + merge_raw_parameter_definitions(context, presentation, raw_properties, + capability_definition.properties, 'properties') if raw_properties: capability_definition._raw['properties'] = raw_properties http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/simple_v1_0/modeling/constraints.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/constraints.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/constraints.py index 7c99eab..9a30cc1 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/constraints.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/constraints.py @@ -15,7 +15,7 @@ import re -from aria.modeling.contraints import NodeTemplateConstraint +from aria.modeling.constraints import NodeTemplateConstraint from aria.modeling.utils import NodeTemplateContainerHolder from aria.modeling.functions import evaluate from aria.parser import implements_specification http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py index 3952785..c0d79e5 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/data_types.py @@ -26,6 +26,7 @@ from aria.parser.validation import Issue from .functions import get_function from ..presentation.types import get_type_by_full_or_shorthand_name + # # DataType # @@ -50,6 +51,7 @@ def get_inherited_constraints(context, presentation): return constraints + def coerce_data_type_value(context, presentation, data_type, entry_schema, constraints, value, # pylint: disable=unused-argument aspect): """ @@ -121,6 +123,7 @@ def coerce_data_type_value(context, presentation, data_type, entry_schema, const return value + def validate_data_type_name(context, presentation): """ Makes sure the complex data type's name is not that of a built-in type. @@ -132,6 +135,7 @@ def validate_data_type_name(context, presentation): % safe_repr(name), locator=presentation._locator, level=Issue.BETWEEN_TYPES) + # # PropertyDefinition, AttributeDefinition, EntrySchema, DataType # @@ -172,6 +176,7 @@ def get_data_type(context, presentation, field_name, allow_none=False): # Try primitive data type return get_primitive_data_type(type_name) + # # PropertyDefinition, EntrySchema # @@ -195,6 +200,7 @@ def get_property_constraints(context, presentation): return constraints + # # ConstraintClause # @@ -310,6 +316,7 @@ def apply_constraint_to_value(context, presentation, constraint_clause, value): return True + # # Repository # @@ -326,6 +333,7 @@ def get_data_type_value(context, presentation, field_name, type_name): locator=presentation._locator, level=Issue.BETWEEN_TYPES) return None + # # Utils # @@ -345,6 +353,7 @@ PRIMITIVE_DATA_TYPES = { 'boolean': bool, 'null': None.__class__} + @implements_specification('3.2.1-3', 'tosca-simple-1.0') def get_primitive_data_type(type_name): """ @@ -358,6 +367,7 @@ def get_primitive_data_type(type_name): return PRIMITIVE_DATA_TYPES.get(type_name) + def get_data_type_name(the_type): """ Returns the name of the type, whether it's a DataType, a primitive type, or another class. @@ -365,6 +375,7 @@ def get_data_type_name(the_type): return the_type._name if hasattr(the_type, '_name') else full_type_name(the_type) + def coerce_value(context, presentation, the_type, entry_schema, constraints, value, aspect=None): # pylint: disable=too-many-return-statements """ Returns the value after it's coerced to its type, reporting validation errors if it cannot be @@ -410,6 +421,7 @@ def coerce_value(context, presentation, the_type, entry_schema, constraints, val # Coerce to primitive type return coerce_to_primitive(context, presentation, the_type, constraints, value, aspect) + def coerce_to_primitive(context, presentation, primitive_type, constraints, value, aspect=None): """ Returns the value after it's coerced to a primitive type, translating exceptions to validation @@ -435,6 +447,7 @@ def coerce_to_primitive(context, presentation, primitive_type, constraints, valu return value + def coerce_to_data_type_class(context, presentation, cls, entry_schema, constraints, value, aspect=None): """ @@ -463,6 +476,7 @@ def coerce_to_data_type_class(context, presentation, cls, entry_schema, constrai return value + def apply_constraints_to_value(context, presentation, constraints, value): """ Applies all constraints to the value. If the value conforms, returns the value. If it does not @@ -478,6 +492,7 @@ def apply_constraints_to_value(context, presentation, constraints, value): value = None return value + def get_container_data_type(presentation): if presentation is None: return None @@ -485,6 +500,7 @@ def get_container_data_type(presentation): return presentation return get_container_data_type(presentation._container) + def report_issue_for_bad_format(context, presentation, the_type, value, aspect, e): if aspect == 'default': aspect = '"default" value' http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py index 3e6aa6f..e04ac4a 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/interfaces.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from aria.utils.collections import merge, deepcopy_with_locators, OrderedDict +from aria.utils.collections import (merge, deepcopy_with_locators, OrderedDict) from aria.parser.presentation import get_locator from aria.parser.validation import Issue -from .properties import (coerce_property_value, convert_property_definitions_to_values) +from .parameters import (coerce_parameter_value, convert_parameter_definitions_to_values) + # # InterfaceType @@ -45,6 +46,7 @@ def get_inherited_operations(context, presentation): return operations + # # InterfaceDefinition # @@ -73,6 +75,7 @@ def get_and_override_input_definitions_from_type(context, presentation): return inputs + def get_and_override_operation_definitions_from_type(context, presentation): """ Returns our operation definitions added on top of those of the interface type, if specified. @@ -96,6 +99,7 @@ def get_and_override_operation_definitions_from_type(context, presentation): return operations + # # NodeType, RelationshipType, GroupType # @@ -124,6 +128,7 @@ def get_inherited_interface_definitions(context, presentation, type_name, for_pr return interfaces + # # NodeTemplate, RelationshipTemplate, GroupTemplate # @@ -186,6 +191,7 @@ def get_template_interfaces(context, presentation, type_name): return template_interfaces + # # Utils # @@ -200,13 +206,14 @@ def convert_interface_definition_from_type_to_template(context, presentation, co raw = convert_interface_definition_from_type_to_raw_template(context, presentation) return InterfaceAssignment(name=presentation._name, raw=raw, container=container) + def convert_interface_definition_from_type_to_raw_template(context, presentation): # pylint: disable=invalid-name raw = OrderedDict() # Copy default values for inputs inputs = presentation._get_inputs(context) if inputs is not None: - raw['inputs'] = convert_property_definitions_to_values(context, inputs) + raw['inputs'] = convert_parameter_definitions_to_values(context, inputs) # Copy operations operations = presentation._get_operations(context) @@ -221,11 +228,12 @@ def convert_interface_definition_from_type_to_raw_template(context, presentation raw[operation_name]['implementation'] = deepcopy_with_locators(implementation._raw) inputs = operation.inputs if inputs is not None: - raw[operation_name]['inputs'] = convert_property_definitions_to_values(context, - inputs) + raw[operation_name]['inputs'] = convert_parameter_definitions_to_values(context, + inputs) return raw + def convert_requirement_interface_definitions_from_type_to_raw_template(context, raw_requirement, # pylint: disable=invalid-name interface_definitions): if not interface_definitions: @@ -240,6 +248,7 @@ def convert_requirement_interface_definitions_from_type_to_raw_template(context, else: raw_requirement['interfaces'][interface_name] = raw_interface + def merge_interface(context, presentation, interface_assignment, our_interface_assignment, interface_definition, interface_name): # Assign/merge interface inputs @@ -282,6 +291,7 @@ def merge_interface(context, presentation, interface_assignment, our_interface_a our_input_assignments, input_definitions, interface_name, operation_name, presentation) + def merge_raw_input_definition(context, the_raw_input, our_input, interface_name, operation_name, presentation, type_name): # Check if we changed the type @@ -305,6 +315,7 @@ def merge_raw_input_definition(context, the_raw_input, our_input, interface_name # Merge merge(the_raw_input, our_input._raw) + def merge_input_definitions(context, inputs, our_inputs, interface_name, operation_name, presentation, type_name): for input_name, our_input in our_inputs.iteritems(): @@ -314,6 +325,7 @@ def merge_input_definitions(context, inputs, our_inputs, interface_name, operati else: inputs[input_name] = our_input._clone(presentation) + def merge_raw_input_definitions(context, raw_inputs, our_inputs, interface_name, operation_name, presentation, type_name): for input_name, our_input in our_inputs.iteritems(): @@ -323,6 +335,7 @@ def merge_raw_input_definitions(context, raw_inputs, our_inputs, interface_name, else: raw_inputs[input_name] = deepcopy_with_locators(our_input._raw) + def merge_raw_operation_definition(context, raw_operation, our_operation, interface_name, presentation, type_name): if not isinstance(our_operation._raw, dict): @@ -353,6 +366,7 @@ def merge_raw_operation_definition(context, raw_operation, our_operation, interf raw_operation['implementation'] = \ deepcopy_with_locators(our_operation._raw['implementation']) + def merge_operation_definitions(context, operations, our_operations, interface_name, presentation, type_name): if not our_operations: @@ -364,6 +378,7 @@ def merge_operation_definitions(context, operations, our_operations, interface_n else: operations[operation_name] = our_operation._clone(presentation) + def merge_raw_operation_definitions(context, raw_operations, our_operations, interface_name, presentation, type_name): for operation_name, our_operation in our_operations.iteritems(): @@ -378,6 +393,7 @@ def merge_raw_operation_definitions(context, raw_operations, our_operations, int else: raw_operations[operation_name] = deepcopy_with_locators(our_operation._raw) + # From either an InterfaceType or an InterfaceDefinition: def merge_interface_definition(context, interface, our_source, presentation, type_name): if hasattr(our_source, 'type'): @@ -408,6 +424,7 @@ def merge_interface_definition(context, interface, our_source, presentation, typ merge_raw_operation_definitions(context, interface._raw, our_operations, our_source._name, presentation, type_name) + def merge_interface_definitions(context, interfaces, our_interfaces, presentation, for_presentation=None): if not our_interfaces: @@ -419,12 +436,14 @@ def merge_interface_definitions(context, interfaces, our_interfaces, presentatio else: interfaces[name] = our_interface._clone(for_presentation) + def merge_interface_definitions_from_their_types(context, interfaces, presentation): for interface in interfaces.itervalues(): the_type = interface._get_type(context) # InterfaceType if the_type is not None: merge_interface_definition(context, interface, the_type, presentation, 'type') + def assign_raw_inputs(context, values, assignments, definitions, interface_name, operation_name, presentation): if not assignments: @@ -454,8 +473,9 @@ def assign_raw_inputs(context, values, assignments, definitions, interface_name, # Note: default value has already been assigned # Coerce value - values['inputs'][input_name] = coerce_property_value(context, assignment, definition, - assignment.value) + values['inputs'][input_name] = coerce_parameter_value(context, assignment, definition, + assignment.value) + def validate_required_inputs(context, presentation, assignment, definition, original_assignment, interface_name, operation_name=None): http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84d8da59/extensions/aria_extension_tosca/simple_v1_0/modeling/parameters.py ---------------------------------------------------------------------- diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/parameters.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/parameters.py new file mode 100644 index 0000000..c910956 --- /dev/null +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/parameters.py @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from aria.utils.collections import (merge, deepcopy_with_locators, OrderedDict) +from aria.utils.formatting import pluralize +from aria.parser.presentation import Value +from aria.parser.validation import Issue + +from .data_types import coerce_value + + +# +# ArtifactType, DataType, CapabilityType, RelationshipType, NodeType, GroupType, PolicyType +# + +def get_inherited_parameter_definitions(context, presentation, field_name, for_presentation=None): + """ + Returns our parameter definitions added on top of those of our parent, if we have one + (recursively). + + Allows overriding all aspects of parent properties except data type. + """ + + # Get definitions from parent + # If we inherit from a primitive, it does not have a parent: + parent = presentation._get_parent(context) if hasattr(presentation, '_get_parent') else None + definitions = get_inherited_parameter_definitions(context, parent, field_name, + for_presentation=presentation) \ + if parent is not None else OrderedDict() + + # Add/merge our definitions + # If we inherit from a primitive, it does not have our field + our_definitions = getattr(presentation, field_name, None) + if our_definitions: + our_definitions_clone = OrderedDict() + for name, our_definition in our_definitions.iteritems(): + our_definitions_clone[name] = our_definition._clone(for_presentation) + our_definitions = our_definitions_clone + merge_parameter_definitions(context, presentation, definitions, our_definitions, field_name) + + for definition in definitions.itervalues(): + definition._reset_method_cache() + + return definitions + + +# +# NodeTemplate, RelationshipTemplate, GroupTemplate, PolicyTemplate +# + +def get_assigned_and_defined_parameter_values(context, presentation, field_name): + """ + Returns the assigned property values while making sure they are defined in our type. + + The property definition's default value, if available, will be used if we did not assign it. + + Makes sure that required properties indeed end up with a value. + """ + + values = OrderedDict() + + the_type = presentation._get_type(context) + field_name_plural = pluralize(field_name) + assignments = getattr(presentation, field_name_plural) + get_fn_name = '_get_{0}'.format(field_name_plural) + definitions = getattr(the_type, get_fn_name)(context) if the_type is not None else None + + # Fill in our assignments, but make sure they are defined + if assignments: + for name, value in assignments.iteritems(): + if (definitions is not None) and (name in definitions): + definition = definitions[name] + values[name] = coerce_parameter_value(context, value, definition, value.value) + else: + context.validation.report('assignment to undefined {0} "{1}" in "{2}"' + .format(field_name, name, presentation._fullname), + locator=value._locator, level=Issue.BETWEEN_TYPES) + + # Fill in defaults from the definitions + if definitions: + for name, definition in definitions.iteritems(): + if values.get(name) is None: + values[name] = coerce_parameter_value(context, presentation, definition, + definition.default) + + validate_required_values(context, presentation, values, definitions) + + return values + + +# +# TopologyTemplate +# + +def get_parameter_values(context, presentation, field_name): + values = OrderedDict() + + parameters = getattr(presentation, field_name) + + # Fill in defaults and values + if parameters: + for name, parameter in parameters.iteritems(): + if values.get(name) is None: + if hasattr(parameter, 'value') and (parameter.value is not None): + # For parameters only: + values[name] = coerce_parameter_value(context, presentation, parameter, + parameter.value) + else: + default = parameter.default if hasattr(parameter, 'default') else None + values[name] = coerce_parameter_value(context, presentation, parameter, default) + + return values + + +# +# Utils +# + +def validate_required_values(context, presentation, values, definitions): + """ + Check if required properties have not been assigned. + """ + + if not definitions: + return + for name, definition in definitions.iteritems(): + if getattr(definition, 'required', False) \ + and ((values is None) or (values.get(name) is None)): + context.validation.report('required property "%s" is not assigned a value in "%s"' + % (name, presentation._fullname), + locator=presentation._get_child_locator('properties'), + level=Issue.BETWEEN_TYPES) + + +def merge_raw_parameter_definition(context, presentation, raw_property_definition, + our_property_definition, field_name, property_name): + # Check if we changed the type + # TODO: allow a sub-type? + type1 = raw_property_definition.get('type') + type2 = our_property_definition.type + if type1 != type2: + context.validation.report( + 'override changes type from "%s" to "%s" for property "%s" in "%s"' + % (type1, type2, property_name, presentation._fullname), + locator=presentation._get_child_locator(field_name, property_name), + level=Issue.BETWEEN_TYPES) + + merge(raw_property_definition, our_property_definition._raw) + + +def merge_raw_parameter_definitions(context, presentation, raw_property_definitions, + our_property_definitions, field_name): + if not our_property_definitions: + return + for property_name, our_property_definition in our_property_definitions.iteritems(): + if property_name in raw_property_definitions: + raw_property_definition = raw_property_definitions[property_name] + merge_raw_parameter_definition(context, presentation, raw_property_definition, + our_property_definition, field_name, property_name) + else: + raw_property_definitions[property_name] = \ + deepcopy_with_locators(our_property_definition._raw) + + +def merge_parameter_definitions(context, presentation, property_definitions, + our_property_definitions, field_name): + if not our_property_definitions: + return + for property_name, our_property_definition in our_property_definitions.iteritems(): + if property_name in property_definitions: + property_definition = property_definitions[property_name] + merge_raw_parameter_definition(context, presentation, property_definition._raw, + our_property_definition, field_name, property_name) + else: + property_definitions[property_name] = our_property_definition + + +# Works on properties, inputs, and parameters +def coerce_parameter_value(context, presentation, definition, value, aspect=None): + the_type = definition._get_type(context) if definition is not None else None + entry_schema = definition.entry_schema if definition is not None else None + constraints = definition._get_constraints(context) \ + if ((definition is not None) and hasattr(definition, '_get_constraints')) else None + value = coerce_value(context, presentation, the_type, entry_schema, constraints, value, aspect) + if (the_type is not None) and hasattr(the_type, '_name'): + type_name = the_type._name + else: + type_name = getattr(definition, 'type', None) + description = getattr(definition, 'description', None) + description = description.value if description is not None else None + return Value(type_name, value, description) + + +def convert_parameter_definitions_to_values(context, definitions): + values = OrderedDict() + for name, definition in definitions.iteritems(): + default = definition.default + values[name] = coerce_parameter_value(context, definition, definition, default) + return values
