This is an automated email from the ASF dual-hosted git repository. exceptionfactory pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push: new a5501baa7f NIFI-13396 Added Python constant Allowable Values to Manifests a5501baa7f is described below commit a5501baa7f25380f424281267d1939584eaf1867 Author: Ferenc Kis <briansolo1...@gmail.com> AuthorDate: Thu Jun 13 10:31:56 2024 +0200 NIFI-13396 Added Python constant Allowable Values to Manifests This closes #8963 Signed-off-by: David Handermann <exceptionfact...@apache.org> --- .../src/main/python/src/nifiapi/documentation.py | 7 ++- .../documentation/PropertyDescription.java | 3 ++ .../main/python/framework/ProcessorInspection.py | 51 +++++++++++++++++----- .../manifest/StandardRuntimeManifestService.java | 28 +++++------- 4 files changed, 61 insertions(+), 28 deletions(-) diff --git a/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py b/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py index 39b031657f..6bce8beb64 100644 --- a/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py +++ b/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py @@ -107,7 +107,8 @@ class PropertyDescription: sensitive: bool = True, default_value: str = None, expression_language_scope: str = None, - controller_service_definition: str = None): + controller_service_definition: str = None, + allowable_values: list[str] = None): self.name = name self.description = description self.display_name = display_name @@ -116,6 +117,7 @@ class PropertyDescription: self.default_value = default_value self.expression_language_scope = expression_language_scope self.controller_service_definition = controller_service_definition + self.allowable_values = allowable_values if allowable_values is not None else [] def getName(self): return self.name @@ -140,3 +142,6 @@ class PropertyDescription: def getControllerServiceDefinition(self): return self.controller_service_definition + + def getAllowableValues(self): + return ArrayList(self.allowable_values) diff --git a/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java b/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java index ca71317cae..0b32ac78c3 100644 --- a/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java +++ b/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java @@ -17,6 +17,7 @@ package org.apache.nifi.python.processor.documentation; +import java.util.List; import org.apache.nifi.python.PythonObjectProxy; public interface PropertyDescription extends PythonObjectProxy { @@ -35,4 +36,6 @@ public interface PropertyDescription extends PythonObjectProxy { String getExpressionLanguageScope(); String getControllerServiceDefinition(); + + List<String> getAllowableValues(); } diff --git a/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py b/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py index cd5fedb24a..2b0b9d56cb 100644 --- a/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py +++ b/nifi-extension-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py @@ -25,6 +25,33 @@ PROCESSOR_INTERFACES = ['org.apache.nifi.python.processor.FlowFileTransform', 'o logger = logging.getLogger("python.ProcessorInspection") +class StringConstantVisitor(ast.NodeVisitor): + def __init__(self): + self.string_assignments = {} + + def visit_Assign(self, node): + if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + string_value = node.value.value + for target in node.targets: + if isinstance(target, ast.Name): + variable_name = target.id + self.string_assignments[variable_name] = string_value + elif isinstance(target, ast.Tuple): + for element in target.elts: + if isinstance(element, ast.Name): + variable_name = element.id + self.string_assignments.append[variable_name] = string_value + self.generic_visit(node) + + +def get_module_string_constants(module_file: str) -> dict: + with open(module_file) as file: + root_node = ast.parse(file.read()) + visitor = StringConstantVisitor() + visitor.visit(root_node) + return visitor.string_assignments + + def get_processor_class_nodes(module_file: str) -> list: with open(module_file) as file: root_node = ast.parse(file.read()) @@ -41,6 +68,7 @@ def get_processor_class_nodes(module_file: str) -> list: def get_processor_details(class_node, module_file, extension_home, dependencies_bundled): # Look for a 'ProcessorDetails' class child_class_nodes = get_class_nodes(class_node) + module_string_constants = get_module_string_constants(module_file) # Get the Java interfaces that it implements interfaces = get_java_interfaces(class_node) @@ -54,7 +82,7 @@ def get_processor_details(class_node, module_file, extension_home, dependencies_ tags = __get_processor_tags(child_class_node) use_cases = get_use_cases(class_node) multi_processor_use_cases = get_multi_processor_use_cases(class_node) - property_descriptions = get_property_descriptions(class_node) + property_descriptions = get_property_descriptions(class_node, module_string_constants) return ExtensionDetails.ExtensionDetails(interfaces=interfaces, type=class_node.name, @@ -184,7 +212,7 @@ def get_processor_configurations(constructor_calls: ast.List) -> list: return configurations -def get_property_descriptions(class_node): +def get_property_descriptions(class_node, module_string_constants): descriptions = [] for element in class_node.body: @@ -200,7 +228,7 @@ def get_property_descriptions(class_node): descriptor_info = {} for keyword in element.value.keywords: key = keyword.arg - value = get_constant_values(keyword.value) + value = get_constant_values(keyword.value, module_string_constants) descriptor_info[key] = value description = PropertyDescription(name=descriptor_info.get('name'), @@ -210,7 +238,8 @@ def get_property_descriptions(class_node): sensitive=replace_null(descriptor_info.get('sensitive'), False), default_value=descriptor_info.get('default_value'), expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'), - controller_service_definition=descriptor_info.get('controller_service_definition')) + controller_service_definition=descriptor_info.get('controller_service_definition'), + allowable_values = descriptor_info.get('allowable_values')) descriptions.append(description) return descriptions @@ -230,22 +259,24 @@ def get_assigned_value(class_node, assignment_id, default_value=None): return default_value -def get_constant_values(val): +def get_constant_values(val, string_constants: dict = {}): if isinstance(val, ast.Constant): return val.value + if isinstance(val, ast.Name): + return string_constants.get(val.id) if isinstance(val, ast.List): - return [get_constant_values(v) for v in val.elts] + return [get_constant_values(v, string_constants) for v in val.elts] if isinstance(val, ast.Dict): keys = val.keys values = val.values - key_values = [get_constant_values(v).strip() for v in keys] - value_values = [get_constant_values(v).strip() for v in values] + key_values = [get_constant_values(v, string_constants).strip() for v in keys] + value_values = [get_constant_values(v, string_constants).strip() for v in values] return dict(zip(key_values, value_values)) if isinstance(val, ast.Attribute): return val.attr if isinstance(val, ast.BinOp) and isinstance(val.op, ast.Add): - left = get_constant_values(val.left) - right = get_constant_values(val.right) + left = get_constant_values(val.left, string_constants) + right = get_constant_values(val.right, string_constants) if left and right: return left + right if left and not right: diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java index a6e2b19ba7..7c0dfce611 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java @@ -20,7 +20,6 @@ import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleDetails; import org.apache.nifi.c2.protocol.component.api.BuildInfo; import org.apache.nifi.c2.protocol.component.api.RuntimeManifest; -import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.extension.manifest.AllowableValue; import org.apache.nifi.extension.manifest.ControllerServiceDefinition; import org.apache.nifi.extension.manifest.ExpressionLanguageScope; @@ -212,8 +211,8 @@ public class StandardRuntimeManifestService implements RuntimeManifestService { property.setRequired(propertyDescription.isRequired()); property.setSensitive(propertyDescription.isSensitive()); - // TODO: Handle Allowable Values property.setControllerServiceDefinition(getManifestControllerServiceDefinition(propertyDescription.getControllerServiceDefinition())); + property.setAllowableValues(getAllowableValues(propertyDescription)); return property; } @@ -228,23 +227,18 @@ public class StandardRuntimeManifestService implements RuntimeManifestService { return definition; } - private static List<AllowableValue> getManifestAllowableValues(final PropertyDescriptor propertyDescriptor) { - if (propertyDescriptor.getAllowableValues() == null) { - return List.of(); - } - - final List<AllowableValue> allowableValues = new ArrayList<>(); - for (final org.apache.nifi.components.AllowableValue allowableValue : propertyDescriptor.getAllowableValues()) { - final AllowableValue manifestAllowableValue = new AllowableValue(); - manifestAllowableValue.setDescription(allowableValue.getDescription()); - manifestAllowableValue.setValue(allowableValue.getValue()); - manifestAllowableValue.setDisplayName(allowableValue.getDisplayName()); - allowableValues.add(manifestAllowableValue); - } - return allowableValues; + private static List<AllowableValue> getAllowableValues(final PropertyDescription propertyDescription) { + return Optional.ofNullable(propertyDescription.getAllowableValues()).orElse(List.of()) + .stream() + .map(value -> { + AllowableValue allowableValue = new AllowableValue(); + allowableValue.setValue(value); + allowableValue.setDisplayName(value); + return allowableValue; + }) + .toList(); } - private static List<UseCase> getUseCases(final PythonProcessorDetails pythonProcessorDetails) { final List<UseCase> useCases = new ArrayList<>(); for (final UseCaseDetails useCaseDetails : pythonProcessorDetails.getUseCases()) {