This is an automated email from the ASF dual-hosted git repository. martinzink pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit d39671da8e3620078d435dfe7a3e73d174abd324 Author: Gabor Gyimesi <[email protected]> AuthorDate: Tue Mar 10 13:12:51 2026 +0100 MINIFICPP-2687 Move Python extension docker tests to modular docker tests This closes #2103 Signed-off-by: Martin Zink <[email protected]> --- .../containers/minifi_container.py | 14 +- .../minifi_test_framework/steps/checking_steps.py | 12 ++ .../src/minifi_test_framework/steps/core_steps.py | 5 + docker/RunBehaveTests.sh | 3 +- docker/test/integration/cluster/ContainerStore.py | 18 --- .../test/integration/cluster/DockerTestCluster.py | 18 --- .../cluster/DockerTestDirectoryBindings.py | 1 - docker/test/integration/cluster/ImageStore.py | 159 -------------------- .../cluster/containers/MinifiContainer.py | 31 ---- .../features/MiNiFi_integration_test_driver.py | 18 --- docker/test/integration/features/environment.py | 5 - .../integration/features/python_script.feature | 30 ---- docker/test/integration/features/steps/steps.py | 25 --- .../minifi/processors/AddPythonAttribute.py | 22 --- .../integration/minifi/processors/ChunkDocument.py | 26 ---- .../minifi/processors/CountingProcessor.py | 24 --- .../minifi/processors/CreateFlowFile.py | 25 --- .../integration/minifi/processors/CreateNothing.py | 25 --- .../EvaluateExpressionLanguageChecker.py | 26 ---- .../ExpressionLanguagePropertyWithValidator.py | 26 ---- .../minifi/processors/FailureWithAttributes.py | 26 ---- .../minifi/processors/FailureWithContent.py | 26 ---- .../minifi/processors/LogDynamicProperties.py | 20 --- .../minifi/processors/LogOnDestructionProcessor.py | 23 --- .../minifi/processors/MoveContentToJson.py | 24 --- .../processors/NifiStyleLogDynamicProperties.py | 25 --- .../integration/minifi/processors/ParseDocument.py | 26 ---- .../processors/ProcessContextInterfaceChecker.py | 26 ---- .../minifi/processors/RelativeImporterProcessor.py | 26 ---- .../minifi/processors/RemoveFlowFile.py | 24 --- .../minifi/processors/RotatingForwarder.py | 26 ---- .../minifi/processors/SetRecordField.py | 26 ---- .../processors/SpecialPropertyTypeChecker.py | 26 ---- .../minifi/processors/TestStateManager.py | 25 --- .../minifi/processors/TransferToOriginal.py | 26 ---- .../integration/resources/python/sleep_forever.py | 23 --- extensions/python/tests/features/environment.py | 124 +++++++++++++++ .../python/tests}/features/python.feature | 167 +++++++++++++-------- .../tests/features/resources}/CreateFlowFile.py | 0 .../tests/features/resources}/CreateNothing.py | 0 .../EvaluateExpressionLanguageChecker.py | 0 .../ExpressionLanguagePropertyWithValidator.py | 0 .../features/resources}/FailureWithAttributes.py | 0 .../features/resources}/FailureWithContent.py | 0 .../features/resources}/LogDynamicProperties.py | 0 .../resources}/NifiStyleLogDynamicProperties.py | 0 .../resources}/ProcessContextInterfaceChecker.py | 0 .../resources}/RelativeImporterProcessor.py | 0 .../tests/features/resources}/RotatingForwarder.py | 0 .../tests/features/resources}/SetRecordField.py | 0 .../resources}/SpecialPropertyTypeChecker.py | 0 .../tests/features/resources}/TestStateManager.py | 0 .../features/resources}/TransferToOriginal.py | 0 .../tests/features/resources}/multiplierutils.py | 0 .../tests/features/resources}/subtractutils.py | 0 extensions/python/tests/features/steps/steps.py | 122 +++++++++++++++ 56 files changed, 386 insertions(+), 938 deletions(-) diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py index 4fc936a9b..c96e5384f 100644 --- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py +++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py @@ -35,6 +35,7 @@ class MinifiContainer(Container): self.properties: dict[str, str] = {} self.log_properties: dict[str, str] = {} self.scenario_id = test_context.scenario_id + self.deploy_timeout_seconds = 20 minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) @@ -85,10 +86,13 @@ class MinifiContainer(Container): finished_str = "MiNiFi started" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), - timeout_seconds=15, + timeout_seconds=self.deploy_timeout_seconds, bail_condition=lambda: self.exited, context=None) + def set_deploy_timeout_seconds(self, timeout_seconds: int): + self.deploy_timeout_seconds = timeout_seconds + def set_property(self, key: str, value: str): self.properties[key] = value @@ -116,9 +120,11 @@ class MinifiContainer(Container): if self.is_fhs: self.properties["nifi.flow.configuration.file"] = "/etc/nifi-minifi-cpp/config.yml" self.properties["nifi.extension.path"] = "/usr/lib64/nifi-minifi-cpp/extensions/*" + self.properties["nifi.python.processor.dir"] = '/var/lib/nifi-minifi-cpp/minifi-python' else: self.properties["nifi.flow.configuration.file"] = "./conf/config.yml" self.properties["nifi.extension.path"] = "../extensions/*" + self.properties["nifi.python.processor.dir"] = '/opt/minifi/minifi-current/minifi-python' self.properties["nifi.administrative.yield.duration"] = "1 sec" self.properties["nifi.bored.yield.duration"] = "100 millis" self.properties["nifi.openssl.fips.support.enable"] = "false" @@ -217,3 +223,9 @@ class MinifiContainer(Container): (code, _) = self.exec_run(["test", "-f", "/tmp/debug.tar.gz"]) return code == 0 + + def add_example_python_processors(self): + run_minifi_cmd = '/opt/minifi/minifi-current/bin/minifi.sh run' if not self.is_fhs else '/usr/bin/minifi' + minifi_python_dir_path = '/opt/minifi/minifi-current/minifi-python' if not self.is_fhs else '/var/lib/nifi-minifi-cpp/minifi-python' + minifi_python_examples = '/opt/minifi/minifi-current/minifi-python-examples' if not self.is_fhs else '/usr/share/doc/nifi-minifi-cpp/pythonprocessor-examples' + self.command = f'sh -c "cp -r {minifi_python_examples} {minifi_python_dir_path}/examples && {run_minifi_cmd}"' diff --git a/behave_framework/src/minifi_test_framework/steps/checking_steps.py b/behave_framework/src/minifi_test_framework/steps/checking_steps.py index 7c546d968..2a0f12f54 100644 --- a/behave_framework/src/minifi_test_framework/steps/checking_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/checking_steps.py @@ -44,6 +44,7 @@ def step_impl(context: MinifiTestContext, container_name: str, content: str, dir @then('a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') +@then("a single file with the content '{content}' is placed in the '{directory}' directory in less than {duration}") def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str): context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') @@ -57,6 +58,7 @@ def step_impl(context: MinifiTestContext, container_name: str, content: str, dir @then('at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') +@then("at least one file with the content '{content}' is placed in the '{directory}' directory in less than {duration}") def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str): context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') @@ -171,6 +173,16 @@ def step_impl(context: MinifiTestContext, directory: str, timeout: str, contents context=context) +@then('files with at least these contents "{contents}" are placed in the "{directory}" directory in less than {timeout}') +def step_impl(context: MinifiTestContext, directory: str, timeout: str, contents: str): + timeout_seconds = humanfriendly.parse_timespan(timeout) + new_contents = contents.replace("\\n", "\n") + contents_arr = new_contents.split(",") + assert wait_for_condition(condition=lambda: all([context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_content(directory, content) for content in contents_arr]), + timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context) + + @then("a file with the JSON content \"{content}\" is placed in the \"{directory}\" directory in less than {duration}") @then("a file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}") def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str): diff --git a/behave_framework/src/minifi_test_framework/steps/core_steps.py b/behave_framework/src/minifi_test_framework/steps/core_steps.py index 5d6f62dd9..ba07d27f5 100644 --- a/behave_framework/src/minifi_test_framework/steps/core_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/core_steps.py @@ -159,3 +159,8 @@ def step_impl(context): @step("{duration} later") def step_impl(context: MinifiTestContext, duration: str): time.sleep(humanfriendly.parse_timespan(duration)) + + +@step("the MiNiFi deployment timeout is set to {timeout_seconds:d} seconds") +def step_impl(context: MinifiTestContext, timeout_seconds: int): + context.get_or_create_default_minifi_container().set_deploy_timeout_seconds(timeout_seconds) diff --git a/docker/RunBehaveTests.sh b/docker/RunBehaveTests.sh index e4f6cda9f..02e31d381 100755 --- a/docker/RunBehaveTests.sh +++ b/docker/RunBehaveTests.sh @@ -209,4 +209,5 @@ exec \ "${docker_dir}/../extensions/lua/tests/features/" \ "${docker_dir}/../extensions/civetweb/tests/features/" \ "${docker_dir}/../extensions/mqtt/tests/features/" \ - "${docker_dir}/../extensions/prometheus/tests/features/" + "${docker_dir}/../extensions/prometheus/tests/features/" \ + "${docker_dir}/../extensions/python/tests/features/" diff --git a/docker/test/integration/cluster/ContainerStore.py b/docker/test/integration/cluster/ContainerStore.py index acf1dd82a..af588fa02 100644 --- a/docker/test/integration/cluster/ContainerStore.py +++ b/docker/test/integration/cluster/ContainerStore.py @@ -191,21 +191,6 @@ class ContainerStore: def enable_sql_in_minifi(self): self.minifi_options.enable_sql = True - def use_nifi_python_processors_with_system_python_packages_installed_in_minifi(self): - self.minifi_options.use_nifi_python_processors_with_system_python_packages_installed = True - - def use_nifi_python_processors_with_virtualenv_in_minifi(self): - self.minifi_options.use_nifi_python_processors_with_virtualenv = True - - def use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi(self): - self.minifi_options.use_nifi_python_processors_with_virtualenv_packages_installed = True - - def remove_python_requirements_txt_in_minifi(self): - self.minifi_options.remove_python_requirements_txt = True - - def use_nifi_python_processors_without_dependencies_in_minifi(self): - self.minifi_options.use_nifi_python_processors_without_dependencies = True - def set_yaml_in_minifi(self): self.minifi_options.config_format = "yaml" @@ -215,9 +200,6 @@ class ContainerStore: def enable_log_metrics_publisher_in_minifi(self): self.minifi_options.enable_log_metrics_publisher = True - def enable_example_minifi_python_processors(self): - self.minifi_options.enable_example_minifi_python_processors = True - def enable_openssl_fips_mode_in_minifi(self): self.minifi_options.enable_openssl_fips_mode = True diff --git a/docker/test/integration/cluster/DockerTestCluster.py b/docker/test/integration/cluster/DockerTestCluster.py index 9f95afacf..8c8904cdd 100644 --- a/docker/test/integration/cluster/DockerTestCluster.py +++ b/docker/test/integration/cluster/DockerTestCluster.py @@ -86,21 +86,6 @@ class DockerTestCluster: def enable_sql_in_minifi(self): self.container_store.enable_sql_in_minifi() - def use_nifi_python_processors_with_system_python_packages_installed_in_minifi(self): - self.container_store.use_nifi_python_processors_with_system_python_packages_installed_in_minifi() - - def use_nifi_python_processors_with_virtualenv_in_minifi(self): - self.container_store.use_nifi_python_processors_with_virtualenv_in_minifi() - - def use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi(self): - self.container_store.use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi() - - def remove_python_requirements_txt_in_minifi(self): - self.container_store.remove_python_requirements_txt_in_minifi() - - def use_nifi_python_processors_without_dependencies_in_minifi(self): - self.container_store.use_nifi_python_processors_without_dependencies_in_minifi() - def set_yaml_in_minifi(self): self.container_store.set_yaml_in_minifi() @@ -110,9 +95,6 @@ class DockerTestCluster: def enable_log_metrics_publisher_in_minifi(self): self.container_store.enable_log_metrics_publisher_in_minifi() - def enable_example_minifi_python_processors(self): - self.container_store.enable_example_minifi_python_processors() - def llama_model_is_downloaded_in_minifi(self): self.container_store.llama_model_is_downloaded_in_minifi() diff --git a/docker/test/integration/cluster/DockerTestDirectoryBindings.py b/docker/test/integration/cluster/DockerTestDirectoryBindings.py index 926378511..80e04c0ec 100644 --- a/docker/test/integration/cluster/DockerTestDirectoryBindings.py +++ b/docker/test/integration/cluster/DockerTestDirectoryBindings.py @@ -54,7 +54,6 @@ class DockerTestDirectoryBindings: # Add resources test_dir = os.environ['TEST_DIRECTORY'] # Based on DockerVerify.sh - shutil.copytree(test_dir + "/resources/python", self.data_directories[self.feature_id]["resources_dir"] + "/python") shutil.copytree(test_dir + "/resources/minifi", self.data_directories[self.feature_id]["minifi_config_dir"], dirs_exist_ok=True) def get_data_directories(self): diff --git a/docker/test/integration/cluster/ImageStore.py b/docker/test/integration/cluster/ImageStore.py index a729cd030..2a21f9d29 100644 --- a/docker/test/integration/cluster/ImageStore.py +++ b/docker/test/integration/cluster/ImageStore.py @@ -23,12 +23,6 @@ from textwrap import dedent import os -class PythonWithDependenciesOptions: - REQUIREMENTS_FILE = 0 - SYSTEM_INSTALLED_PACKAGES = 1 - INLINE_DEFINED_PACKAGES = 2 - - class ImageStore: def __init__(self): self.client = docker.from_env() @@ -47,16 +41,6 @@ class ImageStore: if container_engine == "minifi-cpp-sql": image = self.__build_minifi_cpp_sql_image() - elif container_engine == "minifi-cpp-with-example-python-processors": - image = self.__build_minifi_cpp_image_with_example_minifi_python_processors() - elif container_engine == "minifi-cpp-nifi-python": - image = self.__build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(PythonWithDependenciesOptions.REQUIREMENTS_FILE) - elif container_engine == "minifi-cpp-nifi-python-system-python-packages": - image = self.__build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES) - elif container_engine == "minifi-cpp-nifi-with-inline-python-dependencies": - image = self.__build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES) - elif container_engine == "minifi-cpp-nifi-with-python-without-dependencies": - image = self.__build_minifi_cpp_image_with_nifi_python_processors() elif container_engine == "minifi-cpp-with-llamacpp-model": image = self.__build_minifi_cpp_image_with_llamacpp_model() elif container_engine == "http-proxy": @@ -117,123 +101,6 @@ class ImageStore: return self.__build_image(dockerfile) - def __build_minifi_cpp_image_with_example_minifi_python_processors(self): - dockerfile = dedent("""\ - FROM {base_image} - RUN cp -r {minifi_python_examples_dir} {minifi_python_dir}/examples - """.format(base_image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION, - minifi_python_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_dir_path, - minifi_python_examples_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_examples)) - - return self.__build_image(dockerfile) - - def __build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(self, python_option): - parse_document_url = "https://raw.githubusercontent.com/apache/nifi-python-extensions/refs/heads/main/src/extensions/chunking/ParseDocument.py"; - chunk_document_url = "https://raw.githubusercontent.com/apache/nifi-python-extensions/refs/heads/main/src/extensions/chunking/ChunkDocument.py"; - pip3_install_command = "" - requirements_install_command = "" - additional_cmd = "" - # The following sed command is used to remove the existing dependencies from the ParseDocument and ChunkDocument processors - # /class ProcessorDetails:/,/^$/: Do the following between 'class ProcessorDetails:' and the first empty line (so we don't modify other PropertyDescriptor blocks below) - # /^\s*dependencies\s*=/,/\]\s*$/: Do the following between 'dependencies =' at the start of a line, and ']' at the end of a line - # d: Delete line - parse_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir}/nifi_python_processors/ParseDocument.py && \\'.format(minifi_python_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_dir_path) - chunk_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir}/nifi_python_processors/ChunkDocument.py && \\'.format(minifi_python_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_dir_path) - if python_option == PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES: - if not MinifiContainer.MINIFI_TAG_PREFIX or "bookworm" in MinifiContainer.MINIFI_TAG_PREFIX or "noble" in MinifiContainer.MINIFI_TAG_PREFIX or "trixie" in MinifiContainer.MINIFI_TAG_PREFIX: - additional_cmd = "RUN pip3 install --break-system-packages 'langchain<=0.17.0'" - else: - additional_cmd = "RUN pip3 install 'langchain<=0.17.0'" - elif python_option == PythonWithDependenciesOptions.REQUIREMENTS_FILE: - requirements_install_command = "echo 'langchain<=0.17.0' > {minifi_python_dir}/nifi_python_processors/requirements.txt && \\".format(minifi_python_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_dir_path) - elif python_option == PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES: - parse_document_sed_cmd = parse_document_sed_cmd[:-2] + ' sed -i "s/langchain==[0-9.]\\+/langchain<=0.17.0/" {minifi_python_dir}/nifi_python_processors/ParseDocument.py && \\'.format(minifi_python_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_dir_path) - chunk_document_sed_cmd = 'sed -i "s/\\[\\\'langchain\\\'\\]/\\[\\\'langchain<=0.17.0\\\'\\]/" {minifi_python_dir}/nifi_python_processors/ChunkDocument.py && \\'.format(minifi_python_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_dir_path) - if not MinifiContainer.MINIFI_TAG_PREFIX: - pip3_install_command = "RUN apk --update --no-cache add py3-pip" - dockerfile = dedent("""\ - FROM {base_image} - USER root - {pip3_install_command} - {additional_cmd} - USER minificpp - RUN wget {parse_document_url} --directory-prefix={minifi_python_dir}/nifi_python_processors && \\ - wget {chunk_document_url} --directory-prefix={minifi_python_dir}/nifi_python_processors && \\ - echo 'langchain<=0.17.0' > {minifi_python_dir}/nifi_python_processors/requirements.txt && \\ - {requirements_install_command} - {parse_document_sed_cmd} - {chunk_document_sed_cmd} - python3 -m venv {minifi_python_venv_parent}/venv && \\ - python3 -m venv {minifi_python_venv_parent}/venv-with-langchain && \\ - . {minifi_python_venv_parent}/venv-with-langchain/bin/activate && python3 -m pip install --no-cache-dir "langchain<=0.17.0" && \\ - deactivate - """.format(base_image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION, - pip3_install_command=pip3_install_command, - parse_document_url=parse_document_url, - chunk_document_url=chunk_document_url, - additional_cmd=additional_cmd, - requirements_install_command=requirements_install_command, - parse_document_sed_cmd=parse_document_sed_cmd, - chunk_document_sed_cmd=chunk_document_sed_cmd, - minifi_python_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_dir_path, - minifi_python_venv_parent=MinifiContainer.MINIFI_LOCATIONS.minifi_python_venv_parent)) - return self.__build_image(dockerfile) - - def __build_minifi_cpp_image_with_nifi_python_processors(self): - pip3_install_command = "" - if not MinifiContainer.MINIFI_TAG_PREFIX: - pip3_install_command = "RUN apk --update --no-cache add py3-pip" - dockerfile = dedent("""\ - FROM {base_image} - USER root - {pip3_install_command} - USER minificpp - COPY RotatingForwarder.py {minifi_python_dir}/nifi_python_processors/RotatingForwarder.py - COPY SpecialPropertyTypeChecker.py {minifi_python_dir}/nifi_python_processors/SpecialPropertyTypeChecker.py - COPY ProcessContextInterfaceChecker.py {minifi_python_dir}/nifi_python_processors/ProcessContextInterfaceChecker.py - COPY CreateFlowFile.py {minifi_python_dir}/nifi_python_processors/CreateFlowFile.py - COPY FailureWithAttributes.py {minifi_python_dir}/nifi_python_processors/FailureWithAttributes.py - COPY subtractutils.py {minifi_python_dir}/nifi_python_processors/compute/subtractutils.py - COPY RelativeImporterProcessor.py {minifi_python_dir}/nifi_python_processors/compute/processors/RelativeImporterProcessor.py - COPY multiplierutils.py {minifi_python_dir}/nifi_python_processors/compute/processors/multiplierutils.py - COPY CreateNothing.py {minifi_python_dir}/nifi_python_processors/CreateNothing.py - COPY FailureWithContent.py {minifi_python_dir}/nifi_python_processors/FailureWithContent.py - COPY TransferToOriginal.py {minifi_python_dir}/nifi_python_processors/TransferToOriginal.py - COPY SetRecordField.py {minifi_python_dir}/nifi_python_processors/SetRecordField.py - COPY TestStateManager.py {minifi_python_dir}/nifi_python_processors/TestStateManager.py - COPY NifiStyleLogDynamicProperties.py {minifi_python_dir}/nifi_python_processors/NifiStyleLogDynamicProperties.py - COPY LogDynamicProperties.py {minifi_python_dir}/LogDynamicProperties.py - COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py - COPY EvaluateExpressionLanguageChecker.py {minifi_python_dir}/nifi_python_processors/EvaluateExpressionLanguageChecker.py - RUN python3 -m venv {minifi_python_venv_parent}/venv - """.format(base_image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION, - pip3_install_command=pip3_install_command, - minifi_python_dir=MinifiContainer.MINIFI_LOCATIONS.minifi_python_dir_path, - minifi_python_venv_parent=MinifiContainer.MINIFI_LOCATIONS.minifi_python_venv_parent)) - - def build_full_python_resource_path(resource): - return os.path.join(self.test_dir, "resources", "python", resource) - - return self.__build_image(dockerfile, [ - build_full_python_resource_path("RotatingForwarder.py"), - build_full_python_resource_path("SpecialPropertyTypeChecker.py"), - build_full_python_resource_path("ProcessContextInterfaceChecker.py"), - build_full_python_resource_path("CreateFlowFile.py"), - build_full_python_resource_path("FailureWithAttributes.py"), - build_full_python_resource_path("RelativeImporterProcessor.py"), - build_full_python_resource_path("subtractutils.py"), - build_full_python_resource_path("multiplierutils.py"), - build_full_python_resource_path("CreateNothing.py"), - build_full_python_resource_path("FailureWithContent.py"), - build_full_python_resource_path("TransferToOriginal.py"), - build_full_python_resource_path("SetRecordField.py"), - build_full_python_resource_path("TestStateManager.py"), - build_full_python_resource_path("NifiStyleLogDynamicProperties.py"), - build_full_python_resource_path("LogDynamicProperties.py"), - build_full_python_resource_path("ExpressionLanguagePropertyWithValidator.py"), - build_full_python_resource_path("EvaluateExpressionLanguageChecker.py") - ]) - def __build_minifi_cpp_image_with_llamacpp_model(self): dockerfile = dedent("""\ FROM {base_image} @@ -322,29 +189,3 @@ class ImageStore: except Exception as e: logging.info(e) raise - - def get_minifi_image_python_version(self): - result = self.client.containers.run( - image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION, - command=['python3', '-c', 'import platform; print(platform.python_version())'], - remove=True - ) - - python_ver_str = result.decode('utf-8') - logging.info('MiNiFi python version: %s', python_ver_str) - return tuple(map(int, python_ver_str.split('.'))) - - def is_conda_available_in_minifi_image(self): - container = self.client.containers.create( - image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION, - command=['conda', '--version'], - ) - try: - container.start() - result = container.logs() - container.remove(force=True) - except docker.errors.APIError: - container.remove(force=True) - return False - - return result.decode('utf-8').startswith('conda ') diff --git a/docker/test/integration/cluster/containers/MinifiContainer.py b/docker/test/integration/cluster/containers/MinifiContainer.py index e3675ac81..5031a1704 100644 --- a/docker/test/integration/cluster/containers/MinifiContainer.py +++ b/docker/test/integration/cluster/containers/MinifiContainer.py @@ -29,15 +29,9 @@ class MinifiOptions: def __init__(self): self.enable_provenance = False self.enable_sql = False - self.use_nifi_python_processors_with_system_python_packages_installed = False - self.use_nifi_python_processors_with_virtualenv = False - self.use_nifi_python_processors_with_virtualenv_packages_installed = False - self.remove_python_requirements_txt = False - self.use_nifi_python_processors_without_dependencies = False self.config_format = "json" self.set_ssl_context_properties = False self.enable_log_metrics_publisher = False - self.enable_example_minifi_python_processors = False if "true" in os.environ['MINIFI_FIPS']: self.enable_openssl_fips_mode = True else: @@ -56,9 +50,6 @@ class MinifiLocations: self.properties_path = '/etc/nifi-minifi-cpp/minifi.properties' self.log_properties_path = '/etc/nifi-minifi-cpp/minifi-log.properties' self.uid_properties_path = '/etc/nifi-minifi-cpp/minifi-uid.properties' - self.minifi_python_dir_path = '/var/lib/nifi-minifi-cpp/minifi-python' - self.minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' - self.minifi_python_examples = '/usr/share/doc/nifi-minifi-cpp/pythonprocessor-examples' self.models_path = '/var/lib/nifi-minifi-cpp/models' self.minifi_home = '/var/lib/nifi-minifi-cpp' else: @@ -67,9 +58,6 @@ class MinifiLocations: self.properties_path = '/opt/minifi/minifi-current/conf/minifi.properties' self.log_properties_path = '/opt/minifi/minifi-current/conf/minifi-log.properties' self.uid_properties_path = '/opt/minifi/minifi-current/conf/minifi-uid.properties' - self.minifi_python_dir_path = '/opt/minifi/minifi-current/minifi-python' - self.minifi_python_examples = '/opt/minifi/minifi-current/minifi-python-examples' - self.minifi_python_venv_parent = '/opt/minifi/minifi-current' self.models_path = '/opt/minifi/minifi-current/models' self.minifi_home = '/opt/minifi/minifi-current' @@ -126,7 +114,6 @@ class MinifiContainer(FlowContainer): f.write("nifi.provenance.repository.directory.default={minifi_home}/provenance_repository\n".format(minifi_home=MinifiContainer.MINIFI_LOCATIONS.minifi_home)) f.write("nifi.flowfile.repository.directory.default={minifi_home}/flowfile_repository\n".format(minifi_home=MinifiContainer.MINIFI_LOCATIONS.minifi_home)) f.write("nifi.database.content.repository.directory.default={minifi_home}/content_repository\n".format(minifi_home=MinifiContainer.MINIFI_LOCATIONS.minifi_home)) - f.write("nifi.python.processor.dir={minifi_home}/minifi-python\n".format(minifi_home=MinifiContainer.MINIFI_LOCATIONS.minifi_home)) if self.options.set_ssl_context_properties: f.write("nifi.remote.input.secure=true\n") @@ -146,14 +133,6 @@ class MinifiContainer(FlowContainer): if metrics_publisher_classes: f.write("nifi.metrics.publisher.class=" + ",".join(metrics_publisher_classes) + "\n") - if self.options.use_nifi_python_processors_with_virtualenv or self.options.remove_python_requirements_txt or self.options.use_nifi_python_processors_without_dependencies: - f.write("nifi.python.virtualenv.directory={minifi_python_venv_parent}/venv\n".format(minifi_python_venv_parent=MinifiContainer.MINIFI_LOCATIONS.minifi_python_venv_parent)) - elif self.options.use_nifi_python_processors_with_virtualenv_packages_installed: - f.write("nifi.python.virtualenv.directory={minifi_python_venv_parent}/venv-with-langchain\n".format(minifi_python_venv_parent=MinifiContainer.MINIFI_LOCATIONS.minifi_python_venv_parent)) - - if self.options.use_nifi_python_processors_with_virtualenv or self.options.remove_python_requirements_txt: - f.write("nifi.python.install.packages.automatically=true\n") - if self.options.enable_openssl_fips_mode: f.write("nifi.openssl.fips.support.enable=true\n") else: @@ -175,16 +154,6 @@ class MinifiContainer(FlowContainer): if self.options.enable_sql: image = self.image_store.get_image('minifi-cpp-sql') - elif self.options.enable_example_minifi_python_processors: - image = self.image_store.get_image('minifi-cpp-with-example-python-processors') - elif self.options.use_nifi_python_processors_with_system_python_packages_installed: - image = self.image_store.get_image('minifi-cpp-nifi-python-system-python-packages') - elif self.options.use_nifi_python_processors_with_virtualenv or self.options.use_nifi_python_processors_with_virtualenv_packages_installed: - image = self.image_store.get_image('minifi-cpp-nifi-python') - elif self.options.remove_python_requirements_txt: - image = self.image_store.get_image('minifi-cpp-nifi-with-inline-python-dependencies') - elif self.options.use_nifi_python_processors_without_dependencies: - image = self.image_store.get_image('minifi-cpp-nifi-with-python-without-dependencies') elif self.options.download_llama_model: image = self.image_store.get_image('minifi-cpp-with-llamacpp-model') else: diff --git a/docker/test/integration/features/MiNiFi_integration_test_driver.py b/docker/test/integration/features/MiNiFi_integration_test_driver.py index e6bb48a60..5599dc7f9 100644 --- a/docker/test/integration/features/MiNiFi_integration_test_driver.py +++ b/docker/test/integration/features/MiNiFi_integration_test_driver.py @@ -333,21 +333,6 @@ class MiNiFi_integration_test: def enable_sql_in_minifi(self): self.cluster.enable_sql_in_minifi() - def use_nifi_python_processors_with_system_python_packages_installed_in_minifi(self): - self.cluster.use_nifi_python_processors_with_system_python_packages_installed_in_minifi() - - def use_nifi_python_processors_with_virtualenv_in_minifi(self): - self.cluster.use_nifi_python_processors_with_virtualenv_in_minifi() - - def use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi(self): - self.cluster.use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi() - - def remove_python_requirements_txt_in_minifi(self): - self.cluster.remove_python_requirements_txt_in_minifi() - - def use_nifi_python_processors_without_dependencies_in_minifi(self): - self.cluster.use_nifi_python_processors_without_dependencies_in_minifi() - def set_yaml_in_minifi(self): self.cluster.set_yaml_in_minifi() @@ -360,9 +345,6 @@ class MiNiFi_integration_test: def enable_log_metrics_publisher_in_minifi(self): self.cluster.enable_log_metrics_publisher_in_minifi() - def enable_example_minifi_python_processors(self): - self.cluster.enable_example_minifi_python_processors() - def enable_openssl_fips_mode_in_minifi(self): self.cluster.enable_openssl_fips_mode_in_minifi() diff --git a/docker/test/integration/features/environment.py b/docker/test/integration/features/environment.py index 0b485940b..91ad1cd88 100644 --- a/docker/test/integration/features/environment.py +++ b/docker/test/integration/features/environment.py @@ -48,11 +48,6 @@ def before_scenario(context, scenario): logging.info("Integration test setup at {time:%H:%M:%S.%f}".format(time=datetime.datetime.now())) context.test = MiNiFi_integration_test(context=context, feature_id=context.feature_id) - if "USE_NIFI_PYTHON_PROCESSORS_WITH_LANGCHAIN" in scenario.effective_tags: - if not context.image_store.is_conda_available_in_minifi_image() and context.image_store.get_minifi_image_python_version() < (3, 8, 1): - scenario.skip("NiFi Python processor tests use langchain library which requires Python 3.8.1 or later.") - return - for step in scenario.steps: inject_feature_id(context, step) diff --git a/docker/test/integration/features/python_script.feature b/docker/test/integration/features/python_script.feature deleted file mode 100644 index dfdf2b310..000000000 --- a/docker/test/integration/features/python_script.feature +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -@ENABLE_PYTHON_SCRIPTING -Feature: MiNiFi can execute Python scripts - Background: - Given the content of "/tmp/output" is monitored - - Scenario: ExecuteScript should only allow one Python script running at a time - Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And the scheduling period of the GenerateFlowFile processor is set to "500 ms" - And a ExecuteScript processor with the "Script File" property set to "/tmp/resources/python/sleep_forever.py" - And the "Script Engine" property of the ExecuteScript processor is set to "python" - And the max concurrent tasks attribute of the ExecuteScript processor is set to 3 - And the "success" relationship of the GenerateFlowFile processor is connected to the ExecuteScript - - When all instances start up - Then the Minifi logs contain the following message: "Sleeping forever" 1 times after 5 seconds diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py index 94e0117bb..544a52bf9 100644 --- a/docker/test/integration/features/steps/steps.py +++ b/docker/test/integration/features/steps/steps.py @@ -810,31 +810,6 @@ def step_impl(context, remote_process_group: str): setUpSslContextServiceForRPG(context, remote_process_group) -# Python -@given("python with langchain is installed on the MiNiFi agent {install_mode}") -def step_impl(context, install_mode): - if install_mode == "with required python packages": - context.test.use_nifi_python_processors_with_system_python_packages_installed_in_minifi() - elif install_mode == "with a pre-created virtualenv": - context.test.use_nifi_python_processors_with_virtualenv_in_minifi() - elif install_mode == "with a pre-created virtualenv containing the required python packages": - context.test.use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi() - elif install_mode == "using inline defined Python dependencies to install packages": - context.test.remove_python_requirements_txt_in_minifi() - else: - raise Exception("Unknown python install mode.") - - -@given("python processors without dependencies are present on the MiNiFi agent") -def step_impl(context): - context.test.use_nifi_python_processors_without_dependencies_in_minifi() - - -@given("the example MiNiFi python processors are present") -def step_impl(context): - context.test.enable_example_minifi_python_processors() - - @given("a non-sensitive parameter in the flow config called '{parameter_name}' with the value '{parameter_value}' in the parameter context '{parameter_context_name}'") def step_impl(context, parameter_context_name, parameter_name, parameter_value): container = context.test.acquire_container(context=context, name='minifi-cpp-flow', engine='minifi-cpp') diff --git a/docker/test/integration/minifi/processors/AddPythonAttribute.py b/docker/test/integration/minifi/processors/AddPythonAttribute.py deleted file mode 100644 index 1d651c8de..000000000 --- a/docker/test/integration/minifi/processors/AddPythonAttribute.py +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from ..core.Processor import Processor - - -class AddPythonAttribute(Processor): - def __init__(self, context): - super(AddPythonAttribute, self).__init__(context=context, clazz='AddPythonAttribute', class_prefix='org.apache.nifi.minifi.processors.examples.') diff --git a/docker/test/integration/minifi/processors/ChunkDocument.py b/docker/test/integration/minifi/processors/ChunkDocument.py deleted file mode 100644 index 7dbe170d1..000000000 --- a/docker/test/integration/minifi/processors/ChunkDocument.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class ChunkDocument(Processor): - def __init__(self, context): - super(ChunkDocument, self).__init__( - context=context, - clazz='ChunkDocument', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=['success', 'original', 'failure']) diff --git a/docker/test/integration/minifi/processors/CountingProcessor.py b/docker/test/integration/minifi/processors/CountingProcessor.py deleted file mode 100644 index 2b45636d6..000000000 --- a/docker/test/integration/minifi/processors/CountingProcessor.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from ..core.Processor import Processor - - -class CountingProcessor(Processor): - def __init__(self, context): - super(CountingProcessor, self).__init__(context=context, - clazz='CountingProcessor', - class_prefix='org.apache.nifi.minifi.processors.examples.') diff --git a/docker/test/integration/minifi/processors/CreateFlowFile.py b/docker/test/integration/minifi/processors/CreateFlowFile.py deleted file mode 100644 index 050532981..000000000 --- a/docker/test/integration/minifi/processors/CreateFlowFile.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class CreateFlowFile(Processor): - def __init__(self, context, schedule={'scheduling period': '30 sec'}): - super(CreateFlowFile, self).__init__( - context=context, - clazz='CreateFlowFile', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - schedule=schedule, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/CreateNothing.py b/docker/test/integration/minifi/processors/CreateNothing.py deleted file mode 100644 index 6c1d41e27..000000000 --- a/docker/test/integration/minifi/processors/CreateNothing.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class CreateNothing(Processor): - def __init__(self, context, schedule={'scheduling period': '1 sec'}): - super(CreateNothing, self).__init__( - context=context, - clazz='CreateNothing', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - schedule=schedule, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/EvaluateExpressionLanguageChecker.py b/docker/test/integration/minifi/processors/EvaluateExpressionLanguageChecker.py deleted file mode 100644 index 11f95053c..000000000 --- a/docker/test/integration/minifi/processors/EvaluateExpressionLanguageChecker.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class EvaluateExpressionLanguageChecker(Processor): - def __init__(self, context): - super(EvaluateExpressionLanguageChecker, self).__init__( - context=context, - clazz='EvaluateExpressionLanguageChecker', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py b/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py deleted file mode 100644 index 42bdede34..000000000 --- a/docker/test/integration/minifi/processors/ExpressionLanguagePropertyWithValidator.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class ExpressionLanguagePropertyWithValidator(Processor): - def __init__(self, context): - super(ExpressionLanguagePropertyWithValidator, self).__init__( - context=context, - clazz='ExpressionLanguagePropertyWithValidator', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/FailureWithAttributes.py b/docker/test/integration/minifi/processors/FailureWithAttributes.py deleted file mode 100644 index 296948fde..000000000 --- a/docker/test/integration/minifi/processors/FailureWithAttributes.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class FailureWithAttributes(Processor): - def __init__(self, context): - super(FailureWithAttributes, self).__init__( - context=context, - clazz='FailureWithAttributes', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/FailureWithContent.py b/docker/test/integration/minifi/processors/FailureWithContent.py deleted file mode 100644 index 88384a792..000000000 --- a/docker/test/integration/minifi/processors/FailureWithContent.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class FailureWithContent(Processor): - def __init__(self, context): - super(FailureWithContent, self).__init__( - context=context, - clazz='FailureWithContent', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/LogDynamicProperties.py b/docker/test/integration/minifi/processors/LogDynamicProperties.py deleted file mode 100644 index 359b774af..000000000 --- a/docker/test/integration/minifi/processors/LogDynamicProperties.py +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class LogDynamicProperties(Processor): - def __init__(self, context, schedule={'scheduling period': '1 sec'}): - super(LogDynamicProperties, self).__init__(context=context, clazz='LogDynamicProperties', class_prefix='org.apache.nifi.minifi.processors.', schedule=schedule) diff --git a/docker/test/integration/minifi/processors/LogOnDestructionProcessor.py b/docker/test/integration/minifi/processors/LogOnDestructionProcessor.py deleted file mode 100644 index b05225af0..000000000 --- a/docker/test/integration/minifi/processors/LogOnDestructionProcessor.py +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class LogOnDestructionProcessor(Processor): - def __init__(self, context, schedule={'scheduling period': '2 sec'}): - super(LogOnDestructionProcessor, self).__init__( - context=context, - clazz='LogOnDestructionProcessor', - schedule=schedule) diff --git a/docker/test/integration/minifi/processors/MoveContentToJson.py b/docker/test/integration/minifi/processors/MoveContentToJson.py deleted file mode 100644 index bff6bfb61..000000000 --- a/docker/test/integration/minifi/processors/MoveContentToJson.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from ..core.Processor import Processor - - -class MoveContentToJson(Processor): - def __init__(self, context): - super(MoveContentToJson, self).__init__(context=context, - clazz='MoveContentToJson', - class_prefix='org.apache.nifi.minifi.processors.examples.') diff --git a/docker/test/integration/minifi/processors/NifiStyleLogDynamicProperties.py b/docker/test/integration/minifi/processors/NifiStyleLogDynamicProperties.py deleted file mode 100644 index 8b5fd7780..000000000 --- a/docker/test/integration/minifi/processors/NifiStyleLogDynamicProperties.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class NifiStyleLogDynamicProperties(Processor): - def __init__(self, context, schedule={'scheduling period': '1 sec'}): - super(NifiStyleLogDynamicProperties, self).__init__( - context=context, - clazz='NifiStyleLogDynamicProperties', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - schedule=schedule, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/ParseDocument.py b/docker/test/integration/minifi/processors/ParseDocument.py deleted file mode 100644 index a20167849..000000000 --- a/docker/test/integration/minifi/processors/ParseDocument.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class ParseDocument(Processor): - def __init__(self, context): - super(ParseDocument, self).__init__( - context=context, - clazz='ParseDocument', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=['success', 'original', 'failure']) diff --git a/docker/test/integration/minifi/processors/ProcessContextInterfaceChecker.py b/docker/test/integration/minifi/processors/ProcessContextInterfaceChecker.py deleted file mode 100644 index 37de61f32..000000000 --- a/docker/test/integration/minifi/processors/ProcessContextInterfaceChecker.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class ProcessContextInterfaceChecker(Processor): - def __init__(self, context): - super(ProcessContextInterfaceChecker, self).__init__( - context=context, - clazz='ProcessContextInterfaceChecker', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/RelativeImporterProcessor.py b/docker/test/integration/minifi/processors/RelativeImporterProcessor.py deleted file mode 100644 index b02b3df6a..000000000 --- a/docker/test/integration/minifi/processors/RelativeImporterProcessor.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class RelativeImporterProcessor(Processor): - def __init__(self, context): - super(RelativeImporterProcessor, self).__init__( - context=context, - clazz='RelativeImporterProcessor', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.compute.processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/RemoveFlowFile.py b/docker/test/integration/minifi/processors/RemoveFlowFile.py deleted file mode 100644 index 7625106ff..000000000 --- a/docker/test/integration/minifi/processors/RemoveFlowFile.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from ..core.Processor import Processor - - -class RemoveFlowFile(Processor): - def __init__(self, context): - super(RemoveFlowFile, self).__init__(context=context, - clazz='RemoveFlowFile', - class_prefix='org.apache.nifi.minifi.processors.examples.') diff --git a/docker/test/integration/minifi/processors/RotatingForwarder.py b/docker/test/integration/minifi/processors/RotatingForwarder.py deleted file mode 100644 index 4571538e6..000000000 --- a/docker/test/integration/minifi/processors/RotatingForwarder.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class RotatingForwarder(Processor): - def __init__(self, context): - super(RotatingForwarder, self).__init__( - context=context, - clazz='RotatingForwarder', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/SetRecordField.py b/docker/test/integration/minifi/processors/SetRecordField.py deleted file mode 100644 index 71875b2a4..000000000 --- a/docker/test/integration/minifi/processors/SetRecordField.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class SetRecordField(Processor): - def __init__(self, context): - super(SetRecordField, self).__init__( - context=context, - clazz='SetRecordField', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/SpecialPropertyTypeChecker.py b/docker/test/integration/minifi/processors/SpecialPropertyTypeChecker.py deleted file mode 100644 index 7125b6997..000000000 --- a/docker/test/integration/minifi/processors/SpecialPropertyTypeChecker.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class SpecialPropertyTypeChecker(Processor): - def __init__(self, context): - super(SpecialPropertyTypeChecker, self).__init__( - context=context, - clazz='SpecialPropertyTypeChecker', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/TestStateManager.py b/docker/test/integration/minifi/processors/TestStateManager.py deleted file mode 100644 index 629ea51f3..000000000 --- a/docker/test/integration/minifi/processors/TestStateManager.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class TestStateManager(Processor): - def __init__(self, context, schedule={'scheduling period': '1 sec'}): - super(TestStateManager, self).__init__( - context=context, - clazz='TestStateManager', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - schedule=schedule, - auto_terminate=[]) diff --git a/docker/test/integration/minifi/processors/TransferToOriginal.py b/docker/test/integration/minifi/processors/TransferToOriginal.py deleted file mode 100644 index 6155d4032..000000000 --- a/docker/test/integration/minifi/processors/TransferToOriginal.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from ..core.Processor import Processor - - -class TransferToOriginal(Processor): - def __init__(self, context): - super(TransferToOriginal, self).__init__( - context=context, - clazz='TransferToOriginal', - class_prefix='org.apache.nifi.minifi.processors.nifi_python_processors.', - properties={}, - schedule={'scheduling strategy': 'EVENT_DRIVEN'}, - auto_terminate=[]) diff --git a/docker/test/integration/resources/python/sleep_forever.py b/docker/test/integration/resources/python/sleep_forever.py deleted file mode 100644 index 047f38d10..000000000 --- a/docker/test/integration/resources/python/sleep_forever.py +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import time - - -def onTrigger(context, session): - log.info("Sleeping forever") - while True: - time.sleep(1) diff --git a/extensions/python/tests/features/environment.py b/extensions/python/tests/features/environment.py new file mode 100644 index 000000000..d22adcf27 --- /dev/null +++ b/extensions/python/tests/features/environment.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import docker +import os +from pathlib import Path +from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder +from minifi_test_framework.core.hooks import common_before_scenario +from minifi_test_framework.core.hooks import common_after_scenario + + +def get_minifi_container_image(): + if 'MINIFI_TAG_PREFIX' in os.environ and 'MINIFI_VERSION' in os.environ: + minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] + minifi_version = os.environ['MINIFI_VERSION'] + return 'apacheminificpp:' + minifi_tag_prefix + minifi_version + return "apacheminificpp:behave" + + +def before_all(context): + context.minifi_container_image = get_minifi_container_image() + client: docker.DockerClient = docker.from_env() + is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) + pip3_install_command = "" + minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] if 'MINIFI_TAG_PREFIX' in os.environ else "" + if not minifi_tag_prefix: + pip3_install_command = "RUN apk --update --no-cache add py3-pip" + minifi_python_dir_path = '/var/lib/nifi-minifi-cpp/minifi-python' if is_fhs else '/opt/minifi/minifi-current/minifi-python' + minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' + dockerfile = """ +FROM {base_image} +USER root +{pip3_install_command} +USER minificpp +COPY RotatingForwarder.py {minifi_python_dir}/nifi_python_processors/RotatingForwarder.py +COPY SpecialPropertyTypeChecker.py {minifi_python_dir}/nifi_python_processors/SpecialPropertyTypeChecker.py +COPY ProcessContextInterfaceChecker.py {minifi_python_dir}/nifi_python_processors/ProcessContextInterfaceChecker.py +COPY CreateFlowFile.py {minifi_python_dir}/nifi_python_processors/CreateFlowFile.py +COPY FailureWithAttributes.py {minifi_python_dir}/nifi_python_processors/FailureWithAttributes.py +COPY subtractutils.py {minifi_python_dir}/nifi_python_processors/compute/subtractutils.py +COPY RelativeImporterProcessor.py {minifi_python_dir}/nifi_python_processors/compute/processors/RelativeImporterProcessor.py +COPY multiplierutils.py {minifi_python_dir}/nifi_python_processors/compute/processors/multiplierutils.py +COPY CreateNothing.py {minifi_python_dir}/nifi_python_processors/CreateNothing.py +COPY FailureWithContent.py {minifi_python_dir}/nifi_python_processors/FailureWithContent.py +COPY TransferToOriginal.py {minifi_python_dir}/nifi_python_processors/TransferToOriginal.py +COPY SetRecordField.py {minifi_python_dir}/nifi_python_processors/SetRecordField.py +COPY TestStateManager.py {minifi_python_dir}/nifi_python_processors/TestStateManager.py +COPY NifiStyleLogDynamicProperties.py {minifi_python_dir}/nifi_python_processors/NifiStyleLogDynamicProperties.py +COPY LogDynamicProperties.py {minifi_python_dir}/LogDynamicProperties.py +COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py +COPY EvaluateExpressionLanguageChecker.py {minifi_python_dir}/nifi_python_processors/EvaluateExpressionLanguageChecker.py +RUN python3 -m venv {minifi_python_venv_parent}/venv + """.format(base_image=context.minifi_container_image, + pip3_install_command=pip3_install_command, + minifi_python_dir=minifi_python_dir_path, + minifi_python_venv_parent=minifi_python_venv_parent) + + build_context_path = str(Path(__file__).resolve().parent / "resources") + files_on_context = {} + for filename in os.listdir(build_context_path): + file_path = os.path.join(build_context_path, filename) + with open(file_path, "rb") as f: + files_on_context[filename] = f.read() + + builder = DockerImageBuilder( + image_tag="apacheminificpp-python:latest", + dockerfile_content=dockerfile, + files_on_context=files_on_context + ) + builder.build() + + +def is_conda_available_in_minifi_image(context): + client: docker.DockerClient = docker.from_env() + container = client.containers.create( + image=context.minifi_container_image, + command=['conda', '--version'], + ) + try: + container.start() + result = container.logs() + container.remove(force=True) + except docker.errors.APIError: + container.remove(force=True) + return False + + return result.decode('utf-8').startswith('conda ') + + +def get_minifi_image_python_version(context): + client: docker.DockerClient = docker.from_env() + result = client.containers.run( + image=context.minifi_container_image, + command=['python3', '-c', 'import platform; print(platform.python_version())'], + remove=True + ) + + python_ver_str = result.decode('utf-8') + return tuple(map(int, python_ver_str.split('.'))) + + +def before_scenario(context, scenario): + if "USE_NIFI_PYTHON_PROCESSORS_WITH_LANGCHAIN" in scenario.effective_tags: + if not is_conda_available_in_minifi_image(context) and get_minifi_image_python_version(context) < (3, 8, 1): + scenario.skip("NiFi Python processor tests use langchain library which requires Python 3.8.1 or later.") + return + + common_before_scenario(context, scenario) + context.minifi_container_image = "apacheminificpp-python:latest" + + +def after_scenario(context, scenario): + common_after_scenario(context, scenario) diff --git a/docker/test/integration/features/python.feature b/extensions/python/tests/features/python.feature similarity index 66% rename from docker/test/integration/features/python.feature rename to extensions/python/tests/features/python.feature index 0dad156d8..906782d90 100644 --- a/docker/test/integration/features/python.feature +++ b/extensions/python/tests/features/python.feature @@ -15,16 +15,16 @@ @ENABLE_PYTHON_SCRIPTING Feature: MiNiFi can use python processors in its flows - Background: - Given the content of "/tmp/output" is monitored - Scenario: A MiNiFi instance can update attributes through native python processor Given the example MiNiFi python processors are present And a GenerateFlowFile processor with the "File Size" property set to "0B" - And a AddPythonAttribute processor + And a org.apache.nifi.minifi.processors.examples.AddPythonAttribute processor with the name "AddPythonAttribute" + And AddPythonAttribute is EVENT_DRIVEN And a LogAttribute processor + And LogAttribute is EVENT_DRIVEN And the "success" relationship of the GenerateFlowFile processor is connected to the AddPythonAttribute And the "success" relationship of the AddPythonAttribute processor is connected to the LogAttribute + And AddPythonAttribute's success relationship is auto-terminated When all instances start up Then the Minifi logs contain the following message: "key:Python attribute value:attributevalue" in less than 60 seconds @@ -32,9 +32,10 @@ Feature: MiNiFi can use python processors in its flows Scenario: A MiNiFi instance can handle dynamic properties through native python processor Given a GenerateFlowFile processor with the "File Size" property set to "0B" And a LogDynamicProperties processor with the "Static Property" property set to "static value" + And LogDynamicProperties is EVENT_DRIVEN And the "Dynamic Property" property of the LogDynamicProperties processor is set to "dynamic value" And the "success" relationship of the GenerateFlowFile processor is connected to the LogDynamicProperties - And python processors without dependencies are present on the MiNiFi agent + And LogDynamicProperties's success relationship is auto-terminated When all instances start up Then the Minifi logs contain the following message: "Static Property value: static value" in less than 60 seconds @@ -45,49 +46,66 @@ Feature: MiNiFi can use python processors in its flows Scenario: Native python processor can read empty input stream Given the example MiNiFi python processors are present And a GenerateFlowFile processor with the "File Size" property set to "0B" - And a MoveContentToJson processor + And the scheduling period of the GenerateFlowFile processor is set to "60 sec" + And a org.apache.nifi.minifi.processors.examples.MoveContentToJson processor with the name "MoveContentToJson" + And MoveContentToJson is EVENT_DRIVEN And a PutFile processor with the "Directory" property set to "/tmp/output" + And PutFile is EVENT_DRIVEN And the "success" relationship of the GenerateFlowFile processor is connected to the MoveContentToJson And the "success" relationship of the MoveContentToJson processor is connected to the PutFile + And PutFile's success relationship is auto-terminated When all instances start up - Then a flowfile with the content '{"content": ""}' is placed in the monitored directory in less than 60 seconds + Then a single file with the content '{"content": ""}' is placed in the '/tmp/output' directory in less than 60 seconds Scenario: FlowFile can be removed from session - Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And a RemoveFlowFile processor + Given the example MiNiFi python processors are present + And a GenerateFlowFile processor with the "File Size" property set to "0B" + And a org.apache.nifi.minifi.processors.examples.RemoveFlowFile processor with the name "RemoveFlowFile" + And RemoveFlowFile is EVENT_DRIVEN + And the "success" relationship of the GenerateFlowFile processor is connected to the RemoveFlowFile + And RemoveFlowFile's success relationship is auto-terminated When all instances start up Then the Minifi logs contain the following message: "Removing flow file with UUID" in less than 30 seconds Scenario: Native python processors can be stateful Given the example MiNiFi python processors are present - And a CountingProcessor processor + And a org.apache.nifi.minifi.processors.examples.CountingProcessor processor with the name "CountingProcessor" And the scheduling period of the CountingProcessor processor is set to "100 ms" And a PutFile processor with the "Directory" property set to "/tmp/output" + And PutFile is EVENT_DRIVEN And the "success" relationship of the CountingProcessor processor is connected to the PutFile + And PutFile's success relationship is auto-terminated When all instances start up - Then flowfiles with these contents are placed in the monitored directory in less than 5 seconds: "0,1,2,3,4,5" + Then files with at least these contents "1,2,3,4,5" are placed in the "/tmp/output" directory in less than 5 seconds @USE_NIFI_PYTHON_PROCESSORS_WITH_LANGCHAIN Scenario Outline: MiNiFi C++ can use native NiFi python processors Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a file with filename "test_file.log" and content "test_data" is present in "/tmp/input" - And a ParseDocument processor - And a ChunkDocument processor with the "Chunk Size" property set to "5" + And a org.apache.nifi.minifi.processors.nifi_python_processors.ParseDocument processor with the name "ParseDocument" + And ParseDocument is EVENT_DRIVEN + And a org.apache.nifi.minifi.processors.nifi_python_processors.ChunkDocument processor with the name "ChunkDocument" + And ChunkDocument is EVENT_DRIVEN + And the "Chunk Size" property of the ChunkDocument processor is set to "5" And the "Chunk Overlap" property of the ChunkDocument processor is set to "3" And a PutFile processor with the "Directory" property set to "/tmp/output" + And PutFile is EVENT_DRIVEN And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN And python with langchain is installed on the MiNiFi agent <python_install_mode> And the "success" relationship of the GetFile processor is connected to the ParseDocument And the "success" relationship of the ParseDocument processor is connected to the ChunkDocument And the "success" relationship of the ChunkDocument processor is connected to the PutFile And the "success" relationship of the PutFile processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated + And the MiNiFi deployment timeout is set to 300 seconds When all instances start up - Then at least one flowfile's content match the following regex: '{"text": "test_", "metadata": {"filename": "test_file.log", "uuid": "", "chunk_index": 0, "chunk_count": 3}}' in less than 30 seconds + Then at least one file with the content '{"text": "test_", "metadata": {"filename": "test_file.log", "uuid": "", "chunk_index": 0, "chunk_count": 3}}' is placed in the '/tmp/output' directory in less than 30 seconds And the Minifi logs contain the following message: "key:document.count value:3" in less than 10 seconds Examples: Different python installation modes @@ -98,14 +116,17 @@ Feature: MiNiFi can use python processors in its flows | using inline defined Python dependencies to install packages | Scenario: MiNiFi C++ can use native NiFi source python processors - Given a CreateFlowFile processor + Given a org.apache.nifi.minifi.processors.nifi_python_processors.CreateFlowFile processor with the name "CreateFlowFile" + And the scheduling period of the CreateFlowFile processor is set to "10 seconds" And a PutFile processor with the "Directory" property set to "/tmp/output" + And PutFile is EVENT_DRIVEN And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN And the "space" relationship of the CreateFlowFile processor is connected to the PutFile And the "success" relationship of the PutFile processor is connected to the LogAttribute - And python processors without dependencies are present on the MiNiFi agent + And LogAttribute's success relationship is auto-terminated When the MiNiFi instance starts up - Then a flowfile with the content "Hello World!" is placed in the monitored directory in less than 10 seconds + Then a single file with the content 'Hello World!' is placed in the '/tmp/output' directory in less than 10 seconds And the Minifi logs contain the following message: "key:filename value:" in less than 60 seconds And the Minifi logs contain the following message: "key:type value:space" in less than 60 seconds @@ -115,9 +136,9 @@ Feature: MiNiFi can use python processors in its flows And a file with filename "test_file2.log" and content "test_data_two" is present in "/tmp/input" And a file with filename "test_file3.log" and content "test_data_three" is present in "/tmp/input" And a file with filename "test_file4.log" and content "test_data_four" is present in "/tmp/input" - And a RotatingForwarder processor + And a org.apache.nifi.minifi.processors.nifi_python_processors.RotatingForwarder processor with the name "RotatingForwarder" + And RotatingForwarder is EVENT_DRIVEN And a PutFile processor with the "Directory" property set to "/tmp/output" - And python processors without dependencies are present on the MiNiFi agent And the "success" relationship of the GetFile processor is connected to the RotatingForwarder And the "first" relationship of the RotatingForwarder processor is connected to the PutFile @@ -127,46 +148,55 @@ Feature: MiNiFi can use python processors in its flows When all instances start up - Then flowfiles with these contents are placed in the monitored directory in less than 10 seconds: "test_data_one,test_data_two,test_data_three,test_data_four" + Then files with contents "test_data_one,test_data_two,test_data_three,test_data_four" are placed in the "/tmp/output" directory in less than 10 seconds Scenario: MiNiFi C++ can use special property types including controller services in NiFi native python processors Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And a SpecialPropertyTypeChecker processor + And the scheduling period of the GenerateFlowFile processor is set to "30 sec" + And a org.apache.nifi.minifi.processors.nifi_python_processors.SpecialPropertyTypeChecker processor with the name "SpecialPropertyTypeChecker" + And SpecialPropertyTypeChecker is EVENT_DRIVEN And a PutFile processor with the "Directory" property set to "/tmp/output" - And python processors without dependencies are present on the MiNiFi agent - And a SSL context service is set up for the following processor: "SpecialPropertyTypeChecker" + And PutFile is EVENT_DRIVEN + And an ssl context service is set up for SpecialPropertyTypeChecker And the "success" relationship of the GenerateFlowFile processor is connected to the SpecialPropertyTypeChecker And the "success" relationship of the SpecialPropertyTypeChecker processor is connected to the PutFile + And PutFile's success relationship is auto-terminated When all instances start up - Then one flowfile with the contents "Check successful!" is placed in the monitored directory in less than 30 seconds + Then a single file with the content 'Check successful!' is placed in the '/tmp/output' directory in less than 30 seconds Scenario: NiFi native python processor's ProcessContext interface can be used in MiNiFi C++ Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And a ProcessContextInterfaceChecker processor + And the scheduling period of the GenerateFlowFile processor is set to "30 sec" + And a org.apache.nifi.minifi.processors.nifi_python_processors.ProcessContextInterfaceChecker processor with the name "ProcessContextInterfaceChecker" + And ProcessContextInterfaceChecker is EVENT_DRIVEN And a PutFile processor with the "Directory" property set to "/tmp/output" - And python processors without dependencies are present on the MiNiFi agent + And PutFile is EVENT_DRIVEN And the "success" relationship of the GenerateFlowFile processor is connected to the ProcessContextInterfaceChecker And the "myrelationship" relationship of the ProcessContextInterfaceChecker processor is connected to the PutFile + And PutFile's success relationship is auto-terminated When all instances start up - Then one flowfile with the contents "Check successful!" is placed in the monitored directory in less than 30 seconds + Then a single file with the content 'Check successful!' is placed in the '/tmp/output' directory in less than 30 seconds Scenario: NiFi native python processor can update attributes of a flow file transferred to failure relationship Given a GenerateFlowFile processor with the "File Size" property set to "0B" And a UpdateAttribute processor with the "my.attribute" property set to "my.value" + And UpdateAttribute is EVENT_DRIVEN And the "error.message" property of the UpdateAttribute processor is set to "Old error" - And a FailureWithAttributes processor + And a org.apache.nifi.minifi.processors.nifi_python_processors.FailureWithAttributes processor with the name "FailureWithAttributes" + And FailureWithAttributes is EVENT_DRIVEN And a LogAttribute processor - And python processors without dependencies are present on the MiNiFi agent + And LogAttribute is EVENT_DRIVEN And the "success" relationship of the GenerateFlowFile processor is connected to the UpdateAttribute And the "success" relationship of the UpdateAttribute processor is connected to the FailureWithAttributes And the "failure" relationship of the FailureWithAttributes processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated When all instances start up @@ -175,30 +205,34 @@ Feature: MiNiFi can use python processors in its flows Scenario: NiFi native python processors support relative imports Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And a RelativeImporterProcessor processor + And the scheduling period of the GenerateFlowFile processor is set to "30 sec" + And a org.apache.nifi.minifi.processors.nifi_python_processors.compute.processors.RelativeImporterProcessor processor with the name "RelativeImporterProcessor" + And RelativeImporterProcessor is EVENT_DRIVEN And a PutFile processor with the "Directory" property set to "/tmp/output" - And python processors without dependencies are present on the MiNiFi agent + And PutFile is EVENT_DRIVEN And the "success" relationship of the GenerateFlowFile processor is connected to the RelativeImporterProcessor And the "success" relationship of the RelativeImporterProcessor processor is connected to the PutFile + And PutFile's success relationship is auto-terminated When all instances start up - Then one flowfile with the contents "The final result is 1990" is placed in the monitored directory in less than 30 seconds + Then a single file with the content 'The final result is 1990' is placed in the '/tmp/output' directory in less than 30 seconds Scenario: NiFi native python processor is allowed to be triggered without creating any flow files - Given a CreateNothing processor + Given a org.apache.nifi.minifi.processors.nifi_python_processors.CreateNothing processor with the name "CreateNothing" And a PutFile processor with the "Directory" property set to "/tmp/output" + And PutFile is EVENT_DRIVEN And the "success" relationship of the CreateNothing processor is connected to the PutFile - And python processors without dependencies are present on the MiNiFi agent + And PutFile's success relationship is auto-terminated When the MiNiFi instance starts up - Then no files are placed in the monitored directory in 10 seconds of running time + Then no files are placed in the "/tmp/output" directory in 10 seconds of running time And the Minifi logs do not contain the following message: "Caught Exception during SchedulingAgent::onTrigger of processor CreateNothing" after 1 seconds Scenario: NiFi native python processor cannot specify content of failure result Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And a FailureWithContent processor - And python processors without dependencies are present on the MiNiFi agent + And a org.apache.nifi.minifi.processors.nifi_python_processors.FailureWithContent processor with the name "FailureWithContent" + And FailureWithContent is EVENT_DRIVEN And the "success" relationship of the GenerateFlowFile processor is connected to the FailureWithContent @@ -208,8 +242,8 @@ Feature: MiNiFi can use python processors in its flows Scenario: NiFi native python processor cannot transfer to original relationship Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And a TransferToOriginal processor - And python processors without dependencies are present on the MiNiFi agent + And a org.apache.nifi.minifi.processors.nifi_python_processors.TransferToOriginal processor with the name "TransferToOriginal" + And TransferToOriginal is EVENT_DRIVEN And the "success" relationship of the GenerateFlowFile processor is connected to the TransferToOriginal @@ -221,16 +255,19 @@ Feature: MiNiFi can use python processors in its flows Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a file with the content '{"group": "group1", "name": "John"}\n{"group": "group1", "name": "Jane"}\n{"group": "group2", "name": "Kyle"}\n{"name": "Zoe"}' is present in '/tmp/input' And a file with the content '{"group": "group1", "name": "Steve"}\n{}' is present in '/tmp/input' - And a SetRecordField processor with the "Record Reader" property set to "JsonTreeReader" + And a org.apache.nifi.minifi.processors.nifi_python_processors.SetRecordField processor with the name "SetRecordField" + And SetRecordField is EVENT_DRIVEN + And the "Record Reader" property of the SetRecordField processor is set to "JsonTreeReader" And the "Record Writer" property of the SetRecordField processor is set to "JsonRecordSetWriter" And a JsonTreeReader controller service is set up - And a JsonRecordSetWriter controller service is set up with "Array" output grouping + And a JsonRecordSetWriter controller service is set up and the "Output Grouping" property set to "Array" And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN And the "Log Payload" property of the LogAttribute processor is set to "true" - And python processors without dependencies are present on the MiNiFi agent And the "success" relationship of the GetFile processor is connected to the SetRecordField And the "success" relationship of the SetRecordField processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated When all instances start up @@ -241,62 +278,74 @@ Feature: MiNiFi can use python processors in its flows And the Minifi logs contain the following message: '[{}]' in less than 5 seconds Scenario: MiNiFi C++ can use state manager commands in native NiFi python processors - Given a TestStateManager processor + Given a org.apache.nifi.minifi.processors.nifi_python_processors.TestStateManager processor with the name "TestStateManager" And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN And the "success" relationship of the TestStateManager processor is connected to the LogAttribute - And python processors without dependencies are present on the MiNiFi agent + And LogAttribute's success relationship is auto-terminated When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "key:state_key value:1" in less than 60 seconds - And the Minifi logs contain the following message: "key:state_key value:2" in less than 60 seconds + Then the Minifi logs contain the following message: "key:state_key value:1" in less than 10 seconds + And the Minifi logs contain the following message: "key:state_key value:2" in less than 10 seconds Scenario: MiNiFi C++ can use dynamic properties in native NiFi python processors Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And a NifiStyleLogDynamicProperties processor with the "Static Property" property set to "static value" + And a org.apache.nifi.minifi.processors.nifi_python_processors.NifiStyleLogDynamicProperties processor with the name "NifiStyleLogDynamicProperties" + And NifiStyleLogDynamicProperties is EVENT_DRIVEN + And the "Static Property" property of the NifiStyleLogDynamicProperties processor is set to "static value" And the "Dynamic Property" property of the NifiStyleLogDynamicProperties processor is set to "dynamic value" And the "success" relationship of the GenerateFlowFile processor is connected to the NifiStyleLogDynamicProperties - And python processors without dependencies are present on the MiNiFi agent + And NifiStyleLogDynamicProperties's success relationship is auto-terminated When all instances start up - Then the Minifi logs contain the following message: "Static Property value: static value" in less than 60 seconds - And the Minifi logs contain the following message: "Dynamic Property value: dynamic value" in less than 60 seconds + Then the Minifi logs contain the following message: "Static Property value: static value" in less than 10 seconds + And the Minifi logs contain the following message: "Dynamic Property value: dynamic value" in less than 10 seconds Scenario: MiNiFi C++ allows NiFi python processors to use property validators for expression language enabled properties Given a GenerateFlowFile processor with the "File Size" property set to "0B" - And a ExpressionLanguagePropertyWithValidator processor with the "Integer Property" property set to "42" + And a org.apache.nifi.minifi.processors.nifi_python_processors.ExpressionLanguagePropertyWithValidator processor with the name "ExpressionLanguagePropertyWithValidator" + And ExpressionLanguagePropertyWithValidator is EVENT_DRIVEN + And the "Integer Property" property of the ExpressionLanguagePropertyWithValidator processor is set to "42" And the "success" relationship of the GenerateFlowFile processor is connected to the ExpressionLanguagePropertyWithValidator - And python processors without dependencies are present on the MiNiFi agent + And ExpressionLanguagePropertyWithValidator's success relationship is auto-terminated When all instances start up - Then the Minifi logs contain the following message: "Integer Property value: 42" in less than 60 seconds + Then the Minifi logs contain the following message: "Integer Property value: 42" in less than 10 seconds Scenario: MiNiFi C++ rolls back session if expression language cannot be evaluated as integer in NiFi python processors Given a GenerateFlowFile processor with the "File Size" property set to "0B" And a UpdateAttribute processor with the "my.integer" property set to "invalid" - And a ExpressionLanguagePropertyWithValidator processor with the "Integer Property" property set to "${my.integer}" + And UpdateAttribute is EVENT_DRIVEN + And a org.apache.nifi.minifi.processors.nifi_python_processors.ExpressionLanguagePropertyWithValidator processor with the name "ExpressionLanguagePropertyWithValidator" + And ExpressionLanguagePropertyWithValidator is EVENT_DRIVEN + And the "Integer Property" property of the ExpressionLanguagePropertyWithValidator processor is set to "${my.integer}" And the "success" relationship of the GenerateFlowFile processor is connected to the UpdateAttribute And the "success" relationship of the UpdateAttribute processor is connected to the ExpressionLanguagePropertyWithValidator - And python processors without dependencies are present on the MiNiFi agent + And ExpressionLanguagePropertyWithValidator's success relationship is auto-terminated When all instances start up - Then the Minifi logs contain the following message: "ProcessSession rollback for ExpressionLanguagePropertyWithValidator" in less than 60 seconds + Then the Minifi logs contain the following message: "ProcessSession rollback for ExpressionLanguagePropertyWithValidator" in less than 10 seconds Scenario: MiNiFi C++ can use evaluate expression language expressions correctly using the NiFi python API Given a GenerateFlowFile processor with the "File Size" property set to "0B" + And the scheduling period of the GenerateFlowFile processor is set to "30 seconds" And a UpdateAttribute processor with the "my.attribute" property set to "my.value" - And a EvaluateExpressionLanguageChecker processor + And UpdateAttribute is EVENT_DRIVEN + And a org.apache.nifi.minifi.processors.nifi_python_processors.EvaluateExpressionLanguageChecker processor with the name "EvaluateExpressionLanguageChecker" + And EvaluateExpressionLanguageChecker is EVENT_DRIVEN And the "EL Property" property of the EvaluateExpressionLanguageChecker processor is set to "${my.attribute:toUpper()}" And the "Non EL Property" property of the EvaluateExpressionLanguageChecker processor is set to "non el ${my.attribute}" And the "My Dynamic Property" property of the EvaluateExpressionLanguageChecker processor is set to "Dynamic ${my.attribute}" And a PutFile processor with the "Directory" property set to "/tmp/output" - And python processors without dependencies are present on the MiNiFi agent + And PutFile is EVENT_DRIVEN And the "success" relationship of the GenerateFlowFile processor is connected to the UpdateAttribute And the "success" relationship of the UpdateAttribute processor is connected to the EvaluateExpressionLanguageChecker And the "success" relationship of the EvaluateExpressionLanguageChecker processor is connected to the PutFile + And PutFile's success relationship is auto-terminated When all instances start up - Then one flowfile with the contents "Check successful!" is placed in the monitored directory in less than 30 seconds + Then a single file with the content 'Check successful!' is placed in the '/tmp/output' directory in less than 30 seconds And the Minifi logs contain the following message: "EL Property value: ${my.attribute:toUpper()}" in less than 1 seconds And the Minifi logs contain the following message: "Evaluated EL Property value: MY.VALUE" in less than 1 seconds And the Minifi logs contain the following message: "Non EL Property value: non el ${my.attribute}" in less than 1 seconds diff --git a/docker/test/integration/resources/python/CreateFlowFile.py b/extensions/python/tests/features/resources/CreateFlowFile.py similarity index 100% rename from docker/test/integration/resources/python/CreateFlowFile.py rename to extensions/python/tests/features/resources/CreateFlowFile.py diff --git a/docker/test/integration/resources/python/CreateNothing.py b/extensions/python/tests/features/resources/CreateNothing.py similarity index 100% rename from docker/test/integration/resources/python/CreateNothing.py rename to extensions/python/tests/features/resources/CreateNothing.py diff --git a/docker/test/integration/resources/python/EvaluateExpressionLanguageChecker.py b/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py similarity index 100% rename from docker/test/integration/resources/python/EvaluateExpressionLanguageChecker.py rename to extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py diff --git a/docker/test/integration/resources/python/ExpressionLanguagePropertyWithValidator.py b/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py similarity index 100% rename from docker/test/integration/resources/python/ExpressionLanguagePropertyWithValidator.py rename to extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py diff --git a/docker/test/integration/resources/python/FailureWithAttributes.py b/extensions/python/tests/features/resources/FailureWithAttributes.py similarity index 100% rename from docker/test/integration/resources/python/FailureWithAttributes.py rename to extensions/python/tests/features/resources/FailureWithAttributes.py diff --git a/docker/test/integration/resources/python/FailureWithContent.py b/extensions/python/tests/features/resources/FailureWithContent.py similarity index 100% rename from docker/test/integration/resources/python/FailureWithContent.py rename to extensions/python/tests/features/resources/FailureWithContent.py diff --git a/docker/test/integration/resources/python/LogDynamicProperties.py b/extensions/python/tests/features/resources/LogDynamicProperties.py similarity index 100% rename from docker/test/integration/resources/python/LogDynamicProperties.py rename to extensions/python/tests/features/resources/LogDynamicProperties.py diff --git a/docker/test/integration/resources/python/NifiStyleLogDynamicProperties.py b/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py similarity index 100% rename from docker/test/integration/resources/python/NifiStyleLogDynamicProperties.py rename to extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py diff --git a/docker/test/integration/resources/python/ProcessContextInterfaceChecker.py b/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py similarity index 100% rename from docker/test/integration/resources/python/ProcessContextInterfaceChecker.py rename to extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py diff --git a/docker/test/integration/resources/python/RelativeImporterProcessor.py b/extensions/python/tests/features/resources/RelativeImporterProcessor.py similarity index 100% rename from docker/test/integration/resources/python/RelativeImporterProcessor.py rename to extensions/python/tests/features/resources/RelativeImporterProcessor.py diff --git a/docker/test/integration/resources/python/RotatingForwarder.py b/extensions/python/tests/features/resources/RotatingForwarder.py similarity index 100% rename from docker/test/integration/resources/python/RotatingForwarder.py rename to extensions/python/tests/features/resources/RotatingForwarder.py diff --git a/docker/test/integration/resources/python/SetRecordField.py b/extensions/python/tests/features/resources/SetRecordField.py similarity index 100% rename from docker/test/integration/resources/python/SetRecordField.py rename to extensions/python/tests/features/resources/SetRecordField.py diff --git a/docker/test/integration/resources/python/SpecialPropertyTypeChecker.py b/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py similarity index 100% rename from docker/test/integration/resources/python/SpecialPropertyTypeChecker.py rename to extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py diff --git a/docker/test/integration/resources/python/TestStateManager.py b/extensions/python/tests/features/resources/TestStateManager.py similarity index 100% rename from docker/test/integration/resources/python/TestStateManager.py rename to extensions/python/tests/features/resources/TestStateManager.py diff --git a/docker/test/integration/resources/python/TransferToOriginal.py b/extensions/python/tests/features/resources/TransferToOriginal.py similarity index 100% rename from docker/test/integration/resources/python/TransferToOriginal.py rename to extensions/python/tests/features/resources/TransferToOriginal.py diff --git a/docker/test/integration/resources/python/multiplierutils.py b/extensions/python/tests/features/resources/multiplierutils.py similarity index 100% rename from docker/test/integration/resources/python/multiplierutils.py rename to extensions/python/tests/features/resources/multiplierutils.py diff --git a/docker/test/integration/resources/python/subtractutils.py b/extensions/python/tests/features/resources/subtractutils.py similarity index 100% rename from docker/test/integration/resources/python/subtractutils.py rename to extensions/python/tests/features/resources/subtractutils.py diff --git a/extensions/python/tests/features/steps/steps.py b/extensions/python/tests/features/steps/steps.py new file mode 100644 index 000000000..f65761768 --- /dev/null +++ b/extensions/python/tests/features/steps/steps.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import docker +from behave import given + +from minifi_test_framework.steps import checking_steps # noqa: F401 +from minifi_test_framework.steps import configuration_steps # noqa: F401 +from minifi_test_framework.steps import core_steps # noqa: F401 +from minifi_test_framework.steps import flow_building_steps # noqa: F401 +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder + + +class PythonWithDependenciesOptions: + REQUIREMENTS_FILE = "apacheminificpp-python-requirements-file:latest" + SYSTEM_INSTALLED_PACKAGES = "apacheminificpp-python-system-installed-packages:latest" + INLINE_DEFINED_PACKAGES = "apacheminificpp-python-inline-defined-packages:latest" + + +def build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, python_option): + minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] if 'MINIFI_TAG_PREFIX' in os.environ else "" + client: docker.DockerClient = docker.from_env() + is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) + minifi_python_dir_path = '/var/lib/nifi-minifi-cpp/minifi-python' if is_fhs else '/opt/minifi/minifi-current/minifi-python' + minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' + parse_document_url = "https://raw.githubusercontent.com/apache/nifi-python-extensions/refs/heads/main/src/extensions/chunking/ParseDocument.py"; + chunk_document_url = "https://raw.githubusercontent.com/apache/nifi-python-extensions/refs/heads/main/src/extensions/chunking/ChunkDocument.py"; + pip3_install_command = "" + requirements_install_command = "" + additional_cmd = "" + # The following sed command is used to remove the existing dependencies from the ParseDocument and ChunkDocument processors + # /class ProcessorDetails:/,/^$/: Do the following between 'class ProcessorDetails:' and the first empty line (so we don't modify other PropertyDescriptor blocks below) + # /^\s*dependencies\s*=/,/\]\s*$/: Do the following between 'dependencies =' at the start of a line, and ']' at the end of a line + # d: Delete line + parse_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir}/nifi_python_processors/ParseDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) + chunk_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir}/nifi_python_processors/ChunkDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) + if python_option == PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES: + if not minifi_tag_prefix or "bookworm" in minifi_tag_prefix or "noble" in minifi_tag_prefix or "trixie" in minifi_tag_prefix: + additional_cmd = "RUN pip3 install --break-system-packages 'langchain<=0.17.0'" + else: + additional_cmd = "RUN pip3 install 'langchain<=0.17.0'" + elif python_option == PythonWithDependenciesOptions.REQUIREMENTS_FILE: + requirements_install_command = "echo 'langchain<=0.17.0' > {minifi_python_dir}/nifi_python_processors/requirements.txt && \\".format(minifi_python_dir=minifi_python_dir_path) + elif python_option == PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES: + parse_document_sed_cmd = parse_document_sed_cmd[:-2] + ' sed -i "s/langchain==[0-9.]\\+/langchain<=0.17.0/" {minifi_python_dir}/nifi_python_processors/ParseDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) + chunk_document_sed_cmd = 'sed -i "s/\\[\\\'langchain\\\'\\]/\\[\\\'langchain<=0.17.0\\\'\\]/" {minifi_python_dir}/nifi_python_processors/ChunkDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) + if not minifi_tag_prefix: + pip3_install_command = "RUN apk --update --no-cache add py3-pip" + dockerfile = """\ +FROM {base_image} +USER root +{pip3_install_command} +{additional_cmd} +USER minificpp +RUN wget {parse_document_url} --directory-prefix={minifi_python_dir}/nifi_python_processors && \\ + wget {chunk_document_url} --directory-prefix={minifi_python_dir}/nifi_python_processors && \\ + echo 'langchain<=0.17.0' > {minifi_python_dir}/nifi_python_processors/requirements.txt && \\ + {requirements_install_command} + {parse_document_sed_cmd} + {chunk_document_sed_cmd} + python3 -m venv {minifi_python_venv_parent}/venv && \\ + python3 -m venv {minifi_python_venv_parent}/venv-with-langchain && \\ + . {minifi_python_venv_parent}/venv-with-langchain/bin/activate && python3 -m pip install --no-cache-dir "langchain<=0.17.0" && \\ + deactivate + """.format(base_image=context.minifi_container_image, + pip3_install_command=pip3_install_command, + parse_document_url=parse_document_url, + chunk_document_url=chunk_document_url, + additional_cmd=additional_cmd, + requirements_install_command=requirements_install_command, + parse_document_sed_cmd=parse_document_sed_cmd, + chunk_document_sed_cmd=chunk_document_sed_cmd, + minifi_python_dir=minifi_python_dir_path, + minifi_python_venv_parent=minifi_python_venv_parent) + builder = DockerImageBuilder( + image_tag=python_option, + dockerfile_content=dockerfile, + ) + builder.build() + context.get_or_create_default_minifi_container().image_name = python_option + + +@given("the example MiNiFi python processors are present") +def step_impl(context: MinifiTestContext): + context.get_or_create_default_minifi_container().add_example_python_processors() + + +@given("python with langchain is installed on the MiNiFi agent {install_mode}") +def step_impl(context, install_mode): + client: docker.DockerClient = docker.from_env() + is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) + minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' + if install_mode == "with required python packages": + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES) + context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "false") + elif install_mode == "with a pre-created virtualenv": + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.REQUIREMENTS_FILE) + context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv") + context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "true") + elif install_mode == "with a pre-created virtualenv containing the required python packages": + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.REQUIREMENTS_FILE) + context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv-with-langchain") + context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "false") + elif install_mode == "using inline defined Python dependencies to install packages": + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES) + context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv") + context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "true") + else: + raise Exception("Unknown python install mode.")
