This is an automated email from the ASF dual-hosted git repository. martinzink pushed a commit to branch minifi_behave_impr in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 4bb6d050580655474614696f3e49d1c76c3850d2 Author: Martin Zink <[email protected]> AuthorDate: Wed Jul 8 10:30:49 2026 +0200 MINIFICPP-2883 Support host file copies in docker behave other small improvements --- .../minifi_behave/containers/container_linux.py | 23 +++++++++ behave_framework/src/minifi_behave/core/hooks.py | 55 ++++++++++++++++++++++ .../src/minifi_behave/steps/core_steps.py | 23 +++++++++ .../src/minifi_behave/steps/flow_building_steps.py | 19 ++++++++ .../minifi_rs_playground/features/environment.py | 49 +------------------ 5 files changed, 121 insertions(+), 48 deletions(-) diff --git a/behave_framework/src/minifi_behave/containers/container_linux.py b/behave_framework/src/minifi_behave/containers/container_linux.py index 32fc54119..b8ae10279 100644 --- a/behave_framework/src/minifi_behave/containers/container_linux.py +++ b/behave_framework/src/minifi_behave/containers/container_linux.py @@ -23,6 +23,7 @@ import shlex import tarfile import tempfile import uuid +from collections import defaultdict from typing import TYPE_CHECKING from docker.models.networks import Network @@ -58,6 +59,7 @@ class LinuxContainer(ContainerProtocol): self.files: list[File] = [] self.dirs: list[Directory] = [] self.host_files: list[HostFile] = [] + self._pending_host_copies: list[tuple[str, str]] = [] self.volumes = {} self.command: str | None = command self.entrypoint: str | None = entrypoint @@ -71,6 +73,9 @@ class LinuxContainer(ContainerProtocol): def add_host_file(self, host_path: str, container_path: str, mode: str = "ro"): self.host_files.append(HostFile(container_path, host_path, mode)) + def add_host_file_as_copy(self, host_path: str, container_path: str): + self._pending_host_copies.append((host_path, container_path)) + def add_file_to_running_container(self, content: str, path: str): if not self.container: logger.error("Container is not running. Cannot add file.") @@ -114,6 +119,23 @@ class LinuxContainer(ContainerProtocol): self._write_content_to_file(file_path, None, content) self.volumes[temp_path] = {"bind": directory.path, "mode": directory.mode} + def _configure_volumes_of_host_copies(self): + groups: dict[str, list[tuple[str, str]]] = defaultdict(list) + for host_path, container_path in self._pending_host_copies: + groups[os.path.dirname(container_path)].append( + (host_path, os.path.basename(container_path)) + ) + for container_dir, entries in groups.items(): + temp_path = self._temp_dir.name + container_dir + os.makedirs(temp_path, exist_ok=True) + for host_path, filename in entries: + with open(host_path, "rb") as f: + content = f.read() + self._write_content_to_file( + os.path.join(temp_path, filename), None, content + ) + self.volumes[temp_path] = {"bind": container_dir, "mode": "rw"} + def deploy(self, context: MinifiTestContext | None) -> bool: if self.is_deployed(): logger.info(f"Container '{self.container_name}' is already deployed.") @@ -122,6 +144,7 @@ class LinuxContainer(ContainerProtocol): self._temp_dir = tempfile.TemporaryDirectory() self._configure_volumes_of_container_files() self._configure_volumes_of_container_dirs() + self._configure_volumes_of_host_copies() for host_file in self.host_files: self.volumes[host_file.host_path] = { "bind": host_file.container_path, diff --git a/behave_framework/src/minifi_behave/core/hooks.py b/behave_framework/src/minifi_behave/core/hooks.py index 862ab1310..f79c5d1d8 100644 --- a/behave_framework/src/minifi_behave/core/hooks.py +++ b/behave_framework/src/minifi_behave/core/hooks.py @@ -28,6 +28,10 @@ import docker logger = logging.getLogger(__name__) +from behave_framework.src.minifi_behave.containers.docker_image_builder import ( + DockerImageBuilder, +) + def get_minifi_container_image(): if "MINIFI_TAG_PREFIX" in os.environ and "MINIFI_VERSION" in os.environ: @@ -109,3 +113,54 @@ def common_after_scenario(context: MinifiTestContext, scenario: Scenario): container.clean_up() if hasattr(context, "network"): context.network.remove() + + +def add_extension_to_minifi_container( + extension_name: str, possible_paths: list[str], context: MinifiTestContext +): + new_container_name = f"apacheminificpp:{extension_name}" + is_windows = os.name == "nt" + if is_windows: + lib_filename = f"{extension_name}.dll" + container_extension_dir = ( + "C:/Program Files/ApacheNiFiMiNiFi/nifi-minifi-cpp/extensions" + ) + else: + lib_filename = f"lib{extension_name}.so" + container_extension_dir = "/opt/minifi/minifi-current/extensions/" + + host_path = None + for path in possible_paths: + if os.path.exists(os.path.join(path, lib_filename)): + host_path = os.path.join(path, lib_filename) + break + + assert host_path is not None, ( + f"Could not find {lib_filename} in {[p for p in possible_paths]}" + ) + + with open(host_path, "rb") as f: + lib_content = f.read() + + base_img = get_minifi_container_image() + + if is_windows: + dockerfile = f""" +FROM {base_img} +COPY ["{lib_filename}", "{container_extension_dir}/{lib_filename}"] +""" + else: + dockerfile = f""" +FROM {base_img} +COPY --chown=minificpp:minificpp {lib_filename} {container_extension_dir} +RUN chmod 755 {container_extension_dir}{lib_filename} +""" + + builder = DockerImageBuilder( + image_tag=new_container_name, + dockerfile_content=dockerfile, + files_on_context={lib_filename: lib_content}, + ) + + builder.build() + return new_container_name diff --git a/behave_framework/src/minifi_behave/steps/core_steps.py b/behave_framework/src/minifi_behave/steps/core_steps.py index 15e99115f..75c63d5f8 100644 --- a/behave_framework/src/minifi_behave/steps/core_steps.py +++ b/behave_framework/src/minifi_behave/steps/core_steps.py @@ -175,6 +175,29 @@ def bind_host_resource_file_to_container_path(context: MinifiTestContext, filena ) +@given( + 'a host resource file "{filename}" is copied to the "{container_path}" path in the MiNiFi container "{container_name}"' +) +def copy_host_resource_file_to_container_path_for_container( + context: MinifiTestContext, filename: str, container_path: str, container_name: str +): + path = os.path.join(context.resource_dir, filename) + context.get_or_create_minifi_container(container_name).add_host_file_as_copy( + path, container_path + ) + + +@given( + 'a host resource file "{filename}" is copied to the "{container_path}" path in the MiNiFi container' +) +def copy_host_resource_file_to_container_path( + context: MinifiTestContext, filename: str, container_path: str +): + context.execute_steps( + f'given a host resource file "{filename}" is copied to the "{container_path}" path in the MiNiFi container "{DEFAULT_MINIFI_CONTAINER_NAME}"' + ) + + @step("after {duration} have passed") @step("after {duration} has passed") @step("{duration} later") diff --git a/behave_framework/src/minifi_behave/steps/flow_building_steps.py b/behave_framework/src/minifi_behave/steps/flow_building_steps.py index a7f737a55..736490977 100644 --- a/behave_framework/src/minifi_behave/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_behave/steps/flow_building_steps.py @@ -53,6 +53,9 @@ def transient_flow_with_logondestructionprocessor(context: MinifiTestContext): @given( 'a {processor_type} processor with the name "{processor_name}" and the "{property_name}" property set to "{property_value}"' ) +@given( + 'an {processor_type} processor with the name "{processor_name}" and the "{property_name}" property set to "{property_value}"' +) def processor_with_name_and_property( context: MinifiTestContext, processor_type: str, @@ -65,6 +68,9 @@ def processor_with_name_and_property( context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) +@step( + 'an {processor_type} processor with the "{property_name}" property set to "{property_value}"' +) @step('a {processor_type} processor with the "{property_name}" property set to "{property_value}"') def processor_with_property( context: MinifiTestContext, @@ -77,6 +83,9 @@ def processor_with_property( ) +@step( + 'an {processor_type} processor with the "{property_name}" property set to "{property_value}" in the "{minifi_container_name}" flow' +) @step( 'a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the "{minifi_container_name}" flow' ) @@ -92,6 +101,9 @@ def processor_with_property_in_minifi_flow( context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) +@step( + 'an {processor_type} processor with the "{property_name}" property set to "{property_value}" in the NiFi flow' +) @step('a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the NiFi flow') def processor_with_property_in_nifi_flow( context: MinifiTestContext, @@ -104,6 +116,9 @@ def processor_with_property_in_nifi_flow( context.containers["nifi"].flow_definition.add_processor(processor) +@given( + 'an {processor_type} processor with the name "{processor_name}" in the "{minifi_container_name}" flow' +) @given('a {processor_type} processor with the name "{processor_name}" in the "{minifi_container_name}" flow') def processor_with_name_in_minifi_flow( context: MinifiTestContext, @@ -115,6 +130,7 @@ def processor_with_name_in_minifi_flow( context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) +@given('an {processor_type} processor with the name "{processor_name}"') @given('a {processor_type} processor with the name "{processor_name}"') def processor_with_name(context: MinifiTestContext, processor_type: str, processor_name: str): context.execute_steps( @@ -122,6 +138,7 @@ def processor_with_name(context: MinifiTestContext, processor_type: str, process ) +@given('an {processor_type} processor in the "{minifi_container_name}" flow') @given('a {processor_type} processor in the "{minifi_container_name}" flow') def processor_in_minifi_flow(context: MinifiTestContext, processor_type: str, minifi_container_name: str): processor = Processor(processor_type, processor_type) @@ -129,11 +146,13 @@ def processor_in_minifi_flow(context: MinifiTestContext, processor_type: str, mi @given("a {processor_type} processor in the NiFi flow") +@given("an {processor_type} processor in the NiFi flow") def processor_in_nifi_flow(context: MinifiTestContext, processor_type: str): processor = Processor(processor_type, processor_type) context.containers["nifi"].flow_definition.add_processor(processor) +@given("an {processor_type} processor") @given("a {processor_type} processor") def processor_setup(context: MinifiTestContext, processor_type: str): context.execute_steps(f'given a {processor_type} processor in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') diff --git a/minifi_rust/extensions/minifi_rs_playground/features/environment.py b/minifi_rust/extensions/minifi_rs_playground/features/environment.py index 8be40cabc..8e4e41e3b 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/environment.py +++ b/minifi_rust/extensions/minifi_rs_playground/features/environment.py @@ -17,58 +17,11 @@ import os -from minifi_behave.containers.docker_image_builder import DockerImageBuilder from minifi_behave.core.hooks import ( + add_extension_to_minifi_container, common_after_scenario, common_before_scenario, - get_minifi_container_image, ) -from minifi_behave.core.minifi_test_context import MinifiTestContext - - -def add_extension_to_minifi_container(extension_name: str, possible_paths: list[str], context: MinifiTestContext): - new_container_name = f"apacheminificpp:{extension_name}" - is_windows = os.name == "nt" - if is_windows: - lib_filename = f"{extension_name}.dll" - container_extension_dir = "C:/Program Files/ApacheNiFiMiNiFi/nifi-minifi-cpp/extensions" - else: - lib_filename = f"lib{extension_name}.so" - container_extension_dir = "/opt/minifi/minifi-current/extensions/" - - host_path = None - for path in possible_paths: - if os.path.exists(os.path.join(path, lib_filename)): - host_path = os.path.join(path, lib_filename) - break - - assert host_path is not None, f"Could not find {lib_filename} in {[p for p in possible_paths]}" - - with open(host_path, "rb") as f: - lib_content = f.read() - - base_img = get_minifi_container_image() - - if is_windows: - dockerfile = f""" -FROM {base_img} -COPY ["{lib_filename}", "{container_extension_dir}/{lib_filename}"] -""" - else: - dockerfile = f""" -FROM {base_img} -COPY --chown=minificpp:minificpp {lib_filename} {container_extension_dir} -RUN chmod 755 {container_extension_dir}{lib_filename} -""" - - builder = DockerImageBuilder( - image_tag=new_container_name, - dockerfile_content=dockerfile, - files_on_context={lib_filename: lib_content}, - ) - - builder.build() - return new_container_name def before_all(context):
