This is an automated email from the ASF dual-hosted git repository. martinzink pushed a commit to branch apache-rusty-2 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 553a632a82f73439c171461b5380c5c38501938f Author: Martin Zink <[email protected]> AuthorDate: Fri Oct 17 20:22:19 2025 +0200 MINIFICPP-2718 Windows based docker tests --- behave_framework/pyproject.toml | 11 +- .../{container.py => container_linux.py} | 39 +- .../containers/container_protocol.py | 47 +++ .../containers/container_windows.py | 392 +++++++++++++++++++++ .../containers/http_proxy_container.py | 4 +- .../containers/minifi_container.py | 231 ------------ .../containers/minifi_controller.py | 80 +++++ .../containers/minifi_linux_container.py | 146 ++++++++ .../containers/minifi_protocol.py | 40 +++ .../containers/minifi_win_container.py | 59 ++++ .../containers/nifi_container.py | 13 +- .../src/minifi_test_framework/core/__init__.py | 0 .../src/minifi_test_framework/core/hooks.py | 12 +- .../core/minifi_test_context.py | 34 +- .../src/minifi_test_framework/core/ssl_utils.py | 166 +++------ .../src/minifi_test_framework/steps/__init__.py | 0 .../steps/configuration_steps.py | 12 +- docker/RunBehaveTests.sh | 21 +- docker/installed/win.Dockerfile | 24 ++ .../containers/kinesis_server_container.py | 4 +- .../features/containers/s3_server_container.py | 4 +- .../features/containers/azure_server_container.py | 4 +- .../containers/couchbase_server_container.py | 17 +- .../features/containers/elastic_base_container.py | 4 +- .../features/containers/elasticsearch_container.py | 13 +- .../features/containers/opensearch_container.py | 9 +- .../containers/fake_gcs_server_container.py | 4 +- .../features/containers/grafana_loki_container.py | 13 +- .../features/containers/reverse_proxy_container.py | 4 +- .../features/containers/kafka_server_container.py | 13 +- .../features/containers/mqtt_broker_container.py | 4 +- .../features/containers/opc_ua_server_container.py | 4 +- .../features/containers/prometheus_container.py | 12 +- .../tests/features/containers/splunk_container.py | 13 +- .../containers/postgress_server_container.py | 4 +- .../features/containers/diag_slave_container.py | 4 +- .../tests/features/containers/syslog_container.py | 4 +- .../features/containers/tcp_client_container.py | 4 +- .../tests/features/core_functionality.feature | 6 +- .../tests/features/environment.py | 5 +- .../tests/features/replace_text.feature | 2 +- .../features/steps/minifi_c2_server_container.py | 17 +- .../features/steps/minifi_controller_steps.py | 28 +- .../tests/features/steps/steps.py | 4 +- 44 files changed, 1025 insertions(+), 506 deletions(-) diff --git a/behave_framework/pyproject.toml b/behave_framework/pyproject.toml index c2cc3e4b3..e96c86dff 100644 --- a/behave_framework/pyproject.toml +++ b/behave_framework/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "minifi-test-framework" -version = "0.1.0" +version = "0.2.0" requires-python = ">= 3.10" description = "A testing framework for MiNiFi extensions." dependencies = [ @@ -8,11 +8,14 @@ dependencies = [ "docker==7.1.0", "PyYAML==6.0.3", "humanfriendly==10.0", - "m2crypto==0.46.2", - "pyopenssl==26.0.0", + "cryptography==46.0.5", "pyjks==20.0.0" ] [tool.setuptools] package-dir = {"" = "src"} -packages = ["minifi_test_framework"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["minifi_test_framework*"] + diff --git a/behave_framework/src/minifi_test_framework/containers/container.py b/behave_framework/src/minifi_test_framework/containers/container_linux.py similarity index 93% rename from behave_framework/src/minifi_test_framework/containers/container.py rename to behave_framework/src/minifi_test_framework/containers/container_linux.py index 6beababde..6d49a104d 100644 --- a/behave_framework/src/minifi_test_framework/containers/container.py +++ b/behave_framework/src/minifi_test_framework/containers/container_linux.py @@ -15,29 +15,31 @@ # limitations under the License. # from __future__ import annotations -from typing import TYPE_CHECKING import json import logging import os import shlex -import tempfile import tarfile +import tempfile import uuid - -import docker from docker.models.networks import Network - +from minifi_test_framework.containers.container_protocol import ContainerProtocol from minifi_test_framework.containers.directory import Directory from minifi_test_framework.containers.file import File from minifi_test_framework.containers.host_file import HostFile +from typing import TYPE_CHECKING + +import docker if TYPE_CHECKING: from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class Container: - def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None): +class LinuxContainer(ContainerProtocol): + def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, + entrypoint: str | None = None): + super().__init__() self.image_name: str = image_name self.container_name: str = container_name self.network: Network = network @@ -121,10 +123,11 @@ class Container: pass try: logging.info(f"Creating and starting container '{self.container_name}'...") - self.container = self.client.containers.run( - image=self.image_name, name=self.container_name, ports=self.ports, - environment=self.environment, volumes=self.volumes, network=self.network.name, - command=self.command, entrypoint=self.entrypoint, user=self.user, detach=True) + self.container = self.client.containers.run(image=self.image_name, name=self.container_name, + ports=self.ports, environment=self.environment, + volumes=self.volumes, network=self.network.name, + command=self.command, entrypoint=self.entrypoint, + user=self.user, detach=True) except Exception as e: logging.error(f"Error starting container: {e}") raise @@ -212,7 +215,8 @@ class Container: exit_code, output = self.exec_run("sh -c {}".format(shlex.quote(command))) if exit_code != 0: - logging.debug("While looking for regex %s in directory %s, grep returned exit code %d, output: %s", regex_str, directory_path, exit_code, output) + logging.debug("While looking for regex %s in directory %s, grep returned exit code %d, output: %s", + regex_str, directory_path, exit_code, output) return exit_code == 0 def path_with_content_exists(self, path: str, content: str) -> bool: @@ -376,7 +380,8 @@ class Container: return os.path.join(temp_dir, os.path.basename(directory_path.strip('/'))) except Exception as e: - logging.error(f"Error extracting files from directory path '{directory_path}' from container '{self.container_name}': {e}") + logging.error( + f"Error extracting files from directory path '{directory_path}' from container '{self.container_name}': {e}") return None def _read_files_from_directory(self, directory_path: str) -> list[str] | None: @@ -479,3 +484,11 @@ class Container: return True return False + + def get_memory_usage(self) -> int | None: + exit_code, output = self.exec_run(["awk", "/VmRSS/ { printf \"%d\\n\", $2 }", "/proc/1/status"]) + if exit_code != 0: + return None + memory_usage_in_bytes = int(output.strip()) * 1024 + logging.info(f"{self.container_name} memory usage: {memory_usage_in_bytes} bytes") + return memory_usage_in_bytes diff --git a/behave_framework/src/minifi_test_framework/containers/container_protocol.py b/behave_framework/src/minifi_test_framework/containers/container_protocol.py new file mode 100644 index 000000000..5eb19f088 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/container_protocol.py @@ -0,0 +1,47 @@ +from typing import Protocol, List + +from minifi_test_framework.containers.directory import Directory +from minifi_test_framework.containers.file import File +from minifi_test_framework.containers.host_file import HostFile + + +class ContainerProtocol(Protocol): + image_name: str + container_name: str + dirs: List[Directory] + files: List[File] + host_files: List[HostFile] + + def deploy(self, context) -> bool: + ... + + def clean_up(self): + ... + + def exec_run(self, command): + ... + + def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: + ... + + def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: + ... + + def path_with_content_exists(self, path: str, content: str) -> bool: + ... + + def get_logs(self) -> str: + ... + + @property + def exited(self) -> bool: + ... + + def get_number_of_files(self, directory_path: str) -> int: + ... + + def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool: + ... + + def log_app_output(self) -> bool: + ... diff --git a/behave_framework/src/minifi_test_framework/containers/container_windows.py b/behave_framework/src/minifi_test_framework/containers/container_windows.py new file mode 100644 index 000000000..c1a9c755a --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/container_windows.py @@ -0,0 +1,392 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations +import logging +import os +import tempfile +import base64 +import tarfile +import io +from typing import Union, Optional, Tuple, List, Dict, TYPE_CHECKING + +import docker +from docker.models.networks import Network +from docker.models.containers import Container + +from minifi_test_framework.containers.container_protocol import ContainerProtocol +from minifi_test_framework.containers.directory import Directory +from minifi_test_framework.containers.file import File +from minifi_test_framework.containers.host_file import HostFile + +if TYPE_CHECKING: + from minifi_test_framework.core.minifi_test_context import MinifiTestContext + + +class WindowsContainer(ContainerProtocol): + def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None): + super().__init__() + self.image_name: str = image_name + self.container_name: str = container_name + self.network: Network = network + + self.client: docker.DockerClient = docker.from_env() + self.container: Optional[Container] = None + self.files: List[File] = [] + self.dirs: List[Directory] = [] + self.host_files: List[HostFile] = [] + self.volumes: Dict = {} + self.command: str | None = command + self.entrypoint: str | None = entrypoint + self._temp_dir: Optional[tempfile.TemporaryDirectory] = None + self.ports: Optional[Dict[str, int]] = None + self.environment: List[str] = [] + + def _normalize_path(self, path: str) -> str: + clean_path = path.strip().replace("/", "\\") + if clean_path.startswith("\\"): + clean_path = clean_path[1:] + + # If it doesn't already have a drive letter, assume C: + if ":" not in clean_path: + return f"C:\\{clean_path}" + return clean_path + + def deploy(self, context: MinifiTestContext | None) -> bool: + if self._temp_dir: + self._temp_dir.cleanup() + self._temp_dir = tempfile.TemporaryDirectory() + + for directory in self.dirs: + rel_path = directory.path.strip("/\\") + temp_subdir = os.path.join(self._temp_dir.name, rel_path) + os.makedirs(temp_subdir, exist_ok=True) + + for file_name, content in directory.files.items(): + file_path = os.path.join(temp_subdir, file_name) + with open(file_path, "w", encoding="utf-8") as temp_file: + logging.info(f"writing content into {temp_file.name}") + temp_file.write(content) + + container_bind_path = self._normalize_path(directory.path) + self.volumes[temp_subdir] = { + "bind": container_bind_path, + "mode": directory.mode + } + + for host_file in self.host_files: + container_bind_path = self._normalize_path(host_file.container_path) + self.volumes[host_file.host_path] = {"bind": container_bind_path, "mode": host_file.mode} + + try: + existing_container = self.client.containers.get(self.container_name) + logging.warning(f"Found existing container '{self.container_name}'. Removing it first.") + existing_container.remove(force=True) + except docker.errors.NotFound: + pass + + try: + logging.info(f"Creating and starting container '{self.container_name}'...") + self.container = self.client.containers.create( + image=self.image_name, + name=self.container_name, + ports=self.ports, + environment=self.environment, + volumes=self.volumes, + network=self.network.name, + command=self.command, + entrypoint=self.entrypoint, + detach=True, + tty=False + ) + + self.container.start() + + for file in self.files: + self._copy_content_to_container(file.content, file.path) + + except Exception as e: + logging.error(f"Error starting container: {e}") + self.clean_up() + raise + return True + + def _copy_content_to_container(self, content: str | bytes, target_path: str): + if not self.container: + return + + win_path = self._normalize_path(target_path) + dir_name = os.path.dirname(win_path) + file_name = os.path.basename(win_path) + + self._run_powershell(f"New-Item -ItemType Directory -Force -Path '{dir_name}'") + + tar_stream = io.BytesIO() + with tarfile.open(fileobj=tar_stream, mode='w') as tar: + if isinstance(content, str): + encoded_data = content.encode('utf-8') + else: + encoded_data = content + tarinfo = tarfile.TarInfo(name=file_name) + tarinfo.size = len(encoded_data) + tar.addfile(tarinfo, io.BytesIO(encoded_data)) + + tar_stream.seek(0) + + self.container.put_archive(path=dir_name, data=tar_stream) + + def clean_up(self): + if self._temp_dir: + try: + self._temp_dir.cleanup() + self._temp_dir = None + except Exception as e: + logging.warning(f"Failed to cleanup temp dir: {e}") + + if self.container: + try: + self.container.remove(force=True) + except docker.errors.NotFound: + pass + except Exception as e: + logging.warning(f"Failed to remove container: {e}") + finally: + self.container = None + + def exec_run(self, command: Union[str, list]) -> Tuple[int | None, str]: + logging.debug(f"Running command: {command}") + if self.container: + (code, output) = self.container.exec_run(command, detach=False) + decoded_output = output.decode("utf-8", errors='replace') + logging.debug(f"Result {code}, output: {decoded_output}") + return code, decoded_output + return None, "Container not running." + + def _run_powershell(self, ps_script: str) -> Tuple[int | None, str]: + if not self.container: + return None, "Container not running" + + encoded_command = base64.b64encode(ps_script.encode('utf_16_le')).decode('utf-8') + + cmd_parts = ["powershell", "-NonInteractive", "-NoProfile", "-EncodedCommand", encoded_command] + + return self.exec_run(cmd_parts) + + def not_empty_dir_exists(self, directory_path: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + ps_script = ( + f"if (Test-Path -Path '{win_path}' -PathType Container) {{ " + f" if (Get-ChildItem -Path '{win_path}') {{ exit 0 }} else {{ exit 1 }} " + f"}} else {{ exit 2 }}" + ) + + exit_code, _ = self._run_powershell(ps_script) + return exit_code == 0 + + def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + escaped_content = expected_content.replace("'", "''") + + ps_script = ( + f"if (-not (Test-Path '{win_path}')) {{ exit 2 }}; " + f"$matches = Get-ChildItem -Path '{win_path}' -File -Depth 0 | " + f"Select-String -Pattern '{escaped_content}' -SimpleMatch -List; " + f"if ($matches) {{ exit 0 }} else {{ exit 1 }}" + ) + + exit_code, _ = self._run_powershell(ps_script) + return exit_code == 0 + + def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + escaped_regex = regex_str.replace("'", "''") + + ps_script = ( + f"if (-not (Test-Path '{win_path}')) {{ exit 2 }}; " + f"$matches = Get-ChildItem -Path '{win_path}' -File -Depth 0 | " + f"Select-String -Pattern '{escaped_regex}' -List; " + f"if ($matches) {{ exit 0 }} else {{ exit 1 }}" + ) + + exit_code, _ = self._run_powershell(ps_script) + return exit_code == 0 + + def path_with_content_exists(self, path: str, content: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(path) + + escaped_content = content.replace("'", "''").replace("\n", "\r\n") + + ps_script = ( + f"try {{ " + f" $found = Select-String -Path '{win_path}' -Pattern '^{escaped_content}$'; " + f" if ($found.Count -eq 1) {{ exit 0 }} else {{ exit 1 }} " + f"}} catch {{ exit 2 }}" + ) + + exit_code, output = self._run_powershell(ps_script) + if exit_code != 0: + logging.debug(f"path_with_content_exists failed for {win_path}. Output: {output}") + + return exit_code == 0 + + def directory_has_single_file_with_content(self, directory_path: str, expected_content: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + escaped_content = expected_content.strip().replace("'", "''").replace("\n", "\r\n") + + ps_script = ( + f"$files = Get-ChildItem -Path '{win_path}' -File -Depth 0; " + f"if ($files.Count -ne 1) {{ exit 1 }}; " + f"$actual_content = Get-Content -Path $files[0].FullName -Raw; " + f"if ($actual_content.Trim() -eq '{escaped_content}') {{ exit 0 }} else {{ exit 2 }};" + ) + + exit_code, output = self._run_powershell(ps_script) + if exit_code != 0: + logging.debug(f"Check for single file failed in {win_path}. Output: {output}") + + return exit_code == 0 + + def get_logs(self) -> str: + logging.debug("Getting logs from container '%s'", self.container_name) + if not self.container: + return "" + logs_as_bytes = self.container.logs() + return logs_as_bytes.decode('utf-8', errors='replace') + + def log_app_output(self) -> bool: + logs = self.get_logs() + logging.info("Logs of container '%s':", self.container_name) + for line in logs.splitlines(): + logging.info(line) + return False + + @property + def exited(self) -> bool: + if not self.container: + return False + try: + self.container.reload() + return self.container.status == 'exited' + except docker.errors.NotFound: + self.container = None + return False + except Exception: + return False + + def get_number_of_files(self, directory_path: str) -> int: + if not self.container: + return -1 + + win_path = self._normalize_path(directory_path) + ps_script = f"(Get-ChildItem -Path '{win_path}' -File -Depth 0).Count" + + exit_code, output = self._run_powershell(ps_script) + + if exit_code != 0: + logging.error(f"Error counting files in '{win_path}': {output}") + return -1 + + try: + return int(output.strip()) + except (ValueError, IndexError): + return -1 + + def verify_file_contents(self, directory_path: str, expected_contents: List[str]) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + + ps_list = f"Get-ChildItem -Path '{win_path}' -File -Depth 0 | Select-Object -ExpandProperty FullName" + exit_code, output = self._run_powershell(ps_list) + + if exit_code != 0: + logging.error(f"Error listing files in '{win_path}': {output}") + return False + + actual_filepaths = [path.strip() for path in output.splitlines() if path.strip()] + + if len(actual_filepaths) != len(expected_contents): + return False + + actual_file_contents = [] + for path in actual_filepaths: + ps_read = ( + f"$c = Get-Content -Path '{path}' -Raw; " + f"if ($c) {{ ($c -replace '\\r\\n', '\\n' -replace '\\r', '\\n').Trim() }}" + ) + + exit_code, content = self._run_powershell(ps_read) + if exit_code != 0: + return False + actual_file_contents.append(content) + + normalized_expected = [ + s.strip().replace("\r\n", "\n").replace("\r", "\n") for s in expected_contents + ] + + return sorted(actual_file_contents) == sorted(normalized_expected) + + def nonempty_dir_exists(self, directory_path: str) -> bool: + if not self.container: + return False + + win_path = self._normalize_path(directory_path) + ps_script = ( + f"if ((Test-Path -LiteralPath '{win_path}' -PathType Container) " + f"-and (Get-ChildItem -LiteralPath '{win_path}' -Force | Select-Object -First 1)) " + f"{{ exit 0 }} else {{ exit 1 }}" + ) + + exit_code, _ = self._run_powershell(ps_script) + + return exit_code == 0 + + def directory_contains_file_with_minimum_size(self, directory_path: str, expected_size: int) -> bool: + if not self.container or not self.nonempty_dir_exists(directory_path): + return False + + win_path = self._normalize_path(directory_path) + ps_script = ( + f"Get-ChildItem -LiteralPath '{win_path}' -File " + f"| Where-Object {{ $_.Length -gt {expected_size} }}" + ) + + exit_code, output = self._run_powershell(ps_script) + + if exit_code != 0: + logging.error(f"Error running command to get file sizes: {output}") + return False + + if output and len(output.strip()) > 0: + return True + + return False diff --git a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py index da0cd4c2b..430e2abff 100644 --- a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py +++ b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py @@ -17,13 +17,13 @@ from textwrap import dedent -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class HttpProxy(Container): +class HttpProxy(LinuxContainer): def __init__(self, test_context: MinifiTestContext): dockerfile = dedent("""\ FROM {base_image} diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py deleted file mode 100644 index 8ce0f9b68..000000000 --- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py +++ /dev/null @@ -1,231 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import os -from pathlib import Path -from OpenSSL import crypto - -from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from minifi_test_framework.containers.file import File -from minifi_test_framework.containers.host_file import HostFile -from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition -from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert -from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from .container import Container - - -class MinifiContainer(Container): - def __init__(self, container_name: str, test_context: MinifiTestContext): - super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network) - self.flow_definition = MinifiFlowDefinition() - self.properties: dict[str, str] = {} - self.log_properties: dict[str, str] = {} - self.scenario_id = test_context.scenario_id - self.deploy_timeout_seconds = 20 - - minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) - self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) - self.files.append(File("/tmp/resources/root_ca.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) - self.files.append(File("/tmp/resources/minifi_client.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_client_cert))) - self.files.append(File("/tmp/resources/minifi_client.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_client_key))) - self.files.append(File("/tmp/resources/minifi_merged_cert.crt", - crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_client_cert) + crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_client_key))) - - clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) - self.files.append(File("/tmp/resources/clientuser.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=clientuser_cert))) - self.files.append(File("/tmp/resources/clientuser.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=clientuser_key))) - - minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) - self.files.append(File("/tmp/resources/minifi_server.crt", - crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_server_cert) + crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_server_key))) - - self.is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(self.client.images.get(test_context.minifi_container_image).history()) - if self.is_fhs: - self.minifi_controller_path = '/usr/bin/minifi-controller' - else: - self.minifi_controller_path = '/opt/minifi/minifi-current/bin/minifi-controller' - - self._fill_default_properties() - self._fill_default_log_properties() - - def deploy(self, context: MinifiTestContext | None) -> bool: - flow_config = self.flow_definition.to_yaml() - logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}") - if self.is_fhs: - self.files.append(File("/etc/nifi-minifi-cpp/config.yml", flow_config)) - self.files.append(File("/etc/nifi-minifi-cpp/minifi.properties", self._get_properties_file_content())) - self.files.append( - File("/etc/nifi-minifi-cpp/minifi-log.properties", self._get_log_properties_file_content())) - else: - self.files.append(File("/opt/minifi/minifi-current/conf/config.yml", flow_config)) - self.files.append( - File("/opt/minifi/minifi-current/conf/minifi.properties", self._get_properties_file_content())) - self.files.append(File("/opt/minifi/minifi-current/conf/minifi-log.properties", - self._get_log_properties_file_content())) - - resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller" - self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml"))) - - if not super().deploy(context): - return False - - finished_str = "MiNiFi started" - return wait_for_condition( - condition=lambda: finished_str in self.get_logs(), - timeout_seconds=self.deploy_timeout_seconds, - bail_condition=lambda: self.exited, - context=context) - - def set_deploy_timeout_seconds(self, timeout_seconds: int): - self.deploy_timeout_seconds = timeout_seconds - - def set_property(self, key: str, value: str): - self.properties[key] = value - - def set_log_property(self, key: str, value: str): - self.log_properties[key] = value - - def enable_openssl_fips_mode(self): - self.properties["nifi.openssl.fips.support.enable"] = "true" - - def enable_log_metrics_publisher(self): - self.properties["nifi.metrics.publisher.LogMetricsPublisher.metrics"] = "RepositoryMetrics" - self.properties["nifi.metrics.publisher.LogMetricsPublisher.logging.interval"] = "1s" - self.properties["nifi.metrics.publisher.class"] = "LogMetricsPublisher" - - def fetch_flow_config_from_flow_url(self): - self.properties["nifi.c2.flow.url"] = f"http://minifi-c2-server-{self.scenario_id}:10090/c2/config?class=minifi-test-class"; - - def set_up_ssl_properties(self): - self.properties["nifi.remote.input.secure"] = "true" - self.properties["nifi.security.client.certificate"] = "/tmp/resources/minifi_client.crt" - self.properties["nifi.security.client.private.key"] = "/tmp/resources/minifi_client.key" - self.properties["nifi.security.client.ca.certificate"] = "/tmp/resources/root_ca.crt" - - def _fill_default_properties(self): - if self.is_fhs: - self.properties["nifi.flow.configuration.file"] = "/etc/nifi-minifi-cpp/config.yml" - self.properties["nifi.extension.path"] = "/usr/lib64/nifi-minifi-cpp/extensions/*" - self.properties["nifi.python.processor.dir"] = '/var/lib/nifi-minifi-cpp/minifi-python' - else: - self.properties["nifi.flow.configuration.file"] = "./conf/config.yml" - self.properties["nifi.extension.path"] = "../extensions/*" - self.properties["nifi.python.processor.dir"] = '/opt/minifi/minifi-current/minifi-python' - self.properties["nifi.administrative.yield.duration"] = "1 sec" - self.properties["nifi.bored.yield.duration"] = "100 millis" - self.properties["nifi.openssl.fips.support.enable"] = "false" - self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository" - - def _fill_default_log_properties(self): - self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v" - - self.log_properties["appender.stderr"] = "stderr" - self.log_properties["logger.root"] = "DEBUG, stderr" - self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr" - - def _get_properties_file_content(self): - lines = (f"{key}={value}" for key, value in self.properties.items()) - return "\n".join(lines) - - def _get_log_properties_file_content(self): - lines = (f"{key}={value}" for key, value in self.log_properties.items()) - return "\n".join(lines) - - def get_memory_usage(self) -> int | None: - exit_code, output = self.exec_run(["awk", "/VmRSS/ { printf \"%d\\n\", $2 }", "/proc/1/status"]) - if exit_code != 0: - return None - memory_usage_in_bytes = int(output.strip()) * 1024 - logging.info(f"MiNiFi memory usage: {memory_usage_in_bytes} bytes") - return memory_usage_in_bytes - - def set_controller_socket_properties(self): - self.properties["controller.socket.enable"] = "true" - self.properties["controller.socket.host"] = "localhost" - self.properties["controller.socket.port"] = "9998" - self.properties["controller.socket.local.any.interface"] = "false" - - def update_flow_config_through_controller(self): - self.exec_run([self.minifi_controller_path, "--updateflow", "/tmp/resources/minifi-controller/config.yml"]) - - def updated_config_is_persisted(self) -> bool: - exit_code, output = self.exec_run(["cat", "/opt/minifi/minifi-current/conf/config.yml" if not self.is_fhs else "/etc/nifi-minifi-cpp/config.yml"]) - if exit_code != 0: - logging.error("Failed to read MiNiFi config file to check if updated config is persisted") - return False - return "2f2a3b47-f5ba-49f6-82b5-bc1c86b96f38" in output - - def stop_component_through_controller(self, component: str): - self.exec_run([self.minifi_controller_path, "--stop", component]) - - def start_component_through_controller(self, component: str): - self.exec_run([self.minifi_controller_path, "--start", component]) - - @retry_check(10, 1) - def is_component_running(self, component: str) -> bool: - (code, output) = self.exec_run([self.minifi_controller_path, "--list", "components"]) - return code == 0 and component + ", running: true" in output - - def get_connections(self): - (_, output) = self.exec_run([self.minifi_controller_path, "--list", "connections"]) - connections = [] - for line in output.split('\n'): - if not line.startswith('[') and not line.startswith('Connection Names'): - connections.append(line) - return connections - - @retry_check(10, 1) - def connection_found_through_controller(self, connection: str) -> bool: - return connection in self.get_connections() - - def get_full_connection_count(self) -> int: - (_, output) = self.exec_run([self.minifi_controller_path, "--getfull"]) - for line in output.split('\n'): - if "are full" in line: - return int(line.split(' ')[0]) - return -1 - - def get_connection_size(self, connection: str): - (_, output) = self.exec_run([self.minifi_controller_path, "--getsize", connection]) - for line in output.split('\n'): - if "Size/Max of " + connection in line: - size_and_max = line.split(connection)[1].split('/') - return (int(size_and_max[0].strip()), int(size_and_max[1].strip())) - return (-1, -1) - - def get_manifest(self) -> str: - (_, output) = self.exec_run([self.minifi_controller_path, "--manifest"]) - manifest = "" - for line in output.split('\n'): - if not line.startswith('['): - manifest += line - return manifest - - def create_debug_bundle(self) -> bool: - (code, _) = self.exec_run([self.minifi_controller_path, "--debug", "/tmp"]) - if code != 0: - logging.error("Minifi controller debug command failed with code: %d", code) - return False - - (code, _) = self.exec_run(["test", "-f", "/tmp/debug.tar.gz"]) - return code == 0 - - def add_example_python_processors(self): - run_minifi_cmd = '/opt/minifi/minifi-current/bin/minifi.sh run' if not self.is_fhs else '/usr/bin/minifi' - minifi_python_dir_path = '/opt/minifi/minifi-current/minifi-python' if not self.is_fhs else '/var/lib/nifi-minifi-cpp/minifi-python' - minifi_python_examples = '/opt/minifi/minifi-current/minifi-python-examples' if not self.is_fhs else '/usr/share/doc/nifi-minifi-cpp/pythonprocessor-examples' - self.command = f'sh -c "cp -r {minifi_python_examples} {minifi_python_dir_path}/examples && {run_minifi_cmd}"' diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_controller.py b/behave_framework/src/minifi_test_framework/containers/minifi_controller.py new file mode 100644 index 000000000..22f51d234 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_controller.py @@ -0,0 +1,80 @@ +import logging +from minifi_test_framework.core.helpers import retry_check + + +class MinifiController: + def __init__(self, minifi_container, config_path, bin_path): + self.minifi_container = minifi_container + self.config_path = config_path + self.bin_path = bin_path + + def set_controller_socket_properties(self): + self.minifi_container.properties["controller.socket.enable"] = "true" + self.minifi_container.properties["controller.socket.host"] = "localhost" + self.minifi_container.properties["controller.socket.port"] = "9998" + self.minifi_container.properties["controller.socket.local.any.interface"] = "false" + + def update_flow_config_through_controller(self): + self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--updateflow", "/tmp/resources/minifi-controller/config.yml"]) + + def updated_config_is_persisted(self) -> bool: + exit_code, output = self.minifi_container.exec_run(["cat", self.config_path]) + if exit_code != 0: + logging.error(f"Failed to read MiNiFi config file to check if updated config is persisted: {exit_code} {output}") + return False + return "2f2a3b47-f5ba-49f6-82b5-bc1c86b96f38" in output + + def stop_component_through_controller(self, component: str): + self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--stop", component]) + + def start_component_through_controller(self, component: str): + self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--start", component]) + + @retry_check(max_tries=10, retry_interval_seconds=1) + def is_component_running(self, component: str) -> bool: + (code, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--list", "components"]) + return code == 0 and component + ", running: true" in output + + def get_connections(self): + (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--list", "connections"]) + connections = [] + for line in output.split('\n'): + if not line.startswith('[') and not line.startswith('Connection Names'): + connections.append(line) + return connections + + @retry_check(max_tries=10, retry_interval_seconds=1) + def connection_found_through_controller(self, connection: str) -> bool: + return connection in self.get_connections() + + def get_full_connection_count(self) -> int: + (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--getfull"]) + for line in output.split('\n'): + if "are full" in line: + return int(line.split(' ')[0]) + return -1 + + def get_connection_size(self, connection: str): + (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--getsize", connection]) + for line in output.split('\n'): + if "Size/Max of " + connection in line: + size_and_max = line.split(connection)[1].split('/') + return (int(size_and_max[0].strip()), int(size_and_max[1].strip())) + return (-1, -1) + + def get_manifest(self) -> str: + (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--manifest"]) + manifest = "" + for line in output.split('\n'): + if not line.startswith('['): + manifest += line + return manifest + + def create_debug_bundle(self) -> bool: + (code, _) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--debug", "/tmp"]) + if code != 0: + logging.error("Minifi controller debug command failed with code: %d", code) + return False + + (code, _) = self.minifi_container.exec_run(["test", "-f", "/tmp/debug.tar.gz"]) + return code == 0 diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py new file mode 100644 index 000000000..62311d9ab --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py @@ -0,0 +1,146 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +from minifi_test_framework.containers.file import File +from minifi_test_framework.containers.host_file import HostFile +from minifi_test_framework.core.helpers import wait_for_condition +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, \ + dump_cert, dump_key +from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition +from pathlib import Path + +from .container_linux import LinuxContainer +from .minifi_controller import MinifiController +from .minifi_protocol import MinifiProtocol + + +class NormalDeployment: + def __init__(self): + self.conf_path = "/opt/minifi/minifi-current/conf" + self.bin_path = "/opt/minifi/minifi-current/bin" + self.extension_pattern = "../extensions/*" + self.minifi_python_path = "/opt/minifi/minifi-current/minifi-python" + self.minifi_python_examples_path = "/opt/minifi/minifi-current/minifi-python-examples" + + +class FHSDeployment: + def __init__(self): + self.conf_path = "/etc/nifi-minifi-cpp/" + self.bin_path = "/usr/bin" + self.extension_pattern = "/usr/lib64/nifi-minifi-cpp/extensions/*" + self.minifi_python_path = "/var/lib/nifi-minifi-cpp/minifi-python" + self.minifi_python_examples_path = "/usr/share/doc/nifi-minifi-cpp/pythonprocessor-examples" + + +class MinifiLinuxContainer(LinuxContainer, MinifiProtocol): + def __init__(self, container_name: str, test_context: MinifiTestContext, + deployment: NormalDeployment | FHSDeployment): + super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", + test_context.network) + self.flow_definition = MinifiFlowDefinition() + self.properties: dict[str, str] = {} + self.log_properties: dict[str, str] = {} + self.scenario_id = test_context.scenario_id + self.deploy_timeout_seconds = 20 + self.deployment_type = deployment + + self.controller = MinifiController(self, f"{self.deployment_type.conf_path}/config.yml", + self.deployment_type.bin_path) + + minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key) + self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert))) + self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key))) + self.files.append( + File("/tmp/resources/minifi_merged_cert.crt", dump_cert(minifi_client_cert) + dump_key(minifi_client_key))) + + clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key) + self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert))) + self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key))) + + minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key) + self.files.append( + File("/tmp/resources/minifi_server.crt", dump_cert(cert=minifi_server_cert) + dump_key(minifi_server_key))) + + self._fill_default_properties() + self._fill_default_log_properties() + + def deploy(self, context: MinifiTestContext | None) -> bool: + flow_config = self.flow_definition.to_yaml() + logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}") + self.files.append(File(f"{self.deployment_type.conf_path}/config.yml", flow_config)) + self.files.append( + File(f"{self.deployment_type.conf_path}/minifi.properties", self._get_properties_file_content())) + self.files.append( + File(f"{self.deployment_type.conf_path}/minifi-log.properties", self._get_log_properties_file_content())) + resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller" + self.host_files.append( + HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml"))) + + if not super().deploy(context): + return False + + finished_str = "MiNiFi started" + return wait_for_condition(condition=lambda: finished_str in self.get_logs(), + timeout_seconds=self.deploy_timeout_seconds, bail_condition=lambda: self.exited, + context=context) + + def set_deploy_timeout_seconds(self, timeout_seconds: int): + self.deploy_timeout_seconds = timeout_seconds + + def set_property(self, key: str, value: str): + self.properties[key] = value + + def set_log_property(self, key: str, value: str): + self.log_properties[key] = value + + def _fill_default_properties(self): + self.properties["nifi.flow.configuration.file"] = f"{self.deployment_type.conf_path}/config.yml" + self.properties["nifi.extension.path"] = self.deployment_type.extension_pattern + self.properties["nifi.administrative.yield.duration"] = "1 sec" + self.properties["nifi.bored.yield.duration"] = "100 millis" + self.properties["nifi.openssl.fips.support.enable"] = "false" + self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository" + self.properties["nifi.python.processor.dir"] = self.deployment_type.minifi_python_path + + def _fill_default_log_properties(self): + self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v" + + self.log_properties["appender.stderr"] = "stderr" + self.log_properties["logger.root"] = "DEBUG, stderr" + self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr" + + def _get_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.properties.items()) + return "\n".join(lines) + + def _get_log_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.log_properties.items()) + return "\n".join(lines) + + def add_example_python_processors(self): + run_minifi_cmd = f'{self.deployment_type.bin_path}/minifi.sh run' + self.command = f'sh -c "cp -r {self.deployment_type.minifi_python_examples_path} {self.deployment_type.minifi_python_path}/examples && {run_minifi_cmd}"' diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py b/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py new file mode 100644 index 000000000..4e6d65868 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py @@ -0,0 +1,40 @@ +from typing import Protocol + + +class MinifiProtocol(Protocol): + def set_property(self, key: str, value: str): + ... + + def set_log_property(self, key: str, value: str): + ... + + def set_deploy_timeout_seconds(self, timeout_seconds: int): + ... + + +def set_controller_socket_properties(minifi: MinifiProtocol): + minifi.set_property("controller.socket.enable", "true") + minifi.set_property("controller.socket.host", "localhost") + minifi.set_property("controller.socket.port", "9998") + minifi.set_property("controller.socket.local.any.interface", "false") + + +def enable_openssl_fips_mode(minifi: MinifiProtocol): + minifi.set_property("nifi.openssl.fips.support.enable", "true") + + +def enable_log_metrics_publisher(minifi: MinifiProtocol): + minifi.set_property("nifi.metrics.publisher.LogMetricsPublisher.metrics", "RepositoryMetrics") + minifi.set_property("nifi.metrics.publisher.LogMetricsPublisher.logging.interval", "1s") + minifi.set_property("nifi.metrics.publisher.class", "LogMetricsPublisher") + + +def fetch_flow_config_from_flow_url(minifi: MinifiProtocol, scenario_id: str): + minifi.set_property("nifi.c2.flow.url", f"http://minifi-c2-server-{scenario_id}:10090/c2/config?class=minifi-test-class";) + + +def set_up_ssl_properties(minifi: MinifiProtocol): + minifi.set_property("nifi.remote.input.secure", "true") + minifi.set_property("nifi.security.client.certificate", "/tmp/resources/minifi_client.crt") + minifi.set_property("nifi.security.client.private.key", "/tmp/resources/minifi_client.key") + minifi.set_property("nifi.security.client.ca.certificate", "/tmp/resources/root_ca.crt") diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py new file mode 100644 index 000000000..2e8952cf3 --- /dev/null +++ b/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py @@ -0,0 +1,59 @@ +from typing import Dict + +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition +from minifi_test_framework.containers.directory import Directory +from .container_windows import WindowsContainer +from .minifi_protocol import MinifiProtocol +import logging + + +class MinifiWindowsContainer(WindowsContainer, MinifiProtocol): + def __init__(self, container_name: str, test_context: MinifiTestContext): + super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network) + self.flow_config_str: str = "" + self.flow_definition = MinifiFlowDefinition() + self.properties: Dict[str, str] = {} + self.log_properties: Dict[str, str] = {} + + self._fill_default_properties() + self._fill_default_log_properties() + + def deploy(self, context: MinifiTestContext | None) -> bool: + logging.info(self.flow_definition.to_yaml()) + conf_dir = Directory("\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\conf") + conf_dir.add_file("config.yml", self.flow_definition.to_yaml()) + conf_dir.add_file("minifi.properties", self._get_properties_file_content()) + conf_dir.add_file("minifi-log.properties", self._get_log_properties_file_content()) + + self.dirs.append(conf_dir) + return super().deploy(context) + + def set_property(self, key: str, value: str): + self.properties[key] = value + + def set_log_property(self, key: str, value: str): + self.log_properties[key] = value + + def _fill_default_properties(self): + self.properties["nifi.flow.configuration.file"] = "./conf/config.yml" + self.properties["nifi.extension.path"] = "../extensions/*" + self.properties["nifi.administrative.yield.duration"] = "1 sec" + self.properties["nifi.bored.yield.duration"] = "100 millis" + self.properties["nifi.openssl.fips.support.enable"] = "false" + self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository" + + def _fill_default_log_properties(self): + self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v" + + self.log_properties["appender.stderr"] = "stderr" + self.log_properties["logger.root"] = "DEBUG, stderr" + self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr" + + def _get_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.properties.items()) + return "\n".join(lines) + + def _get_log_properties_file_content(self): + lines = (f"{key}={value}" for key, value in self.log_properties.items()) + return "\n".join(lines) diff --git a/behave_framework/src/minifi_test_framework/containers/nifi_container.py b/behave_framework/src/minifi_test_framework/containers/nifi_container.py index ae0f9d0f8..3d589a8fe 100644 --- a/behave_framework/src/minifi_test_framework/containers/nifi_container.py +++ b/behave_framework/src/minifi_test_framework/containers/nifi_container.py @@ -18,18 +18,19 @@ import gzip import logging import os from pathlib import Path -from OpenSSL import crypto from minifi_test_framework.containers.file import File -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.minifi.nifi_flow_definition import NifiFlowDefinition from minifi_test_framework.containers.host_file import HostFile from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.core.ssl_utils import dump_cert, dump_key -class NifiContainer(Container): + +class NifiContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext, command: list[str] | None = None, use_ssl: bool = False): self.flow_definition = NifiFlowDefinition() name = f"nifi-{test_context.scenario_id}" @@ -78,9 +79,9 @@ class NifiContainer(Container): self.host_files.append(HostFile("/scripts/convert_cert_to_jks.sh", os.path.join(resource_dir, "convert_cert_to_jks.sh"))) nifi_client_cert, nifi_client_key = make_server_cert(common_name=f"nifi-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) - self.files.append(File("/tmp/resources/root_ca.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) - self.files.append(File("/tmp/resources/nifi_client.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=nifi_client_cert))) - self.files.append(File("/tmp/resources/nifi_client.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=nifi_client_key))) + self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/nifi_client.crt", dump_cert(nifi_client_cert))) + self.files.append(File("/tmp/resources/nifi_client.key", dump_key(nifi_client_key))) def deploy(self, context: MinifiTestContext | None) -> bool: flow_config = self.flow_definition.to_json() diff --git a/behave_framework/src/minifi_test_framework/core/__init__.py b/behave_framework/src/minifi_test_framework/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/behave_framework/src/minifi_test_framework/core/hooks.py b/behave_framework/src/minifi_test_framework/core/hooks.py index 3e4e69a95..ea440cc30 100644 --- a/behave_framework/src/minifi_test_framework/core/hooks.py +++ b/behave_framework/src/minifi_test_framework/core/hooks.py @@ -46,6 +46,10 @@ def inject_scenario_id(context: MinifiTestContext, step): def common_before_scenario(context: Context, scenario: Scenario): + if "SUPPORTS_WINDOWS" not in scenario.effective_tags and os.name == 'nt': + scenario.skip("No windows support") + return + if not hasattr(context, "minifi_container_image"): context.minifi_container_image = get_minifi_container_image() @@ -93,6 +97,8 @@ def common_after_scenario(context: MinifiTestContext, scenario: Scenario): with open(scenario_info_path, "w") as f: f.write(header) - for container in context.containers.values(): - container.clean_up() - context.network.remove() + if hasattr(context, 'containers'): + for container in context.containers.values(): + container.clean_up() + if hasattr(context, 'network'): + context.network.remove() diff --git a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py index 959f2e1fa..171aff263 100644 --- a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py +++ b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py @@ -16,32 +16,46 @@ # from __future__ import annotations -from typing import TYPE_CHECKING + +import os +import docker from behave.runner import Context +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey +from cryptography.x509 import Certificate from docker.models.networks import Network -from minifi_test_framework.containers.container import Container -from OpenSSL import crypto +from minifi_test_framework.containers.container_protocol import ContainerProtocol +from minifi_test_framework.containers.minifi_protocol import MinifiProtocol -if TYPE_CHECKING: - from minifi_test_framework.containers.minifi_container import MinifiContainer DEFAULT_MINIFI_CONTAINER_NAME = "minifi-primary" +class MinifiContainer(ContainerProtocol, MinifiProtocol): + pass + + class MinifiTestContext(Context): - containers: dict[str, Container] + containers: dict[str, ContainerProtocol] scenario_id: str network: Network minifi_container_image: str resource_dir: str | None - root_ca_key: crypto.PKey - root_ca_cert: crypto.X509 + root_ca_key: Certificate + root_ca_cert: RSAPrivateKey def get_or_create_minifi_container(self, container_name: str) -> MinifiContainer: if container_name not in self.containers: - from minifi_test_framework.containers.minifi_container import MinifiContainer - self.containers[container_name] = MinifiContainer(container_name, self) + if os.name == 'nt': + from minifi_test_framework.containers.minifi_win_container import MinifiWindowsContainer + minifi_container = MinifiWindowsContainer(container_name, self) + elif 'MINIFI_INSTALLATION_TYPE=FHS' in str(docker.from_env().images.get(self.minifi_container_image).history()): + from minifi_test_framework.containers.minifi_linux_container import MinifiLinuxContainer, FHSDeployment + minifi_container = MinifiLinuxContainer(container_name, self, FHSDeployment()) + else: + from minifi_test_framework.containers.minifi_linux_container import MinifiLinuxContainer, NormalDeployment + minifi_container = MinifiLinuxContainer(container_name, self, NormalDeployment()) + self.containers[container_name] = minifi_container return self.containers[container_name] def get_or_create_default_minifi_container(self) -> MinifiContainer: diff --git a/behave_framework/src/minifi_test_framework/core/ssl_utils.py b/behave_framework/src/minifi_test_framework/core/ssl_utils.py index d39cf9646..e17e9b004 100644 --- a/behave_framework/src/minifi_test_framework/core/ssl_utils.py +++ b/behave_framework/src/minifi_test_framework/core/ssl_utils.py @@ -13,149 +13,79 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime -import time -import logging -import random - -from M2Crypto import X509, EVP, RSA, ASN1 -from OpenSSL import crypto - - -def gen_cert(): - """ - Generate TLS certificate request for testing - """ - - req, key = gen_req() - pub_key = req.get_pubkey() - subject = req.get_subject() - cert = X509.X509() - # noinspection PyTypeChecker - cert.set_serial_number(1) - cert.set_version(2) - cert.set_subject(subject) - t = int(time.time()) - now = ASN1.ASN1_UTCTIME() - now.set_time(t) - now_plus_year = ASN1.ASN1_UTCTIME() - now_plus_year.set_time(t + 60 * 60 * 24 * 365) - cert.set_not_before(now) - cert.set_not_after(now_plus_year) - issuer = X509.X509_Name() - issuer.C = 'US' - issuer.CN = 'minifi-listen' - cert.set_issuer(issuer) - cert.set_pubkey(pub_key) - cert.sign(key, 'sha256') - - return cert, key - - -def rsa_gen_key_callback(): - pass - - -def gen_req(): - """ - Generate TLS certificate request for testing - """ - - logging.info('Generating test certificate request') - key = EVP.PKey() - req = X509.Request() - rsa = RSA.gen_key(1024, 65537, rsa_gen_key_callback) - key.assign_rsa(rsa) - req.set_pubkey(key) - name = req.get_subject() - name.C = 'US' - name.CN = 'minifi-listen' - req.sign(key, 'sha256') - - return req, key +from cryptography import x509 +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey +from cryptography.x509 import Certificate, ExtendedKeyUsage +from cryptography.x509.oid import NameOID -def make_self_signed_cert(common_name): - ca_key = crypto.PKey() - ca_key.generate_key(crypto.TYPE_RSA, 2048) +def gen_cert() -> tuple[Certificate, RSAPrivateKey]: + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - ca_cert = crypto.X509() - ca_cert.set_version(2) - ca_cert.set_serial_number(random.randint(50000000, 100000000)) + subject = issuer = x509.Name([x509.NameAttribute(NameOID.COUNTRY_NAME, u"US"), x509.NameAttribute(NameOID.COMMON_NAME, u"minifi-listen"), ]) - ca_subj = ca_cert.get_subject() - ca_subj.commonName = common_name + cert = x509.CertificateBuilder().subject_name(subject).issuer_name(issuer).public_key(key.public_key()).serial_number( + x509.random_serial_number()).not_valid_before(datetime.datetime.now(datetime.UTC)).not_valid_after( + datetime.datetime.now(datetime.UTC) + datetime.timedelta(days=365)).sign(key, hashes.SHA256()) - ca_cert.add_extensions([ - crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=ca_cert), - ]) - - ca_cert.add_extensions([ - crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert), - ]) - - ca_cert.add_extensions([ - crypto.X509Extension(b"basicConstraints", False, b"CA:TRUE"), - crypto.X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"), - ]) - - ca_cert.set_issuer(ca_subj) - ca_cert.set_pubkey(ca_key) + return cert, key - ca_cert.gmtime_adj_notBefore(0) - ca_cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) - ca_cert.sign(ca_key, 'sha256') +def make_self_signed_cert(common_name: str) -> tuple[Certificate, RSAPrivateKey]: + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - return ca_cert, ca_key + subject = issuer = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, common_name), ]) + cert = x509.CertificateBuilder().subject_name(subject).issuer_name(issuer).public_key(key.public_key()).serial_number( + x509.random_serial_number()).not_valid_before(datetime.datetime.now(datetime.UTC)).not_valid_after( + datetime.datetime.now(datetime.UTC) + datetime.timedelta(days=3650)).add_extension( + x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False, ).add_extension(x509.BasicConstraints(ca=True, path_length=None), + critical=True, ).sign(key, hashes.SHA256()) -def _make_cert(common_name, ca_cert, ca_key, extended_key_usage=None): - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, 2048) + return cert, key - cert = crypto.X509() - cert.set_version(2) - cert.set_serial_number(random.randint(50000000, 100000000)) - client_subj = cert.get_subject() - client_subj.commonName = common_name +def _make_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey, extended_key_usage: ExtendedKeyUsage | None) -> tuple[ + Certificate, RSAPrivateKey]: + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - cert.add_extensions([ - crypto.X509Extension(b"basicConstraints", False, b"CA:FALSE"), - crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=cert), - ]) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, common_name), ]) - extensions = [crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert), - crypto.X509Extension(b"keyUsage", False, b"digitalSignature")] + builder = x509.CertificateBuilder().subject_name(subject).issuer_name(ca_cert.subject).public_key(key.public_key()).serial_number( + x509.random_serial_number()).not_valid_before(datetime.datetime.now(datetime.UTC)).not_valid_after( + datetime.datetime.now(datetime.UTC) + datetime.timedelta(days=3650)).add_extension(x509.BasicConstraints(ca=False, path_length=None), + critical=True, ).add_extension( + x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False, ).add_extension( + x509.SubjectAlternativeName([x509.DNSName(common_name)]), critical=False, ) if extended_key_usage: - extensions.append(crypto.X509Extension(b"extendedKeyUsage", False, extended_key_usage)) - - cert.add_extensions([ - crypto.X509Extension(b"subjectAltName", False, b"DNS.1:" + common_name.encode()) - ]) + builder = builder.add_extension(x509.ExtendedKeyUsage(extended_key_usage), critical=False) - cert.add_extensions(extensions) + cert = builder.sign(ca_key, hashes.SHA256()) + return cert, key - cert.set_issuer(ca_cert.get_subject()) - cert.set_pubkey(key) - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) +def make_client_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: + return _make_cert(common_name, ca_cert, ca_key, x509.ExtendedKeyUsage([x509.OID_CLIENT_AUTH])) - cert.sign(ca_key, 'sha256') - return cert, key +def make_server_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: + return _make_cert(common_name, ca_cert, ca_key, x509.ExtendedKeyUsage(([x509.OID_SERVER_AUTH]))) -def make_client_cert(common_name, ca_cert, ca_key): - return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"clientAuth") +def make_cert_without_extended_usage(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: + return _make_cert(common_name, ca_cert, ca_key, None) -def make_server_cert(common_name, ca_cert, ca_key): - return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"serverAuth") +def dump_cert(cert: Certificate, encoding_type: serialization.Encoding = serialization.Encoding.PEM) -> bytes: + return cert.public_bytes(encoding_type) -def make_cert_without_extended_usage(common_name, ca_cert, ca_key): - return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=None) +def dump_key(key: RSAPrivateKey, encoding_type: serialization.Encoding = serialization.Encoding.PEM) -> bytes: + return key.private_bytes(encoding=encoding_type, format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption()) diff --git a/behave_framework/src/minifi_test_framework/steps/__init__.py b/behave_framework/src/minifi_test_framework/steps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/behave_framework/src/minifi_test_framework/steps/configuration_steps.py b/behave_framework/src/minifi_test_framework/steps/configuration_steps.py index 80f555275..0018eff1d 100644 --- a/behave_framework/src/minifi_test_framework/steps/configuration_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/configuration_steps.py @@ -18,6 +18,10 @@ from behave import step, given from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.containers.minifi_protocol import enable_openssl_fips_mode +from minifi_test_framework.containers.minifi_protocol import set_up_ssl_properties +from minifi_test_framework.containers.minifi_protocol import enable_log_metrics_publisher +from minifi_test_framework.containers.minifi_protocol import fetch_flow_config_from_flow_url @step('MiNiFi configuration "{config_key}" is set to "{config_value}"') @@ -27,7 +31,7 @@ def step_impl(context: MinifiTestContext, config_key: str, config_value: str): @step("log metrics publisher is enabled in MiNiFi") def step_impl(context: MinifiTestContext): - context.get_or_create_default_minifi_container().enable_log_metrics_publisher() + enable_log_metrics_publisher(context.get_or_create_default_minifi_container()) @step('log property "{log_property_key}" is set to "{log_property_value}"') @@ -37,14 +41,14 @@ def step_impl(context: MinifiTestContext, log_property_key: str, log_property_va @given("OpenSSL FIPS mode is enabled in MiNiFi") def step_impl(context: MinifiTestContext): - context.get_or_create_default_minifi_container().enable_openssl_fips_mode() + enable_openssl_fips_mode(context.get_or_create_default_minifi_container()) @given("flow configuration path is set up in flow url property") def step_impl(context: MinifiTestContext): - context.get_or_create_default_minifi_container().fetch_flow_config_from_flow_url() + fetch_flow_config_from_flow_url(context.get_or_create_default_minifi_container(), context.scenario_id) @given("SSL properties are set in MiNiFi") def step_impl(context: MinifiTestContext): - context.get_or_create_default_minifi_container().set_up_ssl_properties() + set_up_ssl_properties(context.get_or_create_default_minifi_container()) diff --git a/docker/RunBehaveTests.sh b/docker/RunBehaveTests.sh index bcfe34640..6492921e2 100755 --- a/docker/RunBehaveTests.sh +++ b/docker/RunBehaveTests.sh @@ -155,22 +155,6 @@ pip install --trusted-host pypi.python.org --upgrade pip setuptools # Install test dependencies echo "Installing test dependencies..." 1>&2 -# hint include/library paths if homewbrew is in use -if brew list 2> /dev/null | grep openssl > /dev/null 2>&1; then - echo "Using homebrew paths for openssl" 1>&2 - LDFLAGS="-L$(brew --prefix [email protected])/lib" - export LDFLAGS - CFLAGS="-I$(brew --prefix [email protected])/include" - export CFLAGS - SWIG_FEATURES="-cpperraswarn -includeall -I$(brew --prefix [email protected])/include" - export SWIG_FEATURES -fi - -if ! command swig -version &> /dev/null; then - echo "Swig could not be found on your system (dependency of m2crypto python library). Please install swig to continue." - exit 1 -fi - pip install -e "${docker_dir}/../behave_framework" export TMPDIR="/tmp/behavex_ci_${RANDOM}" @@ -192,6 +176,9 @@ fi echo "${BEHAVE_OPTS[@]}" -mapfile -t FEATURE_FILES < <(find "${docker_dir}/../extensions" -type f -name '*.feature') +FEATURE_FILES=() +while IFS= read -r -d '' file; do + FEATURE_FILES+=("$file") +done < <(find "${docker_dir}/../extensions" -type f -name '*.feature' -print0) behavex "${BEHAVE_OPTS[@]}" "${FEATURE_FILES[@]}" diff --git a/docker/installed/win.Dockerfile b/docker/installed/win.Dockerfile new file mode 100644 index 000000000..b413b68b9 --- /dev/null +++ b/docker/installed/win.Dockerfile @@ -0,0 +1,24 @@ +#escape=` + +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +LABEL maintainer="Apache NiFi <[email protected]>" + +ARG MSI_SOURCE="nifi-minifi-cpp.msi" + +ENV MINIFI_HOME="C:\Program Files\ApacheNiFiMiNiFi\nifi-minifi-cpp" + +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] + +RUN Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + +RUN choco install git -y +RUN choco install vcredist140 -y + +COPY ${MSI_SOURCE} C:\temp\minifi.msi + +SHELL ["cmd", "/S", "/C"] + +RUN C:\Windows\System32\msiexec.exe /i C:\temp\minifi.msi /qn /norestart /L*V C:\minifi_install.log + +CMD ["C:\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\bin\\minifi.exe"] \ No newline at end of file diff --git a/extensions/aws/tests/features/containers/kinesis_server_container.py b/extensions/aws/tests/features/containers/kinesis_server_container.py index 01b6dced2..d97bb5c83 100644 --- a/extensions/aws/tests/features/containers/kinesis_server_container.py +++ b/extensions/aws/tests/features/containers/kinesis_server_container.py @@ -16,13 +16,13 @@ import logging from pathlib import Path -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition, retry_check from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder -class KinesisServerContainer(Container): +class KinesisServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): builder = DockerImageBuilder( image_tag="minifi-kinesis-mock:latest", diff --git a/extensions/aws/tests/features/containers/s3_server_container.py b/extensions/aws/tests/features/containers/s3_server_container.py index 8f00b2644..2364b2d0c 100644 --- a/extensions/aws/tests/features/containers/s3_server_container.py +++ b/extensions/aws/tests/features/containers/s3_server_container.py @@ -16,12 +16,12 @@ import json import logging -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class S3ServerContainer(Container): +class S3ServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("adobe/s3mock:3.12.0", f"s3-server-{test_context.scenario_id}", test_context.network) self.environment.append("initialBuckets=test_bucket") diff --git a/extensions/azure/tests/features/containers/azure_server_container.py b/extensions/azure/tests/features/containers/azure_server_container.py index 4caeb786b..cb0d2e5eb 100644 --- a/extensions/azure/tests/features/containers/azure_server_container.py +++ b/extensions/azure/tests/features/containers/azure_server_container.py @@ -18,13 +18,13 @@ import logging from docker.errors import ContainerError -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import run_cmd_in_docker_image from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class AzureServerContainer(Container): +class AzureServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("mcr.microsoft.com/azure-storage/azurite:3.35.0", f"azure-storage-server-{test_context.scenario_id}", diff --git a/extensions/couchbase/tests/features/containers/couchbase_server_container.py b/extensions/couchbase/tests/features/containers/couchbase_server_container.py index f1cce4991..3f4f1401f 100644 --- a/extensions/couchbase/tests/features/containers/couchbase_server_container.py +++ b/extensions/couchbase/tests/features/containers/couchbase_server_container.py @@ -15,26 +15,25 @@ import logging -from OpenSSL import crypto from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from minifi_test_framework.core.ssl_utils import make_server_cert -from minifi_test_framework.containers.container import Container +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.file import File from minifi_test_framework.core.minifi_test_context import MinifiTestContext from docker.errors import ContainerError -class CouchbaseServerContainer(Container): +class CouchbaseServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("couchbase:enterprise-7.2.5", f"couchbase-server-{test_context.scenario_id}", test_context.network) couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666)) - couchbase_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert) + couchbase_cert_content = dump_cert(couchbase_cert) self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666)) - couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key) + couchbase_key_content = dump_key(couchbase_key) self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666)) def deploy(self, context: MinifiTestContext | None) -> bool: @@ -66,7 +65,7 @@ class CouchbaseServerContainer(Container): return True - @retry_check(max_tries=12, retry_interval=5) + @retry_check(max_tries=12, retry_interval_seconds=5) def _run_couchbase_cli_command(self, command): (code, output) = self.exec_run(command) if code != 0: @@ -90,7 +89,7 @@ class CouchbaseServerContainer(Container): logging.error(f"Unexpected error while running python command '{command}' in couchbase helper docker: '{e}'") return False - @retry_check(max_tries=15, retry_interval=2) + @retry_check(max_tries=15, retry_interval_seconds=2) def _load_couchbase_certs(self): python_command = f""" import requests diff --git a/extensions/elasticsearch/tests/features/containers/elastic_base_container.py b/extensions/elasticsearch/tests/features/containers/elastic_base_container.py index 5db6e61dc..2287278c4 100644 --- a/extensions/elasticsearch/tests/features/containers/elastic_base_container.py +++ b/extensions/elasticsearch/tests/features/containers/elastic_base_container.py @@ -14,11 +14,11 @@ # limitations under the License. from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class ElasticBaseContainer(Container): +class ElasticBaseContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext, image: str, container_name: str): super().__init__(image, container_name, test_context.network) self.user = None diff --git a/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py b/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py index 037711d75..aec928026 100644 --- a/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py +++ b/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py @@ -19,8 +19,7 @@ import os from elastic_base_container import ElasticBaseContainer from pathlib import Path -from OpenSSL import crypto -from minifi_test_framework.core.ssl_utils import make_server_cert, make_cert_without_extended_usage +from minifi_test_framework.core.ssl_utils import make_server_cert, make_cert_without_extended_usage, dump_cert, dump_key from minifi_test_framework.containers.file import File from minifi_test_framework.containers.host_file import HostFile from minifi_test_framework.core.minifi_test_context import MinifiTestContext @@ -33,19 +32,19 @@ class ElasticsearchContainer(ElasticBaseContainer): http_cert, http_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) transport_cert, transport_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/usr/share/elasticsearch/config/certs/root_ca.crt", root_ca_content, permissions=0o644)) - http_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=http_cert) + http_cert_content = dump_cert(http_cert) self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.crt", http_cert_content, permissions=0o644)) - http_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=http_key) + http_key_content = dump_key(http_key) self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.key", http_key_content, permissions=0o644)) - transport_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=transport_cert) + transport_cert_content = dump_cert(transport_cert) self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.crt", transport_cert_content, permissions=0o644)) - transport_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=transport_key) + transport_key_content = dump_key(transport_key) self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.key", transport_key_content, permissions=0o644)) features_dir = Path(__file__).resolve().parent.parent diff --git a/extensions/elasticsearch/tests/features/containers/opensearch_container.py b/extensions/elasticsearch/tests/features/containers/opensearch_container.py index cdf5008d5..234a26d12 100644 --- a/extensions/elasticsearch/tests/features/containers/opensearch_container.py +++ b/extensions/elasticsearch/tests/features/containers/opensearch_container.py @@ -18,8 +18,7 @@ import logging from elastic_base_container import ElasticBaseContainer from pathlib import Path -from OpenSSL import crypto -from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key from minifi_test_framework.containers.file import File from minifi_test_framework.containers.host_file import HostFile from minifi_test_framework.core.minifi_test_context import MinifiTestContext @@ -31,13 +30,13 @@ class OpensearchContainer(ElasticBaseContainer): admin_pem, admin_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/usr/share/opensearch/config/root-ca.pem", root_ca_content, permissions=0o644)) - admin_pem_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=admin_pem) + admin_pem_content = dump_cert(admin_pem) self.files.append(File("/usr/share/opensearch/config/admin.pem", admin_pem_content, permissions=0o644)) - admin_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=admin_key) + admin_key_content = dump_key(admin_key) self.files.append(File("/usr/share/opensearch/config/admin-key.pem", admin_key_content, permissions=0o644)) features_dir = Path(__file__).resolve().parent.parent diff --git a/extensions/gcp/tests/features/containers/fake_gcs_server_container.py b/extensions/gcp/tests/features/containers/fake_gcs_server_container.py index a9733ba4e..5689065bb 100644 --- a/extensions/gcp/tests/features/containers/fake_gcs_server_container.py +++ b/extensions/gcp/tests/features/containers/fake_gcs_server_container.py @@ -15,12 +15,12 @@ import logging from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.directory import Directory from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class FakeGcsServerContainer(Container): +class FakeGcsServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("fsouza/fake-gcs-server:1.45.1", f"fake-gcs-server-{test_context.scenario_id}", test_context.network, command=f'-scheme http -host fake-gcs-server-{test_context.scenario_id}') diff --git a/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py b/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py index 32f1cf1db..af296c597 100644 --- a/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py +++ b/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py @@ -14,13 +14,12 @@ # limitations under the License. import logging -from OpenSSL import crypto from minifi_test_framework.core.helpers import wait_for_condition, retry_check -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.file import File from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key from docker.errors import ContainerError @@ -30,20 +29,20 @@ class GrafanaLokiOptions: self.enable_multi_tenancy = enable_multi_tenancy -class GrafanaLokiContainer(Container): +class GrafanaLokiContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions): super().__init__("grafana/loki:3.2.1", f"grafana-loki-server-{test_context.scenario_id}", test_context.network) extra_ssl_settings = "" if options.enable_ssl: grafana_loki_cert, grafana_loki_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/etc/loki/root_ca.crt", root_ca_content, permissions=0o644)) - grafana_loki_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=grafana_loki_cert) + grafana_loki_cert_content = dump_cert(grafana_loki_cert) self.files.append(File("/etc/loki/cert.pem", grafana_loki_cert_content, permissions=0o644)) - grafana_loki_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=grafana_loki_key) + grafana_loki_key_content = dump_key(grafana_loki_key) self.files.append(File("/etc/loki/key.pem", grafana_loki_key_content, permissions=0o644)) extra_ssl_settings = """ diff --git a/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py b/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py index 409a9b9e6..43efd49c5 100644 --- a/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py +++ b/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py @@ -14,11 +14,11 @@ # limitations under the License. from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition -class ReverseProxyContainer(Container): +class ReverseProxyContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("minifi-reverse-proxy:latest", f"reverse-proxy-{test_context.scenario_id}", test_context.network) self.environment = [ diff --git a/extensions/kafka/tests/features/containers/kafka_server_container.py b/extensions/kafka/tests/features/containers/kafka_server_container.py index 30073186f..09e35e131 100644 --- a/extensions/kafka/tests/features/containers/kafka_server_container.py +++ b/extensions/kafka/tests/features/containers/kafka_server_container.py @@ -15,17 +15,18 @@ import logging import re + +from cryptography.hazmat.primitives import serialization import jks -from OpenSSL import crypto from minifi_test_framework.core.helpers import wait_for_condition -from minifi_test_framework.core.ssl_utils import make_server_cert -from minifi_test_framework.containers.container import Container +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.file import File from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class KafkaServer(Container): +class KafkaServer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("apache/kafka:4.1.0", f"kafka-server-{test_context.scenario_id}", test_context.network) self.environment.append("KAFKA_NODE_ID=1") @@ -55,7 +56,7 @@ class KafkaServer(Container): kafka_cert, kafka_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, kafka_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, kafka_key), 'rsa_raw') + pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [dump_cert(kafka_cert, encoding_type=serialization.Encoding.DER)], dump_key(kafka_key, encoding_type=serialization.Encoding.DER), 'rsa_raw') server_keystore = jks.KeyStore.new('jks', [pke]) server_keystore_content = server_keystore.saves('abcdefgh') self.files.append(File("/etc/kafka/secrets/kafka.keystore.jks", server_keystore_content, permissions=0o644)) @@ -63,7 +64,7 @@ class KafkaServer(Container): trusted_cert = jks.TrustedCertEntry.new( 'root-ca', # Alias for the certificate - crypto.dump_certificate(crypto.FILETYPE_ASN1, test_context.root_ca_cert) + dump_cert(test_context.root_ca_cert, encoding_type=serialization.Encoding.DER) ) # Create a JKS keystore that will serve as a truststore with the trusted cert entry. diff --git a/extensions/mqtt/tests/features/containers/mqtt_broker_container.py b/extensions/mqtt/tests/features/containers/mqtt_broker_container.py index b309ff309..c4f4b7416 100644 --- a/extensions/mqtt/tests/features/containers/mqtt_broker_container.py +++ b/extensions/mqtt/tests/features/containers/mqtt_broker_container.py @@ -17,14 +17,14 @@ import logging import re from textwrap import dedent -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder from docker.errors import ContainerError -class MqttBrokerContainer(Container): +class MqttBrokerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): dockerfile = dedent("""\ FROM {base_image} diff --git a/extensions/opc/tests/features/containers/opc_ua_server_container.py b/extensions/opc/tests/features/containers/opc_ua_server_container.py index 3dfb4626d..3740b77bd 100644 --- a/extensions/opc/tests/features/containers/opc_ua_server_container.py +++ b/extensions/opc/tests/features/containers/opc_ua_server_container.py @@ -14,12 +14,12 @@ # limitations under the License. from typing import List, Optional -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class OPCUAServerContainer(Container): +class OPCUAServerContainer(LinuxContainer): OPC_SERVER_IMAGE = "lordgamez/open62541:1.5.2" def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None): diff --git a/extensions/prometheus/tests/features/containers/prometheus_container.py b/extensions/prometheus/tests/features/containers/prometheus_container.py index 09053693e..829bd892e 100644 --- a/extensions/prometheus/tests/features/containers/prometheus_container.py +++ b/extensions/prometheus/tests/features/containers/prometheus_container.py @@ -14,15 +14,15 @@ # limitations under the License. import logging -from OpenSSL import crypto -from minifi_test_framework.containers.container import Container from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage from minifi_test_framework.containers.file import File +from minifi_test_framework.core.ssl_utils import dump_cert, dump_key +from minifi_test_framework.containers.container_linux import LinuxContainer -class PrometheusContainer(Container): +class PrometheusContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext, ssl: bool = False): super().__init__("prom/prometheus:v3.9.1", f"prometheus-{test_context.scenario_id}", test_context.network) self.scenario_id = test_context.scenario_id @@ -30,13 +30,13 @@ class PrometheusContainer(Container): if ssl: prometheus_cert, prometheus_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/etc/prometheus/certs/root-ca.pem", root_ca_content, permissions=0o644)) - prometheus_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=prometheus_cert) + prometheus_cert_content = dump_cert(prometheus_cert) self.files.append(File("/etc/prometheus/certs/prometheus.crt", prometheus_cert_content, permissions=0o644)) - prometheus_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=prometheus_key) + prometheus_key_content = dump_key(prometheus_key) self.files.append(File("/etc/prometheus/certs/prometheus.key", prometheus_key_content, permissions=0o644)) extra_ssl_settings = """ diff --git a/extensions/splunk/tests/features/containers/splunk_container.py b/extensions/splunk/tests/features/containers/splunk_container.py index 4106bb63e..9a1488e63 100644 --- a/extensions/splunk/tests/features/containers/splunk_container.py +++ b/extensions/splunk/tests/features/containers/splunk_container.py @@ -15,15 +15,14 @@ import json -from OpenSSL import crypto -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.core.helpers import wait_for_condition, retry_check from minifi_test_framework.containers.file import File -from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key -class SplunkContainer(Container): +class SplunkContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): super().__init__("splunk/splunk:9.2.1-patch2", f"splunk-{test_context.scenario_id}", test_context.network) self.user = None @@ -43,9 +42,9 @@ splunk: self.files.append(File("/tmp/defaults/default.yml", splunk_config_content, mode="rw", permissions=0o644)) splunk_cert, splunk_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - splunk_cert_content = crypto.dump_certificate(crypto.FILETYPE_PEM, splunk_cert) - splunk_key_content = crypto.dump_privatekey(crypto.FILETYPE_PEM, splunk_key) - root_ca_content = crypto.dump_certificate(crypto.FILETYPE_PEM, test_context.root_ca_cert) + splunk_cert_content = dump_cert(splunk_cert) + splunk_key_content = dump_key(splunk_key) + root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/opt/splunk/etc/auth/splunk_cert.pem", splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), permissions=0o644)) self.files.append(File("/opt/splunk/etc/auth/root_ca.pem", root_ca_content.decode(), permissions=0o644)) diff --git a/extensions/sql/tests/features/containers/postgress_server_container.py b/extensions/sql/tests/features/containers/postgress_server_container.py index f1df20e4b..2dea3058e 100644 --- a/extensions/sql/tests/features/containers/postgress_server_container.py +++ b/extensions/sql/tests/features/containers/postgress_server_container.py @@ -17,13 +17,13 @@ import logging from textwrap import dedent -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class PostgresContainer(Container): +class PostgresContainer(LinuxContainer): def __init__(self, context): dockerfile = dedent("""\ FROM {base_image} diff --git a/extensions/standard-processors/tests/features/containers/diag_slave_container.py b/extensions/standard-processors/tests/features/containers/diag_slave_container.py index 27b425958..4741441c4 100644 --- a/extensions/standard-processors/tests/features/containers/diag_slave_container.py +++ b/extensions/standard-processors/tests/features/containers/diag_slave_container.py @@ -18,13 +18,13 @@ import logging from textwrap import dedent -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class DiagSlave(Container): +class DiagSlave(LinuxContainer): def __init__(self, test_context: MinifiTestContext): dockerfile = dedent("""\ FROM panterdsd/diagslave:latest diff --git a/extensions/standard-processors/tests/features/containers/syslog_container.py b/extensions/standard-processors/tests/features/containers/syslog_container.py index f2da956f5..c831af6f1 100644 --- a/extensions/standard-processors/tests/features/containers/syslog_container.py +++ b/extensions/standard-processors/tests/features/containers/syslog_container.py @@ -15,10 +15,10 @@ # limitations under the License. # -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer -class SyslogContainer(Container): +class SyslogContainer(LinuxContainer): def __init__(self, protocol, context): super(SyslogContainer, self).__init__("ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network) self.command = f'/bin/bash -c "echo Syslog {protocol} client started; while true; do logger --{protocol} -n minifi-primary-{context.scenario_id} -P 514 sample_log; sleep 1; done"' diff --git a/extensions/standard-processors/tests/features/containers/tcp_client_container.py b/extensions/standard-processors/tests/features/containers/tcp_client_container.py index 5da752523..74fcf1351 100644 --- a/extensions/standard-processors/tests/features/containers/tcp_client_container.py +++ b/extensions/standard-processors/tests/features/containers/tcp_client_container.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -class TcpClientContainer(Container): +class TcpClientContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): cmd = ( "/bin/sh -c 'apk add netcat-openbsd && " diff --git a/extensions/standard-processors/tests/features/core_functionality.feature b/extensions/standard-processors/tests/features/core_functionality.feature index bab09863f..bae88e4cc 100644 --- a/extensions/standard-processors/tests/features/core_functionality.feature +++ b/extensions/standard-processors/tests/features/core_functionality.feature @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -@CORE +@CORE @SUPPORTS_WINDOWS Feature: Core flow functionalities Test core flow configuration functionalities @@ -33,8 +33,8 @@ Feature: Core flow functionalities When all instances start up - Then at least one file with the content "first_custom_text" is placed in the "/tmp/output" directory in less than 20 seconds - And at least one file with the content "second_custom_text" is placed in the "/tmp/output" directory in less than 20 seconds + Then at least one file with the content "first_custom_text" is placed in the "/tmp/output" directory in less than 200 seconds + And at least one file with the content "second_custom_text" is placed in the "/tmp/output" directory in less than 200 seconds Scenario: A funnel can be used as a terminator Given a GenerateFlowFile processor with the "Data Format" property set to "Text" diff --git a/extensions/standard-processors/tests/features/environment.py b/extensions/standard-processors/tests/features/environment.py index 1edb297a3..a2c3531ce 100644 --- a/extensions/standard-processors/tests/features/environment.py +++ b/extensions/standard-processors/tests/features/environment.py @@ -12,8 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import platform + import docker from minifi_test_framework.core.hooks import common_before_scenario from minifi_test_framework.core.hooks import common_after_scenario @@ -30,6 +31,8 @@ def before_feature(context, feature): def is_minifi_image_alpine_based(context): + if os.name == 'nt': + return False client: docker.DockerClient = docker.from_env() container = client.containers.create( image=context.minifi_container_image, diff --git a/extensions/standard-processors/tests/features/replace_text.feature b/extensions/standard-processors/tests/features/replace_text.feature index f8a3fdb05..b24a0e873 100644 --- a/extensions/standard-processors/tests/features/replace_text.feature +++ b/extensions/standard-processors/tests/features/replace_text.feature @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -@CORE +@CORE @SUPPORTS_WINDOWS Feature: Changing flowfile contents using the ReplaceText processor Scenario Outline: Replace text using Entire text mode diff --git a/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py b/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py index 75c4f4f3b..7f73ae9b1 100644 --- a/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py +++ b/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py @@ -17,32 +17,35 @@ import jks import os -from OpenSSL import crypto -from cryptography.hazmat.primitives.serialization import pkcs12, BestAvailableEncryption, load_pem_private_key + +from cryptography.hazmat.primitives import serialization from cryptography import x509 from pathlib import Path -from minifi_test_framework.containers.container import Container +from cryptography.hazmat.primitives._serialization import BestAvailableEncryption +from cryptography.hazmat.primitives.serialization import load_pem_private_key, pkcs12 +from minifi_test_framework.containers.container_linux import LinuxContainer from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.core.ssl_utils import make_server_cert from minifi_test_framework.containers.file import File from minifi_test_framework.containers.host_file import HostFile +from minifi_test_framework.core.ssl_utils import dump_key, dump_cert -class MinifiC2Server(Container): +class MinifiC2Server(LinuxContainer): def __init__(self, test_context: MinifiTestContext, ssl: bool = False): super().__init__("apache/nifi-minifi-c2:1.27.0", f"minifi-c2-server-{test_context.scenario_id}", test_context.network) if ssl: c2_cert, c2_key = make_server_cert(f"minifi-c2-server-{test_context.scenario_id}", test_context.root_ca_cert, test_context.root_ca_key) - pke = jks.PrivateKeyEntry.new('c2-server-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, c2_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, c2_key), 'rsa_raw') + pke = jks.PrivateKeyEntry.new('c2-server-cert', [dump_cert(c2_cert, encoding_type=serialization.Encoding.DER)], dump_key(c2_key, encoding_type=serialization.Encoding.DER), 'rsa_raw') server_keystore = jks.KeyStore.new('jks', [pke]) server_keystore_content = server_keystore.saves('abcdefgh') self.files.append(File("/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks", server_keystore_content, permissions=0o644)) - private_key_pem = crypto.dump_privatekey(crypto.FILETYPE_PEM, test_context.root_ca_key) + private_key_pem = dump_key(test_context.root_ca_key) private_key = load_pem_private_key(private_key_pem, password=None) - certificate_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, test_context.root_ca_cert) + certificate_pem = dump_cert(test_context.root_ca_cert) certificate = x509.load_pem_x509_certificate(certificate_pem) pkcs12_data = pkcs12.serialize_key_and_certificates( name=None, diff --git a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py index c65d67e0f..940636e72 100644 --- a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py +++ b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py @@ -22,52 +22,52 @@ from minifi_test_framework.core.minifi_test_context import MinifiTestContext @given('controller socket properties are set up') def step_impl(context: MinifiTestContext): - context.get_or_create_default_minifi_container().set_controller_socket_properties() + context.get_or_create_default_minifi_container().controller.set_controller_socket_properties() @when('MiNiFi config is updated through MiNiFi controller') def step_impl(context: MinifiTestContext): - context.get_or_create_default_minifi_container().update_flow_config_through_controller() + context.get_or_create_default_minifi_container().controller.update_flow_config_through_controller() @then('the updated config is persisted') def step_impl(context: MinifiTestContext): - assert context.get_or_create_default_minifi_container().updated_config_is_persisted() + assert context.get_or_create_default_minifi_container().controller.updated_config_is_persisted() @when('the {component} component is stopped through MiNiFi controller') def step_impl(context: MinifiTestContext, component: str): - context.get_or_create_default_minifi_container().stop_component_through_controller(component) + context.get_or_create_default_minifi_container().controller.stop_component_through_controller(component) @when('the {component} component is started through MiNiFi controller') def step_impl(context: MinifiTestContext, component: str): - context.get_or_create_default_minifi_container().start_component_through_controller(component) + context.get_or_create_default_minifi_container().controller.start_component_through_controller(component) @then('the {component} component is not running') def step_impl(context: MinifiTestContext, component: str): - assert not context.get_or_create_default_minifi_container().is_component_running(component) + assert not context.get_or_create_default_minifi_container().controller.is_component_running(component) @then('the {component} component is running') def step_impl(context: MinifiTestContext, component: str): - assert context.get_or_create_default_minifi_container().is_component_running(component) + assert context.get_or_create_default_minifi_container().controller.is_component_running(component) @then('connection \"{connection}\" can be seen through MiNiFi controller') def step_impl(context: MinifiTestContext, connection: str): - assert context.get_or_create_default_minifi_container().connection_found_through_controller(connection) + assert context.get_or_create_default_minifi_container().controller.connection_found_through_controller(connection) @then('{connection_count:d} connections can be seen full through MiNiFi controller') def step_impl(context: MinifiTestContext, connection_count: int): - assert context.get_or_create_default_minifi_container().get_full_connection_count() == connection_count + assert context.get_or_create_default_minifi_container().controller.get_full_connection_count() == connection_count -@retry_check(10, 1) +@retry_check(max_tries=5, retry_interval_seconds=1) def check_connection_size_through_controller(context: MinifiTestContext, connection: str, size: int, max_size: int) -> bool: - return context.get_or_create_default_minifi_container().get_connection_size(connection) == (size, max_size) + return context.get_or_create_default_minifi_container().controller.get_connection_size(connection) == (size, max_size) @then('connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller') @@ -75,9 +75,9 @@ def step_impl(context: MinifiTestContext, connection: str, size: int, max_size: assert check_connection_size_through_controller(context, connection, size, max_size) -@retry_check(10, 1) +@retry_check(max_tries=10, retry_interval_seconds=1) def manifest_can_be_retrieved_through_minifi_controller(context: MinifiTestContext) -> bool: - manifest = context.get_or_create_default_minifi_container().get_manifest() + manifest = context.get_or_create_default_minifi_container().controller.get_manifest() return '"agentManifest": {' in manifest and '"componentManifest": {' in manifest and '"agentType": "cpp"' in manifest @@ -88,4 +88,4 @@ def step_impl(context: MinifiTestContext): @then('debug bundle can be retrieved through MiNiFi controller') def step_impl(context: MinifiTestContext): - assert context.get_or_create_default_minifi_container().create_debug_bundle() + assert context.get_or_create_default_minifi_container().controller.create_debug_bundle() diff --git a/extensions/standard-processors/tests/features/steps/steps.py b/extensions/standard-processors/tests/features/steps/steps.py index 6ef8df1bb..bd82621e7 100644 --- a/extensions/standard-processors/tests/features/steps/steps.py +++ b/extensions/standard-processors/tests/features/steps/steps.py @@ -27,6 +27,8 @@ from minifi_test_framework.core.helpers import wait_for_condition from containers.syslog_container import SyslogContainer from containers.diag_slave_container import DiagSlave from containers.tcp_client_container import TcpClientContainer + +from minifi_test_framework.containers.minifi_protocol import set_up_ssl_properties from minifi_c2_server_container import MinifiC2Server @@ -78,7 +80,7 @@ def step_impl(context: MinifiTestContext): context.get_or_create_default_minifi_container().set_property("nifi.c2.full.heartbeat", "false") context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.class", "minifi-test-class") context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.identifier", "minifi-test-id") - context.get_or_create_default_minifi_container().set_up_ssl_properties() + set_up_ssl_properties(context.get_or_create_default_minifi_container()) @given("a MiNiFi C2 server is set up")
