This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/main by this push: new 7532b4ab1 MINIFICPP-2363 Verify linux package on other distros using the docker test suite 7532b4ab1 is described below commit 7532b4ab1eaefce4b27971b1abb63f38e93a5346 Author: Martin Zink <martinz...@apache.org> AuthorDate: Tue May 7 10:40:52 2024 +0200 MINIFICPP-2363 Verify linux package on other distros using the docker test suite Signed-off-by: Gabor Gyimesi <gamezb...@gmail.com> This closes #1787 --- .github/workflows/ci.yml | 2 +- .github/workflows/clear-actions-cache.yml | 2 +- .github/workflows/gcc13-compat.yml | 6 +- ...python-compatibility.yml => verify-package.yml} | 94 ++++++++++++++++++---- cmake/DockerConfig.cmake | 1 + cmake/VerifyPackageWithDocker.cmake | 53 ++++++++++++ cmake/VerifyPythonCompatibility.cmake | 4 +- docker/python-verify/installed.Dockerfile | 4 +- docker/test/integration/cluster/ImageStore.py | 24 +++++- .../containers/MinifiAsPodInKubernetesCluster.py | 2 +- .../integration/features/azure_storage.feature | 1 + 11 files changed, 164 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79f1eb4d6..7b689287e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -230,7 +230,7 @@ jobs: working-directory: ./build - name: Upload artifact if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: cpack_error path: D:/a/nifi-minifi-cpp/nifi-minifi-cpp/build/_CPack_Packages/win64/WIX/wix.log diff --git a/.github/workflows/clear-actions-cache.yml b/.github/workflows/clear-actions-cache.yml index 7f59e2c1d..6c76da844 100644 --- a/.github/workflows/clear-actions-cache.yml +++ b/.github/workflows/clear-actions-cache.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-20.04 steps: - id: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: clear cache run: | python3 -m venv github_env diff --git a/.github/workflows/gcc13-compat.yml b/.github/workflows/gcc13-compat.yml index ad4855ddb..639f8bade 100644 --- a/.github/workflows/gcc13-compat.yml +++ b/.github/workflows/gcc13-compat.yml @@ -17,7 +17,7 @@ jobs: sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - id: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - id: install_deps run: | sudo add-apt-repository ppa:ubuntu-toolchain-r/test @@ -43,12 +43,12 @@ jobs: if: ${{ failure() && steps.test.conclusion == 'failure' }} run: | if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi - - uses: actions/upload-artifact@v3.1.2 + - uses: actions/upload-artifact@v4 if: ${{ failure() && env.CORES_EXIST == 'true' }} with: name: ubuntu-coredumps path: /var/lib/apport/coredump/ - - uses: actions/upload-artifact@v3.1.2 + - uses: actions/upload-artifact@v4 if: ${{ failure() && env.CORES_EXIST == 'true' }} with: name: ubuntu-binaries diff --git a/.github/workflows/verify-python-compatibility.yml b/.github/workflows/verify-package.yml similarity index 57% rename from .github/workflows/verify-python-compatibility.yml rename to .github/workflows/verify-package.yml index db36235e5..b82f61512 100644 --- a/.github/workflows/verify-python-compatibility.yml +++ b/.github/workflows/verify-package.yml @@ -1,4 +1,4 @@ -name: "MiNiFi-CPP Verify Python Compatibility" +name: "MiNiFi-CPP Verify Package" on: [workflow_dispatch] env: DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ @@ -12,15 +12,15 @@ jobs: timeout-minutes: 180 steps: - id: checkout - uses: actions/checkout@v3 - - id: cache - uses: actions/cache@v3 + uses: actions/checkout@v4 + - name: cache restore + uses: actions/cache/restore@v4 with: path: ~/.ccache - key: rocky-build-py-ccache-${{github.ref}}-${{github.sha}} + key: verify-package-ccache-${{github.ref}}-${{github.sha}} restore-keys: | - rocky-build-py-ccache-${{github.ref}}- - rocky-build-py-ccache-refs/heads/main- + verify-package-ccache-${{github.ref}}- + verify-package-ccache-refs/heads/main- - id: install_deps run: | sudo apt update @@ -35,13 +35,77 @@ jobs: if: always() with: path: ${{ env.CCACHE_DIR }} - key: rocky-build-py-ccache-${{github.ref}}-${{github.sha}} - - uses: actions/upload-artifact@v3 + key: verify-package-ccache-${{github.ref}}-${{github.sha}} + - uses: actions/upload-artifact@v4 with: name: minifi-tar path: build/nifi-minifi-cpp-*-bin-rockylinux.tar.gz if-no-files-found: error + docker-tests: + name: "${{ matrix.platform.name }}" + runs-on: ubuntu-20.04 + timeout-minutes: 180 + needs: rocky-build + strategy: + fail-fast: false + matrix: + platform: [ + { name: "Rocky Linux 8", + id: "rocky8", + build_cmd: "make rocky8_from_rocky_package", + verify_cmd: "make docker-verify-rocky8" }, + { name: "Rocky Linux 9", + id: "rocky9", + build_cmd: "make rocky9_from_rocky_package", + verify_cmd: "make docker-verify-rocky9" }, + { name: "Ubuntu 22.04", + id: "ubuntu_jammy", + build_cmd: "make jammy_from_rocky_package", + verify_cmd: "make docker-verify-jammy" }, + { name: "Ubuntu 24.04", + id: "ubuntu_noble", + build_cmd: "make noble_from_rocky_package", + verify_cmd: "make docker-verify-noble" }, + { name: "Debian 11", + id: "debian_bullseye", + build_cmd: "make bullseye_from_rocky_package", + verify_cmd: "make docker-verify-bullseye" }, + { name: "Debian 12", + id: "debian_bookworm", + build_cmd: "make bookworm_from_rocky_package", + verify_cmd: "make docker-verify-bookworm" }, + ] + + steps: + - id: checkout + uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: minifi-tar + path: build + - id: install_deps + run: | + sudo apt update + sudo apt install -y python3-virtualenv + - run: cd build && cmake ${DOCKER_CMAKE_FLAGS} .. \ + && ${{ matrix.platform.build_cmd }} && ${{ matrix.platform.verify_cmd }} + - name: Test Reporter + if: always() + uses: phoenix-actions/test-reporting@v15 + with: + name: "${{ matrix.platform.name }}" + path: build/behavex_output/behave/*.xml + reporter: java-junit + output-to: 'step-summary' + list-tests: 'failed' + list-suites: 'failed' + - name: Upload artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.platform.id }}_behavex_output + path: build/behavex_output docker-python-tests: name: "${{ matrix.platform.name }}" runs-on: ubuntu-20.04 @@ -69,18 +133,18 @@ jobs: verify_cmd: "make docker-verify-patched_bullseye_py" }, { name: "Ubuntu 22.04 with Anaconda Python", id: "ubuntu22.04_anaconda", - build_cmd: "make jammy_py_from_rocky_package && make conda_jammy_py_docker_from_centos_build", + build_cmd: "make jammy_py_from_rocky_package && make conda_jammy_py_from_rocky_package", verify_cmd: "make docker-verify-conda_jammy_py" }, { name: "Rocky Linux 9 in virtualenv Python", id: "rocky9_venv", - build_cmd: "make rocky9_py_from_rocky_package && make venv_rocky9_py_docker_from_centos_build", + build_cmd: "make rocky9_py_from_rocky_package && make venv_rocky9_py_from_rocky_package", verify_cmd: "make docker-verify-venv_rocky9_py" } ] steps: - id: checkout - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: minifi-tar path: build @@ -89,7 +153,7 @@ jobs: sudo apt update sudo apt install -y python3-virtualenv - run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \ - && ${{ matrix.platform.build_cmd }} && ${{ matrix.platform.verify_cmd }} + && ${{ matrix.platform.build_cmd }} && ${{ matrix.platform.verify_cmd }} - name: Test Reporter if: always() uses: phoenix-actions/test-reporting@v15 @@ -102,7 +166,7 @@ jobs: list-suites: 'failed' - name: Upload artifact if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.platform.id }}_behavex_output path: build/behavex_output diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake index 41c3bab39..e0c902198 100644 --- a/cmake/DockerConfig.cmake +++ b/cmake/DockerConfig.cmake @@ -178,3 +178,4 @@ function(CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE BASE_IMAGE TAG_PREFIX INSTALL_P endfunction() include(VerifyPythonCompatibility) +include(VerifyPackageWithDocker) diff --git a/cmake/VerifyPackageWithDocker.cmake b/cmake/VerifyPackageWithDocker.cmake new file mode 100644 index 000000000..8c926c0e7 --- /dev/null +++ b/cmake/VerifyPackageWithDocker.cmake @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(MINIFI_VERSION_STR ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}) + +function(ADD_PACKAGE_VERIFY TAG_PREFIX) + set(ENABLED_TAGS "CORE") + foreach(MINIFI_OPTION ${MINIFI_OPTIONS}) + string(FIND ${MINIFI_OPTION} "ENABLE" my_index) + if(my_index EQUAL -1) + continue() + elseif(${${MINIFI_OPTION}}) + set(ENABLED_TAGS "${ENABLED_TAGS},${MINIFI_OPTION}") + endif() + endforeach() + + set(DISABLED_TAGS "SKIP_CI,ENABLE_PYTHON_SCRIPTING") + + add_custom_target( + docker-verify-${TAG_PREFIX} + COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS}) +endfunction() + + +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:8 rocky8 "yum install -y wget") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:9 rocky9 "yum install -y wget") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy jammy "apt update \\&\\& apt install -y wget") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:noble noble "apt update \\&\\& apt install -y wget") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bookworm bookworm "apt update \\&\\& apt install -y wget") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bullseye bullseye "apt update \\&\\& apt install -y wget") + +if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features) + ADD_PACKAGE_VERIFY(rocky8) + ADD_PACKAGE_VERIFY(rocky9) + ADD_PACKAGE_VERIFY(jammy) + ADD_PACKAGE_VERIFY(noble) + ADD_PACKAGE_VERIFY(bookworm) + ADD_PACKAGE_VERIFY(bullseye) +endif() diff --git a/cmake/VerifyPythonCompatibility.cmake b/cmake/VerifyPythonCompatibility.cmake index 4eca2be9b..a8b11f5b0 100644 --- a/cmake/VerifyPythonCompatibility.cmake +++ b/cmake/VerifyPythonCompatibility.cmake @@ -35,7 +35,7 @@ endfunction() function(ADD_CONDA_TO_DOCKER TAG_PREFIX) add_custom_target( - conda_${TAG_PREFIX}_docker_from_rocky_build + conda_${TAG_PREFIX}_from_rocky_package COMMAND DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR} -t apacheminificpp:conda_${TAG_PREFIX}-${MINIFI_VERSION_STR} @@ -45,7 +45,7 @@ endfunction() function(ADD_VENV_TO_DOCKER TAG_PREFIX) add_custom_target( - venv_${TAG_PREFIX}_docker_from_rocky_build + venv_${TAG_PREFIX}_from_rocky_package COMMAND DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR} -t apacheminificpp:venv_${TAG_PREFIX}-${MINIFI_VERSION_STR} diff --git a/docker/python-verify/installed.Dockerfile b/docker/python-verify/installed.Dockerfile index aa2c24abc..2b177976f 100644 --- a/docker/python-verify/installed.Dockerfile +++ b/docker/python-verify/installed.Dockerfile @@ -23,8 +23,8 @@ ARG MINIFI_VERSION ARG ARCHIVE_LOCATION ARG INSTALL_PACKAGE_CMD -ARG UID=1000 -ARG GID=1000 +ARG UID=1001 +ARG GID=1001 ENV USER minificpp ENV MINIFI_BASE_DIR /opt/minifi diff --git a/docker/test/integration/cluster/ImageStore.py b/docker/test/integration/cluster/ImageStore.py index 00b1684f4..f8a550fcf 100644 --- a/docker/test/integration/cluster/ImageStore.py +++ b/docker/test/integration/cluster/ImageStore.py @@ -73,13 +73,25 @@ class ImageStore: return image def __build_minifi_cpp_sql_image(self): + if "rocky" in MinifiContainer.MINIFI_TAG_PREFIX: + install_sql_cmd = "dnf -y install postgresql-odbc" + so_location = "psqlodbca.so" + elif "bullseye" in MinifiContainer.MINIFI_TAG_PREFIX or "bookworm" in MinifiContainer.MINIFI_TAG_PREFIX: + install_sql_cmd = "apt -y install odbc-postgresql" + so_location = "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so" + elif "jammy" in MinifiContainer.MINIFI_TAG_PREFIX or "noble" in MinifiContainer.MINIFI_TAG_PREFIX: + install_sql_cmd = "apt -y install odbc-postgresql" + so_location = "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so" + else: + install_sql_cmd = "apk --update --no-cache add psqlodbc" + so_location = "psqlodbca.so" dockerfile = dedent("""\ FROM {base_image} USER root - RUN apk --update --no-cache add psqlodbc + RUN {install_sql_cmd} RUN echo "[PostgreSQL ANSI]" > /odbcinst.ini.template && \ echo "Description=PostgreSQL ODBC driver (ANSI version)" >> /odbcinst.ini.template && \ - echo "Driver=psqlodbca.so" >> /odbcinst.ini.template && \ + echo "Driver={so_location}" >> /odbcinst.ini.template && \ echo "Setup=libodbcpsqlS.so" >> /odbcinst.ini.template && \ echo "Debug=0" >> /odbcinst.ini.template && \ echo "CommLog=1" >> /odbcinst.ini.template && \ @@ -103,7 +115,8 @@ class ImageStore: echo "Password = password" >> /etc/odbc.ini && \ echo "Database = postgres" >> /etc/odbc.ini USER minificpp - """.format(base_image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION)) + """.format(base_image='apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION, + install_sql_cmd=install_sql_cmd, so_location=so_location)) return self.__build_image(dockerfile) @@ -128,7 +141,10 @@ class ImageStore: parse_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}" /opt/minifi/minifi-current/minifi-python/nifi_python_processors/ParseDocument.py && \\' chunk_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}" /opt/minifi/minifi-current/minifi-python/nifi_python_processors/ChunkDocument.py && \\' if python_option == PythonOptions.SYSTEM_INSTALLED_PACKAGES: - additional_cmd = "RUN pip3 install --break-system-packages 'langchain<=0.17.0'" + if not MinifiContainer.MINIFI_TAG_PREFIX: + additional_cmd = "RUN pip3 install --break-system-packages 'langchain<=0.17.0'" + else: + additional_cmd = "RUN pip3 install 'langchain<=0.17.0'" elif python_option == PythonOptions.REQUIREMENTS_FILE: requirements_install_command = "echo 'langchain<=0.17.0' > /opt/minifi/minifi-current/minifi-python/nifi_python_processors/requirements.txt && \\" elif python_option == PythonOptions.INLINE_DEFINED_PACKAGES: diff --git a/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py b/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py index beecb9a83..59488cc2f 100644 --- a/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py +++ b/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py @@ -43,7 +43,7 @@ class MinifiAsPodInKubernetesCluster(MinifiContainer): self.kubernetes_proxy = kubernetes_proxy docker_client = docker.from_env() - minifi_image = docker_client.images.get(MinifiAsPodInKubernetesCluster.MINIFI_IMAGE_NAME + ':' + os.environ['MINIFI_VERSION']) + minifi_image = docker_client.images.get(MinifiAsPodInKubernetesCluster.MINIFI_IMAGE_NAME + ':' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION) minifi_image.tag(MinifiAsPodInKubernetesCluster.MINIFI_IMAGE_NAME, MinifiAsPodInKubernetesCluster.MINIFI_IMAGE_TAG) def _create_container_config_dir(self, config_dir): diff --git a/docker/test/integration/features/azure_storage.feature b/docker/test/integration/features/azure_storage.feature index 9338da83f..5bd431c11 100644 --- a/docker/test/integration/features/azure_storage.feature +++ b/docker/test/integration/features/azure_storage.feature @@ -29,6 +29,7 @@ Feature: Sending data from MiNiFi-C++ to an Azure storage server And a PutFile processor with the "Directory" property set to "/tmp/output" And the "success" relationship of the GetFile processor is connected to the PutAzureBlobStorage And the "success" relationship of the PutAzureBlobStorage processor is connected to the PutFile + And the "failure" relationship of the PutAzureBlobStorage processor is connected to the PutAzureBlobStorage And an Azure storage server is set up