This is an automated email from the ASF dual-hosted git repository.
lordgamez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 7532b4ab1 MINIFICPP-2363 Verify linux package on other distros using
the docker test suite
7532b4ab1 is described below
commit 7532b4ab1eaefce4b27971b1abb63f38e93a5346
Author: Martin Zink <martinz...@apache.org>
AuthorDate: Tue May 7 10:40:52 2024 +0200
MINIFICPP-2363 Verify linux package on other distros using the docker test
suite
Signed-off-by: Gabor Gyimesi <gamezb...@gmail.com>
This closes #1787
---
.github/workflows/ci.yml | 2 +-
.github/workflows/clear-actions-cache.yml | 2 +-
.github/workflows/gcc13-compat.yml | 6 +-
...python-compatibility.yml => verify-package.yml} | 94 ++++++++++++++++++----
cmake/DockerConfig.cmake | 1 +
cmake/VerifyPackageWithDocker.cmake | 53 ++++++++++++
cmake/VerifyPythonCompatibility.cmake | 4 +-
docker/python-verify/installed.Dockerfile | 4 +-
docker/test/integration/cluster/ImageStore.py | 24 +++++-
.../containers/MinifiAsPodInKubernetesCluster.py | 2 +-
.../integration/features/azure_storage.feature | 1 +
11 files changed, 164 insertions(+), 29 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 79f1eb4d6..7b689287e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -230,7 +230,7 @@ jobs:
working-directory: ./build
- name: Upload artifact
if: failure()
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: cpack_error
path:
D:/a/nifi-minifi-cpp/nifi-minifi-cpp/build/_CPack_Packages/win64/WIX/wix.log
diff --git a/.github/workflows/clear-actions-cache.yml
b/.github/workflows/clear-actions-cache.yml
index 7f59e2c1d..6c76da844 100644
--- a/.github/workflows/clear-actions-cache.yml
+++ b/.github/workflows/clear-actions-cache.yml
@@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-20.04
steps:
- id: checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: clear cache
run: |
python3 -m venv github_env
diff --git a/.github/workflows/gcc13-compat.yml
b/.github/workflows/gcc13-compat.yml
index ad4855ddb..639f8bade 100644
--- a/.github/workflows/gcc13-compat.yml
+++ b/.github/workflows/gcc13-compat.yml
@@ -17,7 +17,7 @@ jobs:
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- id: checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- id: install_deps
run: |
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
@@ -43,12 +43,12 @@ jobs:
if: ${{ failure() && steps.test.conclusion == 'failure' }}
run: |
if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo
"CORES_EXIST=true" >> $GITHUB_ENV; fi
- - uses: actions/upload-artifact@v3.1.2
+ - uses: actions/upload-artifact@v4
if: ${{ failure() && env.CORES_EXIST == 'true' }}
with:
name: ubuntu-coredumps
path: /var/lib/apport/coredump/
- - uses: actions/upload-artifact@v3.1.2
+ - uses: actions/upload-artifact@v4
if: ${{ failure() && env.CORES_EXIST == 'true' }}
with:
name: ubuntu-binaries
diff --git a/.github/workflows/verify-python-compatibility.yml
b/.github/workflows/verify-package.yml
similarity index 57%
rename from .github/workflows/verify-python-compatibility.yml
rename to .github/workflows/verify-package.yml
index db36235e5..b82f61512 100644
--- a/.github/workflows/verify-python-compatibility.yml
+++ b/.github/workflows/verify-package.yml
@@ -1,4 +1,4 @@
-name: "MiNiFi-CPP Verify Python Compatibility"
+name: "MiNiFi-CPP Verify Package"
on: [workflow_dispatch]
env:
DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS=
-DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON
-DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
@@ -12,15 +12,15 @@ jobs:
timeout-minutes: 180
steps:
- id: checkout
- uses: actions/checkout@v3
- - id: cache
- uses: actions/cache@v3
+ uses: actions/checkout@v4
+ - name: cache restore
+ uses: actions/cache/restore@v4
with:
path: ~/.ccache
- key: rocky-build-py-ccache-${{github.ref}}-${{github.sha}}
+ key: verify-package-ccache-${{github.ref}}-${{github.sha}}
restore-keys: |
- rocky-build-py-ccache-${{github.ref}}-
- rocky-build-py-ccache-refs/heads/main-
+ verify-package-ccache-${{github.ref}}-
+ verify-package-ccache-refs/heads/main-
- id: install_deps
run: |
sudo apt update
@@ -35,13 +35,77 @@ jobs:
if: always()
with:
path: ${{ env.CCACHE_DIR }}
- key: rocky-build-py-ccache-${{github.ref}}-${{github.sha}}
- - uses: actions/upload-artifact@v3
+ key: verify-package-ccache-${{github.ref}}-${{github.sha}}
+ - uses: actions/upload-artifact@v4
with:
name: minifi-tar
path: build/nifi-minifi-cpp-*-bin-rockylinux.tar.gz
if-no-files-found: error
+ docker-tests:
+ name: "${{ matrix.platform.name }}"
+ runs-on: ubuntu-20.04
+ timeout-minutes: 180
+ needs: rocky-build
+ strategy:
+ fail-fast: false
+ matrix:
+ platform: [
+ { name: "Rocky Linux 8",
+ id: "rocky8",
+ build_cmd: "make rocky8_from_rocky_package",
+ verify_cmd: "make docker-verify-rocky8" },
+ { name: "Rocky Linux 9",
+ id: "rocky9",
+ build_cmd: "make rocky9_from_rocky_package",
+ verify_cmd: "make docker-verify-rocky9" },
+ { name: "Ubuntu 22.04",
+ id: "ubuntu_jammy",
+ build_cmd: "make jammy_from_rocky_package",
+ verify_cmd: "make docker-verify-jammy" },
+ { name: "Ubuntu 24.04",
+ id: "ubuntu_noble",
+ build_cmd: "make noble_from_rocky_package",
+ verify_cmd: "make docker-verify-noble" },
+ { name: "Debian 11",
+ id: "debian_bullseye",
+ build_cmd: "make bullseye_from_rocky_package",
+ verify_cmd: "make docker-verify-bullseye" },
+ { name: "Debian 12",
+ id: "debian_bookworm",
+ build_cmd: "make bookworm_from_rocky_package",
+ verify_cmd: "make docker-verify-bookworm" },
+ ]
+
+ steps:
+ - id: checkout
+ uses: actions/checkout@v4
+ - uses: actions/download-artifact@v4
+ with:
+ name: minifi-tar
+ path: build
+ - id: install_deps
+ run: |
+ sudo apt update
+ sudo apt install -y python3-virtualenv
+ - run: cd build && cmake ${DOCKER_CMAKE_FLAGS} .. \
+ && ${{ matrix.platform.build_cmd }} && ${{
matrix.platform.verify_cmd }}
+ - name: Test Reporter
+ if: always()
+ uses: phoenix-actions/test-reporting@v15
+ with:
+ name: "${{ matrix.platform.name }}"
+ path: build/behavex_output/behave/*.xml
+ reporter: java-junit
+ output-to: 'step-summary'
+ list-tests: 'failed'
+ list-suites: 'failed'
+ - name: Upload artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ matrix.platform.id }}_behavex_output
+ path: build/behavex_output
docker-python-tests:
name: "${{ matrix.platform.name }}"
runs-on: ubuntu-20.04
@@ -69,18 +133,18 @@ jobs:
verify_cmd: "make docker-verify-patched_bullseye_py" },
{ name: "Ubuntu 22.04 with Anaconda Python",
id: "ubuntu22.04_anaconda",
- build_cmd: "make jammy_py_from_rocky_package && make
conda_jammy_py_docker_from_centos_build",
+ build_cmd: "make jammy_py_from_rocky_package && make
conda_jammy_py_from_rocky_package",
verify_cmd: "make docker-verify-conda_jammy_py" },
{ name: "Rocky Linux 9 in virtualenv Python",
id: "rocky9_venv",
- build_cmd: "make rocky9_py_from_rocky_package && make
venv_rocky9_py_docker_from_centos_build",
+ build_cmd: "make rocky9_py_from_rocky_package && make
venv_rocky9_py_from_rocky_package",
verify_cmd: "make docker-verify-venv_rocky9_py" }
]
steps:
- id: checkout
- uses: actions/checkout@v3
- - uses: actions/download-artifact@v3
+ uses: actions/checkout@v4
+ - uses: actions/download-artifact@v4
with:
name: minifi-tar
path: build
@@ -89,7 +153,7 @@ jobs:
sudo apt update
sudo apt install -y python3-virtualenv
- run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON
-DDOCKER_BUILD_ONLY=ON .. \
- && ${{ matrix.platform.build_cmd }} && ${{
matrix.platform.verify_cmd }}
+ && ${{ matrix.platform.build_cmd }} && ${{
matrix.platform.verify_cmd }}
- name: Test Reporter
if: always()
uses: phoenix-actions/test-reporting@v15
@@ -102,7 +166,7 @@ jobs:
list-suites: 'failed'
- name: Upload artifact
if: failure()
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: ${{ matrix.platform.id }}_behavex_output
path: build/behavex_output
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index 41c3bab39..e0c902198 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -178,3 +178,4 @@ function(CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE BASE_IMAGE
TAG_PREFIX INSTALL_P
endfunction()
include(VerifyPythonCompatibility)
+include(VerifyPackageWithDocker)
diff --git a/cmake/VerifyPackageWithDocker.cmake
b/cmake/VerifyPackageWithDocker.cmake
new file mode 100644
index 000000000..8c926c0e7
--- /dev/null
+++ b/cmake/VerifyPackageWithDocker.cmake
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(MINIFI_VERSION_STR
${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH})
+
+function(ADD_PACKAGE_VERIFY TAG_PREFIX)
+ set(ENABLED_TAGS "CORE")
+ foreach(MINIFI_OPTION ${MINIFI_OPTIONS})
+ string(FIND ${MINIFI_OPTION} "ENABLE" my_index)
+ if(my_index EQUAL -1)
+ continue()
+ elseif(${${MINIFI_OPTION}})
+ set(ENABLED_TAGS "${ENABLED_TAGS},${MINIFI_OPTION}")
+ endif()
+ endforeach()
+
+ set(DISABLED_TAGS "SKIP_CI,ENABLE_PYTHON_SCRIPTING")
+
+ add_custom_target(
+ docker-verify-${TAG_PREFIX}
+ COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh
--image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ${ENABLED_TAGS}
--tags_to_exclude=${DISABLED_TAGS}
--parallel_processes=${DOCKER_VERIFY_THREADS})
+endfunction()
+
+
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:8 rocky8 "yum install -y
wget")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:9 rocky9 "yum install -y
wget")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy jammy "apt update \\&\\&
apt install -y wget")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:noble noble "apt update \\&\\&
apt install -y wget")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bookworm bookworm "apt update
\\&\\& apt install -y wget")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bullseye bullseye "apt update
\\&\\& apt install -y wget")
+
+if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
+ ADD_PACKAGE_VERIFY(rocky8)
+ ADD_PACKAGE_VERIFY(rocky9)
+ ADD_PACKAGE_VERIFY(jammy)
+ ADD_PACKAGE_VERIFY(noble)
+ ADD_PACKAGE_VERIFY(bookworm)
+ ADD_PACKAGE_VERIFY(bullseye)
+endif()
diff --git a/cmake/VerifyPythonCompatibility.cmake
b/cmake/VerifyPythonCompatibility.cmake
index 4eca2be9b..a8b11f5b0 100644
--- a/cmake/VerifyPythonCompatibility.cmake
+++ b/cmake/VerifyPythonCompatibility.cmake
@@ -35,7 +35,7 @@ endfunction()
function(ADD_CONDA_TO_DOCKER TAG_PREFIX)
add_custom_target(
- conda_${TAG_PREFIX}_docker_from_rocky_build
+ conda_${TAG_PREFIX}_from_rocky_package
COMMAND DOCKER_BUILDKIT=1 docker build
--build-arg
BASE_IMAGE=apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR}
-t apacheminificpp:conda_${TAG_PREFIX}-${MINIFI_VERSION_STR}
@@ -45,7 +45,7 @@ endfunction()
function(ADD_VENV_TO_DOCKER TAG_PREFIX)
add_custom_target(
- venv_${TAG_PREFIX}_docker_from_rocky_build
+ venv_${TAG_PREFIX}_from_rocky_package
COMMAND DOCKER_BUILDKIT=1 docker build
--build-arg
BASE_IMAGE=apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR}
-t apacheminificpp:venv_${TAG_PREFIX}-${MINIFI_VERSION_STR}
diff --git a/docker/python-verify/installed.Dockerfile
b/docker/python-verify/installed.Dockerfile
index aa2c24abc..2b177976f 100644
--- a/docker/python-verify/installed.Dockerfile
+++ b/docker/python-verify/installed.Dockerfile
@@ -23,8 +23,8 @@ ARG MINIFI_VERSION
ARG ARCHIVE_LOCATION
ARG INSTALL_PACKAGE_CMD
-ARG UID=1000
-ARG GID=1000
+ARG UID=1001
+ARG GID=1001
ENV USER minificpp
ENV MINIFI_BASE_DIR /opt/minifi
diff --git a/docker/test/integration/cluster/ImageStore.py
b/docker/test/integration/cluster/ImageStore.py
index 00b1684f4..f8a550fcf 100644
--- a/docker/test/integration/cluster/ImageStore.py
+++ b/docker/test/integration/cluster/ImageStore.py
@@ -73,13 +73,25 @@ class ImageStore:
return image
def __build_minifi_cpp_sql_image(self):
+ if "rocky" in MinifiContainer.MINIFI_TAG_PREFIX:
+ install_sql_cmd = "dnf -y install postgresql-odbc"
+ so_location = "psqlodbca.so"
+ elif "bullseye" in MinifiContainer.MINIFI_TAG_PREFIX or "bookworm" in
MinifiContainer.MINIFI_TAG_PREFIX:
+ install_sql_cmd = "apt -y install odbc-postgresql"
+ so_location = "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so"
+ elif "jammy" in MinifiContainer.MINIFI_TAG_PREFIX or "noble" in
MinifiContainer.MINIFI_TAG_PREFIX:
+ install_sql_cmd = "apt -y install odbc-postgresql"
+ so_location = "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so"
+ else:
+ install_sql_cmd = "apk --update --no-cache add psqlodbc"
+ so_location = "psqlodbca.so"
dockerfile = dedent("""\
FROM {base_image}
USER root
- RUN apk --update --no-cache add psqlodbc
+ RUN {install_sql_cmd}
RUN echo "[PostgreSQL ANSI]" > /odbcinst.ini.template && \
echo "Description=PostgreSQL ODBC driver (ANSI version)"
>> /odbcinst.ini.template && \
- echo "Driver=psqlodbca.so" >> /odbcinst.ini.template && \
+ echo "Driver={so_location}" >> /odbcinst.ini.template && \
echo "Setup=libodbcpsqlS.so" >> /odbcinst.ini.template && \
echo "Debug=0" >> /odbcinst.ini.template && \
echo "CommLog=1" >> /odbcinst.ini.template && \
@@ -103,7 +115,8 @@ class ImageStore:
echo "Password = password" >> /etc/odbc.ini && \
echo "Database = postgres" >> /etc/odbc.ini
USER minificpp
- """.format(base_image='apacheminificpp:' +
MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION))
+ """.format(base_image='apacheminificpp:' +
MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION,
+ install_sql_cmd=install_sql_cmd,
so_location=so_location))
return self.__build_image(dockerfile)
@@ -128,7 +141,10 @@ class ImageStore:
parse_document_sed_cmd = 'sed -i "/class
ProcessorDetails:/,/^$/{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}"
/opt/minifi/minifi-current/minifi-python/nifi_python_processors/ParseDocument.py
&& \\'
chunk_document_sed_cmd = 'sed -i "/class
ProcessorDetails:/,/^$/{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}"
/opt/minifi/minifi-current/minifi-python/nifi_python_processors/ChunkDocument.py
&& \\'
if python_option == PythonOptions.SYSTEM_INSTALLED_PACKAGES:
- additional_cmd = "RUN pip3 install --break-system-packages
'langchain<=0.17.0'"
+ if not MinifiContainer.MINIFI_TAG_PREFIX:
+ additional_cmd = "RUN pip3 install --break-system-packages
'langchain<=0.17.0'"
+ else:
+ additional_cmd = "RUN pip3 install 'langchain<=0.17.0'"
elif python_option == PythonOptions.REQUIREMENTS_FILE:
requirements_install_command = "echo 'langchain<=0.17.0' >
/opt/minifi/minifi-current/minifi-python/nifi_python_processors/requirements.txt
&& \\"
elif python_option == PythonOptions.INLINE_DEFINED_PACKAGES:
diff --git
a/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py
b/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py
index beecb9a83..59488cc2f 100644
---
a/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py
+++
b/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py
@@ -43,7 +43,7 @@ class MinifiAsPodInKubernetesCluster(MinifiContainer):
self.kubernetes_proxy = kubernetes_proxy
docker_client = docker.from_env()
- minifi_image =
docker_client.images.get(MinifiAsPodInKubernetesCluster.MINIFI_IMAGE_NAME + ':'
+ os.environ['MINIFI_VERSION'])
+ minifi_image =
docker_client.images.get(MinifiAsPodInKubernetesCluster.MINIFI_IMAGE_NAME + ':'
+ MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION)
minifi_image.tag(MinifiAsPodInKubernetesCluster.MINIFI_IMAGE_NAME,
MinifiAsPodInKubernetesCluster.MINIFI_IMAGE_TAG)
def _create_container_config_dir(self, config_dir):
diff --git a/docker/test/integration/features/azure_storage.feature
b/docker/test/integration/features/azure_storage.feature
index 9338da83f..5bd431c11 100644
--- a/docker/test/integration/features/azure_storage.feature
+++ b/docker/test/integration/features/azure_storage.feature
@@ -29,6 +29,7 @@ Feature: Sending data from MiNiFi-C++ to an Azure storage
server
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the GetFile processor is connected to
the PutAzureBlobStorage
And the "success" relationship of the PutAzureBlobStorage processor is
connected to the PutFile
+ And the "failure" relationship of the PutAzureBlobStorage processor is
connected to the PutAzureBlobStorage
And an Azure storage server is set up
