URL: https://github.com/freeipa/freeipa/pull/5792 Author: abbra Title: #5792: [Backport][ipa-4-9] azure: bump F32->F34 Action: opened
PR body: """ This PR was opened automatically because PR #5790 was pushed to master and backport to ipa-4-9 is required. """ To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/5792/head:pr5792 git checkout pr5792
From 607a3e16eb2857e94ed6cf42ebc2ef79f9b5d224 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Wed, 5 May 2021 17:33:50 +0300 Subject: [PATCH 01/25] azure: bump F32->F34 Fixes: https://pagure.io/freeipa/issue/8848 --- ipatests/azure/Dockerfiles/Dockerfile.build.fedora | 2 +- ipatests/azure/templates/variables-fedora.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ipatests/azure/Dockerfiles/Dockerfile.build.fedora b/ipatests/azure/Dockerfiles/Dockerfile.build.fedora index 523aedf0ba7..e8df3da5fa2 100644 --- a/ipatests/azure/Dockerfiles/Dockerfile.build.fedora +++ b/ipatests/azure/Dockerfiles/Dockerfile.build.fedora @@ -1,4 +1,4 @@ -FROM fedora:32 +FROM fedora:34 MAINTAINER [FreeIPA Developers freeipa-devel@lists.fedorahosted.org] ENV container=docker LANG=en_US.utf8 LANGUAGE=en_US.utf8 LC_ALL=en_US.utf8 diff --git a/ipatests/azure/templates/variables-fedora.yml b/ipatests/azure/templates/variables-fedora.yml index 115389a84d3..222b8f6b384 100644 --- a/ipatests/azure/templates/variables-fedora.yml +++ b/ipatests/azure/templates/variables-fedora.yml @@ -1,7 +1,7 @@ variables: IPA_PLATFORM: fedora # the Docker public image to build IPA packages (rpms) - DOCKER_BUILD_IMAGE: 'fedora:32' + DOCKER_BUILD_IMAGE: 'fedora:34' # the Dockerfile to build Docker image for running IPA tests DOCKER_DOCKERFILE: ${{ format('Dockerfile.build.{0}', variables.IPA_PLATFORM) }} From d520cb3e22d104ab4fea5294f6752a18148e2631 Mon Sep 17 00:00:00 2001 From: Alexander Bokovoy <aboko...@redhat.com> Date: Wed, 10 Mar 2021 10:08:34 +0200 Subject: [PATCH 02/25] freeipa.spec: do not use jsl for linting on Fedora 34+ jsl package is orphaned in Fedora 34+ as it cannot be built. Related: https://pagure.io/freeipa/issue/8847 Signed-off-by: Alexander Bokovoy <aboko...@redhat.com> --- freeipa.spec.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freeipa.spec.in b/freeipa.spec.in index 6968afc6f22..f84d108a4e3 100755 --- a/freeipa.spec.in +++ b/freeipa.spec.in @@ -49,7 +49,7 @@ # lint is not executed during rpmbuild # %%global with_lint 1 %if %{with lint} - %global linter_options --enable-pylint --with-jslint --enable-rpmlint + %global linter_options --enable-pylint --without-jslint --enable-rpmlint %else %global linter_options --disable-pylint --without-jslint --disable-rpmlint %endif @@ -325,7 +325,10 @@ BuildRequires: python3-m2r # %if %{with lint} BuildRequires: git +%if 0%{?fedora} < 34 +# jsl is orphaned in Fedora 34+ BuildRequires: jsl +%endif BuildRequires: nss-tools BuildRequires: rpmlint BuildRequires: softhsm From 815f5a271a7090cf9ccf7fb86b4e5d69ee462126 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Wed, 5 May 2021 19:53:37 +0300 Subject: [PATCH 03/25] azure: Collect systemd boot log If an error occured while containers setup phase then no logs will be collected and it is hard(impossible?) to debug such issues on remote Azure host. With this change in case of such error all the container's journals will be collected in `systemd_boot_logs`. --- ipatests/azure/scripts/azure-run-tests.sh | 37 +++++++++++++++++------ ipatests/azure/templates/test-jobs.yml | 1 + 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/ipatests/azure/scripts/azure-run-tests.sh b/ipatests/azure/scripts/azure-run-tests.sh index b549e6271d7..9be044cc62e 100755 --- a/ipatests/azure/scripts/azure-run-tests.sh +++ b/ipatests/azure/scripts/azure-run-tests.sh @@ -1,5 +1,7 @@ #!/bin/bash -eux +set -o pipefail + if [ $# -ne 1 ]; then echo "Docker environment ID is not provided" exit 1 @@ -52,20 +54,30 @@ if [ "$IPA_TESTS_TYPE" == "base" ]; then IPA_TESTS_REPLICAS="0" fi -function compose_execute() { - # execute given command within every container of compose +# path to env dir outside from container +project_dir="${IPA_TESTS_ENV_WORKING_DIR}/${IPA_TESTS_ENV_NAME}" + +# path for journal if containers setup fails +SYSTEMD_BOOT_LOG="${project_dir}/systemd_boot_logs" - local containers="${PROJECT_ID}_master_1" +BASH_CMD="/bin/bash --noprofile --norc" + +function containers() { + local _containers="${PROJECT_ID}_master_1" # build list of replicas for i in $(seq 1 1 "$IPA_TESTS_REPLICAS"); do - containers+=" ${PROJECT_ID}_replica_${i}" + _containers+=" ${PROJECT_ID}_replica_${i}" done # build list of clients for i in $(seq 1 1 "$IPA_TESTS_CLIENTS"); do - containers+=" ${PROJECT_ID}_client_${i}" + _containers+=" ${PROJECT_ID}_client_${i}" done + printf "$_containers" +} - for container in $containers; do +function compose_execute() { + # execute given command within every container of compose + for container in $(containers); do docker exec -t \ "$container" \ "$@" \ @@ -74,7 +86,6 @@ function compose_execute() { done } -project_dir="${IPA_TESTS_ENV_WORKING_DIR}/${IPA_TESTS_ENV_NAME}" ln -sfr \ "${IPA_TESTS_DOCKERFILES}/docker-compose.yml" \ "$project_dir"/ @@ -107,7 +118,15 @@ IPA_TESTS_ENV_NAME="$IPA_TESTS_ENV_NAME" \ IPA_TEST_CONFIG_TEMPLATE="${BUILD_REPOSITORY_LOCALPATH}/ipatests/azure/templates/ipa-test-config-template.yaml" \ IPA_TESTS_REPO_PATH="$IPA_TESTS_REPO_PATH" \ IPA_TESTS_DOMAIN="$IPA_TESTS_DOMAIN" \ -python3 setup_containers.py +python3 setup_containers.py || \ + { mkdir -p "$SYSTEMD_BOOT_LOG"; + for container in $(containers); do + docker exec -t "$container" \ + $BASH_CMD -eu \ + -c 'journalctl -b --no-pager' > "${SYSTEMD_BOOT_LOG}/systemd_boot_${container}.log"; + done + exit 1; + } # path to runner within container tests_runner="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/azure-run-${IPA_TESTS_TYPE}-tests.sh" @@ -123,7 +142,7 @@ tests_result=1 --env IPA_TESTS_TO_IGNORE="$IPA_TESTS_TO_IGNORE" \ --env IPA_TESTS_ARGS="$IPA_TESTS_ARGS" \ "$IPA_TESTS_CONTROLLER" \ - /bin/bash --noprofile --norc \ + $BASH_CMD \ -eux "$tests_runner" && tests_result=0 ; } || tests_result=$? echo "Report disk usage" diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index 01c3daf1b70..b913c763a0e 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -150,6 +150,7 @@ steps: !*/*.yml !*/*.yaml !*/*.log + !*/systemd_boot_logs/*.log !*.tar.gz EOF cat "$artifacts_ignore_path" From 943b9652fc4c384dd87c70ed4b5595285ef749fb Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 6 May 2021 19:52:21 +0300 Subject: [PATCH 04/25] azure: Enforce multi-user.target as default systemd's target This may speed up boot process. For example, 'fedora:34' set graphical.target as default, while multi-user one will be more appropriate. --- ipatests/azure/Dockerfiles/Dockerfile.build.fedora | 3 ++- ipatests/azure/Dockerfiles/Dockerfile.build.rawhide | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ipatests/azure/Dockerfiles/Dockerfile.build.fedora b/ipatests/azure/Dockerfiles/Dockerfile.build.fedora index e8df3da5fa2..b6890b51805 100644 --- a/ipatests/azure/Dockerfiles/Dockerfile.build.fedora +++ b/ipatests/azure/Dockerfiles/Dockerfile.build.fedora @@ -23,7 +23,8 @@ RUN echo 'deltarpm = false' >> /etc/dnf/dnf.conf \ && echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config \ && systemctl enable sshd \ && for i in /usr/lib/systemd/system/*-domainname.service; \ - do sed -i 's#^ExecStart=/#ExecStart=-/#' $i ; done + do sed -i 's#^ExecStart=/#ExecStart=-/#' $i ; done \ + && systemctl set-default multi-user.target STOPSIGNAL RTMIN+3 VOLUME ["/freeipa", "/run", "/tmp"] diff --git a/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide b/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide index 47cb81e864d..ba97f2c6704 100644 --- a/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide +++ b/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide @@ -25,7 +25,8 @@ RUN echo 'deltarpm = false' >> /etc/dnf/dnf.conf \ && echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config \ && systemctl enable sshd \ && for i in /usr/lib/systemd/system/*-domainname.service; \ - do sed -i 's#^ExecStart=/#ExecStart=-/#' $i ; done + do sed -i 's#^ExecStart=/#ExecStart=-/#' $i ; done \ + && systemctl set-default multi-user.target STOPSIGNAL RTMIN+3 VOLUME ["/freeipa", "/run", "/tmp"] From cf2e8ad775287cbad850215322ac74d26d81a78f Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 6 May 2021 20:24:07 +0300 Subject: [PATCH 05/25] azure: Wait for systemd booted The calling of systemd's utils during systemd boot may lead to unpredictable results. For example, if DBus(dbus-broker) service is not started then DBus request goes nowhere and eventually will be timeouted. So, it's safer to wait fully booted system. --- ipatests/azure/scripts/setup_containers.py | 52 ++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/ipatests/azure/scripts/setup_containers.py b/ipatests/azure/scripts/setup_containers.py index c81577ecdd5..71963a2c8eb 100644 --- a/ipatests/azure/scripts/setup_containers.py +++ b/ipatests/azure/scripts/setup_containers.py @@ -1,10 +1,19 @@ +from __future__ import annotations + +from datetime import datetime import logging import os import subprocess +import time import docker from jinja2 import Template +from typing import NamedTuple, TYPE_CHECKING + +if TYPE_CHECKING: + from typing import List, Tuple, Union + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") IPA_TESTS_ENV_WORKING_DIR = os.environ.get("IPA_TESTS_ENV_WORKING_DIR") @@ -25,6 +34,11 @@ IPA_TEST_CONFIG = "ipa-test-config.yaml" +class ExecRunReturn(NamedTuple): # pylint: disable=inherit-non-class #3876 + exit_code: int + output: Tuple[bytes, bytes] + + class Container: """ Represents Docker container @@ -62,17 +76,19 @@ def ipv6(self): return self._ipv6 - def execute(self, args): + def execute( + self, args: Union[str, List[str]], raiseonerr: bool = True + ) -> ExecRunReturn: """ Exec an arbitrary command within container """ dcont = self.dclient.containers.get(self.name) logging.info("%s: run: %s", dcont.name, args) - result = dcont.exec_run(args, demux=True) + result: ExecRunReturn = dcont.exec_run(args, demux=True) if result.output[0] is not None: logging.info("%s: %s", dcont.name, result.output[0]) logging.info("%s: result: %s", dcont.name, result.exit_code) - if result.exit_code: + if result.exit_code and raiseonerr: logging.error("stderr: %s", result.output[1].decode()) raise subprocess.CalledProcessError( result.exit_code, args, result.output[1] @@ -230,6 +246,29 @@ def setup_container_overrides(self): self.execute_all(["systemctl", "daemon-reload"]) + def wait_systemd_target_reached( + self, + target_name: str = "default.target", + startup_timeout: int = 180, + ) -> None: + RETRY_DELAY_SEC = 5 + cmd = ["systemctl", "is-active", "--quiet", target_name] + for cont in self.containers: + reached = False + start = datetime.today() + while not reached: + result = cont.execute(cmd, raiseonerr=False) + reached = result.exit_code == 0 + if not reached: + elapsed = int((datetime.today() - start).total_seconds()) + if elapsed > startup_timeout: + raise RuntimeError( + f"Systemd's target '{target_name}' wasn't reached " + f"in container: '{cont.name}'\n" + f"stderr: {result.output[1].decode('utf-8')}" + ) + time.sleep(RETRY_DELAY_SEC) + class Controller(Container): """ @@ -243,6 +282,12 @@ def __init__(self, contr_type=IPA_CONTROLLER_TYPE): def append(self, containers_group): self.containers_groups.append(containers_group) + def wait_systemd_target_reached( + self, target_name: str = "multi-user.target" + ) -> None: + for containers_group in self.containers_groups: + containers_group.wait_systemd_target_reached(target_name) + def setup_ssh(self): """ Generate ssh key pair and copy public part to all containers @@ -368,6 +413,7 @@ def setup_container_overrides(self): controller.append(clients) controller.append(replicas) +controller.wait_systemd_target_reached() controller.setup_ssh() controller.setup_hosts() controller.setup_hostname() From 9213a53e22d1332f4605821685e94644fd1da30b Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 6 May 2021 20:54:51 +0300 Subject: [PATCH 06/25] azure: Remove no longer needed repo libseccomp2 2.5.1 is on focal-updates(Ubuntu 20.04LTS): https://packages.ubuntu.com/focal-updates/libseccomp2 --- ipatests/azure/templates/test-jobs.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index b913c763a0e..51fd81ae2c2 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -13,11 +13,7 @@ steps: moreutils \ rng-tools \ systemd-coredump \ - python3-docker \ - software-properties-common - sudo add-apt-repository -y ppa:abbra/freeipa-libseccomp - sudo apt-get update - sudo apt-get install -y libseccomp2 + python3-docker displayName: Install Host's tests requirements - script: | From 4106f9ca606f658edde7034c3f0bc903718ca3f3 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 6 May 2021 22:19:31 +0300 Subject: [PATCH 07/25] azure: Mask systemd-resolved The initial value of NS of resolv.conf is 127.0.0.11, this is the embedded NS of docker-compose. The disabling of this feature is not currently supported by Docker. On startup systemd-resolved caches the /etc/resolv.conf (docker-compose version), which is later modified by setup_containers.py script. This results in resolving error occurs: ```console [root@replica1 /]# getent ahosts master1.ipa.test ... can't resolve [root@replica1 /]# grep 'hosts:' /etc/nsswitch.conf hosts: files myhostname resolve [!UNAVAIL=return] dns [root@replica1 /]# resolvectl status Global LLMNR setting: resolve MulticastDNS setting: no DNSOverTLS setting: no DNSSEC setting: no DNSSEC supported: no Current DNS Server: 127.0.0.11 DNS Servers: 127.0.0.11 Fallback DNS Servers: 1.1.1.1 8.8.8.8 1.0.0.1 8.8.4.4 2606:4700:4700::1111 2001:4860:4860::8888 2606:4700:4700::1001 2001:4860:4860::8844 ``` According to docs: https://www.freedesktop.org/software/systemd/man/systemd-resolved.service.html#/etc/resolv.conf our case is 4(managed by other packages). So, restart of systemd-resolved is enough for its re-initialization, but not for services that already received DNS results. To speed up the overall process and to no restart each service which wants internet connection(or wait until service retries connection) systemd-resolved is masked. --- ipatests/azure/Dockerfiles/Dockerfile.build.fedora | 1 + ipatests/azure/Dockerfiles/Dockerfile.build.rawhide | 1 + 2 files changed, 2 insertions(+) diff --git a/ipatests/azure/Dockerfiles/Dockerfile.build.fedora b/ipatests/azure/Dockerfiles/Dockerfile.build.fedora index b6890b51805..afc31fe4ef3 100644 --- a/ipatests/azure/Dockerfiles/Dockerfile.build.fedora +++ b/ipatests/azure/Dockerfiles/Dockerfile.build.fedora @@ -24,6 +24,7 @@ RUN echo 'deltarpm = false' >> /etc/dnf/dnf.conf \ && systemctl enable sshd \ && for i in /usr/lib/systemd/system/*-domainname.service; \ do sed -i 's#^ExecStart=/#ExecStart=-/#' $i ; done \ + && { systemctl mask systemd-resolved ||: ; } \ && systemctl set-default multi-user.target STOPSIGNAL RTMIN+3 diff --git a/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide b/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide index ba97f2c6704..70ad9e804bf 100644 --- a/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide +++ b/ipatests/azure/Dockerfiles/Dockerfile.build.rawhide @@ -26,6 +26,7 @@ RUN echo 'deltarpm = false' >> /etc/dnf/dnf.conf \ && systemctl enable sshd \ && for i in /usr/lib/systemd/system/*-domainname.service; \ do sed -i 's#^ExecStart=/#ExecStart=-/#' $i ; done \ + && { systemctl mask systemd-resolved ||: ; } \ && systemctl set-default multi-user.target STOPSIGNAL RTMIN+3 From 7770c39bcbc5c1ba144690d88ee0a896f6d095d9 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Fri, 7 May 2021 19:20:45 +0300 Subject: [PATCH 08/25] ipatests: Update expectations for test_detect_container Since https://github.com/systemd/systemd/pull/17902/commits/a4a9a6f7c6e9cd9e219c56d08434a04bc2f395ff systemd improves the detection of Docker and Podman containers based on the presence of files-markers. ```console [slev@test systemd]$ git describe --contains --tags a4a9a6f7c6e9cd9e219c56d08434a04bc2f395ff v248-rc1~155^2~1 ``` Note: on Azure unit tests are run as non-privileged user in non-systemd inited container. This worked on F32 because: ```console [root@6d2aad38f62c /]# rpm -q systemd systemd-245.9-1.fc32.x86_64 ``` So, actual comparison in test was `assert None == None`. But F34 has: ```console [root@1ff1325f5a61 /]# rpm -q systemd systemd-248-2.fc34.x86_64 ``` So, the test's expectations should be updated. Unfortunately, this is incompatible with older versions of systemd (< v248). See https://github.com/systemd/systemd/pull/17902 for details. --- ipatests/test_ipaplatform/test_tasks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ipatests/test_ipaplatform/test_tasks.py b/ipatests/test_ipaplatform/test_tasks.py index bc1e37ebf2b..23f4fb3680f 100644 --- a/ipatests/test_ipaplatform/test_tasks.py +++ b/ipatests/test_ipaplatform/test_tasks.py @@ -47,10 +47,10 @@ def test_detect_container(): k, v = item.split('=', 1) if k == 'container': container = v + elif os.path.isfile("/run/.containerenv"): + container = "podman" + elif os.path.isfile("/.dockerenv"): + container = "docker" detected = tasks.detect_container() - if container == 'oci': - # systemd doesn't know about podman - assert detected in {'container-other', container} - else: - assert detected == container + assert detected == container From 744b9475bfe20d1ae84edb10070cf8be35d2b5f4 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Sat, 8 May 2021 12:32:52 +0300 Subject: [PATCH 09/25] azure: Add workaround for PhantomJS against OpenSSL 1.1.1 WebUI unit tests fail with: ``` PhantomJS threw an error:ERROR >> Auto configuration failed 0 [ >> 'Auto configuration failed', >> '140613066520384:error:25066067:DSO support routines:DLFCN_LOAD:could not load the shared library:dso_dlfcn.c:185:filename(libssl_conf.so): libssl_conf.so: cannot open shared object file: No such file or directory', >> '140613066520384:error:25070067:DSO support routines:DSO_load:could not load the shared library:dso_lib.c:244:', >> '140613066520384:error:0E07506E:configuration file routines:MODULE_LOAD_DSO:error loading dso:conf_mod.c:285:module=ssl_conf, path=ssl_conf', >> '140613066520384:error:0E076071:configuration file routines:MODULE_RUN:unknown module name:conf_mod.c:222:module=ssl_conf' >> ] ... Warning: PhantomJS exited unexpectedly with exit code 1. Use --force to continue. Aborted due to warnings. ``` See https://github.com/wch/webshot/pull/93 for details. --- ipatests/azure/azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ipatests/azure/azure-pipelines.yml b/ipatests/azure/azure-pipelines.yml index 11c55be25c1..46527cbd346 100644 --- a/ipatests/azure/azure-pipelines.yml +++ b/ipatests/azure/azure-pipelines.yml @@ -126,6 +126,9 @@ jobs: - script: | set -e echo "Running WebUI unit tests" + # PhantomJS is not compatible with OpenSSL 1.1.1 + # https://github.com/wch/webshot/pull/93 + export OPENSSL_CONF=whatever cd $(builddir)/install/ui/js/libs && make cd $(builddir)/install/ui && npm install cd $(builddir)/install/ui && node_modules/grunt/bin/grunt --verbose test From b0e16805a3dc96757c0e4be1a027ee96d11f06e7 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Sat, 8 May 2021 07:46:56 +0300 Subject: [PATCH 10/25] azure: Warn about memory issues The nonzero number of memory/memory+Swap usage hits limits may indicate the possible env instability(crashes, random failures, etc.). > memory.failcnt # show the number of memory usage hits limits memory.memsw.failcnt # show the number of memory+Swap hits limits --- ipatests/azure/scripts/azure-run-tests.sh | 32 ++++++++++++++++------- ipatests/azure/templates/test-jobs.yml | 15 +++++++++++ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/ipatests/azure/scripts/azure-run-tests.sh b/ipatests/azure/scripts/azure-run-tests.sh index 9be044cc62e..47fc06651f5 100755 --- a/ipatests/azure/scripts/azure-run-tests.sh +++ b/ipatests/azure/scripts/azure-run-tests.sh @@ -149,16 +149,28 @@ echo "Report disk usage" compose_execute df -h echo "Report memory statistics" -files=" -/sys/fs/cgroup/memory/memory.memsw.failcnt -/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes -/sys/fs/cgroup/memory/memory.memsw.max_usage_in_bytes -/sys/fs/cgroup/memory/memory.failcnt -/sys/fs/cgroup/memory/memory.max_usage_in_bytes -/sys/fs/cgroup/memory/memory.limit_in_bytes -/proc/sys/vm/swappiness -" -compose_execute head -n 1 $files +files='/sys/fs/cgroup/memory/memory.memsw.failcnt \ +/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes \ +/sys/fs/cgroup/memory/memory.memsw.max_usage_in_bytes \ +/sys/fs/cgroup/memory/memory.failcnt \ +/sys/fs/cgroup/memory/memory.max_usage_in_bytes \ +/sys/fs/cgroup/memory/memory.limit_in_bytes \ +/proc/sys/vm/swappiness \ +' + +MEMORY_STATS_PATH="$project_dir/memory.stats" +compose_execute $BASH_CMD -eu -c \ + "for file in $files; do printf '%s=%s\n' \"\$file\" \"\$(head -n 1 \$file)\" ; done" > "$MEMORY_STATS_PATH" + +sed -E -n \ + 's/(.*): .*(memory\.(memsw\.)?failcnt)=([0-9]+)/\1 \2 \4/p' \ + "$MEMORY_STATS_PATH" | \ +tr -d '\r' | \ +while read -r container memtype failcnt; do + if [ "$failcnt" -gt 0 ]; then + grep "^$container.*memory\..*" "$MEMORY_STATS_PATH" >> "$project_dir/memory.warnings" + fi +done pushd "$project_dir" BUILD_REPOSITORY_LOCALPATH="$BUILD_REPOSITORY_LOCALPATH" \ diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index 51fd81ae2c2..16b225cb6d3 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -63,6 +63,20 @@ steps: condition: succeededOrFailed() displayName: Host's memory statistics +- script: | + set -eu + function emit_warning() { + printf "##vso[task.logissue type=warning]%s\n" "$1" + } + + for memory_warn in $(find ${IPA_TESTS_ENV_WORKING_DIR}/*/ -maxdepth 1 -name memory.warnings); + do + env_name="$(basename $(dirname $memory_warn))" + emit_warning "Test env '$env_name' has high memory usage: $(echo '' && cat $memory_warn)" + done + condition: succeededOrFailed() + displayName: Check memory consumption + - script: | set -eu HOST_JOURNAL=host_journal.log @@ -147,6 +161,7 @@ steps: !*/*.yaml !*/*.log !*/systemd_boot_logs/*.log + !*/memory.stats !*.tar.gz EOF cat "$artifacts_ignore_path" From b45d161312f1038db6505e4eac6b481ef06890df Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 13 May 2021 18:16:04 +0300 Subject: [PATCH 11/25] BIND: Setup logging - allow BIND's logging customization - preconfig logging with ISC recommendations: https://kb.isc.org/docs/aa-01526 Fixes: https://pagure.io/freeipa/issue/8856 --- install/share/Makefile.am | 1 + .../share/bind.ipa-logging-ext.conf.template | 91 +++++++++++++++++++ install/share/bind.named.conf.template | 2 + ipaplatform/base/paths.py | 4 + ipaplatform/debian/paths.py | 1 + ipaplatform/fedora_container/paths.py | 3 + ipaplatform/rhel_container/paths.py | 3 + ipaplatform/suse/paths.py | 1 + ipaserver/install/bindinstance.py | 7 +- ipaserver/install/ipa_backup.py | 1 + ipatests/test_integration/test_upgrade.py | 26 +++++- 11 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 install/share/bind.ipa-logging-ext.conf.template diff --git a/install/share/Makefile.am b/install/share/Makefile.am index 6c4c3f0ede1..0f1a6975fc3 100644 --- a/install/share/Makefile.am +++ b/install/share/Makefile.am @@ -46,6 +46,7 @@ dist_app_DATA = \ kerberos.ldif \ bind.ipa-ext.conf.template \ bind.ipa-options-ext.conf.template \ + bind.ipa-logging-ext.conf.template \ bind.named.conf.template \ bind.openssl.cnf.template \ bind.openssl.cryptopolicy.cnf.template \ diff --git a/install/share/bind.ipa-logging-ext.conf.template b/install/share/bind.ipa-logging-ext.conf.template new file mode 100644 index 00000000000..1270e4d1896 --- /dev/null +++ b/install/share/bind.ipa-logging-ext.conf.template @@ -0,0 +1,91 @@ +channel named { + file "${NAMED_DATA_DIR}named.log" versions 10 size 20M; + severity info; + print-time yes; + print-category yes; + print-severity yes; +}; + +channel security { + file "${NAMED_DATA_DIR}security.log" versions 10 size 20M; + severity info; + print-time yes; + print-severity yes; +}; + +channel dnssec { + file "${NAMED_DATA_DIR}dnssec.log" versions 10 size 20M; + severity info; + print-time yes; + print-severity yes; +}; + +channel resolver { + file "${NAMED_DATA_DIR}resolver.log" versions 10 size 20M; + severity info; + print-time yes; + print-severity yes; +}; + +channel query_log { + file "${NAMED_DATA_DIR}query.log" versions 10 size 80M; + severity info; + print-time yes; + print-severity yes; +}; + +channel query_error { + file "${NAMED_DATA_DIR}query_errors.log" versions 10 size 20M; + severity info; + print-time yes; + print-severity yes; +}; + +channel lame_servers { + file "${NAMED_DATA_DIR}lame-servers.log" versions 10 size 20M; + severity info; + print-time yes; + print-severity yes; +}; + +channel capacity { + file "${NAMED_DATA_DIR}capacity.log" versions 10 size 20M; + severity info; + print-time yes; + print-severity yes; +}; + +channel database { + file "${NAMED_DATA_DIR}database.log" versions 10 size 20M; + severity info; + print-time yes; + print-severity yes; +}; + +channel update { + file "${NAMED_DATA_DIR}update.log" versions 10 size 10M; + severity info; + print-time yes; + print-severity yes; +}; + +category default { default_syslog; named; }; +category general { default_syslog; named; }; +category security { security; }; +category queries { query_log; }; +category query-errors { query_error; }; +category lame-servers { lame_servers; }; +category dnssec { dnssec; }; +category edns-disabled { default_syslog; resolver; }; +category config { default_syslog; named; }; +category resolver { resolver; }; +category cname { resolver; }; +category spill { capacity; }; +category rate-limit { capacity; }; +category database { database; }; +category client { default_syslog; named; }; +category network { default_syslog; named; }; +category unmatched { named; }; +category delegation-only { named; }; +category update { default_syslog; update; }; +category update-security { default_syslog; update; }; diff --git a/install/share/bind.named.conf.template b/install/share/bind.named.conf.template index 55e8142ebc2..3e1cfe612e6 100644 --- a/install/share/bind.named.conf.template +++ b/install/share/bind.named.conf.template @@ -4,6 +4,7 @@ * * * - $NAMED_CUSTOM_OPTIONS_CONF (for options) + * - $NAMED_LOGGING_OPTIONS_CONF (for logging options) * - $NAMED_CUSTOM_CONF (all other settings) */ @@ -37,6 +38,7 @@ logging { severity dynamic; print-time yes; }; + include "$NAMED_LOGGING_OPTIONS_CONF"; }; ${NAMED_ZONE_COMMENT}zone "." IN { diff --git a/ipaplatform/base/paths.py b/ipaplatform/base/paths.py index 683f875da73..91423b33276 100644 --- a/ipaplatform/base/paths.py +++ b/ipaplatform/base/paths.py @@ -86,12 +86,16 @@ class BasePathNamespace: NAMED_CONF = "/etc/named.conf" NAMED_CONF_BAK = "/etc/named.conf.ipa-backup" NAMED_CUSTOM_CONF = "/etc/named/ipa-ext.conf" + NAMED_LOGGING_OPTIONS_CONF = "/etc/named/ipa-logging-ext.conf" NAMED_CUSTOM_OPTIONS_CONF = "/etc/named/ipa-options-ext.conf" NAMED_CONF_SRC = '/usr/share/ipa/bind.named.conf.template' NAMED_CUSTOM_CONF_SRC = '/usr/share/ipa/bind.ipa-ext.conf.template' NAMED_CUSTOM_OPTIONS_CONF_SRC = ( '/usr/share/ipa/bind.ipa-options-ext.conf.template' ) + NAMED_LOGGING_OPTIONS_CONF_SRC = ( + "/usr/share/ipa/bind.ipa-logging-ext.conf.template" + ) NAMED_VAR_DIR = "/var/named" NAMED_KEYTAB = "/etc/named.keytab" NAMED_RFC1912_ZONES = "/etc/named.rfc1912.zones" diff --git a/ipaplatform/debian/paths.py b/ipaplatform/debian/paths.py index 176e651e7fb..7cfed089704 100644 --- a/ipaplatform/debian/paths.py +++ b/ipaplatform/debian/paths.py @@ -36,6 +36,7 @@ class DebianPathNamespace(BasePathNamespace): NAMED_CONF_BAK = "/etc/bind/named.conf.ipa-backup" NAMED_CUSTOM_CONF = "/etc/bind/ipa-ext.conf" NAMED_CUSTOM_OPTIONS_CONF = "/etc/bind/ipa-options-ext.conf" + NAMED_LOGGING_OPTIONS_CONF = "/etc/bind/ipa-logging-ext.conf" NAMED_VAR_DIR = "/var/cache/bind" NAMED_KEYTAB = "/etc/bind/named.keytab" NAMED_RFC1912_ZONES = "/etc/bind/named.conf.default-zones" diff --git a/ipaplatform/fedora_container/paths.py b/ipaplatform/fedora_container/paths.py index b6eb87bd24f..bdfd085d638 100644 --- a/ipaplatform/fedora_container/paths.py +++ b/ipaplatform/fedora_container/paths.py @@ -20,6 +20,9 @@ class FedoraContainerPathNamespace(FedoraPathNamespace): NAMED_CUSTOM_OPTIONS_CONF = data( FedoraPathNamespace.NAMED_CUSTOM_OPTIONS_CONF ) + NAMED_LOGGING_OPTIONS_CONF = data( + FedoraPathNamespace.NAMED_LOGGING_OPTIONS_CONF + ) NSSWITCH_CONF = data(FedoraPathNamespace.NSSWITCH_CONF) PKI_CONFIGURATION = data(FedoraPathNamespace.PKI_CONFIGURATION) SAMBA_DIR = data(FedoraPathNamespace.SAMBA_DIR) diff --git a/ipaplatform/rhel_container/paths.py b/ipaplatform/rhel_container/paths.py index 55cbc2cafc7..d1635702911 100644 --- a/ipaplatform/rhel_container/paths.py +++ b/ipaplatform/rhel_container/paths.py @@ -20,6 +20,9 @@ class RHELContainerPathNamespace(RHELPathNamespace): NAMED_CUSTOM_OPTIONS_CONF = data( RHELPathNamespace.NAMED_CUSTOM_OPTIONS_CONF ) + NAMED_LOGGING_OPTIONS_CONF = data( + RHELPathNamespace.NAMED_LOGGING_OPTIONS_CONF + ) NSSWITCH_CONF = data(RHELPathNamespace.NSSWITCH_CONF) PKI_CONFIGURATION = data(RHELPathNamespace.PKI_CONFIGURATION) SAMBA_DIR = data(RHELPathNamespace.SAMBA_DIR) diff --git a/ipaplatform/suse/paths.py b/ipaplatform/suse/paths.py index 383f191db03..5abcf8c30bc 100644 --- a/ipaplatform/suse/paths.py +++ b/ipaplatform/suse/paths.py @@ -27,6 +27,7 @@ class SusePathNamespace(BasePathNamespace): HTTPD_PASSWORD_CONF = "/etc/apache2/ipa/password.conf" NAMED_CUSTOM_CONF = "/etc/named.d/ipa-ext.conf" NAMED_CUSTOM_OPTIONS_CONF = "/etc/named.d/ipa-options-ext.conf" + NAMED_LOGGING_OPTIONS_CONF = "/etc/named.d/ipa-logging-ext.conf" NAMED_VAR_DIR = "/var/lib/named" NAMED_MANAGED_KEYS_DIR = "/var/lib/named/dyn" OPENSSL_DIR = "/etc/ssl" diff --git a/ipaserver/install/bindinstance.py b/ipaserver/install/bindinstance.py index 19941cd0075..a5aef03d418 100644 --- a/ipaserver/install/bindinstance.py +++ b/ipaserver/install/bindinstance.py @@ -884,6 +884,7 @@ def _setup_sub_dict(self): NAMED_CONF=paths.NAMED_CONF, NAMED_CUSTOM_CONF=paths.NAMED_CUSTOM_CONF, NAMED_CUSTOM_OPTIONS_CONF=paths.NAMED_CUSTOM_OPTIONS_CONF, + NAMED_LOGGING_OPTIONS_CONF=paths.NAMED_LOGGING_OPTIONS_CONF, NAMED_DATA_DIR=constants.NAMED_DATA_DIR, NAMED_ZONE_COMMENT=constants.NAMED_ZONE_COMMENT, NAMED_DNSSEC_VALIDATION=self._get_dnssec_validation(), @@ -1082,7 +1083,11 @@ def setup_named_conf(self, backup=False): ( paths.NAMED_CUSTOM_OPTIONS_CONF_SRC, paths.NAMED_CUSTOM_OPTIONS_CONF - ) + ), + ( + paths.NAMED_LOGGING_OPTIONS_CONF_SRC, + paths.NAMED_LOGGING_OPTIONS_CONF, + ), ) for src, dest in user_configs: if not os.path.exists(dest): diff --git a/ipaserver/install/ipa_backup.py b/ipaserver/install/ipa_backup.py index 055851865c5..2904c9e2e79 100644 --- a/ipaserver/install/ipa_backup.py +++ b/ipaserver/install/ipa_backup.py @@ -125,6 +125,7 @@ class Backup(admintool.AdminTool): paths.NAMED_CONF, paths.NAMED_CUSTOM_CONF, paths.NAMED_CUSTOM_OPTIONS_CONF, + paths.NAMED_LOGGING_OPTIONS_CONF, paths.NAMED_KEYTAB, paths.RESOLV_CONF, paths.SYSCONFIG_PKI_TOMCAT, diff --git a/ipatests/test_integration/test_upgrade.py b/ipatests/test_integration/test_upgrade.py index f884f75d972..82fe29fc64c 100644 --- a/ipatests/test_integration/test_upgrade.py +++ b/ipatests/test_integration/test_upgrade.py @@ -164,7 +164,12 @@ def get_named_confs(self): paths.NAMED_CUSTOM_OPTIONS_CONF, encoding="utf-8" ) print(opt_conf) - return named_conf, custom_conf, opt_conf + + log_conf = self.master.get_file_contents( + paths.NAMED_LOGGING_OPTIONS_CONF, encoding="utf-8" + ) + print(log_conf) + return named_conf, custom_conf, opt_conf, log_conf @pytest.mark.skip_if_platform( "debian", reason="Debian does not use crypto policy" @@ -176,17 +181,20 @@ def test_named_conf_crypto_policy(self): assert paths.NAMED_CRYPTO_POLICY_FILE in named_conf def test_current_named_conf(self): - named_conf, custom_conf, opt_conf = self.get_named_confs() - # verify that both includes are present exactly one time + named_conf, custom_conf, opt_conf, log_conf = self.get_named_confs() + # verify that all includes are present exactly one time inc_opt_conf = f'include "{paths.NAMED_CUSTOM_OPTIONS_CONF}";' assert named_conf.count(inc_opt_conf) == 1 inc_custom_conf = f'include "{paths.NAMED_CUSTOM_CONF}";' assert named_conf.count(inc_custom_conf) == 1 + inc_log_conf = f'include "{paths.NAMED_LOGGING_OPTIONS_CONF}";' + assert named_conf.count(inc_log_conf) == 1 assert "dnssec-validation yes;" in opt_conf assert "dnssec-validation" not in named_conf assert custom_conf + assert log_conf def test_update_named_conf_simple(self): # remove files to force a migration @@ -196,13 +204,15 @@ def test_update_named_conf_simple(self): "-f", paths.NAMED_CUSTOM_CONF, paths.NAMED_CUSTOM_OPTIONS_CONF, + paths.NAMED_LOGGING_OPTIONS_CONF, ] ) self.master.run_command(['ipa-server-upgrade']) - named_conf, custom_conf, opt_conf = self.get_named_confs() + named_conf, custom_conf, opt_conf, log_conf = self.get_named_confs() # not empty assert custom_conf.strip() + assert log_conf.strip() # has dnssec-validation enabled in option config assert "dnssec-validation yes;" in opt_conf assert "dnssec-validation" not in named_conf @@ -212,6 +222,8 @@ def test_update_named_conf_simple(self): assert named_conf.count(inc_opt_conf) == 1 inc_custom_conf = f'include "{paths.NAMED_CUSTOM_CONF}";' assert named_conf.count(inc_custom_conf) == 1 + inc_log_conf = f'include "{paths.NAMED_LOGGING_OPTIONS_CONF}";' + assert named_conf.count(inc_log_conf) == 1 def test_update_named_conf_old(self): # remove files to force a migration @@ -221,6 +233,7 @@ def test_update_named_conf_old(self): "-f", paths.NAMED_CUSTOM_CONF, paths.NAMED_CUSTOM_OPTIONS_CONF, + paths.NAMED_LOGGING_OPTIONS_CONF, ] ) # dump an old named conf to verify migration @@ -233,10 +246,11 @@ def test_update_named_conf_old(self): # upgrade self.master.run_command(['ipa-server-upgrade']) - named_conf, custom_conf, opt_conf = self.get_named_confs() + named_conf, custom_conf, opt_conf, log_conf = self.get_named_confs() # not empty assert custom_conf.strip() + assert log_conf.strip() # dnssec-validation is migrated as "disabled" from named.conf assert "dnssec-validation no;" in opt_conf assert "dnssec-validation" not in named_conf @@ -246,6 +260,8 @@ def test_update_named_conf_old(self): assert named_conf.count(inc_opt_conf) == 1 inc_custom_conf = f'include "{paths.NAMED_CUSTOM_CONF}";' assert named_conf.count(inc_custom_conf) == 1 + inc_log_conf = f'include "{paths.NAMED_LOGGING_OPTIONS_CONF}";' + assert named_conf.count(inc_log_conf) == 1 def test_admin_root_alias_upgrade_CVE_2020_10747(self): # Test upgrade for CVE-2020-10747 fix From 8d92b902158a61a35f6587e2419e5851a5f94633 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 13 May 2021 18:31:51 +0300 Subject: [PATCH 12/25] ipatests: Setup and collect BIND logs For Base/XMLRPC tests BIND's logs are already collected. --- .../azure/scripts/azure-run-base-tests.sh | 5 +++ ipatests/azure/scripts/variables.sh | 2 ++ ipatests/pytest_ipa/integration/__init__.py | 3 ++ ipatests/pytest_ipa/integration/tasks.py | 31 +++++++++++++++++++ 4 files changed, 41 insertions(+) diff --git a/ipatests/azure/scripts/azure-run-base-tests.sh b/ipatests/azure/scripts/azure-run-base-tests.sh index 7b538671e55..ee2b36c5649 100755 --- a/ipatests/azure/scripts/azure-run-base-tests.sh +++ b/ipatests/azure/scripts/azure-run-base-tests.sh @@ -48,6 +48,11 @@ if [ "$install_result" -eq 0 ] ; then sed -ri "s/mode = production/mode = developer/" /etc/ipa/default.conf systemctl restart "$HTTPD_SYSTEMD_NAME" + # debugging for BIND + sed -i "s/severity info;/severity debug;/" "$BIND_LOGGING_OPTIONS_CONF" + cat "$BIND_LOGGING_OPTIONS_CONF" + systemctl restart "$BIND_SYSTEMD_NAME" + firewalld_cmd --add-service={freeipa-ldap,freeipa-ldaps,dns} echo ${server_password} | kinit admin && ipa ping diff --git a/ipatests/azure/scripts/variables.sh b/ipatests/azure/scripts/variables.sh index 21181bf757d..6b855def8cc 100755 --- a/ipatests/azure/scripts/variables.sh +++ b/ipatests/azure/scripts/variables.sh @@ -7,6 +7,8 @@ HTTPD_BASEDIR='/etc/httpd' HTTPD_ALIASDIR="${HTTPD_BASEDIR}/alias" BIND_BASEDIR='/var/named' BIND_DATADIR="${BIND_BASEDIR}/data" +BIND_SYSTEMD_NAME='named.service' +BIND_LOGGING_OPTIONS_CONF='/etc/named/ipa-logging-ext.conf' function firewalld_cmd() { :; } diff --git a/ipatests/pytest_ipa/integration/__init__.py b/ipatests/pytest_ipa/integration/__init__.py index 0a195378cb5..c25bcd84841 100644 --- a/ipatests/pytest_ipa/integration/__init__.py +++ b/ipatests/pytest_ipa/integration/__init__.py @@ -35,6 +35,7 @@ from ipapython import ipautil from ipaplatform.paths import paths +from ipaplatform.constants import constants from . import fips from .config import Config from .env_config import get_global_config @@ -43,6 +44,8 @@ logger = logging.getLogger(__name__) CLASS_LOGFILES = [ + # BIND logs + os.path.join(paths.NAMED_VAR_DIR, constants.NAMED_DATA_DIR), # dirsrv logs paths.VAR_LOG_DIRSRV, # IPA install logs diff --git a/ipatests/pytest_ipa/integration/tasks.py b/ipatests/pytest_ipa/integration/tasks.py index e443aa015e0..be552237142 100755 --- a/ipatests/pytest_ipa/integration/tasks.py +++ b/ipatests/pytest_ipa/integration/tasks.py @@ -377,6 +377,7 @@ def install_master(host, setup_dns=True, setup_kra=False, setup_adtrust=False, setup_sssd_debugging(host) kinit_admin(host) if setup_dns: + setup_named_debugging(host) # fixup DNS zone default TTL for IPA DNS zone # For tests we should not wait too long set_default_ttl_for_ipa_dns_zone(host, raiseonerr=raiseonerr) @@ -515,6 +516,8 @@ def install_replica(master, replica, setup_ca=True, setup_dns=False, enable_ds_audit_log(replica, 'on') setup_sssd_debugging(replica) kinit_admin(replica) + if setup_dns: + setup_named_debugging(replica) else: fw.disable_services(fw_services) return result @@ -810,6 +813,34 @@ def setup_sssd_debugging(host): clear_sssd_cache(host) +def setup_named_debugging(host): + """ + Sets debug level to debug in each section of custom logging config + """ + + host.run_command( + [ + "sed", + "-i", + "s/severity info;/severity debug;/", + paths.NAMED_LOGGING_OPTIONS_CONF, + ], + ) + host.run_command(["cat", paths.NAMED_LOGGING_OPTIONS_CONF]) + result = host.run_command( + [ + "python3", + "-c", + ( + "from ipaplatform.services import knownservices; " + "print(knownservices.named.systemd_name)" + ), + ] + ) + service_name = result.stdout_text.strip() + host.run_command(["systemctl", "restart", service_name]) + + @contextmanager def remote_sssd_config(host): """Context manager for editing sssd config file on a remote host. From 334524da75046241c08227e087adba08f79aba1e Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Fri, 14 May 2021 11:35:46 +0300 Subject: [PATCH 13/25] azure: Run Base and XMLRPC tests is isolated network The tests in these envs make DNS requests to wild(internet) NSs, though usually tests assume the opposite making requests to `test.` zone. This makes CI unstable and dependent on wild resolvers and logically wrong. In future there can be tests which may want to check BIND as resolver(cache) for external networks. In this case such tests should be placed on not isolated mode. By default, a test env is not isolated from internet(as it was before), but it may be a good idea to change this default in future. --- ipatests/azure/Dockerfiles/docker-compose.yml | 1 + .../azure/azure_definitions/base-fedora.yml | 2 ++ .../azure/scripts/azure-run-base-tests.sh | 20 +++++++++++++++++-- ipatests/azure/scripts/azure-run-tests.sh | 6 ++++++ ipatests/azure/scripts/generate-matrix.py | 3 +++ 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/ipatests/azure/Dockerfiles/docker-compose.yml b/ipatests/azure/Dockerfiles/docker-compose.yml index 3bfd1a14c21..a6fc4e13c44 100644 --- a/ipatests/azure/Dockerfiles/docker-compose.yml +++ b/ipatests/azure/Dockerfiles/docker-compose.yml @@ -56,3 +56,4 @@ networks: driver: default config: - subnet: ${IPA_IPV6_SUBNET} + internal: ${IPA_NETWORK_INTERNAL} diff --git a/ipatests/azure/azure_definitions/base-fedora.yml b/ipatests/azure/azure_definitions/base-fedora.yml index f9cc3b92aa0..a5ae194002b 100644 --- a/ipatests/azure/azure_definitions/base-fedora.yml +++ b/ipatests/azure/azure_definitions/base-fedora.yml @@ -13,6 +13,7 @@ vms: - test_xmlrpc/test_dns_plugin.py args: "-k 'not test_dns_soa'" type: base + isolated: "true" - container_job: xmlrpc tests: @@ -20,3 +21,4 @@ vms: ignore: - test_xmlrpc/test_dns_plugin.py type: base + isolated: "true" diff --git a/ipatests/azure/scripts/azure-run-base-tests.sh b/ipatests/azure/scripts/azure-run-base-tests.sh index ee2b36c5649..fce1b7a047e 100755 --- a/ipatests/azure/scripts/azure-run-base-tests.sh +++ b/ipatests/azure/scripts/azure-run-base-tests.sh @@ -28,13 +28,29 @@ server_password=Secret123 echo "Installing FreeIPA master for the domain ${IPA_TESTS_DOMAIN} and realm ${IPA_TESTS_REALM}" +case "$IPA_NETWORK_INTERNAL" in + true ) + AUTO_FORWARDERS='--no-forwarders' + ;; + + false ) + AUTO_FORWARDERS='--auto-forwarders' + ;; + + * ) + echo "Unsupported value for IPA_NETWORK_INTERNAL: '$IPA_NETWORK_INTERNAL'" + exit 1 + ;; +esac + install_result=1 { ipa-server-install -U \ --domain "$IPA_TESTS_DOMAIN" \ --realm "$IPA_TESTS_REALM" \ -p "$server_password" -a "$server_password" \ - --setup-dns --setup-kra --auto-forwarders && install_result=0 ; } || \ - install_result=$? + --setup-dns --setup-kra \ + $AUTO_FORWARDERS \ + && install_result=0 ; } || install_result=$? rm -rf "$IPA_TESTS_LOGSDIR" mkdir "$IPA_TESTS_LOGSDIR" diff --git a/ipatests/azure/scripts/azure-run-tests.sh b/ipatests/azure/scripts/azure-run-tests.sh index 47fc06651f5..661185aa026 100755 --- a/ipatests/azure/scripts/azure-run-tests.sh +++ b/ipatests/azure/scripts/azure-run-tests.sh @@ -44,6 +44,9 @@ IPA_TESTS_REPLICAS="${!IPA_TESTS_REPLICAS_VARNAME:-0}" IPA_TESTS_CONTROLLER="${PROJECT_ID}_master_1" IPA_TESTS_LOGSDIR="${IPA_TESTS_REPO_PATH}/ipa_envs/${IPA_TESTS_ENV_NAME}/${CI_RUNNER_LOGS_DIR}" +IPA_TESTS_NETWORK_INTERNAL_VARNAME="IPA_TESTS_NETWORK_INTERNAL_${PROJECT_ID}" +IPA_NETWORK_INTERNAL="${!IPA_TESTS_NETWORK_INTERNAL_VARNAME:-false}" + IPA_TESTS_DOMAIN="${IPA_TESTS_DOMAIN:-ipa.test}" # bash4 IPA_TESTS_REALM="${IPA_TESTS_DOMAIN^^}" @@ -102,6 +105,7 @@ pushd "$project_dir" BUILD_REPOSITORY_LOCALPATH="$BUILD_REPOSITORY_LOCALPATH" \ IPA_DOCKER_IMAGE="${IPA_DOCKER_IMAGE:-freeipa-azure-builder}" \ IPA_NETWORK="${IPA_NETWORK:-ipanet}" \ +IPA_NETWORK_INTERNAL="$IPA_NETWORK_INTERNAL" \ IPA_IPV6_SUBNET="2001:db8:1:${PROJECT_ID}::/64" \ docker-compose -p "$PROJECT_ID" up \ --scale replica="$IPA_TESTS_REPLICAS" \ @@ -141,6 +145,7 @@ tests_result=1 --env IPA_TESTS_TO_RUN="$IPA_TESTS_TO_RUN" \ --env IPA_TESTS_TO_IGNORE="$IPA_TESTS_TO_IGNORE" \ --env IPA_TESTS_ARGS="$IPA_TESTS_ARGS" \ + --env IPA_NETWORK_INTERNAL="$IPA_NETWORK_INTERNAL" \ "$IPA_TESTS_CONTROLLER" \ $BASH_CMD \ -eux "$tests_runner" && tests_result=0 ; } || tests_result=$? @@ -176,6 +181,7 @@ pushd "$project_dir" BUILD_REPOSITORY_LOCALPATH="$BUILD_REPOSITORY_LOCALPATH" \ IPA_DOCKER_IMAGE="${IPA_DOCKER_IMAGE:-freeipa-azure-builder}" \ IPA_NETWORK="${IPA_NETWORK:-ipanet}" \ +IPA_NETWORK_INTERNAL="$IPA_NETWORK_INTERNAL" \ IPA_IPV6_SUBNET="2001:db8:1:${PROJECT_ID}::/64" \ docker-compose -p "$PROJECT_ID" down popd diff --git a/ipatests/azure/scripts/generate-matrix.py b/ipatests/azure/scripts/generate-matrix.py index 56a8af97f3a..03d97aa7dc2 100644 --- a/ipatests/azure/scripts/generate-matrix.py +++ b/ipatests/azure/scripts/generate-matrix.py @@ -28,6 +28,9 @@ jobs[f'ipa_tests_type_{job_id}'] = vm_job.get( 'type', 'integration') jobs[f'ipa_tests_args_{job_id}'] = vm_job.get('args', '') + jobs[f'ipa_tests_network_internal_{job_id}'] = vm_job.get( + 'isolated', 'false' + ) containers = vm_job.get('containers') replicas = 0 From f0f1656a0d265ff56ade2af005aa844c4b7419ad Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Sat, 15 May 2021 14:53:46 +0300 Subject: [PATCH 14/25] ipatests: Handle network-isolated mode Since the dns plugin's tests have no access to wild resolvers nobody answer such requests but authoritative NS. --- ipatests/test_xmlrpc/test_dns_plugin.py | 143 ++++++++++++++++++------ 1 file changed, 111 insertions(+), 32 deletions(-) diff --git a/ipatests/test_xmlrpc/test_dns_plugin.py b/ipatests/test_xmlrpc/test_dns_plugin.py index 41a70c4519c..6cd75d0caf6 100644 --- a/ipatests/test_xmlrpc/test_dns_plugin.py +++ b/ipatests/test_xmlrpc/test_dns_plugin.py @@ -83,6 +83,18 @@ zone2_rname = u'root.%s' % zone2_absolute zone2_rname_dnsname = DNSName(zone2_rname) +zone2_sub = "sub.%s" % zone2_absolute +zone2_sub_ns = "ns1.%s" % zone2_sub +zone2_sub_ns_dnsname = DNSName(zone2_sub_ns) +zone2_sub_rname = "root.%s" % zone2_sub +zone2_sub_rname_dnsname = DNSName(zone2_sub_rname) + +zone2_sub_dnsname = DNSName(zone2_sub) +zone2_sub_absolute_dnsname = DNSName(zone2_sub) +zone2_sub_dn = DN( + ("idnsname", zone2_sub), api.env.container_dns, api.env.basedn +) + zone3 = u'zone3.test' zone3_dnsname = DNSName(zone3) zone3_absolute = u'%s.' % zone3 @@ -97,6 +109,13 @@ zone3_rname = u'root.%s' % zone3_absolute zone3_rname_dnsname = DNSName(zone3_rname) +zone3a = "zone3a.test" +zone3a_absolute = "%s." % zone3a +zone3a_rname = "root.%s" % zone3a_absolute + +zone3a_sub = "sub.%s" % zone3a_absolute +zone3a_sub_rname = "root.%s" % zone3a_sub + zone3_ns2_arec = u'ns2' zone3_ns2_arec_dnsname = DNSName(zone3_ns2_arec) zone3_ns2_arec_dn = DN(('idnsname',zone3_ns2_arec), zone3_dn) @@ -261,8 +280,8 @@ ptr_revzone3_dn = DN(('idnsname',cnamerev), revzone3_classless2_dn) ptr_revzone3_hostname = zone3_ns2 -relnxname = u'does-not-exist-test' -absnxname = u'does.not.exist.test.' +relnxname = "does-not-exist-test" +absnxname = "does.not.exist.test." arec1 = u'172.16.29.111' arec2 = u'172.31.254.222' @@ -449,10 +468,26 @@ def dns_setup(self, declarative_setup): pass cleanup_commands = [ - ('dnszone_del', [zone1, zone2, zone3, zone4, zone5, revzone1, revzone2, - revzone3_classless1, revzone3_classless2, - idnzone1, revidnzone1], - {'continue': True}), + ( + "dnszone_del", + [ + zone1, + zone2, + zone2_sub, + zone3, + zone3a, + zone3a_sub, + zone4, + zone5, + revzone1, + revzone2, + revzone3_classless1, + revzone3_classless2, + idnzone1, + revidnzone1, + ], + {'continue': True}, + ), ('dnsconfig_mod', [], {'idnsforwarders' : None, 'idnsforwardpolicy' : None, 'idnsallowsyncptr' : None, @@ -541,35 +576,50 @@ def dns_setup(self, declarative_setup): ), dict( - desc='Try to create a zone with nonexistent NS entry', + desc=f"Parent zone for {zone2} tests", command=( 'dnszone_add', [zone2], { - 'idnssoamname': zone2_ns, 'idnssoarname': zone2_rname, } ), - expected=errors.NotFound(reason='Nameserver \'%s\' does not have a corresponding A/AAAA record' % (zone2_ns)), + expected=lambda x, y: True, + ), + + dict( + desc="Try to create a zone with nonexistent NS entry", + command=( + "dnszone_add", [zone2_sub], { + "idnssoamname": zone2_sub_ns, + "idnssoarname": zone2_sub_rname, + } + ), + expected=errors.NotFound( + reason=( + "Nameserver '%s' does not have a corresponding A/AAAA " + "record" % zone2_sub_ns + ), + ), ), dict( desc='Create a zone with nonexistent NS entry with --force', command=( - 'dnszone_add', [zone2], { - 'idnssoamname': zone2_ns, - 'idnssoarname': zone2_rname, - 'force': True, + "dnszone_add", [zone2_sub], { + "idnssoamname": zone2_sub_ns, + "idnssoarname": zone2_sub_rname, + "force": True, } ), expected={ - 'value': zone2_absolute_dnsname, + 'value': zone2_sub_absolute_dnsname, 'summary': None, 'result': { - 'dn': zone2_dn, - 'idnsname': [zone2_absolute_dnsname], + 'dn': zone2_sub_dn, + 'idnsname': [zone2_sub_absolute_dnsname], 'idnszoneactive': [u'TRUE'], - 'idnssoamname': [zone2_ns_dnsname], + 'idnssoamname': [zone2_sub_ns_dnsname], 'nsrecord': nameservers, - 'idnssoarname': [zone2_rname_dnsname], + 'idnssoarname': [zone2_sub_rname_dnsname], 'idnssoaserial': [fuzzy_digits], 'idnssoarefresh': [fuzzy_digits], 'idnssoaretry': [fuzzy_digits], @@ -1479,8 +1529,17 @@ def dns_setup(self, declarative_setup): dict( desc='Try to add unresolvable absolute NS record to %r using dnsrecord_add' % (name_ns), - command=('dnsrecord_add', [zone1, name_ns], {'nsrecord': absnxname}), - expected=errors.NotFound(reason=u"Nameserver '%s' does not have a corresponding A/AAAA record" % absnxname), + command=( + "dnsrecord_add", + [zone1, name_ns], + {"nsrecord": "notexisted.%s" % zone1_absolute}, + ), + expected=errors.NotFound( + reason=( + "Nameserver '%s' does not have a corresponding A/AAAA " + "record" % "notexisted.%s" % zone1_absolute + ), + ), ), dict( @@ -2140,29 +2199,49 @@ def dns_setup(self, declarative_setup): % zone1_permission) ), + dict( + desc=f"Parent zone for {zone3a} tests", + command=( + "dnszone_add", [zone3a], { + "idnssoarname": zone3a_rname, + } + ), + expected=lambda x, y: True, + ), dict( - desc='Try to create zone %r with relative nameserver' % zone3, + desc="Try to create zone %r with relative nameserver" % zone3a_sub, command=( - 'dnszone_add', [zone3], { - 'idnssoamname': u'ns', - 'idnssoarname': zone3_rname, + "dnszone_add", [zone3a_sub], { + "idnssoamname": "ns", + "idnssoarname": zone3a_sub_rname, } ), - expected=errors.NotFound(reason=u"Nameserver 'ns.%s' does not have a corresponding A/AAAA record" - % zone3_absolute) + expected=errors.NotFound( + reason=( + "Nameserver 'ns.%s' does not have a corresponding A/AAAA " + "record" % zone3a_sub + ) + ), ), dict( - desc='Try to create zone %r with nameserver in the zone itself' % zone3, + desc=( + "Try to create zone %r with nameserver in the zone itself" + % zone3a_sub + ), command=( - 'dnszone_add', [zone3], { - 'idnssoamname': zone3_absolute, - 'idnssoarname': zone3_rname, + "dnszone_add", [zone3a_sub], { + "idnssoamname": zone3a_sub, + "idnssoarname": zone3a_sub_rname, } ), - expected=errors.NotFound(reason=u"Nameserver '%s' does not have a corresponding A/AAAA record" - % zone3_absolute) + expected=errors.NotFound( + reason=( + "Nameserver '%s' does not have a corresponding A/AAAA " + "record" % zone3a_sub + ) + ), ), From 1aac3aa8df9a28c708144c2bda1d4cf1597de8c0 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Sat, 15 May 2021 14:52:50 +0300 Subject: [PATCH 15/25] dnsutil: Improvements for IPA DNS Resolver - check only IPv6 address of local NS if specified - increase request timeout(2sec is too small, BIND resolver's default 10sec) --- ipapython/dnsutil.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/ipapython/dnsutil.py b/ipapython/dnsutil.py index a5b3491cb54..4abe5101f87 100644 --- a/ipapython/dnsutil.py +++ b/ipapython/dnsutil.py @@ -97,9 +97,19 @@ def __init__(self, *args, **kwargs): ) def reset_ipa_defaults(self): + """ + BIND's default timeout for resolver is 10sec. + If that changes then it causes Timeout (instead of SERVFAIL) + exception for dnspython if BIND under high load. So, let's make + it the same + operation time. + + dnspython default is 2sec + """ + self.timeout = 10 + 2 + + # dnspython default is 5sec + self.lifetime = min(self.timeout * len(self.nameservers) * 2, 45) self.use_search_by_default = True - # the default is 5sec - self.lifetime = 15 def reset(self): super().reset() @@ -118,6 +128,22 @@ def _resolve_address(self, ip_address, *args, **kwargs): **kwargs, ) + def read_resolv_conf(self, *args, **kwargs): + """ + dnspython tries nameservers sequentially(not parallel). + IPA controlled BIND always listen on IPv6 if available, + so no need to send requests to both IPv4 and IPv6 endpoints + of the same NS(though BIND handles this). + """ + super().read_resolv_conf(*args, **kwargs) + # deduplicate + nameservers = list(dict.fromkeys(self.nameservers)) + ipv6_loopback = "::1" + ipv4_loopback = "127.0.0.1" + if ipv6_loopback in nameservers and ipv4_loopback in nameservers: + nameservers.remove(ipv4_loopback) + self.nameservers = nameservers + class DNSZoneAlreadyExists(dns.exception.DNSException): supp_kwargs = {'zone', 'ns'} From e55df05598719d54f0df3f752fdb57c71ec7b6d8 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Fri, 14 May 2021 19:02:36 +0300 Subject: [PATCH 16/25] dns: get_reverse_zone: Ignore resolver's timeout The DNS server may not process a query in a its internal timeout for a some reason or don't answer for a query at all. This may indicate a high load on DNS server. For example, if IPA DNS server is configured with 'none' forward policy (read as resolver), then SERVFAIL/Timeout errors will be normal until the hot cache for zones. Resolver's timeout in turn, indicates that it queried a server, but didn't received an answer in specified timeout. Related: https://pagure.io/freeipa/issue/7397 --- ipaserver/plugins/dns.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ipaserver/plugins/dns.py b/ipaserver/plugins/dns.py index eb21e3cee87..acb16613cd3 100644 --- a/ipaserver/plugins/dns.py +++ b/ipaserver/plugins/dns.py @@ -555,6 +555,13 @@ def get_reverse_zone(ipaddr): 'All nameservers failed to answer the query ' 'for DNS reverse zone %(revdns)s') % dict(revdns=revdns) ) + except dns.resolver.Timeout: + raise errors.NotFound( + reason=_( + "No answers could be found in the specified lifetime " + "for DNS reverse zone %(revdns)s" + ) % dict(revdns=revdns) + ) try: api.Command['dnszone_show'](revzone) From 8d61f83f2f3437250d07f4af8d52e224e7fec7ef Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 28 Jan 2021 12:38:37 +0300 Subject: [PATCH 17/25] pytest: Show extra summary information for all except passed tests By default pytest reports in summary section about tests failures and errors. It will be helpful to see skipped, xfailed and xpassed tests. Signed-off-by: Stanislav Levin <s...@altlinux.org> --- ipatests/azure/scripts/azure-run-base-tests.sh | 1 + ipatests/azure/scripts/azure-run-integration-tests.sh | 1 + tox.ini | 3 +++ 3 files changed, 5 insertions(+) diff --git a/ipatests/azure/scripts/azure-run-base-tests.sh b/ipatests/azure/scripts/azure-run-base-tests.sh index fce1b7a047e..e32a67db029 100755 --- a/ipatests/azure/scripts/azure-run-base-tests.sh +++ b/ipatests/azure/scripts/azure-run-base-tests.sh @@ -85,6 +85,7 @@ if [ "$install_result" -eq 0 ] ; then --logging-level=debug \ --logfile-dir="$IPA_TESTS_LOGSDIR" \ --verbose \ + -ra \ --with-xunit \ $IPA_TESTS_ARGS \ $IPA_TESTS_TO_IGNORE \ diff --git a/ipatests/azure/scripts/azure-run-integration-tests.sh b/ipatests/azure/scripts/azure-run-integration-tests.sh index 0a08c2e0022..2042419cbc9 100755 --- a/ipatests/azure/scripts/azure-run-integration-tests.sh +++ b/ipatests/azure/scripts/azure-run-integration-tests.sh @@ -16,6 +16,7 @@ tests_result=1 --logfile-dir="$IPA_TESTS_LOGSDIR" \ --with-xunit \ --verbose \ + -ra \ $IPA_TESTS_ARGS \ $IPA_TESTS_TO_IGNORE \ $IPA_TESTS_TO_RUN && tests_result=0 ; } || \ diff --git a/tox.ini b/tox.ini index 006e3f0dcb9..9eebd9b8d9c 100644 --- a/tox.ini +++ b/tox.ini @@ -59,3 +59,6 @@ ignore = E203, E402, E231, W503, E731, E741 max-line-length = 80 # exclude auto-generated remote plugins exclude=.git,.venv,build,_build,rpmbuild,2_49,2_114,2_156,2_164 + +[pytest] +addopts = -ra -v From 52725739ca26cd9fca6e884448aa190bc5a604b2 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Wed, 19 May 2021 12:03:06 +0300 Subject: [PATCH 18/25] ipatests: Ignore warnings on failed to read files on tarring There are tons of useless warnings about missing files on collecting logs, such as: ``` tar: /var/log/ipaserver-kra-install.log: Warning: Cannot stat: No such file or directory tar: /var/log/ipaepn.log: Warning: Cannot stat: No such file or directory tar: /etc/NetworkManager/NetworkManager.conf: Warning: Cannot stat: No such file or directory tar: /var/log/ipabackup.log: Warning: Cannot stat: No such file or directory tar: /var/log/iparestore.log: Warning: Cannot stat: No such file or directory ... ``` Since `--ignore-failed-read` option is passed to tar the caller doesn't care about not readable(mostly missing) files and these warnings may be filtered out. This improves the readability of test logs. --- ipatests/azure/scripts/azure-run-base-tests.sh | 1 + ipatests/azure/templates/test-jobs.yml | 2 +- ipatests/pytest_ipa/integration/__init__.py | 14 +++++++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ipatests/azure/scripts/azure-run-base-tests.sh b/ipatests/azure/scripts/azure-run-base-tests.sh index e32a67db029..3d5346f17d8 100755 --- a/ipatests/azure/scripts/azure-run-base-tests.sh +++ b/ipatests/azure/scripts/azure-run-base-tests.sh @@ -14,6 +14,7 @@ function collect_logs() { printf "Collecting logs\n" journalctl -b --no-pager > systemd_journal.log tar --ignore-failed-read -czf "$out_file" \ + --warning=no-failed-read \ /var/log/dirsrv \ "$HTTPD_LOGDIR" \ /var/log/ipa* \ diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index 16b225cb6d3..e0e25c63364 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -93,7 +93,7 @@ steps: printf "SECCOMP:\n" grep ' SECCOMP ' "$HOST_JOURNAL" && \ emit_warning "There are reported SECCOMP syscalls. Please, check the logs." - tar --ignore-failed-read -czf "$HOST_JOURNAL_PATH" "$HOST_JOURNAL" + tar -czf "$HOST_JOURNAL_PATH" "$HOST_JOURNAL" condition: succeededOrFailed() displayName: Host's systemd journal diff --git a/ipatests/pytest_ipa/integration/__init__.py b/ipatests/pytest_ipa/integration/__init__.py index c25bcd84841..b6c5ca92462 100644 --- a/ipatests/pytest_ipa/integration/__init__.py +++ b/ipatests/pytest_ipa/integration/__init__.py @@ -218,9 +218,17 @@ def collect_logs(name, logs_dict, logfile_dir=None, beakerlib_plugin=None): tmpname = cmd.stdout_text.strip() # Tar up the logs on the remote server cmd = host.run_command( - ['tar', 'cJvf', tmpname, '--ignore-failed-read', - '--dereference'] + logs, - log_stdout=False, raiseonerr=False) + [ + "tar", + "cJvf", + tmpname, + "--ignore-failed-read", + "--warning=no-failed-read", + "--dereference", + ] + logs, + log_stdout=False, + raiseonerr=False, + ) if cmd.returncode: logger.warning('Could not collect all requested logs') # fetch tar file From d2c721e411cfc6772b382dfd20cc49190d418eb9 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Wed, 19 May 2021 12:17:23 +0300 Subject: [PATCH 19/25] ipatests: Suppress list trust or certificates There are tons of useless information in test's runner log on server uninstallation about list trust and certificates, such as: ``` RUN ['trust', 'list'] pkcs11:id=%D2%87%B4%E3%DF%37%27%93%55%F6%56%EA%81%E5%36%CC%8C%1E%3F%BD;type=cert type: certificate label: ACCVRAIZ1 trust: anchor category: authority pkcs11:id=%F7%7D%C5%FD%C4%E8%9A%1B%77%64%A7%F5%1D%A0%CC%BF%87%60%9A%6D;type=cert type: certificate label: AC RAIZ FNMT-RCM trust: anchor category: authority pkcs11:id=%52%D8%88%3A%C8%9F%78%66%ED%89%F3%7B%38%70%94%C9%02%02%36%D0;type=cert type: certificate label: Actalis Authentication Root CA trust: anchor category: authority ... ``` This improves the readability of test logs. --- ipatests/pytest_ipa/integration/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ipatests/pytest_ipa/integration/tasks.py b/ipatests/pytest_ipa/integration/tasks.py index be552237142..61402bc0d11 100755 --- a/ipatests/pytest_ipa/integration/tasks.py +++ b/ipatests/pytest_ipa/integration/tasks.py @@ -1069,7 +1069,9 @@ def uninstall_master(host, ignore_topology_disconnect=True, raiseonerr=False).returncode == 1 assert host.run_command(['test', '-f', paths.IPA_P11_KIT], raiseonerr=False).returncode == 1 - assert "IPA CA" not in host.run_command(['trust', 'list']).stdout_text + assert "IPA CA" not in host.run_command( + ["trust", "list"], log_stdout=False + ).stdout_text if clean: Firewall(host).disable_services(["freeipa-ldap", "freeipa-ldaps", From 82d96b2780c7a3d25c10d7d6fb1407a37970fc9a Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Wed, 19 May 2021 18:47:16 +0300 Subject: [PATCH 20/25] azure: Collect installed packages The list of installed packages may be useful for checking the versions of packages for analysis. Previously, only the newly installed packages can be observed on Build phase. This is convenient for experienced users of PR-CI. Note: the read-only access provided for non-master containers to be able to execute Azure scripts. The logs are still collected only on controller. Only RPM-based collection is implemented for Fedora. By default nothing is collected. Users may want to override `installed_packages` function in the corresponding `ipatests/azure/scripts/variables-DISTRO.sh`. --- ipatests/azure/Dockerfiles/docker-compose.yml | 2 + ipatests/azure/scripts/azure-run-tests.sh | 44 ++++++++++++++++++- ipatests/azure/scripts/variables-fedora.sh | 4 ++ ipatests/azure/scripts/variables.sh | 2 + ipatests/azure/templates/test-jobs.yml | 1 + 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/ipatests/azure/Dockerfiles/docker-compose.yml b/ipatests/azure/Dockerfiles/docker-compose.yml index a6fc4e13c44..5a8c613abdc 100644 --- a/ipatests/azure/Dockerfiles/docker-compose.yml +++ b/ipatests/azure/Dockerfiles/docker-compose.yml @@ -28,6 +28,7 @@ services: mem_limit: "1900m" volumes: - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd + - ${BUILD_REPOSITORY_LOCALPATH}:${IPA_TESTS_REPO_PATH}:ro networks: - ${IPA_NETWORK} @@ -42,6 +43,7 @@ services: mem_limit: "536870912" volumes: - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd + - ${BUILD_REPOSITORY_LOCALPATH}:${IPA_TESTS_REPO_PATH}:ro # nfs server - ./exports:/exports - /lib/modules:/lib/modules:ro diff --git a/ipatests/azure/scripts/azure-run-tests.sh b/ipatests/azure/scripts/azure-run-tests.sh index 661185aa026..567d8996c39 100755 --- a/ipatests/azure/scripts/azure-run-tests.sh +++ b/ipatests/azure/scripts/azure-run-tests.sh @@ -44,6 +44,11 @@ IPA_TESTS_REPLICAS="${!IPA_TESTS_REPLICAS_VARNAME:-0}" IPA_TESTS_CONTROLLER="${PROJECT_ID}_master_1" IPA_TESTS_LOGSDIR="${IPA_TESTS_REPO_PATH}/ipa_envs/${IPA_TESTS_ENV_NAME}/${CI_RUNNER_LOGS_DIR}" +# path to azure scripts inside container +IPA_TESTS_SCRIPTS_IN="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" +# path to azure scripts outside of container +IPA_TESTS_SCRIPTS_OUT="${BUILD_REPOSITORY_LOCALPATH}/${IPA_TESTS_SCRIPTS}" + IPA_TESTS_NETWORK_INTERNAL_VARNAME="IPA_TESTS_NETWORK_INTERNAL_${PROJECT_ID}" IPA_NETWORK_INTERNAL="${!IPA_TESTS_NETWORK_INTERNAL_VARNAME:-false}" @@ -51,6 +56,7 @@ IPA_TESTS_DOMAIN="${IPA_TESTS_DOMAIN:-ipa.test}" # bash4 IPA_TESTS_REALM="${IPA_TESTS_DOMAIN^^}" + # for base tests only 1 master is needed even if another was specified if [ "$IPA_TESTS_TYPE" == "base" ]; then IPA_TESTS_CLIENTS="0" @@ -63,6 +69,9 @@ project_dir="${IPA_TESTS_ENV_WORKING_DIR}/${IPA_TESTS_ENV_NAME}" # path for journal if containers setup fails SYSTEMD_BOOT_LOG="${project_dir}/systemd_boot_logs" +# path to directory where to dump list of packages outside of container +IPA_INSTALLED_PKGS_DIR="${project_dir}/installed_packages" + BASH_CMD="/bin/bash --noprofile --norc" function containers() { @@ -132,12 +141,43 @@ python3 setup_containers.py || \ exit 1; } +# collect list of all the installed packages +mkdir -p "$IPA_INSTALLED_PKGS_DIR" + +# controller +docker exec -t \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_SCRIPTS_IN}" \ + --env IPA_PLATFORM="$IPA_PLATFORM" \ + "$IPA_TESTS_CONTROLLER" \ + $BASH_CMD -eu \ + -c \ + "source '${IPA_TESTS_SCRIPTS_IN}/variables.sh' && \ + echo '# Controller container: $IPA_TESTS_CONTROLLER' && \ + echo '# IPA platform: '\$IPA_PLATFORM && \ + installed_packages \ + " > "${IPA_INSTALLED_PKGS_DIR}/packages_controller_${IPA_TESTS_CONTROLLER}.log" + +# workers +for container in $(containers); do + docker exec -t \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_SCRIPTS_IN}" \ + --env IPA_PLATFORM="$IPA_PLATFORM" \ + "$container" \ + $BASH_CMD -eu \ + -c \ + "source '${IPA_TESTS_SCRIPTS_IN}/variables.sh' && \ + echo '# Container: $container' && \ + echo '# IPA platform: '\$IPA_PLATFORM && \ + installed_packages \ + " > "${IPA_INSTALLED_PKGS_DIR}/packages_${container}.log" +done + # path to runner within container -tests_runner="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/azure-run-${IPA_TESTS_TYPE}-tests.sh" +tests_runner="${IPA_TESTS_SCRIPTS_IN}/azure-run-${IPA_TESTS_TYPE}-tests.sh" tests_result=1 { docker exec -t \ - --env IPA_TESTS_SCRIPTS="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_SCRIPTS_IN}" \ --env IPA_PLATFORM="$IPA_PLATFORM" \ --env IPA_TESTS_DOMAIN="$IPA_TESTS_DOMAIN" \ --env IPA_TESTS_REALM="$IPA_TESTS_REALM" \ diff --git a/ipatests/azure/scripts/variables-fedora.sh b/ipatests/azure/scripts/variables-fedora.sh index 41d7d888ffe..62600c9b08d 100755 --- a/ipatests/azure/scripts/variables-fedora.sh +++ b/ipatests/azure/scripts/variables-fedora.sh @@ -5,3 +5,7 @@ function firewalld_cmd() { firewall-cmd $@ } + +function installed_packages() { + rpm -qa | sort +} diff --git a/ipatests/azure/scripts/variables.sh b/ipatests/azure/scripts/variables.sh index 6b855def8cc..1efb2adeae2 100755 --- a/ipatests/azure/scripts/variables.sh +++ b/ipatests/azure/scripts/variables.sh @@ -12,5 +12,7 @@ BIND_LOGGING_OPTIONS_CONF='/etc/named/ipa-logging-ext.conf' function firewalld_cmd() { :; } +function installed_packages() { :; } + # this should be the last to override base variables with platform specific source "$IPA_TESTS_SCRIPTS/variables-${IPA_PLATFORM}.sh" diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index e0e25c63364..b8c5bcb93ea 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -161,6 +161,7 @@ steps: !*/*.yaml !*/*.log !*/systemd_boot_logs/*.log + !*/installed_packages/*.log !*/memory.stats !*.tar.gz EOF From 3ca3a1c7728f1ad38ed71d45a8d5bb614b84a029 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Wed, 19 May 2021 00:51:36 +0300 Subject: [PATCH 21/25] ipatests: dnssec: Add alternative approach for checking chain of trust drill is currently broken on F34. Fortunately, there are another tools for checking DNSSEC trust. One of them is `delv`: > delv is a tool for sending DNS queries and validating the results, using the same internal resolver and validator logic as named. delv sends to a specified name server all queries needed to fetch and validate the requested data; this includes the original requested query, subsequent queries to follow CNAME or DNAME chains, queries for DNSKEY, and DS records to establish a chain of trust for DNSSEC validation. It does not perform iterative resolution, but simulates the behavior of a name server configured for DNSSEC validating and forwarding. Related: https://pagure.io/freeipa/issue/8793 Signed-off-by: Stanislav Levin <s...@altlinux.org> --- ipatests/test_integration/test_dnssec.py | 109 +++++++++++++++++++++-- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/ipatests/test_integration/test_dnssec.py b/ipatests/test_integration/test_dnssec.py index bae16120a7a..e1bfc48b5a3 100644 --- a/ipatests/test_integration/test_dnssec.py +++ b/ipatests/test_integration/test_dnssec.py @@ -4,17 +4,22 @@ from __future__ import absolute_import +import base64 import logging import re import subprocess import time +import textwrap import dns.dnssec import dns.name +import pytest +import yaml from ipatests.test_integration.base import IntegrationTest from ipatests.pytest_ipa.integration import tasks from ipatests.pytest_ipa.integration.firewall import Firewall +from ipaplatform.tasks import tasks as platform_tasks from ipaplatform.paths import paths from ipapython.dnsutil import DNSResolver @@ -350,11 +355,7 @@ def test_sign_root_zone(self): self.replicas[0].ip, root_zone, timeout=300 ), "Zone %s is not signed (replica)" % root_zone - def test_chain_of_trust(self): - """ - Validate signed DNS records, using our own signed root zone - :return: - """ + def test_delegation(self): dnszone_add_dnssec(self.master, example_test_zone) # delegation @@ -419,6 +420,10 @@ def test_chain_of_trust(self): rtype="DS" ), "No DS record of '%s' returned from replica" % example_test_zone + def test_chain_of_trust_drill(self): + """ + Validate signed DNS records, using our own signed root zone + """ # extract DSKEY from root zone ans = resolve_with_dnssec(self.master.ip, root_zone, rtype="DNSKEY") @@ -462,6 +467,100 @@ def test_chain_of_trust(self): self.master.run_command(args) self.replicas[0].run_command(args) + def test_chain_of_trust_delv(self): + """ + Validate signed DNS records, using our own signed root zone + """ + INITIAL_KEY_FMT = '%s initial-key %d %d %d "%s";' + + # delv reports its version on stderr + delv_version = self.master.run_command( + ["delv", "-v"] + ).stderr_text.rstrip().replace("delv ", "") + assert delv_version + + delv_version_parsed = platform_tasks.parse_ipa_version(delv_version) + if delv_version_parsed < platform_tasks.parse_ipa_version("9.16"): + pytest.skip( + f"Requires delv >= 9.16(+yaml), installed: '{delv_version}'" + ) + + # extract DSKEY from root zone + ans = resolve_with_dnssec( + self.master.ip, root_zone, rtype="DNSKEY" + ) + dnskey_rrset = ans.response.get_rrset( + ans.response.answer, + dns.name.from_text(root_zone), + dns.rdataclass.IN, + dns.rdatatype.DNSKEY, + ) + assert dnskey_rrset, "No DNSKEY records received" + + # export trust keys for root zone + initial_keys = [] + for key_rdata in dnskey_rrset: + if key_rdata.flags != 257: + continue # it is not KSK + + initial_keys.append( + INITIAL_KEY_FMT % ( + root_zone, + key_rdata.flags, + key_rdata.protocol, + key_rdata.algorithm, + base64.b64encode(key_rdata.key).decode("utf-8"), + ) + ) + + assert initial_keys, "No KSK returned from the root zone" + + trust_anchors = textwrap.dedent( + """\ + trust-anchors {{ + {initial_key} + }}; + """ + ).format(initial_key="\n".join(initial_keys)) + logger.debug("Root zone trust-anchors: %s", trust_anchors) + + # set trusted anchor for our root zone + for host in [self.master, self.replicas[0]]: + host.put_file_contents(paths.DNSSEC_TRUSTED_KEY, trust_anchors) + + # verify signatures + args = [ + "delv", + "+yaml", + "+nosplit", + "+vtrace", + "@127.0.0.1", + example_test_zone, + "-a", + paths.DNSSEC_TRUSTED_KEY, + "SOA", + ] + + # delv puts trace info on stderr + for host in [self.master, self.replicas[0]]: + result = host.run_command(args) + yaml_data = yaml.safe_load(result.stdout_text) + + query_name_abs = dns.name.from_text(example_test_zone) + root_zone_name = dns.name.from_text(root_zone) + query_name_rel = query_name_abs.relativize( + root_zone_name + ).to_text() + assert yaml_data["query_name"] == query_name_rel + assert yaml_data["status"] == "success" + + assert len(yaml_data["records"]) == 1 + fully_validated = yaml_data["records"][0]["fully_validated"] + fully_validated.sort() + assert len(fully_validated) == 2 + assert f"{example_test_zone} 1 IN RRSIG SOA" in fully_validated[0] + assert f"{example_test_zone} 1 IN SOA" in fully_validated[1] + def test_servers_use_localhost_as_dns(self): # check that localhost is set as DNS server for host in [self.master, self.replicas[0]]: From cdff9577ee6c1d7a7403134d632baca457dad670 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Thu, 20 May 2021 15:38:39 +0300 Subject: [PATCH 22/25] azure: Warn about extra and missing gating tests compared to PR-CI --- ipatests/azure/azure-pipelines.yml | 2 ++ ipatests/azure/scripts/gating_compare.py | 46 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 ipatests/azure/scripts/gating_compare.py diff --git a/ipatests/azure/azure-pipelines.yml b/ipatests/azure/azure-pipelines.yml index 46527cbd346..edf26ad77f8 100644 --- a/ipatests/azure/azure-pipelines.yml +++ b/ipatests/azure/azure-pipelines.yml @@ -57,6 +57,8 @@ jobs: definition: 'ipatests/azure/azure_definitions/base.yml' displayName: Generate Matrix for Base tests name: base_matrix + - script: python3 $(IPA_TESTS_SCRIPTS)/gating_compare.py + displayName: Check for consistency with PR-CI - job: Lint pool: diff --git a/ipatests/azure/scripts/gating_compare.py b/ipatests/azure/scripts/gating_compare.py new file mode 100644 index 00000000000..77439e7e45d --- /dev/null +++ b/ipatests/azure/scripts/gating_compare.py @@ -0,0 +1,46 @@ +import yaml + +PRCI_GATING = "ipatests/prci_definitions/gating.yaml" +AZURE_GATING = "ipatests/azure/azure_definitions/gating.yml" + +prci_tests = [] +azure_tests = [] + +SKIP_IN_AZURE_LIST = [ + "test_integration/test_authselect.py", # requires external DNS +] + +EXTRA_AZURE_LIST = [] + +with open(PRCI_GATING) as f: + prci_gating = yaml.safe_load(f) + for task in prci_gating["jobs"].values(): + job = task["job"] + if job["class"] == "RunPytest": + prci_tests.extend(job["args"]["test_suite"].split()) + + prci_tests.sort() + +with open(AZURE_GATING) as f: + azure_gating = yaml.safe_load(f) + for vm_jobs in azure_gating["vms"]: + for job in vm_jobs["vm_jobs"]: + azure_tests.extend(job["tests"]) + + azure_tests.sort() + +missing_in_azure = set(prci_tests) - set(azure_tests + SKIP_IN_AZURE_LIST) +if missing_in_azure: + print( + "##vso[task.logissue type=warning]" + "Missing gating tests in Azure Pipelines, compared to PR-CI", + missing_in_azure, + ) + +extra_in_azure = set(azure_tests) - set(prci_tests + EXTRA_AZURE_LIST) +if extra_in_azure: + print( + "##vso[task.logissue type=warning]" + "Extra gating tests in Azure Pipelines, compared to PR-CI", + extra_in_azure, + ) From 3486ce5da3293aac97f8548f3a26c045fea4a8dd Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Sat, 8 May 2021 12:55:09 +0300 Subject: [PATCH 23/25] azure: Re-balance tests envs --- .../azure/azure_definitions/gating-fedora.yml | 60 +++++++++++++------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/ipatests/azure/azure_definitions/gating-fedora.yml b/ipatests/azure/azure_definitions/gating-fedora.yml index 62ded1885a6..7c86e76dab0 100644 --- a/ipatests/azure/azure_definitions/gating-fedora.yml +++ b/ipatests/azure/azure_definitions/gating-fedora.yml @@ -21,10 +21,6 @@ vms: tests: - test_integration/test_external_ca.py::TestExternalCAInstall - - container_job: membermanager - tests: - - test_integration/test_membermanager.py - - vm_jobs: - container_job: InstallDNSSECFirst containers: @@ -50,13 +46,6 @@ vms: tests: - test_integration/test_external_ca.py::TestExternalCAProfileScenarios - # requires external DNS configuration, this is not set up yet - # - container_job: authselect - # containers: - # clients: 1 - # tests: - # - test_integration/test_authselect.py - - vm_jobs: - container_job: sudo containers: @@ -77,10 +66,6 @@ vms: - test_integration/test_topologies.py - test_integration/test_testconfig.py - - container_job: external_ca_SelfExternalSelf - tests: - - test_integration/test_external_ca.py::TestSelfExternalSelf - - container_job: external_ca_ExternalCAConstraints containers: clients: 1 @@ -88,11 +73,22 @@ vms: - test_integration/test_external_ca.py::TestExternalCAConstraints - vm_jobs: - # Requires more memory per container and fails reliably - # - container_job: commands - # tests: - # - test_integration/test_commands.py + - container_job: commands + containers: + replicas: 1 + clients: 1 + tests: + - test_integration/test_commands.py + + - container_job: upgrade + tests: + - test_integration/test_upgrade.py + - container_job: membermanager + tests: + - test_integration/test_membermanager.py + +- vm_jobs: - container_job: ServerReplicaCALessToCAFull tests: - test_integration/test_caless.py::TestServerReplicaCALessToCAFull @@ -105,8 +101,34 @@ vms: tests: - test_integration/test_replica_promotion.py::TestSubCAkeyReplication + - container_job: adtrust_install + tests: + - test_integration/test_adtrust_install.py + containers: + replicas: 1 + +- vm_jobs: - container_job: advise containers: clients: 1 tests: - test_integration/test_advise.py + + - container_job: cert + tests: + - test_integration/test_cert.py + containers: + replicas: 1 + clients: 1 + + - container_job: external_ca_SelfExternalSelf + tests: + - test_integration/test_external_ca.py::TestSelfExternalSelf + + # requires external DNS configuration, this is not set up yet + # - container_job: authselect + # containers: + # clients: 1 + # tests: + # - test_integration/test_authselect.py + # From 4313310bd4fdc258a4f3783763fc5e4239e4f230 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Fri, 21 May 2021 00:33:55 +0300 Subject: [PATCH 24/25] azure: coredump: Wait for systemd fully booted Otherwise, 'Check for coredumps' task fails with: ``` Verifying : samba-debugsource-2:4.14.4-0.fc34.x86_64 20/20 [Errno 2] No such file or directory: '/var/lib/dnf/rpmdb_lock.pid' Finishing: Check for coredumps ``` This is due to systemd-tmpfiles(not ready yet). --- ipatests/azure/scripts/wait-for-systemd.sh | 9 +++++++++ ipatests/azure/templates/test-jobs.yml | 5 +++++ 2 files changed, 14 insertions(+) create mode 100755 ipatests/azure/scripts/wait-for-systemd.sh diff --git a/ipatests/azure/scripts/wait-for-systemd.sh b/ipatests/azure/scripts/wait-for-systemd.sh new file mode 100755 index 00000000000..d598734430f --- /dev/null +++ b/ipatests/azure/scripts/wait-for-systemd.sh @@ -0,0 +1,9 @@ +#!/bin/bash -eux + +for i in $(seq 35) +do + systemctl is-active --quiet default.target && exit 0 + sleep 5 +done + +exit 1 diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index b8c5bcb93ea..dadb2e78268 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -128,6 +128,11 @@ steps: --name "$CONTAINER_COREDUMP" freeipa-azure-builder docker start "$CONTAINER_COREDUMP" + docker exec -t \ + "$CONTAINER_COREDUMP" \ + /bin/bash --noprofile --norc -eux \ + "${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/wait-for-systemd.sh" + docker exec -t \ --env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \ --env IPA_TESTS_SCRIPTS="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" \ From dd98c37293af63bbfb206b258870aac112313796 Mon Sep 17 00:00:00 2001 From: Stanislav Levin <s...@altlinux.org> Date: Mon, 10 May 2021 03:22:06 +0300 Subject: [PATCH 25/25] azure: Make it possible to adjust Docker resources per test env --- ipatests/azure/Dockerfiles/docker-compose.yml | 9 ++- .../azure/azure_definitions/base-fedora.yml | 11 +++ .../azure/azure_definitions/gating-fedora.yml | 76 ++++++++++++++++++- ipatests/azure/scripts/azure-run-tests.sh | 34 +++++++++ ipatests/azure/scripts/generate-matrix.py | 19 +++++ 5 files changed, 142 insertions(+), 7 deletions(-) diff --git a/ipatests/azure/Dockerfiles/docker-compose.yml b/ipatests/azure/Dockerfiles/docker-compose.yml index 5a8c613abdc..dc3707effa4 100644 --- a/ipatests/azure/Dockerfiles/docker-compose.yml +++ b/ipatests/azure/Dockerfiles/docker-compose.yml @@ -8,7 +8,8 @@ services: security_opt: - apparmor:unconfined - seccomp:./seccomp.json - mem_limit: "1900m" + mem_limit: "${IPA_TESTS_SERVER_MEM_LIMIT}" + memswap_limit: "${IPA_TESTS_SERVER_MEMSWAP_LIMIT}" volumes: - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd - ./ipa-test-config.yaml:/root/.ipa/ipa-test-config.yaml:ro @@ -25,7 +26,8 @@ services: security_opt: - apparmor:unconfined - seccomp:./seccomp.json - mem_limit: "1900m" + mem_limit: "${IPA_TESTS_REPLICA_MEM_LIMIT}" + memswap_limit: "${IPA_TESTS_REPLICA_MEMSWAP_LIMIT}" volumes: - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd - ${BUILD_REPOSITORY_LOCALPATH}:${IPA_TESTS_REPO_PATH}:ro @@ -40,7 +42,8 @@ services: security_opt: - apparmor:unconfined - seccomp:./seccomp.json - mem_limit: "536870912" + mem_limit: "${IPA_TESTS_CLIENT_MEM_LIMIT}" + memswap_limit: "${IPA_TESTS_CLIENT_MEMSWAP_LIMIT}" volumes: - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd - ${BUILD_REPOSITORY_LOCALPATH}:${IPA_TESTS_REPO_PATH}:ro diff --git a/ipatests/azure/azure_definitions/base-fedora.yml b/ipatests/azure/azure_definitions/base-fedora.yml index a5ae194002b..d32f76d2ada 100644 --- a/ipatests/azure/azure_definitions/base-fedora.yml +++ b/ipatests/azure/azure_definitions/base-fedora.yml @@ -1,3 +1,14 @@ +default_resources: + server: + mem_limit: "1800m" + memswap_limit: "2500m" + replica: + mem_limit: "1800m" + memswap_limit: "2500m" + client: + mem_limit: "512m" + memswap_limit: "1000m" + vms: - vm_jobs: - container_job: base diff --git a/ipatests/azure/azure_definitions/gating-fedora.yml b/ipatests/azure/azure_definitions/gating-fedora.yml index 7c86e76dab0..70c15834c5c 100644 --- a/ipatests/azure/azure_definitions/gating-fedora.yml +++ b/ipatests/azure/azure_definitions/gating-fedora.yml @@ -1,6 +1,22 @@ +default_resources: + server: + mem_limit: "2200m" + memswap_limit: "3000m" + replica: + mem_limit: "2200m" + memswap_limit: "3000m" + client: + mem_limit: "512m" + memswap_limit: "1000m" + vms: - vm_jobs: - container_job: InstallMaster + containers: + resources: + server: + mem_limit: "3200m" + memswap_limit: "4800m" tests: - test_integration/test_installation.py::TestInstallMaster @@ -14,6 +30,13 @@ vms: containers: replicas: 1 clients: 1 + resources: + replica: + mem_limit: "2200m" + memswap_limit: "3300m" + client: + mem_limit: "768m" + memswap_limit: "1024m" tests: - test_integration/test_forced_client_reenrollment.py @@ -25,12 +48,20 @@ vms: - container_job: InstallDNSSECFirst containers: replicas: 1 + resources: + replica: + mem_limit: "2400m" + memswap_limit: "3500m" tests: - test_integration/test_dnssec.py::TestInstallDNSSECFirst - container_job: simple_replication containers: replicas: 1 + resources: + replica: + mem_limit: "2400m" + memswap_limit: "3500m" tests: - test_integration/test_simple_replication.py @@ -50,6 +81,10 @@ vms: - container_job: sudo containers: clients: 1 + resources: + server: + mem_limit: "2600m" + memswap_limit: "3900m" tests: - test_integration/test_sudo.py @@ -57,6 +92,13 @@ vms: containers: replicas: 1 clients: 1 + resources: + replica: + mem_limit: "2200m" + memswap_limit: "3300m" + client: + mem_limit: "768m" + memswap_limit: "1024m" tests: - test_integration/test_external_ca.py::TestExternalCA @@ -69,6 +111,10 @@ vms: - container_job: external_ca_ExternalCAConstraints containers: clients: 1 + resources: + client: + mem_limit: "768m" + memswap_limit: "1024m" tests: - test_integration/test_external_ca.py::TestExternalCAConstraints @@ -77,13 +123,16 @@ vms: containers: replicas: 1 clients: 1 + resources: + server: + mem_limit: "3500m" + memswap_limit: "4000m" + client: + mem_limit: "768m" + memswap_limit: "1024m" tests: - test_integration/test_commands.py - - container_job: upgrade - tests: - - test_integration/test_upgrade.py - - container_job: membermanager tests: - test_integration/test_membermanager.py @@ -111,6 +160,10 @@ vms: - container_job: advise containers: clients: 1 + resources: + client: + mem_limit: "768m" + memswap_limit: "1024m" tests: - test_integration/test_advise.py @@ -120,11 +173,26 @@ vms: containers: replicas: 1 clients: 1 + resources: + replica: + mem_limit: "2300m" + memswap_limit: "3300m" - container_job: external_ca_SelfExternalSelf tests: - test_integration/test_external_ca.py::TestSelfExternalSelf +- vm_jobs: + - container_job: upgrade + containers: + resources: + server: + mem_limit: "2200m" + memswap_limit: "3300m" + tests: + - test_integration/test_upgrade.py + + # requires external DNS configuration, this is not set up yet # - container_job: authselect # containers: diff --git a/ipatests/azure/scripts/azure-run-tests.sh b/ipatests/azure/scripts/azure-run-tests.sh index 567d8996c39..e2291d14961 100755 --- a/ipatests/azure/scripts/azure-run-tests.sh +++ b/ipatests/azure/scripts/azure-run-tests.sh @@ -52,6 +52,28 @@ IPA_TESTS_SCRIPTS_OUT="${BUILD_REPOSITORY_LOCALPATH}/${IPA_TESTS_SCRIPTS}" IPA_TESTS_NETWORK_INTERNAL_VARNAME="IPA_TESTS_NETWORK_INTERNAL_${PROJECT_ID}" IPA_NETWORK_INTERNAL="${!IPA_TESTS_NETWORK_INTERNAL_VARNAME:-false}" +# Docker resources +# mem_limit +IPA_TESTS_SERVER_MEM_LIMIT_VARNAME="IPA_TESTS_SERVER_MEM_LIMIT_${PROJECT_ID}" +IPA_TESTS_SERVER_MEM_LIMIT="${!IPA_TESTS_SERVER_MEM_LIMIT_VARNAME:-2000m}" + +IPA_TESTS_REPLICA_MEM_LIMIT_VARNAME="IPA_TESTS_REPLICA_MEM_LIMIT_${PROJECT_ID}" +IPA_TESTS_REPLICA_MEM_LIMIT="${!IPA_TESTS_REPLICA_MEM_LIMIT_VARNAME:-2000m}" + +IPA_TESTS_CLIENT_MEM_LIMIT_VARNAME="IPA_TESTS_CLIENT_MEM_LIMIT_${PROJECT_ID}" +IPA_TESTS_CLIENT_MEM_LIMIT="${!IPA_TESTS_CLIENT_MEM_LIMIT_VARNAME:-512m}" + +# memswap_limit +IPA_TESTS_SERVER_MEMSWAP_LIMIT_VARNAME="IPA_TESTS_SERVER_MEMSWAP_LIMIT_${PROJECT_ID}" +IPA_TESTS_SERVER_MEMSWAP_LIMIT="${!IPA_TESTS_SERVER_MEMSWAP_LIMIT_VARNAME:-2500m}" + +IPA_TESTS_REPLICA_MEMSWAP_LIMIT_VARNAME="IPA_TESTS_REPLICA_MEMSWAP_LIMIT_${PROJECT_ID}" +IPA_TESTS_REPLICA_MEMSWAP_LIMIT="${!IPA_TESTS_REPLICA_MEMSWAP_LIMIT_VARNAME:-2500m}" + +IPA_TESTS_CLIENT_MEMSWAP_LIMIT_VARNAME="IPA_TESTS_CLIENT_MEMSWAP_LIMIT_${PROJECT_ID}" +IPA_TESTS_CLIENT_MEMSWAP_LIMIT="${!IPA_TESTS_CLIENT_MEMSWAP_LIMIT_VARNAME:-768m}" +# + IPA_TESTS_DOMAIN="${IPA_TESTS_DOMAIN:-ipa.test}" # bash4 IPA_TESTS_REALM="${IPA_TESTS_DOMAIN^^}" @@ -116,6 +138,12 @@ IPA_DOCKER_IMAGE="${IPA_DOCKER_IMAGE:-freeipa-azure-builder}" \ IPA_NETWORK="${IPA_NETWORK:-ipanet}" \ IPA_NETWORK_INTERNAL="$IPA_NETWORK_INTERNAL" \ IPA_IPV6_SUBNET="2001:db8:1:${PROJECT_ID}::/64" \ +IPA_TESTS_SERVER_MEM_LIMIT="$IPA_TESTS_SERVER_MEM_LIMIT" \ +IPA_TESTS_REPLICA_MEM_LIMIT="$IPA_TESTS_REPLICA_MEM_LIMIT" \ +IPA_TESTS_CLIENT_MEM_LIMIT="$IPA_TESTS_CLIENT_MEM_LIMIT" \ +IPA_TESTS_SERVER_MEMSWAP_LIMIT="$IPA_TESTS_SERVER_MEMSWAP_LIMIT" \ +IPA_TESTS_REPLICA_MEMSWAP_LIMIT="$IPA_TESTS_REPLICA_MEMSWAP_LIMIT" \ +IPA_TESTS_CLIENT_MEMSWAP_LIMIT="$IPA_TESTS_CLIENT_MEMSWAP_LIMIT" \ docker-compose -p "$PROJECT_ID" up \ --scale replica="$IPA_TESTS_REPLICAS" \ --scale client="$IPA_TESTS_CLIENTS" \ @@ -223,6 +251,12 @@ IPA_DOCKER_IMAGE="${IPA_DOCKER_IMAGE:-freeipa-azure-builder}" \ IPA_NETWORK="${IPA_NETWORK:-ipanet}" \ IPA_NETWORK_INTERNAL="$IPA_NETWORK_INTERNAL" \ IPA_IPV6_SUBNET="2001:db8:1:${PROJECT_ID}::/64" \ +IPA_TESTS_SERVER_MEM_LIMIT="$IPA_TESTS_SERVER_MEM_LIMIT" \ +IPA_TESTS_REPLICA_MEM_LIMIT="$IPA_TESTS_REPLICA_MEM_LIMIT" \ +IPA_TESTS_CLIENT_MEM_LIMIT="$IPA_TESTS_CLIENT_MEM_LIMIT" \ +IPA_TESTS_SERVER_MEMSWAP_LIMIT="$IPA_TESTS_SERVER_MEMSWAP_LIMIT" \ +IPA_TESTS_REPLICA_MEMSWAP_LIMIT="$IPA_TESTS_REPLICA_MEMSWAP_LIMIT" \ +IPA_TESTS_CLIENT_MEMSWAP_LIMIT="$IPA_TESTS_CLIENT_MEMSWAP_LIMIT" \ docker-compose -p "$PROJECT_ID" down popd diff --git a/ipatests/azure/scripts/generate-matrix.py b/ipatests/azure/scripts/generate-matrix.py index 03d97aa7dc2..fb596edb547 100644 --- a/ipatests/azure/scripts/generate-matrix.py +++ b/ipatests/azure/scripts/generate-matrix.py @@ -1,4 +1,6 @@ import argparse +import copy +import pprint import json import yaml @@ -13,6 +15,7 @@ with open(args.azure_template) as f: data = yaml.safe_load(f) + default_resources = data["default_resources"] matrix_jobs = {} for vm in data['vms']: vm_jobs = vm['vm_jobs'] @@ -33,14 +36,28 @@ ) containers = vm_job.get('containers') + cont_resources = copy.deepcopy(default_resources) replicas = 0 clients = 0 if containers: replicas = containers.get('replicas', 0) clients = containers.get('clients', 0) + + resources = containers.get("resources") + if resources: + for cont in ["server", "replica", "client"]: + cont_resources[cont].update( + resources.get(cont, {}) + ) + jobs[f'ipa_tests_replicas_{job_id}'] = replicas jobs[f'ipa_tests_clients_{job_id}'] = clients + for cont in ["server", "replica", "client"]: + for res in ["mem_limit", "memswap_limit"]: + key = f"ipa_tests_{cont}_{res}_{job_id}" + jobs[key] = cont_resources[cont][res] + if len(vm_jobs) > args.max_azure_env_jobs: raise ValueError( f"Number of defined jobs:{len(vm_jobs)} within VM:'{job_name}'" @@ -49,5 +66,7 @@ if job_name in matrix_jobs: raise ValueError(f"Environment names should be unique:{job_name}") matrix_jobs[job_name] = jobs + + pprint.pprint(matrix_jobs) print("##vso[task.setVariable variable=matrix;isOutput=true]" + json.dumps(matrix_jobs))
_______________________________________________ FreeIPA-devel mailing list -- freeipa-devel@lists.fedorahosted.org To unsubscribe send an email to freeipa-devel-le...@lists.fedorahosted.org Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/ List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedorahosted.org/archives/list/freeipa-devel@lists.fedorahosted.org Do not reply to spam on the list, report it: https://pagure.io/fedora-infrastructure
