URL: https://github.com/freeipa/freeipa/pull/4049 Author: frasertweedale Title: #4049: Fix nightly CI regressions in CA-less tests Action: opened
PR body: """ ``` 118eadef6 (Fraser Tweedale, 19 minutes ago) Fix test regressions caused by certificate validation changes Some integration tests (that were enabled in nightly CI but not PR-CI) are failing due to changes in the error messages. Update the error message assertions to get these tests going again. Part of: https://pagure.io/freeipa/issue/8142 2e863efb5 (Fraser Tweedale, 30 minutes ago) ipatests: assert_error: allow regexp match Enhance the assert_error subroutine to provide regular expression matching against the command's stderr output, in additional to substring match. Part of: https://pagure.io/freeipa/issue/8142 ``` """ To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/4049/head:pr4049 git checkout pr4049
From 2e863efb52aeb2f1acfde05585462a386fc105d3 Mon Sep 17 00:00:00 2001 From: Fraser Tweedale <ftwee...@redhat.com> Date: Tue, 17 Dec 2019 14:48:35 +1100 Subject: [PATCH 1/3] ipatests: assert_error: allow regexp match Enhance the assert_error subroutine to provide regular expression matching against the command's stderr output, in additional to substring match. Part of: https://pagure.io/freeipa/issue/8142 --- ipatests/pytest_ipa/integration/tasks.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ipatests/pytest_ipa/integration/tasks.py b/ipatests/pytest_ipa/integration/tasks.py index 4b9947cc9e..6b7362a150 100644 --- a/ipatests/pytest_ipa/integration/tasks.py +++ b/ipatests/pytest_ipa/integration/tasks.py @@ -1604,9 +1604,19 @@ def upload_temp_contents(host, contents, encoding='utf-8'): return tmpname -def assert_error(result, stderr_text, returncode=None): - "Assert that `result` command failed and its stderr contains `stderr_text`" - assert stderr_text in result.stderr_text, result.stderr_text +def assert_error(result, pattern, returncode=None): + """ + Assert that `result` command failed and its stderr contains ``pattern``. + ``pattern`` may be a ``str`` or a ``re.Pattern`` (regular expression). + + """ + if isinstance(pattern, re.Pattern): + assert pattern.search(result.stderr_text), \ + f"pattern {pattern} not found in stderr {result.stderr_text!r}" + else: + assert stderr_text in result.stderr_text, \ + f"substring {pattern} not found in stderr {result.stderr_text!r}" + if returncode is not None: assert result.returncode == returncode else: From 118eadef6c4779b0a75a7ac6a986eb9497fc03c4 Mon Sep 17 00:00:00 2001 From: Fraser Tweedale <ftwee...@redhat.com> Date: Tue, 17 Dec 2019 14:59:19 +1100 Subject: [PATCH 2/3] Fix test regressions caused by certificate validation changes Some integration tests (that were enabled in nightly CI but not PR-CI) are failing due to changes in the error messages. Update the error message assertions to get these tests going again. Part of: https://pagure.io/freeipa/issue/8142 --- ipatests/test_integration/test_caless.py | 69 +++++++++++++----------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/ipatests/test_integration/test_caless.py b/ipatests/test_integration/test_caless.py index ab0c886a65..928f5cf742 100644 --- a/ipatests/test_integration/test_caless.py +++ b/ipatests/test_integration/test_caless.py @@ -22,6 +22,7 @@ import functools import logging import os +import re import tempfile import shutil import glob @@ -50,7 +51,6 @@ assert_error = tasks.assert_error NSS_INVALID_FMT = "certutil: certificate is invalid: %s" -CERT_EXPIRED_MSG = NSS_INVALID_FMT % "Peer's Certificate has expired." BAD_USAGE_MSG = NSS_INVALID_FMT % ("Certificate key usage inadequate for " "attempted operation.") @@ -576,10 +576,12 @@ def test_expired_http(self): result = self.install_server(http_pkcs12='http.p12', dirsrv_pkcs12='dirsrv.p12') - assert_error(result, - 'The server certificate in {dir}/http.p12 is not valid: ' - '{err}'.format(dir=self.master.config.test_dir, - err=CERT_EXPIRED_MSG)) + + pattern = re.compile( + r'The server certificate in {dir}/http\.p12 is not valid: ' + '.*has expired'.format(dir=re.escape(self.master.config.test_dir)) + ) + assert_error(result, pattern) @server_install_teardown def test_expired_ds(self): @@ -591,10 +593,12 @@ def test_expired_ds(self): result = self.install_server(http_pkcs12='http.p12', dirsrv_pkcs12='dirsrv.p12') - assert_error(result, - 'The server certificate in {dir}/dirsrv.p12 is not ' - 'valid: {err}'.format(dir=self.master.config.test_dir, - err=CERT_EXPIRED_MSG)) + + pattern = re.compile( + r'The server certificate in {dir}/dirsrv\.p12 is not valid: ' + '.*has expired'.format(dir=re.escape(self.master.config.test_dir)) + ) + assert_error(result, pattern) @server_install_teardown def test_http_bad_usage(self): @@ -918,24 +922,28 @@ def test_expired_http(self): result = self.prepare_replica(http_pkcs12='http.p12', dirsrv_pkcs12='dirsrv.p12') - assert_error(result, - 'The server certificate in {dir}/http.p12 is not ' - 'valid: {err}'.format(dir=self.master.config.test_dir, - err=CERT_EXPIRED_MSG)) + + pattern = re.compile( + r'The server certificate in {dir}/http\.p12 is not valid: ' + '.*has expired'.format(dir=re.escape(self.master.config.test_dir)) + ) + assert_error(result, pattern) @replica_install_teardown def test_expired_ds(self): "IPA replica install with expired DS certificate" - self.create_pkcs12('ca1/replica-expired', filename='http.p12') - self.create_pkcs12('ca1/replica', filename='dirsrv.p12') + self.create_pkcs12('ca1/replica', filename='http.p12') + self.create_pkcs12('ca1/replica-expired', filename='dirsrv.p12') result = self.prepare_replica(http_pkcs12='http.p12', dirsrv_pkcs12='dirsrv.p12') - assert_error(result, - 'The server certificate in {dir}/http.p12 is not ' - 'valid: {err}'.format(dir=self.master.config.test_dir, - err=CERT_EXPIRED_MSG)) + + pattern = re.compile( + r'The server certificate in {dir}/dirsrv\.p12 is not valid: ' + '.*has expired'.format(dir=re.escape(self.master.config.test_dir)) + ) + assert_error(result, pattern) @replica_install_teardown def test_http_bad_usage(self): @@ -1330,21 +1338,20 @@ def test_invalid_ds_cn(self): 'The server certificate in server.p12 is not valid: ' 'invalid for server %s' % self.master.hostname) - def test_expired_http(self): - "Install new expired HTTP certificate" + def _test_expired_service_cert(self, w_or_d): + """Install new expired HTTP/DS certificate.""" + result = self.certinstall(w_or_d, 'ca1/server-expired') + pattern = re.compile( + r'The server certificate in server\.p12 is not valid: ' + '.*has expired' + ) + assert_error(result, pattern) - result = self.certinstall('w', 'ca1/server-expired') - assert_error(result, - 'The server certificate in server.p12 is not valid: {err}' - .format(err=CERT_EXPIRED_MSG)) + def test_expired_http(self): + self._test_expired_service_cert('w') def test_expired_ds(self): - "Install new expired DS certificate" - - result = self.certinstall('d', 'ca1/server-expired') - assert_error(result, - 'The server certificate in server.p12 is not valid: {err}' - .format(err=CERT_EXPIRED_MSG)) + self._test_expired_service_cert('d') def test_http_bad_usage(self): "Install new HTTP certificate with invalid key usage" From c76b29aa9479f5db20da510d8726015b4852eeff Mon Sep 17 00:00:00 2001 From: Fraser Tweedale <ftwee...@redhat.com> Date: Tue, 17 Dec 2019 15:06:06 +1100 Subject: [PATCH 3/3] temp commit --- .freeipa-pr-ci.yaml | 2 +- ipatests/prci_definitions/temp_commit.yaml | 32 +++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.freeipa-pr-ci.yaml b/.freeipa-pr-ci.yaml index abcf8c5b63..8065669008 120000 --- a/.freeipa-pr-ci.yaml +++ b/.freeipa-pr-ci.yaml @@ -1 +1 @@ -ipatests/prci_definitions/gating.yaml \ No newline at end of file +ipatests/prci_definitions/temp_commit.yaml \ No newline at end of file diff --git a/ipatests/prci_definitions/temp_commit.yaml b/ipatests/prci_definitions/temp_commit.yaml index 5afbe014f9..0a6c22edc2 100644 --- a/ipatests/prci_definitions/temp_commit.yaml +++ b/ipatests/prci_definitions/temp_commit.yaml @@ -57,14 +57,38 @@ jobs: timeout: 1800 topology: *build - fedora-latest/temp_commit: + fedora-latest/test_caless_TestCertInstall: requires: [fedora-latest/build] priority: 50 job: class: RunPytest args: build_url: '{fedora-latest/build_url}' - test_suite: test_integration/test_REPLACEME.py + test_suite: test_integration/test_caless.py::TestCertInstall template: *ci-master-latest - timeout: 3600 - topology: *master_1repl_1client + timeout: 5400 + topology: *master_1repl + + fedora-latest/test_caless_TestServerInstall: + requires: [fedora-latest/build] + priority: 50 + job: + class: RunPytest + args: + build_url: '{fedora-latest/build_url}' + test_suite: test_integration/test_caless.py::TestServerInstall + template: *ci-master-latest + timeout: 12000 + topology: *master_1repl + + fedora-latest/test_caless_TestReplicaInstall: + requires: [fedora-latest/build] + priority: 50 + job: + class: RunPytest + args: + build_url: '{fedora-latest/build_url}' + test_suite: test_integration/test_caless.py::TestReplicaInstall + template: *ci-master-latest + timeout: 5400 + topology: *master_1repl
_______________________________________________ FreeIPA-devel mailing list -- freeipa-devel@lists.fedorahosted.org To unsubscribe send an email to freeipa-devel-le...@lists.fedorahosted.org Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/ List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedorahosted.org/archives/list/freeipa-devel@lists.fedorahosted.org
