URL: https://github.com/freeipa/freeipa/pull/2592 Author: wladich Title: #2592: [Backport][ipa-4-7] ipatests: add test for ipa-restore in multi-master configuration Action: opened
PR body: """ This PR is manual backport of https://github.com/freeipa/freeipa/pull/2588 please wait for CI before pushing and do not forget about backport to branches specified with labels if needed. In case of questions or problems contact @wladich who is author of the original PR. """ To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/2592/head:pr2592 git checkout pr2592
From 476a24409484510e9fbe2fb197a40f6e6a1dcbe1 Mon Sep 17 00:00:00 2001 From: Sergey Orlov <sor...@redhat.com> Date: Wed, 7 Nov 2018 11:23:05 +0100 Subject: [PATCH] ipatests: add test for ipa-restore in multi-master configuration Test ensures that after ipa-restore on the master, the replica can be re-synchronized and a new replica can be created. https://pagure.io/freeipa/issue/7455 --- ipatests/pytest_ipa/integration/tasks.py | 52 +++-- .../test_backup_and_restore.py | 189 +++++++++++++++--- 2 files changed, 195 insertions(+), 46 deletions(-) diff --git a/ipatests/pytest_ipa/integration/tasks.py b/ipatests/pytest_ipa/integration/tasks.py index 7a944c5eee..fe23ebad2a 100644 --- a/ipatests/pytest_ipa/integration/tasks.py +++ b/ipatests/pytest_ipa/integration/tasks.py @@ -1104,19 +1104,26 @@ def _entries_to_ldif(entries): return io.getvalue() -def wait_for_replication(ldap, timeout=30): - """Wait until updates on all replication agreements are done (or failed) +def wait_for_replication(ldap, timeout=30, + target_status_re=r'^0 |^Error \(0\) ', + raise_on_timeout=False): + """Wait for all replication agreements to reach desired state + With defaults waits until updates on all replication agreements are + done (or failed) and exits without exception :param ldap: LDAP client autenticated with necessary rights to read the mapping tree :param timeout: Maximum time to wait, in seconds + :param target_status_re: Regexp of status to wait for + :param raise_on_timeout: if True, raises AssertionError if status not + reached in specified time Note that this waits for updates originating on this host, not those coming from other hosts. """ logger.debug('Waiting for replication to finish') - for i in range(timeout): - time.sleep(1) + start = time.time() + while True: status_attr = 'nsds5replicaLastUpdateStatus' progress_attr = 'nsds5replicaUpdateInProgress' entries = ldap.get_entries( @@ -1124,25 +1131,24 @@ def wait_for_replication(ldap, timeout=30): filter='(objectclass=nsds5replicationagreement)', attrs_list=[status_attr, progress_attr]) logger.debug('Replication agreements: \n%s', _entries_to_ldif(entries)) - if any( - not ( - # older DS format - e.single_value[status_attr].startswith('0 ') or - # newer DS format - e.single_value[status_attr].startswith('Error (0) ') - ) - for e in entries - ): - logger.error('Replication error') - continue + statuses = [entry.single_value[status_attr] for entry in entries] + wrong_statuses = [s for s in statuses + if not re.match(target_status_re, s)] if any(e.single_value[progress_attr] == 'TRUE' for e in entries): - logger.debug('Replication in progress (waited %s/%ss)', - i, timeout) + msg = 'Replication not finished' + logger.debug(msg) + elif wrong_statuses: + msg = 'Unexpected replication status: %s' % wrong_statuses[0] + logger.debug(msg) else: logger.debug('Replication finished') + return + if time.time() - start > timeout: + logger.error('Giving up wait for replication to finish') + if raise_on_timeout: + raise AssertionError(msg) break - else: - logger.error('Giving up wait for replication to finish') + time.sleep(1) def wait_for_cleanallruv_tasks(ldap, timeout=30): @@ -1538,3 +1544,11 @@ def strip_cert_header(pem): return s.group(1) else: return pem + + +def user_add(host, login): + host.run_command([ + "ipa", "user-add", login, + "--first", "test", + "--last", "user" + ]) diff --git a/ipatests/test_integration/test_backup_and_restore.py b/ipatests/test_integration/test_backup_and_restore.py index 063f1d0e67..71d692dde7 100644 --- a/ipatests/test_integration/test_backup_and_restore.py +++ b/ipatests/test_integration/test_backup_and_restore.py @@ -24,6 +24,7 @@ import re import contextlib from tempfile import NamedTemporaryFile +import pytest from ipaplatform.constants import constants from ipaplatform.paths import paths @@ -137,6 +138,8 @@ def restore_checker(host): yield + tasks.kinit_admin(host) + for (check, assert_func), expected in zip(CHECKS, results): logger.info('Checking result for %s', check.__name__) got = check(host) @@ -164,6 +167,24 @@ def backup(host): raise AssertionError('Backup directory not found in output') +@pytest.yield_fixture(scope="function") +def cert_sign_request(request): + master = request.instance.master + hosts = [master] + request.instance.replicas + csrs = {} + for host in hosts: + request_path = host.run_command(['mktemp']).stdout_text.strip() + openssl_command = [ + 'openssl', 'req', '-new', '-nodes', '-out', request_path, + '-subj', '/CN=' + master.hostname + ] + host.run_command(openssl_command) + csrs[host.hostname] = request_path + yield csrs + for host in hosts: + host.run_command(['rm', csrs[host.hostname]]) + + class TestBackupAndRestore(IntegrationTest): topology = 'star' @@ -446,36 +467,92 @@ def test_full_backup_reinstall_restore_with_vault(self): class TestBackupAndRestoreWithReplica(IntegrationTest): - """Regression test for https://pagure.io/freeipa/issue/7234"""; - num_replicas = 1 + """Regression tests for issues 7234 and 7455 + + https://pagure.io/freeipa/issue/7234 + - check that oddjobd service is started after restore + - check new replica setup after restore + https://pagure.io/freeipa/issue/7455 + check that after restore and replication reinitialization + - users and CA data are at state before backup + - CA can be installed on existing replica + - new replica with CA can be setup + """ + num_replicas = 2 topology = "star" @classmethod def install(cls, mh): + cls.replica1 = cls.replicas[0] + cls.replica2 = cls.replicas[1] if cls.domain_level is None: domain_level = cls.master.config.domain_level else: domain_level = cls.domain_level - - if cls.topology is None: - return - else: - tasks.install_topo( - cls.topology, cls.master, [], - cls.clients, domain_level - ) - - def test_full_backup_and_restore_with_replica(self): - replica = self.replicas[0] + # Configure only master and one replica. + # Replica is configured without CA + tasks.install_topo( + cls.topology, cls.master, [cls.replica1], + cls.clients, domain_level, + setup_replica_cas=False + ) + + def get_users(self, host): + res = host.run_command(['ipa', 'user-find']) + users = set() + for line in res.stdout_text.splitlines(): + k, _unused, v = line.strip().partition(': ') + if k == 'User login': + users.add(v) + return users + + def check_replication_error(self, host): + status = r'Error \(19\) Replication error acquiring replica: ' \ + 'Replica has different database generation ID' + tasks.wait_for_replication( + host.ldap_connect(), target_status_re=status, + raise_on_timeout=True) + + def check_replication_success(self, host): + status = r'Error \(0\) Replica acquired successfully: ' \ + 'Incremental update succeeded' + tasks.wait_for_replication( + host.ldap_connect(), target_status_re=status, + raise_on_timeout=True) + + def request_test_service_cert(self, host, request_path, + expect_connection_error=False): + res = host.run_command([ + 'ipa', 'cert-request', '--principal=TEST/' + self.master.hostname, + request_path + ], raiseonerr=not expect_connection_error) + if expect_connection_error: + assert (1 == res.returncode and + '[Errno 111] Connection refused' in res.stderr_text) + + def test_full_backup_and_restore_with_replica(self, cert_sign_request): + # check prerequisites + self.check_replication_success(self.master) + self.check_replication_success(self.replica1) + + self.master.run_command( + ['ipa', 'service-add', 'TEST/' + self.master.hostname]) + + tasks.user_add(self.master, 'test1_master') + tasks.user_add(self.replica1, 'test1_replica') with restore_checker(self.master): backup_path = backup(self.master) - logger.info("Backup path for %s is %s", self.master, backup_path) + # change data after backup + self.master.run_command(['ipa', 'user-del', 'test1_master']) + self.replica1.run_command(['ipa', 'user-del', 'test1_replica']) + tasks.user_add(self.master, 'test2_master') + tasks.user_add(self.replica1, 'test2_replica') - self.master.run_command([ - "ipa-server-install", "--uninstall", "-U" - ]) + # simulate master crash + self.master.run_command(['ipactl', 'stop']) + tasks.uninstall_master(self.master, clean=False) logger.info("Stopping and disabling oddjobd service") self.master.run_command([ @@ -485,18 +562,76 @@ def test_full_backup_and_restore_with_replica(self): "systemctl", "disable", "oddjobd" ]) - self.master.run_command( - ["ipa-restore", backup_path], - stdin_text='yes' - ) + self.master.run_command(['ipa-restore', '-U', backup_path]) - status = self.master.run_command([ - "systemctl", "status", "oddjobd" - ]) - assert "active (running)" in status.stdout_text + status = self.master.run_command([ + "systemctl", "status", "oddjobd" + ]) + assert "active (running)" in status.stdout_text + + # replication should not work after restoration + # create users to force master and replica to try to replicate + tasks.user_add(self.master, 'test3_master') + tasks.user_add(self.replica1, 'test3_replica') + self.check_replication_error(self.master) + self.check_replication_error(self.replica1) + assert {'admin', 'test1_master', 'test1_replica', 'test3_master'} == \ + self.get_users(self.master) + assert {'admin', 'test2_master', 'test2_replica', 'test3_replica'} == \ + self.get_users(self.replica1) + + # reestablish and check replication + self.replica1.run_command(['ipa-replica-manage', 're-initialize', + '--from', self.master.hostname]) + # create users to force master and replica to try to replicate + tasks.user_add(self.master, 'test4_master') + tasks.user_add(self.replica1, 'test4_replica') + self.check_replication_success(self.master) + self.check_replication_success(self.replica1) + assert {'admin', 'test1_master', 'test1_replica', + 'test3_master', 'test4_master', 'test4_replica'} == \ + self.get_users(self.master) + assert {'admin', 'test1_master', 'test1_replica', + 'test3_master', 'test4_master', 'test4_replica'} == \ + self.get_users(self.replica1) + + # CA on master should be accesible from master and replica + self.request_test_service_cert( + self.master, cert_sign_request[self.master.hostname]) + self.request_test_service_cert( + self.replica1, cert_sign_request[self.replica1.hostname]) + + # replica should not be able to sign certificates without CA on master + self.master.run_command(['ipactl', 'stop']) + try: + self.request_test_service_cert( + self.replica1, cert_sign_request[self.replica1.hostname], + expect_connection_error=True) + finally: + self.master.run_command(['ipactl', 'start']) - tasks.install_replica(self.master, replica) - check_replication(self.master, replica, "testuser1") + tasks.install_ca(self.replica1) + + # now replica should be able to sign certificates without CA on master + self.master.run_command(['ipactl', 'stop']) + self.request_test_service_cert( + self.replica1, cert_sign_request[self.replica1.hostname]) + self.master.run_command(['ipactl', 'start']) + + # check installation of new replica + tasks.install_replica(self.master, self.replica2, setup_ca=True) + check_replication(self.master, self.replica2, "testuser") + + # new replica should be able to sign certificates without CA on master + # and old replica + self.master.run_command(['ipactl', 'stop']) + self.replica1.run_command(['ipactl', 'stop']) + try: + self.request_test_service_cert( + self.replica2, cert_sign_request[self.replica2.hostname]) + finally: + self.replica1.run_command(['ipactl', 'start']) + self.master.run_command(['ipactl', 'start']) class TestUserRootFilesOwnershipPermission(IntegrationTest):
_______________________________________________ FreeIPA-devel mailing list -- freeipa-devel@lists.fedorahosted.org To unsubscribe send an email to freeipa-devel-le...@lists.fedorahosted.org Fedora Code of Conduct: https://getfedora.org/code-of-conduct.html List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedorahosted.org/archives/list/freeipa-devel@lists.fedorahosted.org
