Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package crmsh for openSUSE:Factory checked
in at 2022-11-23 09:48:02
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/crmsh (Old)
and /work/SRC/openSUSE:Factory/.crmsh.new.1597 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "crmsh"
Wed Nov 23 09:48:02 2022 rev:267 rq:1037344 version:4.4.1+20221122.102a8e11
Changes:
--------
--- /work/SRC/openSUSE:Factory/crmsh/crmsh.changes 2022-11-16
15:43:54.979958006 +0100
+++ /work/SRC/openSUSE:Factory/.crmsh.new.1597/crmsh.changes 2022-11-23
09:48:27.999154228 +0100
@@ -1,0 +2,21 @@
+Tue Nov 22 14:36:03 UTC 2022 - xli...@suse.com
+
+- Update to version 4.4.1+20221122.102a8e11:
+ * Dev: workflows: add behave test `healthcheck`
+ * Dev: behave: add functional test for previous changes
+ * Dev: upgradeutil: change the format of seq from int to major.minor
+ * Dev: unittest: move tests to test_healthcheck
+ * Dev: bootstrap: fix passwordless ssh authentication for hacluster
automatically when a new node is joining the cluster (bsc#1201785)
+ * Dev: refactor: extract healthcheck module from upgradeutil
+ * Fix: testcases: fix shadow cib tests for previous changes.
+ * Fix: testcases: add no_reg option for utils.list_cluster_nodes
+ * Dev: unittest: add new tests for upgradeutil
+ * Dev: upgradeutil: automated init ssh passwordless auth for hacluster after
upgrading (bsc#1201785)
+
+-------------------------------------------------------------------
+Tue Nov 22 10:19:09 UTC 2022 - nicholas.y...@suse.com
+
+- Update to version 4.4.1+20221122.20aa6e8e:
+ * Dev: workflows: update actions version
+
+-------------------------------------------------------------------
Old:
----
crmsh-4.4.1+20221116.4faefec3.tar.bz2
New:
----
crmsh-4.4.1+20221122.102a8e11.tar.bz2
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ crmsh.spec ++++++
--- /var/tmp/diff_new_pack.BKQzyS/_old 2022-11-23 09:48:28.779158298 +0100
+++ /var/tmp/diff_new_pack.BKQzyS/_new 2022-11-23 09:48:28.783158319 +0100
@@ -36,7 +36,7 @@
Summary: High Availability cluster command-line interface
License: GPL-2.0-or-later
Group: %{pkg_group}
-Version: 4.4.1+20221116.4faefec3
+Version: 4.4.1+20221122.102a8e11
Release: 0
URL: http://crmsh.github.io
Source0: %{name}-%{version}.tar.bz2
++++++ _servicedata ++++++
--- /var/tmp/diff_new_pack.BKQzyS/_old 2022-11-23 09:48:28.847158653 +0100
+++ /var/tmp/diff_new_pack.BKQzyS/_new 2022-11-23 09:48:28.851158674 +0100
@@ -9,7 +9,7 @@
</service>
<service name="tar_scm">
<param name="url">https://github.com/ClusterLabs/crmsh.git</param>
- <param
name="changesrevision">4faefec3264baba80b0f2bd59aa718280062f7c2</param>
+ <param
name="changesrevision">5baaacb0a20a8ed89adaa403a50dacde998688f6</param>
</service>
</servicedata>
(No newline at EOF)
++++++ crmsh-4.4.1+20221116.4faefec3.tar.bz2 ->
crmsh-4.4.1+20221122.102a8e11.tar.bz2 ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/crmsh-4.4.1+20221116.4faefec3/.github/workflows/crmsh-ci.yml
new/crmsh-4.4.1+20221122.102a8e11/.github/workflows/crmsh-ci.yml
--- old/crmsh-4.4.1+20221116.4faefec3/.github/workflows/crmsh-ci.yml
2022-11-16 04:36:27.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/.github/workflows/crmsh-ci.yml
2022-11-22 15:17:41.000000000 +0100
@@ -18,9 +18,9 @@
jobs:
general_check:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: check data-manifest
run: |
./update-data-manifest.sh
@@ -33,16 +33,16 @@
}
unit_test:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ['3.6', '3.8', '3.10']
fail-fast: false
timeout-minutes: 5
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up Python
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
@@ -54,10 +54,10 @@
tox -v -e${{ matrix.python-version }}
functional_test_crm_report_bugs:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for crm_report
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -65,10 +65,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF crm_report_bugs`
functional_test_bootstrap_bugs:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for bootstrap bugs
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -76,10 +76,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF bootstrap_bugs`
functional_test_bootstrap_common:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for bootstrap common
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -87,10 +87,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF bootstrap_init_join_remove`
functional_test_bootstrap_options:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for bootstrap options
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -98,10 +98,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF bootstrap_options`
functional_test_qdevice_setup_remove:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for qdevice setup and remove
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -109,10 +109,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF qdevice_setup_remove`
functional_test_qdevice_options:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for qdevice options
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -120,10 +120,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF qdevice_options`
functional_test_qdevice_validate:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for qdevice validate
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -131,10 +131,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF qdevice_validate`
functional_test_qdevice_user_case:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for qdevice user case
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -142,10 +142,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF qdevice_usercase`
functional_test_resource_subcommand:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for resource subcommand
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -153,10 +153,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF resource_failcount resource_set`
functional_test_configure_sublevel:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for configure sublevel bugs
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -164,10 +164,10 @@
$DOCKER_SCRIPT `$GET_INDEX_OF configure_bugs`
functional_test_constraints_bugs:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for constraints bugs
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
@@ -175,21 +175,32 @@
$DOCKER_SCRIPT `$GET_INDEX_OF constraints_bugs`
functional_test_geo_cluster:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: functional test for geo cluster
run: |
echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
sudo systemctl restart docker.service
$DOCKER_SCRIPT `$GET_INDEX_OF geo_setup`
+ functional_test_healthcheck:
+ runs-on: ubuntu-20.04
+ timeout-minutes: 40
+ steps:
+ - uses: actions/checkout@v3
+ - name: functional test for healthcheck
+ run: |
+ echo '{ "exec-opts": ["native.cgroupdriver=systemd"] }' | sudo tee
/etc/docker/daemon.json
+ sudo systemctl restart docker.service
+ $DOCKER_SCRIPT `$GET_INDEX_OF healthcheck`
+
original_regression_test:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: original regression test
run: |
$DOCKER_SCRIPT `$GET_INDEX_OF "regression test"`
@@ -209,11 +220,12 @@
functional_test_configure_sublevel,
functional_test_constraints_bugs,
functional_test_geo_cluster,
+ functional_test_healthcheck,
original_regression_test]
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 10
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: delivery process
if: github.repository == 'ClusterLabs/crmsh' && github.ref ==
'refs/heads/master' && github.event_name == 'push'
run: |
@@ -229,10 +241,10 @@
submit:
needs: delivery
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 10
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: submit process
if: github.repository == 'ClusterLabs/crmsh' && github.ref ==
'refs/heads/master' && github.event_name == 'push'
run: |
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/crmsh-4.4.1+20221116.4faefec3/crmsh/bootstrap.py
new/crmsh-4.4.1+20221122.102a8e11/crmsh/bootstrap.py
--- old/crmsh-4.4.1+20221116.4faefec3/crmsh/bootstrap.py 2022-11-16
04:36:27.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/crmsh/bootstrap.py 2022-11-22
15:17:41.000000000 +0100
@@ -25,6 +25,7 @@
from pathlib import Path
from contextlib import contextmanager
from . import config
+from . import upgradeutil
from . import utils
from . import xmlutil
from .cibconfig import mkset_obj, cib_factory
@@ -1251,6 +1252,10 @@
_context.sbd_manager.sbd_init()
+def init_upgradeutil():
+ upgradeutil.force_set_local_upgrade_seq()
+
+
def init_ocfs2():
"""
OCFS2 configure process
@@ -1426,7 +1431,14 @@
if user == "root":
copy_ssh_key(public_key, user, remote_node)
else:
- append_to_remote_file(public_key, remote_node, authorized_file)
+ try:
+ append_to_remote_file(public_key, remote_node, authorized_file)
+ except ValueError:
+ utils.get_stdout_or_raise_error(
+ '/usr/bin/env python3 -m crmsh.healthcheck fix-cluster
PasswordlessHaclusterAuthenticationFeature',
+ remote_node,
+ )
+ append_to_remote_file(public_key, remote_node, authorized_file)
if add:
configure_ssh_key(remote=remote_node)
@@ -2049,6 +2061,7 @@
init_corosync()
init_remote_auth()
init_sbd()
+ init_upgradeutil()
lock_inst = lock.Lock()
try:
@@ -2120,6 +2133,7 @@
cluster_node = prompt_for_string("IP address or hostname of
existing node (e.g.: 192.168.1.1)", ".+")
_context.cluster_node = cluster_node
+ init_upgradeutil()
utils.ping_node(cluster_node)
join_ssh(cluster_node)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/crmsh-4.4.1+20221116.4faefec3/crmsh/healthcheck.py
new/crmsh-4.4.1+20221122.102a8e11/crmsh/healthcheck.py
--- old/crmsh-4.4.1+20221116.4faefec3/crmsh/healthcheck.py 1970-01-01
01:00:00.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/crmsh/healthcheck.py 2022-11-22
15:17:41.000000000 +0100
@@ -0,0 +1,271 @@
+import logging
+import argparse
+import os
+import os.path
+import parallax
+import subprocess
+import sys
+import typing
+
+import crmsh.parallax
+import crmsh.utils
+
+
+logger = logging.getLogger(__name__)
+
+
+class Feature:
+ _feature_registry = dict()
+
+ def __init_subclass__(cls, **kwargs):
+ super().__init_subclass__(**kwargs)
+ Feature._feature_registry[cls.__name__.rsplit('.', 1)[-1]] = cls
+
+ @staticmethod
+ def get_feature_by_name(name: str):
+ return Feature._feature_registry[name]
+
+ def check_quick(self) -> bool:
+ raise NotImplementedError
+
+ def check_local(self, nodes: typing.Iterable[str]) -> bool:
+ """Check whether the feature is functional on local node."""
+ raise NotImplementedError
+
+ def check_cluster(self, nodes: typing.Iterable[str]) -> bool:
+ """Check whether the feature is functional on the cluster."""
+ raise NotImplementedError
+
+ def fix_local(self, nodes: typing.Iterable[str], ask:
typing.Callable[[str], None]) -> None:
+ """Fix the feature on local node.
+
+ At least one of fix_local and fix_cluster should be implemented. If
fix_local is not implemented, this method
+ will be run on each node.
+ """
+ raise NotImplementedError
+
+ def fix_cluster(self, nodes: typing.Iterable[str], ask:
typing.Callable[[str], None]) -> None:
+ """Fix the feature on the cluster.
+
+ At least one of fix_local and fix_cluster should be implemented. If
this method is not implemented, fix_local
+ will be run on each node.
+ """
+ raise NotImplementedError
+
+
+class FixFailure(Exception):
+ pass
+
+
+class AskDeniedByUser(Exception):
+ pass
+
+
+def feature_quick_check(feature: Feature):
+ return feature.check_quick()
+
+
+def feature_local_check(feature: Feature, nodes: typing.Iterable[str]):
+ try:
+ if not feature.check_quick():
+ return False
+ except NotImplementedError:
+ pass
+ return feature.check_local(nodes)
+
+
+def feature_full_check(feature: Feature, nodes: typing.Iterable[str]) -> bool:
+ try:
+ if not feature.check_quick():
+ return False
+ except NotImplementedError:
+ pass
+ try:
+ if not feature.check_local(nodes):
+ return False
+ except NotImplementedError:
+ pass
+ try:
+ return feature.check_cluster(nodes)
+ except NotImplementedError:
+ results = _parallax_run(
+ nodes,
+ '/usr/bin/env python3 -m crmsh.healthcheck check-local {}'.format(
+ feature.__class__.__name__.rsplit('.', 1)[-1],
+ )
+ )
+ return all(rc == 0 for rc, _, _ in results.values())
+
+
+def feature_fix(feature: Feature, nodes: typing.Iterable[str], ask:
typing.Callable[[str], None]) -> None:
+ try:
+ return feature.fix_cluster(nodes, ask)
+ except NotImplementedError:
+ results = _parallax_run(
+ nodes,
+ '/usr/bin/env python3 -m crmsh.healthcheck fix-local {}'.format(
+ feature.__class__.__name__.rsplit('.', 1)[-1],
+ )
+ )
+ if any(rc != 0 for rc, _, _ in results.values()):
+ raise FixFailure
+
+
+class PasswordlessHaclusterAuthenticationFeature(Feature):
+ SSH_DIR = os.path.expanduser('~hacluster/.ssh')
+ KEY_TYPES = ['ed25519', 'ecdsa', 'rsa']
+
+ def check_quick(self) -> bool:
+ for key_type in self.KEY_TYPES:
+ try:
+ os.stat('{}/{}'.format(self.SSH_DIR, key_type))
+ os.stat('{}/{}.pub'.format(self.SSH_DIR, key_type))
+ return True
+ except FileNotFoundError:
+ pass
+ return False
+
+ def check_local(self, nodes: typing.Iterable[str]) -> bool:
+ try:
+ for node in nodes:
+ subprocess.check_call(
+ ['sudo', 'su', '-', 'hacluster', '-c', 'ssh hacluster@{}
true'.format(node)],
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ return True
+ except subprocess.CalledProcessError:
+ return False
+
+ def fix_cluster(self, nodes: typing.Iterable[str], ask:
typing.Callable[[str], None]) -> None:
+ logger.debug("setup passwordless ssh authentication for user
hacluster")
+ try:
+ nodes_without_keys = [
+ node for node, result in
+ _parallax_run(
+ nodes,
+ '[ -f ~hacluster/.ssh/id_rsa ] || [ -f
~hacluster/.ssh/id_ecdsa ] || [ -f ~hacluster/.ssh/id_ed25519 ]'
+ ).items()
+ if result[0] != 0
+ ]
+ except parallax.Error:
+ raise FixFailure()
+ if nodes_without_keys:
+ ask("Setup passwordless ssh authentication for user hacluster?")
+ if len(nodes_without_keys) == len(nodes):
+ # pick one node to run init ssh on it
+ init_node = nodes_without_keys[0]
+ # and run join ssh on other nodes
+ join_nodes = list()
+ join_nodes.extend(nodes)
+ join_nodes.remove(init_node)
+ join_target_node = init_node
+ else:
+ nodes_with_keys = set(nodes) - set(nodes_without_keys)
+ # no need to init ssh
+ init_node = None
+ join_nodes = nodes_without_keys
+ # pick one node as join target
+ join_target_node = next(iter(nodes_with_keys))
+ if init_node is not None:
+ try:
+ crmsh.parallax.parallax_call([init_node], 'crm cluster
init ssh -y')
+ except ValueError as e:
+ logger.error('Failed to initialize passwordless ssh
authentication on node %s.', init_node, exc_info=e)
+ raise FixFailure from None
+ try:
+ for node in join_nodes:
+ crmsh.parallax.parallax_call([node], 'crm cluster join ssh
-c {} -y'.format(join_target_node))
+ except ValueError as e:
+ logger.error('Failed to initialize passwordless ssh
authentication.', exc_info=e)
+ raise FixFailure from None
+
+
+def _parallax_run(nodes: str, cmd: str) -> typing.Dict[str, typing.Tuple[int,
bytes, bytes]]:
+ parallax_options = parallax.Options()
+ parallax_options.ssh_options = ['StrictHostKeyChecking=no',
'ConnectTimeout=10']
+ ret = dict()
+ for node, result in parallax.run(nodes, cmd, parallax_options).items():
+ if isinstance(result, parallax.Error):
+ logger.warning("SSH connection to remote node %s failed.", node,
exc_info=result)
+ raise result
+ ret[node] = result
+ return ret
+
+
+def main_check_local(args) -> int:
+ try:
+ feature = Feature.get_feature_by_name(args.feature)()
+ nodes = crmsh.utils.list_cluster_nodes(no_reg=True)
+ if nodes:
+ if feature_local_check(feature, nodes):
+ return 0
+ else:
+ return 1
+ except KeyError:
+ logger.error('No such feature: %s.', args.feature)
+ return 2
+
+
+def main_fix_local(args) -> int:
+ try:
+ feature = Feature.get_feature_by_name(args.feature)()
+ nodes = crmsh.utils.list_cluster_nodes(no_reg=True)
+ if nodes:
+ if args.yes:
+ def ask(msg): return True
+ else:
+ def ask(msg): return crmsh.utils.ask('Healthcheck: fix: ' +
msg)
+ if args.without_check or not feature_local_check(feature, nodes):
+ feature.fix_local(nodes, ask)
+ return 0
+ except KeyError:
+ logger.error('No such feature: %s.', args.feature)
+ return 2
+
+
+def main_fix_cluster(args) -> int:
+ try:
+ feature = Feature.get_feature_by_name(args.feature)()
+ nodes = crmsh.utils.list_cluster_nodes(no_reg=True)
+ if nodes:
+ if args.yes:
+ def ask(msg): return True
+ else:
+ def ask(msg): return crmsh.utils.ask('Healthcheck: fix: ' +
msg)
+ if args.without_check or not feature_full_check(feature, nodes):
+ feature_fix(feature, nodes, ask)
+ return 0
+ except KeyError:
+ logger.error('No such feature: %s.', args.feature)
+ return 2
+
+
+def main() -> int:
+ # This entrance is for internal programmatic use only.
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers()
+
+ check_local_parser = subparsers.add_parser('check-local')
+ check_local_parser.add_argument('feature')
+ check_local_parser.set_defaults(func=main_check_local)
+
+ fix_cluster_parser = subparsers.add_parser('fix-local')
+ fix_cluster_parser.add_argument('--yes', action='store_true')
+ fix_cluster_parser.add_argument('--without-check', action='store_true')
+ fix_cluster_parser.add_argument('feature')
+ fix_cluster_parser.set_defaults(func=main_fix_local)
+
+ fix_cluster_parser = subparsers.add_parser('fix-cluster')
+ fix_cluster_parser.add_argument('--yes', action='store_true')
+ fix_cluster_parser.add_argument('--without-check', action='store_true')
+ fix_cluster_parser.add_argument('feature')
+ fix_cluster_parser.set_defaults(func=main_fix_cluster)
+
+ args = parser.parse_args()
+ return args.func(args)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/crmsh-4.4.1+20221116.4faefec3/crmsh/main.py
new/crmsh-4.4.1+20221122.102a8e11/crmsh/main.py
--- old/crmsh-4.4.1+20221116.4faefec3/crmsh/main.py 2022-11-16
04:36:27.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/crmsh/main.py 2022-11-22
15:17:41.000000000 +0100
@@ -11,6 +11,7 @@
from . import constants
from . import clidisplay
from . import term
+from . import upgradeutil
from . import utils
from . import userdir
@@ -371,6 +372,7 @@
if options.profile:
return profile_run(context, user_args)
else:
+ upgradeutil.upgrade_if_needed()
return main_input_loop(context, user_args)
except KeyboardInterrupt:
print("Ctrl-C, leaving")
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/crmsh-4.4.1+20221116.4faefec3/crmsh/upgradeutil.py
new/crmsh-4.4.1+20221122.102a8e11/crmsh/upgradeutil.py
--- old/crmsh-4.4.1+20221116.4faefec3/crmsh/upgradeutil.py 1970-01-01
01:00:00.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/crmsh/upgradeutil.py 2022-11-22
15:17:41.000000000 +0100
@@ -0,0 +1,172 @@
+import logging
+import os.path
+import typing
+
+import parallax
+import sys
+
+import crmsh.healthcheck
+import crmsh.parallax
+import crmsh.utils
+
+
+# pump this seq when upgrade check need to be run
+CURRENT_UPGRADE_SEQ = (1, 0)
+DATA_DIR = os.path.expanduser('~hacluster/crmsh')
+SEQ_FILE_PATH = DATA_DIR + '/upgrade_seq'
+# touch this file to force a upgrade process
+FORCE_UPGRADE_FILE_PATH = DATA_DIR + '/upgrade_forced'
+
+
+VERSION_FEATURES = {
+ (1, 0): [crmsh.healthcheck.PasswordlessHaclusterAuthenticationFeature]
+}
+
+
+logger = logging.getLogger(__name__)
+
+
+class _SkipUpgrade(Exception):
+ pass
+
+
+def _parse_upgrade_seq(s: bytes) -> typing.Tuple[int, int]:
+ parts = s.split(b'.', 1)
+ if len(parts) != 2:
+ raise ValueError('Invalid upgrade seq {}'.format(s))
+ major = int(parts[0])
+ minor = int(parts[1])
+ return major, minor
+
+
+def _format_upgrade_seq(s: typing.Tuple[int, int]) -> str:
+ return '.'.join((str(x) for x in s))
+
+
+def _get_file_content(path, default=None):
+ try:
+ with open(path, 'rb') as f:
+ return f.read()
+ except FileNotFoundError:
+ return default
+
+
+def _parallax_run(nodes: str, cmd: str) -> typing.Dict[str, typing.Tuple[int,
bytes, bytes]]:
+ parallax_options = parallax.Options()
+ parallax_options.ssh_options = ['StrictHostKeyChecking=no',
'ConnectTimeout=10']
+ ret = dict()
+ for node, result in parallax.run(nodes, cmd, parallax_options).items():
+ if isinstance(result, parallax.Error):
+ logger.warning("SSH connection to remote node %s failed.", node,
exc_info=result)
+ raise result
+ ret[node] = result
+ return ret
+
+
+def _is_upgrade_needed(nodes):
+ """decide whether upgrading is needed by checking local sequence file"""
+ needed = False
+ try:
+ os.stat(FORCE_UPGRADE_FILE_PATH)
+ needed = True
+ except FileNotFoundError:
+ pass
+ if not needed:
+ try:
+ local_seq = _parse_upgrade_seq(_get_file_content(SEQ_FILE_PATH,
b'').strip())
+ except ValueError:
+ local_seq = (0, 0)
+ needed = CURRENT_UPGRADE_SEQ > local_seq
+ return needed
+
+
+def _is_cluster_target_seq_consistent(nodes):
+ cmd = '/usr/bin/env python3 -m crmsh.upgradeutil get-seq'
+ try:
+ results = list(_parallax_run(nodes, cmd).values())
+ except parallax.Error as e:
+ raise _SkipUpgrade() from None
+ try:
+ return all(CURRENT_UPGRADE_SEQ == _parse_upgrade_seq(stdout.strip())
if rc == 0 else False for rc, stdout, stderr in results)
+ except ValueError as e:
+ logger.warning("Remote command '%s' returns unexpected output: %s",
cmd, results, exc_info=e)
+ return False
+
+
+def _get_minimal_seq_in_cluster(nodes) -> typing.Tuple[int, int]:
+ try:
+ return min(
+ _parse_upgrade_seq(stdout.strip()) if rc == 0 else (0, 0)
+ for rc, stdout, stderr in _parallax_run(nodes, 'cat
{}'.format(SEQ_FILE_PATH)).values()
+ )
+ except ValueError:
+ return 0, 0
+
+
+def _upgrade(nodes, seq):
+ def ask(msg: str):
+ if not crmsh.utils.ask('Upgrade of crmsh configuration: ' + msg):
+ raise crmsh.healthcheck.AskDeniedByUser()
+ try:
+ for key in VERSION_FEATURES.keys():
+ if seq < key <= CURRENT_UPGRADE_SEQ:
+ for feature_class in VERSION_FEATURES[key]:
+ feature = feature_class()
+ if crmsh.healthcheck.feature_full_check(feature, nodes):
+ logger.info("Upgrade: feature %s is already
functional.")
+ else:
+ logger.debug("Upgrade: fixing feature %s...")
+ crmsh.healthcheck.feature_fix(feature, nodes, ask)
+ logger.info("Upgrade of crmsh configuration succeeded.")
+ except crmsh.healthcheck.AskDeniedByUser:
+ raise _SkipUpgrade() from None
+
+
+def upgrade_if_needed():
+ nodes = crmsh.utils.list_cluster_nodes(no_reg=True)
+ if nodes and _is_upgrade_needed(nodes):
+ logger.info("crmsh version is newer than its configuration.
Configuration upgrade is needed.")
+ try:
+ if not _is_cluster_target_seq_consistent(nodes):
+ logger.warning("crmsh version is inconsistent in cluster.")
+ raise _SkipUpgrade()
+ seq = _get_minimal_seq_in_cluster(nodes)
+ logger.debug(
+ "Upgrading crmsh configuration from seq %s to %s.",
+ seq, _format_upgrade_seq(CURRENT_UPGRADE_SEQ),
+ )
+ _upgrade(nodes, seq)
+ except _SkipUpgrade:
+ logger.warning("Upgrade of crmsh configuration skipped.")
+ return
+ crmsh.parallax.parallax_call(
+ nodes,
+ "mkdir -p '{}' && echo '{}' > '{}'".format(
+ DATA_DIR,
+ _format_upgrade_seq(CURRENT_UPGRADE_SEQ),
+ SEQ_FILE_PATH,
+ ),
+ )
+ crmsh.parallax.parallax_call(nodes, 'rm -f
{}'.format(FORCE_UPGRADE_FILE_PATH))
+ logger.debug("Upgrade of crmsh configuration finished.", seq)
+
+
+def force_set_local_upgrade_seq():
+ """Create the upgrade sequence file and set it to CURRENT_UPGRADE_SEQ.
+
+ It should only be used when initializing new cluster nodes."""
+ try:
+ os.mkdir(DATA_DIR)
+ except FileExistsError:
+ pass
+ with open(SEQ_FILE_PATH, 'w', encoding='ascii') as f:
+ print(_format_upgrade_seq(CURRENT_UPGRADE_SEQ), file=f)
+
+
+def main():
+ if sys.argv[1] == 'get-seq':
+ print(_format_upgrade_seq(CURRENT_UPGRADE_SEQ))
+
+
+if __name__ == '__main__':
+ main()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/crmsh-4.4.1+20221116.4faefec3/crmsh/utils.py
new/crmsh-4.4.1+20221122.102a8e11/crmsh/utils.py
--- old/crmsh-4.4.1+20221116.4faefec3/crmsh/utils.py 2022-11-16
04:36:27.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/crmsh/utils.py 2022-11-22
15:17:41.000000000 +0100
@@ -1703,13 +1703,13 @@
print("{}\n".format(out))
-def list_cluster_nodes():
+def list_cluster_nodes(no_reg=False):
'''
Returns a list of nodes in the cluster.
'''
from . import xmlutil
cib = None
- rc, out, err = get_stdout_stderr(constants.CIB_QUERY)
+ rc, out, err = get_stdout_stderr(constants.CIB_QUERY, no_reg=no_reg)
# When cluster service running
if rc == 0:
cib = etree.fromstring(out)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/crmsh-4.4.1+20221116.4faefec3/data-manifest
new/crmsh-4.4.1+20221122.102a8e11/data-manifest
--- old/crmsh-4.4.1+20221116.4faefec3/data-manifest 2022-11-16
04:36:27.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/data-manifest 2022-11-22
15:17:41.000000000 +0100
@@ -75,6 +75,7 @@
test/features/crm_report_bugs.feature
test/features/environment.py
test/features/geo_setup.feature
+test/features/healthcheck.feature
test/features/ocfs2.feature
test/features/qdevice_options.feature
test/features/qdevice_setup_remove.feature
@@ -186,6 +187,7 @@
test/unittests/test_crashtest_utils.py
test/unittests/test_gv.py
test/unittests/test_handles.py
+test/unittests/test_healthcheck.py
test/unittests/test_lock.py
test/unittests/test_objset.py
test/unittests/test_ocfs2.py
@@ -197,6 +199,7 @@
test/unittests/test_scripts.py
test/unittests/test_time.py
test/unittests/test_ui_cluster.py
+test/unittests/test_upgradeuitl.py
test/unittests/test_utils.py
test/unittests/test_watchdog.py
test/unittests/test_xmlutil.py
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/crmsh-4.4.1+20221116.4faefec3/test/crm-interface
new/crmsh-4.4.1+20221122.102a8e11/test/crm-interface
--- old/crmsh-4.4.1+20221116.4faefec3/test/crm-interface 2022-11-16
04:36:27.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/test/crm-interface 2022-11-22
15:17:41.000000000 +0100
@@ -67,6 +67,10 @@
}
crm_filesession() {
local _file=`mktemp`
+ $CRM_NO_REG -c $CIB<<EOF
+configure
+delete node1
+EOF
$CRM -c $CIB configure save xml $_file
CIB_file=$_file $CRM <<EOF
`cat`
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/crmsh-4.4.1+20221116.4faefec3/test/features/healthcheck.feature
new/crmsh-4.4.1+20221122.102a8e11/test/features/healthcheck.feature
--- old/crmsh-4.4.1+20221116.4faefec3/test/features/healthcheck.feature
1970-01-01 01:00:00.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/test/features/healthcheck.feature
2022-11-22 15:17:41.000000000 +0100
@@ -0,0 +1,27 @@
+@healthcheck
+Feature: healthcheck detect and fix problems in a crmsh deployment
+
+ Tag @clean means need to stop cluster service if the service is available
+ Need nodes: hanode1 hanode2 hanode3
+
+ Background: Setup a two nodes cluster
+ Given Cluster service is "stopped" on "hanode1"
+ And Cluster service is "stopped" on "hanode2"
+ And Cluster service is "stopped" on "hanode3"
+ When Run "crm cluster init -y" on "hanode1"
+ Then Cluster service is "started" on "hanode1"
+ And Show cluster status on "hanode1"
+ When Run "crm cluster join -c hanode1 -y" on "hanode2"
+ Then Cluster service is "started" on "hanode2"
+ And Online nodes are "hanode1 hanode2"
+ And Show cluster status on "hanode1"
+
+ @clean
+ Scenario: a new node joins when directory ~hacluster/.ssh is removed from
cluster
+ When Run "rm -rf ~hacluster/.ssh" on "hanode1"
+ And Run "rm -rf ~hacluster/.ssh" on "hanode2"
+ And Run "crm cluster join -c hanode1 -y" on "hanode3"
+ Then Cluster service is "started" on "hanode3"
+ And File "~hacluster/.ssh/id_rsa" exists on "hanode1"
+ And File "~hacluster/.ssh/id_rsa" exists on "hanode2"
+ And File "~hacluster/.ssh/id_rsa" exists on "hanode3"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/crmsh-4.4.1+20221116.4faefec3/test/features/steps/step_implementation.py
new/crmsh-4.4.1+20221122.102a8e11/test/features/steps/step_implementation.py
---
old/crmsh-4.4.1+20221116.4faefec3/test/features/steps/step_implementation.py
2022-11-16 04:36:27.000000000 +0100
+++
new/crmsh-4.4.1+20221122.102a8e11/test/features/steps/step_implementation.py
2022-11-22 15:17:41.000000000 +0100
@@ -404,3 +404,8 @@
resource.setrlimit(resource.RLIMIT_NOFILE, (50, 50))
for i in range(51):
parallax.parallax_call([node], "true")
+
+
+@then('File "{path}" exists on "{node}"')
+def step_impl(context, path, node):
+ parallax.parallax_call([node], '[ -f {} ]'.format(path))
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/crmsh-4.4.1+20221116.4faefec3/test/unittests/test_healthcheck.py
new/crmsh-4.4.1+20221122.102a8e11/test/unittests/test_healthcheck.py
--- old/crmsh-4.4.1+20221116.4faefec3/test/unittests/test_healthcheck.py
1970-01-01 01:00:00.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/test/unittests/test_healthcheck.py
2022-11-22 15:17:41.000000000 +0100
@@ -0,0 +1,75 @@
+import unittest
+from unittest import mock
+import sys
+
+from crmsh import healthcheck
+
+
+class _Py37MockCallShim:
+ def __init__(self, mock_call):
+ self._mock_call = mock_call
+
+ def __getattr__(self, item):
+ f = getattr(self._mock_call, item)
+
+ def g(*args, **kwargs):
+ return f(*((self._mock_call, ) + args[1:]), **kwargs)
+ return g
+
+ @property
+ def args(self):
+ if sys.version_info.major == 3 and sys.version_info.major < 8:
+ return self._mock_call[0]
+ else:
+ return self._mock_call.args
+
+ @property
+ def kwargs(self):
+ def args(self):
+ if sys.version_info.major == 3 and sys.version_info.major < 8:
+ return self._mock_call[1]
+ else:
+ return self._mock_call.kwargs
+
+
+class TestPasswordlessHaclusterAuthenticationFeature(unittest.TestCase):
+ @mock.patch('crmsh.parallax.parallax_call')
+ @mock.patch('crmsh.utils.ask')
+ @mock.patch('crmsh.healthcheck._parallax_run')
+ def test_upgrade_partially_initialized(self, mock_parallax_run, mock_ask,
mock_parallax_call: mock.MagicMock):
+ nodes = ['node-{}'.format(i) for i in range(1, 6)]
+ return_value = {'node-{}'.format(i): (0, b'', b'') for i in range(1,
4)}
+ return_value.update({'node-{}'.format(i): (1, b'', b'') for i in
range(4, 6)})
+ mock_parallax_run.return_value = return_value
+ mock_ask.return_value = True
+
healthcheck.feature_fix(healthcheck.PasswordlessHaclusterAuthenticationFeature(),
nodes, mock_ask)
+
self.assertFalse(any(_Py37MockCallShim(call_args).args[1].startswith('crm
cluster init ssh') for call_args in mock_parallax_call.call_args_list))
+ self.assertEqual(
+ {'node-{}'.format(i) for i in range(4, 6)},
+ set(
+ _Py37MockCallShim(call_args).args[0][0] for call_args in
mock_parallax_call.call_args_list
+ if _Py37MockCallShim(call_args).args[1].startswith('crm
cluster join ssh')
+ ),
+ )
+
+ @mock.patch('crmsh.parallax.parallax_call')
+ @mock.patch('crmsh.utils.ask')
+ @mock.patch('crmsh.healthcheck._parallax_run')
+ def test_upgrade_clean(self, mock_parallax_run, mock_ask,
mock_parallax_call: mock.MagicMock):
+ nodes = ['node-{}'.format(i) for i in range(1, 6)]
+ mock_parallax_run.return_value = {node: (1, b'', b'') for node in
nodes}
+ mock_ask.return_value = True
+
healthcheck.feature_fix(healthcheck.PasswordlessHaclusterAuthenticationFeature(),
nodes, mock_ask)
+ self.assertEqual(
+ 1, len([
+ True for call_args in mock_parallax_call.call_args_list
+ if _Py37MockCallShim(call_args).args[1].startswith('crm
cluster init ssh')
+ ])
+ )
+ self.assertEqual(
+ len(nodes) - 1,
+ len([
+ True for call_args in mock_parallax_call.call_args_list
+ if _Py37MockCallShim(call_args).args[1].startswith('crm
cluster join ssh')
+ ]),
+ )
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/crmsh-4.4.1+20221116.4faefec3/test/unittests/test_upgradeuitl.py
new/crmsh-4.4.1+20221122.102a8e11/test/unittests/test_upgradeuitl.py
--- old/crmsh-4.4.1+20221116.4faefec3/test/unittests/test_upgradeuitl.py
1970-01-01 01:00:00.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/test/unittests/test_upgradeuitl.py
2022-11-22 15:17:41.000000000 +0100
@@ -0,0 +1,54 @@
+import os
+import sys
+import unittest
+from unittest import mock
+
+from crmsh import upgradeutil
+
+
+class TestUpgradeCondition(unittest.TestCase):
+ @mock.patch('crmsh.upgradeutil._get_file_content')
+ @mock.patch('os.stat')
+ def test_is_upgrade_needed_by_force_upgrade(self, mock_stat:
mock.MagicMock, mock_get_file_content):
+ mock_stat.return_value = mock.Mock(os.stat_result)
+ mock_get_file_content.return_value = b''
+ self.assertTrue(upgradeutil._is_upgrade_needed(['node-1', 'node-2']))
+
+ @mock.patch('crmsh.upgradeutil._get_file_content')
+ @mock.patch('os.stat')
+ def test_is_upgrade_needed_by_non_existent_seq(
+ self,
+ mock_stat: mock.MagicMock,
+ mock_get_file_content: mock.MagicMock,
+ ):
+ mock_stat.side_effect = FileNotFoundError()
+ mock_get_file_content.return_value = b''
+ self.assertTrue(upgradeutil._is_upgrade_needed(['node-1', 'node-2']))
+
+ @mock.patch('crmsh.upgradeutil.CURRENT_UPGRADE_SEQ')
+ @mock.patch('crmsh.upgradeutil._get_file_content')
+ @mock.patch('os.stat')
+ def test_is_upgrade_needed_by_seq_less_than_expected(
+ self,
+ mock_stat,
+ mock_get_file_content,
+ mock_current_upgrade_seq: mock.MagicMock,
+ ):
+ mock_stat.side_effect = FileNotFoundError()
+ mock_get_file_content.return_value = b'0.1\n'
+ mock_current_upgrade_seq.__gt__.return_value = True
+ self.assertTrue(upgradeutil._is_upgrade_needed(['node-1', 'node-2']))
+
+ @mock.patch('crmsh.upgradeutil.CURRENT_UPGRADE_SEQ')
+ @mock.patch('crmsh.upgradeutil._get_file_content')
+ @mock.patch('os.stat')
+ def test_is_upgrade_needed_by_seq_not_less_than_expected(
+ self,
+ mock_stat,
+ mock_get_file_content,
+ mock_current_upgrade_seq: mock.MagicMock,
+ ):
+ mock_stat.side_effect = FileNotFoundError()
+ mock_get_file_content.return_value = b'1.0\n'
+ mock_current_upgrade_seq.__gt__.return_value = False
+ self.assertFalse(upgradeutil._is_upgrade_needed(['node-1', 'node-2']))
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/crmsh-4.4.1+20221116.4faefec3/test/unittests/test_utils.py
new/crmsh-4.4.1+20221122.102a8e11/test/unittests/test_utils.py
--- old/crmsh-4.4.1+20221116.4faefec3/test/unittests/test_utils.py
2022-11-16 04:36:27.000000000 +0100
+++ new/crmsh-4.4.1+20221122.102a8e11/test/unittests/test_utils.py
2022-11-22 15:17:41.000000000 +0100
@@ -1560,7 +1560,18 @@
mock_etree.return_value = None
res = utils.list_cluster_nodes()
assert res is None
- mock_run.assert_called_once_with(constants.CIB_QUERY)
+ mock_run.assert_called_once_with(constants.CIB_QUERY, no_reg=False)
+ mock_etree.assert_called_once_with("data")
+
+
+@mock.patch('crmsh.utils.etree.fromstring')
+@mock.patch('crmsh.utils.get_stdout_stderr')
+def test_list_cluster_nodes_none_no_reg(mock_run, mock_etree):
+ mock_run.return_value = (0, "data", None)
+ mock_etree.return_value = None
+ res = utils.list_cluster_nodes(no_reg=True)
+ assert res is None
+ mock_run.assert_called_once_with(constants.CIB_QUERY, no_reg=True)
mock_etree.assert_called_once_with("data")
@@ -1573,7 +1584,7 @@
mock_isfile.return_value = False
res = utils.list_cluster_nodes()
assert res is None
- mock_run.assert_called_once_with(constants.CIB_QUERY)
+ mock_run.assert_called_once_with(constants.CIB_QUERY, no_reg=False)
mock_env.assert_called_once_with("CIB_file", constants.CIB_RAW_FILE)
mock_isfile.assert_called_once_with(constants.CIB_RAW_FILE)
@@ -1597,7 +1608,7 @@
res = utils.list_cluster_nodes()
assert res == ["node2"]
- mock_run.assert_called_once_with(constants.CIB_QUERY)
+ mock_run.assert_called_once_with(constants.CIB_QUERY, no_reg=False)
mock_env.assert_called_once_with("CIB_file", constants.CIB_RAW_FILE)
mock_isfile.assert_called_once_with(constants.CIB_RAW_FILE)
mock_file2elem.assert_called_once_with(constants.CIB_RAW_FILE)
