Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package borgmatic for openSUSE:Factory checked in at 2023-03-13 12:40:43 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/borgmatic (Old) and /work/SRC/openSUSE:Factory/.borgmatic.new.31432 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "borgmatic" Mon Mar 13 12:40:43 2023 rev:37 rq:1071077 version:1.7.8 Changes: -------- --- /work/SRC/openSUSE:Factory/borgmatic/borgmatic.changes 2023-02-27 12:55:51.947606436 +0100 +++ /work/SRC/openSUSE:Factory/.borgmatic.new.31432/borgmatic.changes 2023-03-13 12:41:24.383888796 +0100 @@ -1,0 +2,24 @@ +Tue Mar 7 20:08:12 UTC 2023 - Dirk Müller <dmuel...@suse.com> + +- update to 1.7.8: + * #620: With the "create" action and the "--list" ("--files") + flag, only show excluded files at verbosity 2. + * #621: Add optional authentication to the ntfy monitoring + hook. + * With the "create" action, only one of "--list" ("--files") + and "--progress" flags can be used. + * This lines up with the new behavior in Borg 2.0.0b5. + * Internally support new Borg 2.0.0b5 "--filter" status + characters / item flags for the "create" action. + * Fix the "create" action with the "--dry-run" flag querying + for databases when a PostgreSQL/MySQL "all" database is + configured. Now, these queries are skipped due to the dry run. + * Add "--repository" flag to the "rcreate" action to optionally + select one configured repository to create. + * Add "--progress" flag to the "transfer" action, new in Borg + 2.0.0b5. + * Add "checkpoint_volume" configuration option to creates + checkpoints every specified number of bytes during a + long-running backup, new in Borg 2.0.0b5. + +------------------------------------------------------------------- Old: ---- borgmatic-1.7.7.tar.gz New: ---- borgmatic-1.7.8.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ borgmatic.spec ++++++ --- /var/tmp/diff_new_pack.lQ3EmZ/_old 2023-03-13 12:41:25.299893570 +0100 +++ /var/tmp/diff_new_pack.lQ3EmZ/_new 2023-03-13 12:41:25.303893590 +0100 @@ -17,7 +17,7 @@ Name: borgmatic -Version: 1.7.7 +Version: 1.7.8 Release: 0 Summary: Automation tool for borgbackup License: GPL-3.0-only ++++++ borgmatic-1.7.7.tar.gz -> borgmatic-1.7.8.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/NEWS new/borgmatic-1.7.8/NEWS --- old/borgmatic-1.7.7/NEWS 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/NEWS 2023-03-03 00:34:22.000000000 +0100 @@ -1,3 +1,19 @@ +1.7.8 + * #620: With the "create" action and the "--list" ("--files") flag, only show excluded files at + verbosity 2. + * #621: Add optional authentication to the ntfy monitoring hook. + * With the "create" action, only one of "--list" ("--files") and "--progress" flags can be used. + This lines up with the new behavior in Borg 2.0.0b5. + * Internally support new Borg 2.0.0b5 "--filter" status characters / item flags for the "create" + action. + * Fix the "create" action with the "--dry-run" flag querying for databases when a PostgreSQL/MySQL + "all" database is configured. Now, these queries are skipped due to the dry run. + * Add "--repository" flag to the "rcreate" action to optionally select one configured repository to + create. + * Add "--progress" flag to the "transfer" action, new in Borg 2.0.0b5. + * Add "checkpoint_volume" configuration option to creates checkpoints every specified number of + bytes during a long-running backup, new in Borg 2.0.0b5. + 1.7.7 * #642: Add MySQL database hook "add_drop_database" configuration option to control whether dumped MySQL databases get dropped right before restore. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/actions/rcreate.py new/borgmatic-1.7.8/borgmatic/actions/rcreate.py --- old/borgmatic-1.7.7/borgmatic/actions/rcreate.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/actions/rcreate.py 2023-03-03 00:34:22.000000000 +0100 @@ -1,6 +1,7 @@ import logging import borgmatic.borg.rcreate +import borgmatic.config.validate logger = logging.getLogger(__name__) @@ -17,6 +18,11 @@ ''' Run the "rcreate" action for the given repository. ''' + if rcreate_arguments.repository and not borgmatic.config.validate.repositories_match( + repository, rcreate_arguments.repository + ): + return + logger.info('{}: Creating repository'.format(repository)) borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/borg/create.py new/borgmatic-1.7.8/borgmatic/borg/create.py --- old/borgmatic-1.7.7/borgmatic/borg/create.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/borg/create.py 2023-03-03 00:34:22.000000000 +0100 @@ -196,6 +196,27 @@ ) +def make_list_filter_flags(local_borg_version, dry_run): + ''' + Given the local Borg version and whether this is a dry run, return the corresponding flags for + passing to "--list --filter". The general idea is that excludes are shown for a dry run or when + the verbosity is debug. + ''' + base_flags = 'AME' + show_excludes = logger.isEnabledFor(logging.DEBUG) + + if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version): + if show_excludes or dry_run: + return f'{base_flags}+-' + else: + return base_flags + + if show_excludes: + return f'{base_flags}x-' + else: + return f'{base_flags}-' + + DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' @@ -275,7 +296,7 @@ paths = tuple( path_line.split(' ', 1)[1] for path_line in paths_output.split('\n') - if path_line and path_line.startswith('- ') + if path_line and path_line.startswith('- ') or path_line.startswith('+ ') ) return tuple( @@ -337,11 +358,13 @@ expand_home_directories(location_config.get('exclude_patterns')) ) checkpoint_interval = storage_config.get('checkpoint_interval', None) + checkpoint_volume = storage_config.get('checkpoint_volume', None) chunker_params = storage_config.get('chunker_params', None) compression = storage_config.get('compression', None) upload_rate_limit = storage_config.get('upload_rate_limit', None) umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) + list_filter_flags = make_list_filter_flags(local_borg_version, dry_run) files_cache = location_config.get('files_cache') archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT) extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '') @@ -381,6 +404,7 @@ + make_pattern_flags(location_config, pattern_file.name if pattern_file else None) + make_exclude_flags(location_config, exclude_file.name if exclude_file else None) + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ()) + + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ()) + (('--chunker-params', chunker_params) if chunker_params else ()) + (('--compression', compression) if compression else ()) + upload_ratelimit_flags @@ -399,7 +423,11 @@ + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--list', '--filter', 'AMEx-') if list_files and not json and not progress else ()) + + ( + ('--list', '--filter', list_filter_flags) + if list_files and not json and not progress + else () + ) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_archive_flags(repository, archive_name_format, local_borg_version) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/borg/feature.py new/borgmatic-1.7.8/borgmatic/borg/feature.py --- old/borgmatic-1.7.7/borgmatic/borg/feature.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/borg/feature.py 2023-03-03 00:34:22.000000000 +0100 @@ -14,6 +14,7 @@ RLIST = 8 RINFO = 9 MATCH_ARCHIVES = 10 + EXCLUDED_FILES_MINUS = 11 FEATURE_TO_MINIMUM_BORG_VERSION = { @@ -27,6 +28,7 @@ Feature.RLIST: parse_version('2.0.0a2'), # borg rlist Feature.RINFO: parse_version('2.0.0a2'), # borg rinfo Feature.MATCH_ARCHIVES: parse_version('2.0.0b3'), # borg --match-archives + Feature.EXCLUDED_FILES_MINUS: parse_version('2.0.0b5'), # --list --filter uses "-" for excludes } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/borg/transfer.py new/borgmatic-1.7.8/borgmatic/borg/transfer.py --- old/borgmatic-1.7.7/borgmatic/borg/transfer.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/borg/transfer.py 2023-03-03 00:34:22.000000000 +0100 @@ -2,7 +2,7 @@ import borgmatic.logger from borgmatic.borg import environment, flags -from borgmatic.execute import execute_command +from borgmatic.execute import DO_NOT_CAPTURE, execute_command logger = logging.getLogger(__name__) @@ -28,6 +28,7 @@ + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) + + (('--progress',) if transfer_arguments.progress else ()) + ( flags.make_flags( 'match-archives', transfer_arguments.match_archives or transfer_arguments.archive @@ -45,6 +46,7 @@ return execute_command( full_command, output_log_level=logging.ANSWER, + output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None, borg_local_path=local_path, extra_environment=environment.make_environment(storage_config), ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/commands/arguments.py new/borgmatic-1.7.8/borgmatic/commands/arguments.py --- old/borgmatic-1.7.7/borgmatic/commands/arguments.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/commands/arguments.py 2023-03-03 00:34:22.000000000 +0100 @@ -248,6 +248,10 @@ help='Path to an existing Borg repository whose key material should be reused (Borg 2.x+ only)', ) rcreate_group.add_argument( + '--repository', + help='Path of the new repository to create (must be already specified in a borgmatic configuration file), defaults to the configured repository if there is only one', + ) + rcreate_group.add_argument( '--copy-crypt-key', action='store_true', help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key (Borg 2.x+ only)', @@ -293,6 +297,12 @@ help='Upgrader type used to convert the transfered data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion', ) transfer_group.add_argument( + '--progress', + default=False, + action='store_true', + help='Display progress as each archive is transferred', + ) + transfer_group.add_argument( '-a', '--match-archives', '--glob-archives', @@ -833,6 +843,11 @@ 'The --excludes flag has been replaced with exclude_patterns in configuration.' ) + if 'create' in arguments and arguments['create'].list_files and arguments['create'].progress: + raise ValueError( + 'With the create action, only one of --list (--files) and --progress flags can be used.' + ) + if ( ('list' in arguments and 'rinfo' in arguments and arguments['list'].json) or ('list' in arguments and 'info' in arguments and arguments['list'].json) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/config/schema.yaml new/borgmatic-1.7.8/borgmatic/config/schema.yaml --- old/borgmatic-1.7.7/borgmatic/config/schema.yaml 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/config/schema.yaml 2023-03-03 00:34:22.000000000 +0100 @@ -240,6 +240,16 @@ for details. Defaults to checkpoints every 1800 seconds (30 minutes). example: 1800 + checkpoint_volume: + type: integer + description: | + Number of backed up bytes between each checkpoint during a + long-running backup. Only supported with Borg 2+. See + https://borgbackup.readthedocs.io/en/stable/faq.html + for details. Defaults to only time-based checkpointing (see + "checkpoint_interval") instead of volume-based + checkpointing. + example: 1048576 chunker_params: type: string description: | @@ -1029,6 +1039,16 @@ description: | The address of your self-hosted ntfy.sh instance. example: https://ntfy.your-domain.com + username: + type: string + description: | + The username used for authentication. + example: testuser + password: + type: string + description: | + The password used for authentication. + example: fakepassword start: type: object properties: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/hooks/mysql.py new/borgmatic-1.7.8/borgmatic/hooks/mysql.py --- old/borgmatic-1.7.7/borgmatic/hooks/mysql.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/hooks/mysql.py 2023-03-03 00:34:22.000000000 +0100 @@ -24,7 +24,7 @@ SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, extra_environment, log_prefix, dry_run_label): +def database_names_to_dump(database, extra_environment, log_prefix, dry_run): ''' Given a requested database config, return the corresponding sequence of database names to dump. In the case of "all", query for the names of databases on the configured host and return them, @@ -32,6 +32,8 @@ ''' if database['name'] != 'all': return (database['name'],) + if dry_run: + return () show_command = ( ('mysql',) @@ -43,9 +45,7 @@ + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) - logger.debug( - '{}: Querying for "all" MySQL databases to dump{}'.format(log_prefix, dry_run_label) - ) + logger.debug(f'{log_prefix}: Querying for "all" MySQL databases to dump') show_output = execute_command_and_capture_output( show_command, extra_environment=extra_environment ) @@ -125,9 +125,13 @@ dump_path = make_dump_path(location_config) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None dump_database_names = database_names_to_dump( - database, extra_environment, log_prefix, dry_run_label + database, extra_environment, log_prefix, dry_run ) + if not dump_database_names: + if dry_run: + continue + raise ValueError('Cannot find any MySQL databases to dump.') if database['name'] == 'all' and database.get('format'): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/hooks/ntfy.py new/borgmatic-1.7.8/borgmatic/hooks/ntfy.py --- old/borgmatic-1.7.7/borgmatic/hooks/ntfy.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/hooks/ntfy.py 2023-03-03 00:34:22.000000000 +0100 @@ -56,14 +56,30 @@ 'X-Tags': state_config.get('tags'), } + username = hook_config.get('username') + password = hook_config.get('password') + + auth = None + if (username and password) is not None: + auth = requests.auth.HTTPBasicAuth(username, password) + logger.info(f'{config_filename}: Using basic auth with user {username} for ntfy') + elif username is not None: + logger.warning( + f'{config_filename}: Password missing for ntfy authentication, defaulting to no auth' + ) + elif password is not None: + logger.warning( + f'{config_filename}: Username missing for ntfy authentication, defaulting to no auth' + ) + if not dry_run: logging.getLogger('urllib3').setLevel(logging.ERROR) try: - response = requests.post(f'{base_url}/{topic}', headers=headers) + response = requests.post(f'{base_url}/{topic}', headers=headers, auth=auth) if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Ntfy error: {error}') + logger.warning(f'{config_filename}: ntfy error: {error}') def destroy_monitor( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/borgmatic/hooks/postgresql.py new/borgmatic-1.7.8/borgmatic/hooks/postgresql.py --- old/borgmatic-1.7.7/borgmatic/hooks/postgresql.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/borgmatic/hooks/postgresql.py 2023-03-03 00:34:22.000000000 +0100 @@ -43,7 +43,7 @@ EXCLUDED_DATABASE_NAMES = ('template0', 'template1') -def database_names_to_dump(database, extra_environment, log_prefix, dry_run_label): +def database_names_to_dump(database, extra_environment, log_prefix, dry_run): ''' Given a requested database config, return the corresponding sequence of database names to dump. In the case of "all" when a database format is given, query for the names of databases on the @@ -56,6 +56,8 @@ return (requested_name,) if not database.get('format'): return ('all',) + if dry_run: + return () list_command = ( ('psql', '--list', '--no-password', '--csv', '--tuples-only') @@ -64,9 +66,7 @@ + (('--username', database['username']) if 'username' in database else ()) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) ) - logger.debug( - '{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label) - ) + logger.debug(f'{log_prefix}: Querying for "all" PostgreSQL databases to dump') list_output = execute_command_and_capture_output( list_command, extra_environment=extra_environment ) @@ -99,10 +99,13 @@ extra_environment = make_extra_environment(database) dump_path = make_dump_path(location_config) dump_database_names = database_names_to_dump( - database, extra_environment, log_prefix, dry_run_label + database, extra_environment, log_prefix, dry_run ) if not dump_database_names: + if dry_run: + continue + raise ValueError('Cannot find any PostgreSQL databases to dump.') for database_name in dump_database_names: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/docs/how-to/upgrade.md new/borgmatic-1.7.8/docs/how-to/upgrade.md --- old/borgmatic-1.7.7/docs/how-to/upgrade.md 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/docs/how-to/upgrade.md 2023-03-03 00:34:22.000000000 +0100 @@ -169,12 +169,21 @@ Borg 1 repository so that the subsequent data transfer can work. The `--encryption` value above selects the same chunk ID algorithm (`blake2`) -used in Borg 1, thereby making deduplication work across transferred archives -and new archives. Note that `repokey-blake2-chacha20-poly1305` may be faster -than `repokey-blake2-aes-ocb` on certain platforms like ARM64. Read about -[Borg encryption -modes](https://borgbackup.readthedocs.io/en/2.0.0b4/usage/rcreate.html#encryption-mode-tldr) -for the menu of available encryption modes. +commonly used in Borg 1, thereby making deduplication work across transferred +archives and new archives. + +If you get an error about "You must keep the same ID hash" from Borg, that +means the encryption value you specified doesn't correspond to your source +repository's chunk ID algorithm. In that case, try not using `blake2`: + +```bash +borgmatic rcreate --verbosity 1 --encryption repokey-aes-ocb \ + --source-repository original.borg --repository upgraded.borg +``` + +Read about [Borg encryption +modes](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/rcreate.html#encryption-mode-tldr) +for more details. To transfer data from your original Borg 1 repository to your newly created Borg 2 repository: @@ -196,7 +205,7 @@ Note that by omitting the `--upgrader` flag, you can also do archive transfers between related Borg 2 repositories without upgrading, even down to individual archives. For more on that functionality, see the [Borg transfer -documentation](https://borgbackup.readthedocs.io/en/2.0.0b4/usage/transfer.html). +documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/transfer.html). That's it! Now you can use your new Borg 2 repository as normal with borgmatic. If you've got multiple repositories, repeat the above process for diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/setup.py new/borgmatic-1.7.8/setup.py --- old/borgmatic-1.7.7/setup.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/setup.py 2023-03-03 00:34:22.000000000 +0100 @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.7' +VERSION = '1.7.8' setup( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/tests/integration/commands/test_arguments.py new/borgmatic-1.7.8/tests/integration/commands/test_arguments.py --- old/borgmatic-1.7.7/tests/integration/commands/test_arguments.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/tests/integration/commands/test_arguments.py 2023-03-03 00:34:22.000000000 +0100 @@ -422,6 +422,13 @@ module.parse_arguments('--list', 'rcreate') +def test_parse_arguments_disallows_list_with_progress_for_create_action(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('create', '--list', '--progress') + + def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/tests/unit/actions/test_rcreate.py new/borgmatic-1.7.8/tests/unit/actions/test_rcreate.py --- old/borgmatic-1.7.7/tests/unit/actions/test_rcreate.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/tests/unit/actions/test_rcreate.py 2023-03-03 00:34:22.000000000 +0100 @@ -5,10 +5,39 @@ def test_run_rcreate_does_not_raise(): flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.rcreate).should_receive('create_repository') arguments = flexmock( encryption_mode=flexmock(), source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_dirs=flexmock(), + ) + + module.run_rcreate( + repository='repo', + storage={}, + local_borg_version=None, + rcreate_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_rcreate_bails_if_repository_does_not_match(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( + False + ) + flexmock(module.borgmatic.borg.rcreate).should_receive('create_repository').never() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/tests/unit/borg/test_create.py new/borgmatic-1.7.8/tests/unit/borg/test_create.py --- old/borgmatic-1.7.7/tests/unit/borg/test_create.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/tests/unit/borg/test_create.py 2023-03-03 00:34:22.000000000 +0100 @@ -284,6 +284,48 @@ assert exclude_flags == () +def test_make_list_filter_flags_with_debug_and_feature_available_includes_plus_and_minus(): + flexmock(module.logger).should_receive('isEnabledFor').and_return(True) + flexmock(module.feature).should_receive('available').and_return(True) + + assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME+-' + + +def test_make_list_filter_flags_with_info_and_feature_available_omits_plus_and_minus(): + flexmock(module.logger).should_receive('isEnabledFor').and_return(False) + flexmock(module.feature).should_receive('available').and_return(True) + + assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME' + + +def test_make_list_filter_flags_with_debug_and_feature_available_and_dry_run_includes_plus_and_minus(): + flexmock(module.logger).should_receive('isEnabledFor').and_return(True) + flexmock(module.feature).should_receive('available').and_return(True) + + assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=True) == 'AME+-' + + +def test_make_list_filter_flags_with_info_and_feature_available_and_dry_run_includes_plus_and_minus(): + flexmock(module.logger).should_receive('isEnabledFor').and_return(False) + flexmock(module.feature).should_receive('available').and_return(True) + + assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=True) == 'AME+-' + + +def test_make_list_filter_flags_with_debug_and_feature_not_available_includes_x(): + flexmock(module.logger).should_receive('isEnabledFor').and_return(True) + flexmock(module.feature).should_receive('available').and_return(False) + + assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AMEx-' + + +def test_make_list_filter_flags_with_info_and_feature_not_available_omits_x(): + flexmock(module.logger).should_receive('isEnabledFor').and_return(False) + flexmock(module.feature).should_receive('available').and_return(False) + + assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME-' + + def test_collect_borgmatic_source_directories_set_when_directory_exists(): flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.os.path).should_receive('expanduser') @@ -358,7 +400,7 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_list(): flexmock(module).should_receive('execute_command_and_capture_output').and_return( - 'Processing files ...\n- /foo\n- /bar\n- /baz' + 'Processing files ...\n- /foo\n+ /bar\n- /baz' ) flexmock(module).should_receive('special_file').and_return(True) flexmock(module).should_receive('any_parent_directories').and_return(False) @@ -374,7 +416,7 @@ def test_collect_special_file_paths_excludes_requested_directories(): flexmock(module).should_receive('execute_command_and_capture_output').and_return( - '- /foo\n- /bar\n- /baz' + '+ /foo\n- /bar\n- /baz' ) flexmock(module).should_receive('special_file').and_return(True) flexmock(module).should_receive('any_parent_directories').and_return(False).and_return( @@ -392,7 +434,7 @@ def test_collect_special_file_paths_excludes_non_special_files(): flexmock(module).should_receive('execute_command_and_capture_output').and_return( - '- /foo\n- /bar\n- /baz' + '+ /foo\n+ /bar\n+ /baz' ) flexmock(module).should_receive('special_file').and_return(True).and_return(False).and_return( True @@ -423,6 +465,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -464,6 +507,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -509,6 +553,7 @@ flexmock(module).should_receive('write_pattern_file').and_return( flexmock(name='/tmp/patterns') ).and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(pattern_flags) @@ -553,6 +598,7 @@ flexmock(module).should_receive('write_pattern_file').and_return(None).and_return( flexmock(name='/tmp/excludes') ) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -594,6 +640,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -636,6 +683,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -676,6 +724,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -718,6 +767,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -758,6 +808,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -801,6 +852,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -844,6 +896,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -874,6 +927,48 @@ ) +def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()) + flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'create', '--checkpoint-volume', '1024') + REPO_ARCHIVE_WITH_PATHS, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + + module.create_archive( + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'checkpoint_volume': 1024}, + local_borg_version='1.2.3', + ) + + def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -885,6 +980,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -926,6 +1022,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -972,6 +1069,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(feature_available) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1015,6 +1113,7 @@ ) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1057,6 +1156,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1104,6 +1204,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(feature_available) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1146,6 +1247,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1205,6 +1307,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1258,6 +1361,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(feature_available) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1311,6 +1415,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(feature_available) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1353,6 +1458,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1395,6 +1501,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1437,6 +1544,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1479,6 +1587,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1520,6 +1629,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1561,6 +1671,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1603,6 +1714,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1612,7 +1724,7 @@ ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create', '--list', '--filter', 'AMEx-') + REPO_ARCHIVE_WITH_PATHS, + ('borg', 'create', '--list', '--filter', 'FOO') + REPO_ARCHIVE_WITH_PATHS, output_log_level=module.borgmatic.logger.ANSWER, output_file=None, borg_local_path='borg', @@ -1645,6 +1757,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1688,6 +1801,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1731,6 +1845,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1791,6 +1906,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(flexmock(name='/tmp/excludes')) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module.logger).should_receive('warning').twice() @@ -1857,6 +1973,7 @@ flexmock(module).should_receive('write_pattern_file').and_return(None).and_return( flexmock(name='/excludes') ) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1921,6 +2038,7 @@ ('special',) ) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -1981,6 +2099,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2022,6 +2141,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2064,6 +2184,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2106,6 +2227,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2148,6 +2270,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2189,6 +2312,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2231,6 +2355,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2273,6 +2398,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2314,6 +2440,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) @@ -2356,6 +2483,7 @@ flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.feature).should_receive('available').and_return(True) flexmock(module).should_receive('ensure_files_readable') flexmock(module).should_receive('make_pattern_flags').and_return(()) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/tests/unit/borg/test_transfer.py new/borgmatic-1.7.8/tests/unit/borg/test_transfer.py --- old/borgmatic-1.7.7/tests/unit/borg/test_transfer.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/tests/unit/borg/test_transfer.py 2023-03-03 00:34:22.000000000 +0100 @@ -18,6 +18,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -27,7 +28,9 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives=None, source_repository=None), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), ) @@ -44,6 +47,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--repo', 'repo', '--dry-run'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -53,7 +57,9 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives=None, source_repository=None), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), ) @@ -67,6 +73,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -76,7 +83,9 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives=None, source_repository=None), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), ) @@ -90,6 +99,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--debug', '--show-rc', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -100,7 +110,9 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives=None, source_repository=None), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), ) @@ -117,6 +129,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--match-archives', 'archive', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -126,7 +139,9 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive='archive', match_archives=None, source_repository=None), + transfer_arguments=flexmock( + archive='archive', progress=None, match_archives=None, source_repository=None + ), ) @@ -143,6 +158,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--match-archives', 'sh:foo*', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -152,7 +168,9 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives='sh:foo*', source_repository=None), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives='sh:foo*', source_repository=None + ), ) @@ -166,6 +184,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg2', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg2', extra_environment=None, ) @@ -175,7 +194,9 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives=None, source_repository=None), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), local_path='borg2', ) @@ -193,6 +214,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--remote-path', 'borg2', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -202,7 +224,9 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives=None, source_repository=None), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), remote_path='borg2', ) @@ -221,6 +245,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--lock-wait', '5', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -230,7 +255,35 @@ repository='repo', storage_config=storage_config, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives=None, source_repository=None), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + ) + + +def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--progress', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.DO_NOT_CAPTURE, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository='repo', + storage_config={}, + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=True, match_archives=None, source_repository=None + ), ) @@ -248,6 +301,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', flag_name, 'value', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -258,7 +312,11 @@ storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( - archive=None, match_archives=None, source_repository=None, **{argument_name: 'value'} + archive=None, + progress=None, + match_archives=None, + source_repository=None, + **{argument_name: 'value'}, ), ) @@ -275,6 +333,7 @@ flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--repo', 'repo', '--other-repo', 'other'), output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, borg_local_path='borg', extra_environment=None, ) @@ -284,5 +343,7 @@ repository='repo', storage_config={}, local_borg_version='2.3.4', - transfer_arguments=flexmock(archive=None, match_archives=None, source_repository='other'), + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository='other' + ), ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/tests/unit/hooks/test_mysql.py new/borgmatic-1.7.8/tests/unit/hooks/test_mysql.py --- old/borgmatic-1.7.7/tests/unit/hooks/test_mysql.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/tests/unit/hooks/test_mysql.py 2023-03-03 00:34:22.000000000 +0100 @@ -9,26 +9,36 @@ def test_database_names_to_dump_passes_through_name(): extra_environment = flexmock() log_prefix = '' - dry_run_label = '' names = module.database_names_to_dump( - {'name': 'foo'}, extra_environment, log_prefix, dry_run_label + {'name': 'foo'}, extra_environment, log_prefix, dry_run=False ) assert names == ('foo',) +def test_database_names_to_dump_bails_for_dry_run(): + extra_environment = flexmock() + log_prefix = '' + flexmock(module).should_receive('execute_command_and_capture_output').never() + + names = module.database_names_to_dump( + {'name': 'all'}, extra_environment, log_prefix, dry_run=True + ) + + assert names == () + + def test_database_names_to_dump_queries_mysql_for_database_names(): extra_environment = flexmock() log_prefix = '' - dry_run_label = '' flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'), extra_environment=extra_environment, ).and_return('foo\nbar\nmysql\n').once() names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, log_prefix, dry_run_label + {'name': 'all'}, extra_environment, log_prefix, dry_run=False ) assert names == ('foo', 'bar') @@ -323,7 +333,6 @@ def test_dump_databases_errors_for_missing_all_databases(): databases = [{'name': 'all'}] - process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/localhost/all' @@ -331,7 +340,18 @@ flexmock(module).should_receive('database_names_to_dump').and_return(()) with pytest.raises(ValueError): - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) + + +def test_dump_databases_does_not_error_for_missing_all_databases_with_dry_run(): + databases = [{'name': 'all'}] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.dump).should_receive('make_database_dump_filename').and_return( + 'databases/localhost/all' + ) + flexmock(module).should_receive('database_names_to_dump').and_return(()) + + assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == [] def test_restore_database_dump_runs_mysql_to_restore(): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/tests/unit/hooks/test_ntfy.py new/borgmatic-1.7.8/tests/unit/hooks/test_ntfy.py --- old/borgmatic-1.7.7/tests/unit/hooks/test_ntfy.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/tests/unit/hooks/test_ntfy.py 2023-03-03 00:34:22.000000000 +0100 @@ -38,6 +38,7 @@ flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(module.monitor.State.FAIL), + auth=None, ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -45,6 +46,51 @@ ) +def test_ping_monitor_with_auth_hits_hosted_ntfy_on_fail(): + hook_config = { + 'topic': topic, + 'username': 'testuser', + 'password': 'fakepassword', + } + flexmock(module.requests).should_receive('post').with_args( + f'{default_base_url}/{topic}', + headers=return_default_message_headers(module.monitor.State.FAIL), + auth=module.requests.auth.HTTPBasicAuth('testuser', 'fakepassword'), + ).and_return(flexmock(ok=True)).once() + + module.ping_monitor( + hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + ) + + +def test_ping_monitor_auth_with_no_username_warning(): + hook_config = {'topic': topic, 'password': 'fakepassword'} + flexmock(module.requests).should_receive('post').with_args( + f'{default_base_url}/{topic}', + headers=return_default_message_headers(module.monitor.State.FAIL), + auth=None, + ).and_return(flexmock(ok=True)).once() + flexmock(module.logger).should_receive('warning').once() + + module.ping_monitor( + hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + ) + + +def test_ping_monitor_auth_with_no_password_warning(): + hook_config = {'topic': topic, 'username': 'testuser'} + flexmock(module.requests).should_receive('post').with_args( + f'{default_base_url}/{topic}', + headers=return_default_message_headers(module.monitor.State.FAIL), + auth=None, + ).and_return(flexmock(ok=True)).once() + flexmock(module.logger).should_receive('warning').once() + + module.ping_monitor( + hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + ) + + def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): hook_config = {'topic': topic} flexmock(module.requests).should_receive('post').never() @@ -76,6 +122,7 @@ flexmock(module.requests).should_receive('post').with_args( f'{custom_base_url}/{topic}', headers=return_default_message_headers(module.monitor.State.FAIL), + auth=None, ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -95,7 +142,7 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic, 'fail': custom_message_config} flexmock(module.requests).should_receive('post').with_args( - f'{default_base_url}/{topic}', headers=custom_message_headers, + f'{default_base_url}/{topic}', headers=custom_message_headers, auth=None ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -108,6 +155,7 @@ flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(module.monitor.State.START), + auth=None, ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -124,6 +172,7 @@ flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(module.monitor.State.FAIL), + auth=None, ).and_raise(module.requests.exceptions.ConnectionError) flexmock(module.logger).should_receive('warning').once() @@ -145,6 +194,7 @@ flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(module.monitor.State.FAIL), + auth=None, ).and_return(response) flexmock(module.logger).should_receive('warning').once() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/borgmatic-1.7.7/tests/unit/hooks/test_postgresql.py new/borgmatic-1.7.8/tests/unit/hooks/test_postgresql.py --- old/borgmatic-1.7.7/tests/unit/hooks/test_postgresql.py 2023-02-21 00:32:47.000000000 +0100 +++ new/borgmatic-1.7.8/tests/unit/hooks/test_postgresql.py 2023-03-03 00:34:22.000000000 +0100 @@ -9,19 +9,32 @@ def test_database_names_to_dump_passes_through_individual_database_name(): database = {'name': 'foo'} - assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ('foo',) + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + 'foo', + ) def test_database_names_to_dump_passes_through_individual_database_name_with_format(): database = {'name': 'foo', 'format': 'custom'} - assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ('foo',) + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + 'foo', + ) def test_database_names_to_dump_passes_through_all_without_format(): database = {'name': 'all'} - assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ('all',) + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + 'all', + ) + + +def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): + database = {'name': 'all', 'format': 'custom'} + flexmock(module).should_receive('execute_command_and_capture_output').never() + + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=True) == () def test_database_names_to_dump_with_all_and_format_lists_databases(): @@ -30,7 +43,7 @@ 'foo,test,\nbar,test,"stuff and such"' ) - assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ( + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -53,7 +66,7 @@ extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ( + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -66,7 +79,7 @@ extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ( + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -79,7 +92,7 @@ extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ( + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -91,7 +104,9 @@ 'foo,test,\ntemplate0,test,blah' ) - assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == ('foo',) + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + 'foo', + ) def test_dump_databases_runs_pg_dump_for_each_database(): @@ -139,6 +154,15 @@ module.dump_databases(databases, 'test.yaml', {}, dry_run=False) +def test_dump_databases_does_not_raise_when_no_database_names_to_dump(): + databases = [{'name': 'foo'}, {'name': 'bar'}] + flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(()) + + module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == [] + + def test_dump_databases_with_duplicate_dump_skips_pg_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})