From e32e636cff9910f660c5bcaa4c2cb2e9586f41b9 Mon Sep 17 00:00:00 2001 From: Israel Barth Rubio Date: Tue, 14 Jan 2025 19:49:35 +0000 Subject: [PATCH v1] pg_combinebackup: add support for hard links Up to now, pg_combinebackup reconstructs incremental files, if needed, otherwise copy them from any of the input backups to the output directory. That copy mecanism can use different methods, depending on the argument specified by the user. This commit adds support for a new "copy method": hard links (-k/--link). When using that mode, instead of copying unmodified files from the input backups to the output directory, pg_combinebackup creates the files as hard links from the output directory to the input backups. The new link method might speed up the reconstruction of the synthetic backup (no file copy) and reduce disk usage taken by the synthetic backup. The benefits depend on the modification pattern of files in PGDATA between backups, imposed by the workload on Postgres. This feature requires that the input backups plus the output directory are in the same file system. Also, caution is required from the user when modifying or starting the cluster from a synthetic backup, as that might invalidate one or more of the input backups. Signed-off-by: Israel Barth Rubio --- doc/src/sgml/ref/pg_combinebackup.sgml | 56 ++++++++++++++++++++- src/bin/pg_combinebackup/copy_file.c | 25 +++++++++ src/bin/pg_combinebackup/copy_file.h | 1 + src/bin/pg_combinebackup/pg_combinebackup.c | 18 ++++++- src/bin/pg_combinebackup/t/003_timeline.pl | 35 +++++++++++-- 5 files changed, 128 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/ref/pg_combinebackup.sgml b/doc/src/sgml/ref/pg_combinebackup.sgml index 091982f62a..49be0217c5 100644 --- a/doc/src/sgml/ref/pg_combinebackup.sgml +++ b/doc/src/sgml/ref/pg_combinebackup.sgml @@ -137,13 +137,38 @@ PostgreSQL documentation + + + + + + Use hard links instead of copying files to the synthetic backup. + Reconstruction of the synthetic backup might be faster (no file copying) + and use less disk space. + + + + Requires that the input backups and the output directory are in the + same file system. + + + + If a backup manifest is not available or does not contain checksum of + the right type, file cloning will be used to copy the file, but the + file will be also read block-by-block for the checksum calculation. + + + + Use efficient file cloning (also known as reflinks on some systems) instead of copying files to the new data directory, - which can result in near-instantaneous copying of the data files. + which can result in near-instantaneous copying of the data files, giving + the speed advantages of / + while leaving the input backups untouched. @@ -167,7 +192,8 @@ PostgreSQL documentation Perform regular file copy. This is the default. (See also - and .) + , and + /.) @@ -308,6 +334,32 @@ PostgreSQL documentation + + Notes + + + When pg_combinebackup is run with + /, the output synthetic backup may + share several files with the input backups because of the hard links it + creates. Any changes performed on files that were linked between any of the + input backups and the synthetic backup, will be reflected on both backups. + + + + + Modifying files or starting a cluster from a synthetic backup that was + reconstructed with /, might + invalidate one or more of the inputs backups that were used to reconstruct + it. + + + + It is recommended to move the synthetic backup to another file system or + machine before performing changes and/or starting the cluster. + + + + See Also diff --git a/src/bin/pg_combinebackup/copy_file.c b/src/bin/pg_combinebackup/copy_file.c index 4e27814839..8687e23fde 100644 --- a/src/bin/pg_combinebackup/copy_file.c +++ b/src/bin/pg_combinebackup/copy_file.c @@ -40,6 +40,9 @@ static void copy_file_copyfile(const char *src, const char *dst, pg_checksum_context *checksum_ctx); #endif +static void copy_file_link(const char *src, const char *dest, + pg_checksum_context *checksum_ctx); + /* * Copy a regular file, optionally computing a checksum, and emitting * appropriate debug messages. But if we're in dry-run mode, then just emit @@ -93,6 +96,10 @@ copy_file(const char *src, const char *dst, strategy_implementation = copy_file_copyfile; break; #endif + case COPY_METHOD_LINK: + strategy_name = "link"; + strategy_implementation = copy_file_link; + break; } if (dry_run) @@ -304,3 +311,21 @@ copy_file_copyfile(const char *src, const char *dst, checksum_file(src, checksum_ctx); } #endif /* WIN32 */ + +/* + * copy_file_link + * Hard-links a file from src to dest. + * + * If needed, also reads the file and calculates the checksum. + */ +static void +copy_file_link(const char *src, const char *dest, + pg_checksum_context *checksum_ctx) +{ + if (link(src, dest) < 0) + pg_fatal("error while linking file from \"%s\" to \"%s\": %m", + src, dest); + + /* if needed, calculate checksum of the file */ + checksum_file(src, checksum_ctx); +} diff --git a/src/bin/pg_combinebackup/copy_file.h b/src/bin/pg_combinebackup/copy_file.h index 92f104115b..5a8517629c 100644 --- a/src/bin/pg_combinebackup/copy_file.h +++ b/src/bin/pg_combinebackup/copy_file.h @@ -25,6 +25,7 @@ typedef enum CopyMethod #ifdef WIN32 COPY_METHOD_COPYFILE, #endif + COPY_METHOD_LINK, } CopyMethod; extern void copy_file(const char *src, const char *dst, diff --git a/src/bin/pg_combinebackup/pg_combinebackup.c b/src/bin/pg_combinebackup/pg_combinebackup.c index 5864ec574f..76f7a8704e 100644 --- a/src/bin/pg_combinebackup/pg_combinebackup.c +++ b/src/bin/pg_combinebackup/pg_combinebackup.c @@ -135,6 +135,7 @@ main(int argc, char *argv[]) {"no-sync", no_argument, NULL, 'N'}, {"output", required_argument, NULL, 'o'}, {"tablespace-mapping", required_argument, NULL, 'T'}, + {"link", no_argument, NULL, 'k'}, {"manifest-checksums", required_argument, NULL, 1}, {"no-manifest", no_argument, NULL, 2}, {"sync-method", required_argument, NULL, 3}, @@ -172,7 +173,7 @@ main(int argc, char *argv[]) opt.copy_method = COPY_METHOD_COPY; /* process command-line options */ - while ((c = getopt_long(argc, argv, "dnNo:T:", + while ((c = getopt_long(argc, argv, "dnNo:T:k", long_options, &optindex)) != -1) { switch (c) @@ -193,6 +194,9 @@ main(int argc, char *argv[]) case 'T': add_tablespace_mapping(&opt, optarg); break; + case 'k': + opt.copy_method = COPY_METHOD_LINK; + break; case 1: if (!pg_checksum_parse_type(optarg, &opt.manifest_checksums)) @@ -424,6 +428,17 @@ main(int argc, char *argv[]) } } + /* Warn about the possibility of compromising the backups, when link mode */ + if (opt.copy_method == COPY_METHOD_LINK) + { + pg_log_warning("Modifying files or starting the cluster from the " + "synthetic backup might invalidate one or more of the " + "input backups"); + pg_log_warning_hint("It is recommended to move the synthetic backup to " + "another file system or machine before performing " + "changes and/or starting the cluster"); + } + /* It's a success, so don't remove the output directories. */ reset_directory_cleanup_list(); exit(0); @@ -766,6 +781,7 @@ help(const char *progname) printf(_(" -o, --output=DIRECTORY output directory\n")); printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n" " relocate tablespace in OLDDIR to NEWDIR\n")); + printf(_(" -k, --link link files instead of copying\n")); printf(_(" --clone clone (reflink) files instead of copying\n")); printf(_(" --copy copy files (default)\n")); printf(_(" --copy-file-range copy using copy_file_range() system call\n")); diff --git a/src/bin/pg_combinebackup/t/003_timeline.pl b/src/bin/pg_combinebackup/t/003_timeline.pl index ad2dd04872..09d37a604a 100644 --- a/src/bin/pg_combinebackup/t/003_timeline.pl +++ b/src/bin/pg_combinebackup/t/003_timeline.pl @@ -70,6 +70,20 @@ $node2->command_ok( ], "incremental backup from node2"); +# Verify all the backups at this point, when mode is --link. +# In that case, the next init_from_backup runs pg_combinebackup with --link and +# then changes the linked postgresql.conf, causing future executions of +# pg_verifybackup to fail because of a mismatch in the size of that file. +if ($mode eq "--link") +{ + for my $backup_name (qw(backup1 backup2 backup3)) + { + $node1->command_ok( + [ 'pg_verifybackup', $node1->backup_dir . '/' . $backup_name ], + "verify backup $backup_name"); + } +} + # Restore the incremental backup and use it to create a new node. my $node3 = PostgreSQL::Test::Cluster->new('node3'); $node3->init_from_backup( @@ -89,12 +103,25 @@ select string_agg(a::text, ':'), string_agg(b, ':') from mytable; EOM is($result, '1:2:4:5|aardvark:beetle:dingo:elephant'); -# Let's also verify all the backups. +# Let's also verify all the backups, now for any mode. for my $backup_name (qw(backup1 backup2 backup3)) { - $node1->command_ok( - [ 'pg_verifybackup', $node1->backup_dir . '/' . $backup_name ], - "verify backup $backup_name"); + my $cmd = [ 'pg_verifybackup', $node1->backup_dir . '/' . $backup_name ]; + my $msg = "verify backup $backup_name"; + + # We expect verify backup3 to fail, when mode is --link. For more details, + # refer to a previous comment related with pg_verifybackup execution. + if ($mode eq "--link" && $backup_name eq "backup3") + { + $node1->command_fails_like( + $cmd, + qr/"postgresql\.conf" has size \d+ on disk but size \d+ in the manifest/, + $msg); + } + else + { + $node1->command_ok($cmd, $msg); + } } # OK, that's all. -- 2.43.5