Hi,
I will work out this one with postgres -C and come back till the next commitfest. I found that something similar is already used in pg_ctl and there is a mechanism for finding valid executables in exec.c. So it does not seem to be a big deal at the first sight.
I have reworked the patch, please find new version attached. It is 3 times as smaller than the previous one and now touches pg_rewind's code only. Tests are also slightly refactored in order to remove duplicated code. Execution of postgres -C is used for restore_command retrieval (if -r is passed) as being suggested. Otherwise everything works as before.
Andres, Alvaro does it make sense now? Regards -- Alexey Kondratov Postgres Professional https://www.postgrespro.com Russian Postgres Company
>From 4c8f5c228e089e7e72835ae5c409a5bc8425ab15 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov <kondratov.alek...@gmail.com> Date: Tue, 19 Feb 2019 19:14:53 +0300 Subject: [PATCH v4] pg_rewind: options to use restore_command from command line or cluster config --- doc/src/sgml/ref/pg_rewind.sgml | 30 ++++- src/bin/pg_rewind/parsexlog.c | 161 +++++++++++++++++++++++++- src/bin/pg_rewind/pg_rewind.c | 98 +++++++++++++++- src/bin/pg_rewind/pg_rewind.h | 7 +- src/bin/pg_rewind/t/001_basic.pl | 4 +- src/bin/pg_rewind/t/002_databases.pl | 4 +- src/bin/pg_rewind/t/003_extrafiles.pl | 4 +- src/bin/pg_rewind/t/RewindTest.pm | 84 +++++++++++++- 8 files changed, 372 insertions(+), 20 deletions(-) diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index 53a64ee29e..0c2441afa7 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -67,8 +67,10 @@ PostgreSQL documentation ancestor. In the typical failover scenario where the target cluster was shut down soon after the divergence, this is not a problem, but if the target cluster ran for a long time after the divergence, the old WAL - files might no longer be present. In that case, they can be manually - copied from the WAL archive to the <filename>pg_wal</filename> directory, or + files might no longer be present. In that case, they can be automatically + copied by <application>pg_rewind</application> from the WAL archive to the + <filename>pg_wal</filename> directory if either <literal>-r</literal> or + <literal>-R</literal> option is specified, or fetched on startup by configuring <xref linkend="guc-primary-conninfo"/> or <xref linkend="guc-restore-command"/>. The use of <application>pg_rewind</application> is not limited to failover, e.g. a standby @@ -200,6 +202,30 @@ PostgreSQL documentation </listitem> </varlistentry> + <varlistentry> + <term><option>-r</option></term> + <term><option>--use-postgresql-conf</option></term> + <listitem> + <para> + Use restore_command in the <filename>postgresql.conf</filename> to + retreive missing in the target <filename>pg_wal</filename> directory + WAL files from the WAL archive. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-R <replaceable class="parameter">restore_command</replaceable></option></term> + <term><option>--restore-command=<replaceable class="parameter">restore_command</replaceable></option></term> + <listitem> + <para> + Specifies the restore_command to use for retrieval of the missing + in the target <filename>pg_wal</filename> directory WAL files from + the WAL archive. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><option>--debug</option></term> <listitem> diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index e19c265cbb..5978ec9b99 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -12,6 +12,7 @@ #include "postgres_fe.h" #include <unistd.h> +#include <sys/stat.h> #include "pg_rewind.h" #include "filemap.h" @@ -45,6 +46,7 @@ static char xlogfpath[MAXPGPATH]; typedef struct XLogPageReadPrivate { const char *datadir; + const char *restoreCommand; int tliIndex; } XLogPageReadPrivate; @@ -53,6 +55,9 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); +static int RestoreArchivedWAL(const char *path, const char *xlogfname, + off_t expectedSize, const char *restoreCommand); + /* * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of @@ -60,7 +65,7 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, */ void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, - XLogRecPtr endpoint) + XLogRecPtr endpoint, const char *restore_command) { XLogRecord *record; XLogReaderState *xlogreader; @@ -69,6 +74,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; + private.restoreCommand = restore_command; xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, &private); if (xlogreader == NULL) @@ -156,7 +162,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, - XLogRecPtr *lastchkptredo) + XLogRecPtr *lastchkptredo, const char *restoreCommand) { /* Walk backwards, starting from the given record */ XLogRecord *record; @@ -181,6 +187,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; + private.restoreCommand = restoreCommand; xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, &private); if (xlogreader == NULL) @@ -291,9 +298,30 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (xlogreadfd < 0) { - printf(_("could not open file \"%s\": %s\n"), xlogfpath, - strerror(errno)); - return -1; + /* + * If we have no restore_command to execute, then exit. + */ + if (private->restoreCommand == NULL) + { + printf(_("could not open file \"%s\": %s\n"), xlogfpath, + strerror(errno)); + return -1; + } + + /* + * Since we have restore_command to execute, then try to retreive + * missing WAL file from the archive. + */ + xlogreadfd = RestoreArchivedWAL(private->datadir, + xlogfname, + WalSegSz, + private->restoreCommand); + + if (xlogreadfd < 0) + return -1; + else + pg_log(PG_DEBUG, "using restored from archive version of file \"%s\"\n", + xlogfpath); } } @@ -409,3 +437,126 @@ extractPageInfo(XLogReaderState *record) process_block_change(forknum, rnode, blkno); } } + +/* + * Attempt to retrieve the specified file from off-line archival storage. + * If successful return a file descriptor of restored WAL file, else + * return -1. + * + * For fixed-size files, the caller may pass the expected size as an + * additional crosscheck on successful recovery. If the file size is not + * known, set expectedSize = 0. + * + * This is a simplified and adapted to frontend version of + * RestoreArchivedFile function from transam/xlogarchive.c + */ +static int +RestoreArchivedWAL(const char *path, const char *xlogfname, + off_t expectedSize, const char *restoreCommand) +{ + char xlogpath[MAXPGPATH], + xlogRestoreCmd[MAXPGPATH], + *dp, + *endp; + const char *sp; + int rc, + xlogfd; + struct stat stat_buf; + + snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname); + + /* + * Construct the command to be executed. + */ + dp = xlogRestoreCmd; + endp = xlogRestoreCmd + MAXPGPATH - 1; + *endp = '\0'; + + for (sp = restoreCommand; *sp; sp++) + { + if (*sp == '%') + { + switch (sp[1]) + { + case 'p': + /* %p: relative path of target file */ + sp++; + StrNCpy(dp, xlogpath, endp - dp); + make_native_path(dp); + dp += strlen(dp); + break; + case 'f': + /* %f: filename of desired file */ + sp++; + StrNCpy(dp, xlogfname, endp - dp); + dp += strlen(dp); + break; + case 'r': + /* %r: filename of last restartpoint */ + pg_fatal("restore_command with %%r cannot be used during rewind process.\n"); + break; + case '%': + /* convert %% to a single % */ + sp++; + if (dp < endp) + *dp++ = *sp; + break; + default: + /* otherwise treat the % as not special */ + if (dp < endp) + *dp++ = *sp; + break; + } + } + else + { + if (dp < endp) + *dp++ = *sp; + } + } + *dp = '\0'; + + /* + * Execute restore_command, which should copy + * the missing WAL file from archival storage. + */ + rc = system(xlogRestoreCmd); + + if (rc == 0) + { + /* + * Command apparently succeeded, but let's make sure the file is + * really there now and has the correct size. + */ + if (stat(xlogpath, &stat_buf) == 0) + { + if (expectedSize > 0 && stat_buf.st_size != expectedSize) + { + printf(_("archive file \"%s\" has wrong size: %lu instead of %lu, %s"), + xlogfname, (unsigned long) stat_buf.st_size, + (unsigned long) expectedSize, strerror(errno)); + } + else + { + xlogfd = open(xlogpath, O_RDONLY | PG_BINARY, 0); + + if (xlogfd < 0) + printf(_("could not open restored from archive file \"%s\": %s\n"), + xlogpath, strerror(errno)); + else + return xlogfd; + } + } + else + { + /* Stat failed */ + printf(_("could not stat file \"%s\": %s"), + xlogpath, strerror(errno)); + } + } + + printf(_("could not restore file \"%s\" from archive: %s\n"), + xlogfname, strerror(errno)); + + return -1; +} diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 7ccde5c87f..9da578ec42 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -30,6 +30,8 @@ #include "getopt_long.h" #include "storage/bufpage.h" +#define MAX_RESTORE_COMMAND 1024 + static void usage(const char *progname); static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli, @@ -52,11 +54,13 @@ int WalSegSz; char *datadir_target = NULL; char *datadir_source = NULL; char *connstr_source = NULL; +char *restore_command = NULL; bool debug = false; bool showprogress = false; bool dry_run = false; bool do_sync = true; +bool restore_wals = false; /* Target history */ TimeLineHistoryEntry *targetHistory; @@ -75,6 +79,9 @@ usage(const char *progname) printf(_(" -N, --no-sync do not wait for changes to be written\n")); printf(_(" safely to disk\n")); printf(_(" -P, --progress write progress messages\n")); + printf(_(" -r, --use-postgresql-conf use restore_command in the postgresql.conf to\n")); + printf(_(" retreive WALs from archive\n")); + printf(_(" -R, --restore-command=COMMAND restore_command\n")); printf(_(" --debug write a lot of debug messages\n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" -?, --help show this help, then exit\n")); @@ -94,6 +101,8 @@ main(int argc, char **argv) {"dry-run", no_argument, NULL, 'n'}, {"no-sync", no_argument, NULL, 'N'}, {"progress", no_argument, NULL, 'P'}, + {"use-postgresql-conf", no_argument, NULL, 'r'}, + {"restore-command", required_argument, NULL, 'R'}, {"debug", no_argument, NULL, 3}, {NULL, 0, NULL, 0} }; @@ -129,7 +138,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "D:nNP", long_options, &option_index)) != -1) + while ((c = getopt_long(argc, argv, "D:nNPR:r", long_options, &option_index)) != -1) { switch (c) { @@ -141,6 +150,10 @@ main(int argc, char **argv) showprogress = true; break; + case 'r': + restore_wals = true; + break; + case 'n': dry_run = true; break; @@ -157,6 +170,10 @@ main(int argc, char **argv) datadir_target = pg_strdup(optarg); break; + case 'R': + restore_command = pg_strdup(optarg); + break; + case 1: /* --source-pgdata */ datadir_source = pg_strdup(optarg); break; @@ -223,6 +240,78 @@ main(int argc, char **argv) umask(pg_mode_mask); + if (restore_command != NULL) + { + if (restore_wals) + { + fprintf(stderr, _("%s: conflicting options: both -r and -R are specified\n"), + progname); + fprintf(stderr, _("You must run %s with either -r/--use-postgresql-conf " + "or -R/--restore-command.\n"), progname); + exit(1); + } + + pg_log(PG_DEBUG, "using command line restore_command=\'%s\'.\n", restore_command); + } + else if (restore_wals) + { + int rc; + char postgres_exec_path[MAXPGPATH], + postgres_cmd[MAXPGPATH], + cmd_output[MAX_RESTORE_COMMAND]; + FILE *output_fp; + + /* Find postgres executable. */ + rc = find_other_exec(argv[0], "postgres", + PG_BACKEND_VERSIONSTR, + postgres_exec_path); + + if (rc < 0) + { + char full_path[MAXPGPATH]; + + if (find_my_exec(argv[0], full_path) < 0) + strlcpy(full_path, progname, sizeof(full_path)); + + if (rc == -1) + fprintf(stderr, + _("the program \"postgres\" is needed by %s " + "but was not found in the\n" + "same directory as \"%s\".\n" + "Check your installation.\n"), + progname, full_path); + else + fprintf(stderr, + _("the program \"postgres\" was found by \"%s\"\n" + "but was not the same version as %s.\n" + "Check your installation.\n"), + full_path, progname); + exit(1); + } + + /* Build a command to execute for restore_command GUC retrieval if set. */ + snprintf(postgres_cmd, sizeof(postgres_cmd), "%s -D %s -C restore_command", + postgres_exec_path, datadir_target); + + if ((output_fp = popen(postgres_cmd, "r")) == NULL || + fgets(cmd_output, sizeof(cmd_output), output_fp) == NULL) + pg_fatal("could not get restore_command using %s: %s\n", + postgres_cmd, strerror(errno)); + + pclose(output_fp); + + /* Remove trailing newline */ + if (strchr(cmd_output, '\n') != NULL) + *strchr(cmd_output, '\n') = '\0'; + + if (!strcmp(cmd_output, "")) + pg_fatal("restore_command is not set on the target cluster\n"); + + restore_command = pg_strdup(cmd_output); + + pg_log(PG_DEBUG, "using config variable restore_command=\'%s\'.\n", restore_command); + } + /* Connect to remote server */ if (connstr_source) libpqConnect(connstr_source); @@ -294,9 +383,8 @@ main(int argc, char **argv) exit(0); } - findLastCheckpoint(datadir_target, divergerec, - lastcommontliIndex, - &chkptrec, &chkpttli, &chkptredo); + findLastCheckpoint(datadir_target, divergerec, lastcommontliIndex, + &chkptrec, &chkpttli, &chkptredo, restore_command); printf(_("rewinding from last common checkpoint at %X/%X on timeline %u\n"), (uint32) (chkptrec >> 32), (uint32) chkptrec, chkpttli); @@ -319,7 +407,7 @@ main(int argc, char **argv) */ pg_log(PG_PROGRESS, "reading WAL in target\n"); extractPageMap(datadir_target, chkptrec, lastcommontliIndex, - ControlFile_target.checkPoint); + ControlFile_target.checkPoint, restore_command); filemap_finalize(); if (showprogress) diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index 83b2898b8b..08a753475c 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -32,11 +32,10 @@ extern int targetNentries; /* in parsexlog.c */ extern void extractPageMap(const char *datadir, XLogRecPtr startpoint, - int tliIndex, XLogRecPtr endpoint); + int tliIndex, XLogRecPtr endpoint, const char *restoreCommand); extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr, - int tliIndex, - XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, - XLogRecPtr *lastchkptredo); + int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, + XLogRecPtr *lastchkptredo, const char *restoreCommand); extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex); diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.pl index 115192170e..8a6fa33016 100644 --- a/src/bin/pg_rewind/t/001_basic.pl +++ b/src/bin/pg_rewind/t/001_basic.pl @@ -1,7 +1,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 10; +use Test::More tests => 20; use FindBin; use lib $FindBin::RealBin; @@ -106,5 +106,7 @@ in master, before promotion # Run the test in both modes run_test('local'); run_test('remote'); +run_test('archive'); +run_test('archive_conf'); exit(0); diff --git a/src/bin/pg_rewind/t/002_databases.pl b/src/bin/pg_rewind/t/002_databases.pl index 6dc05720a1..99ea821cfe 100644 --- a/src/bin/pg_rewind/t/002_databases.pl +++ b/src/bin/pg_rewind/t/002_databases.pl @@ -1,7 +1,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 6; +use Test::More tests => 12; use FindBin; use lib $FindBin::RealBin; @@ -62,5 +62,7 @@ template1 # Run the test in both modes. run_test('local'); run_test('remote'); +run_test('archive'); +run_test('archive_conf'); exit(0); diff --git a/src/bin/pg_rewind/t/003_extrafiles.pl b/src/bin/pg_rewind/t/003_extrafiles.pl index c4040bd562..24cec256de 100644 --- a/src/bin/pg_rewind/t/003_extrafiles.pl +++ b/src/bin/pg_rewind/t/003_extrafiles.pl @@ -3,7 +3,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 4; +use Test::More tests => 8; use File::Find; @@ -90,5 +90,7 @@ sub run_test # Run the test in both modes. run_test('local'); run_test('remote'); +run_test('archive'); +run_test('archive_conf'); exit(0); diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 85cae7e47b..4560fe4430 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -39,7 +39,9 @@ use Carp; use Config; use Exporter 'import'; use File::Copy; -use File::Path qw(rmtree); +use File::Glob ':bsd_glob'; +use File::Path qw(remove_tree make_path); +use File::Spec::Functions qw(catdir catfile); use IPC::Run qw(run); use PostgresNode; use TestLib; @@ -197,6 +199,38 @@ sub promote_standby return; } +# Moves WAL files to the temporary location and returns restore_command +# to get them back. +sub move_wals +{ + my $tmp_dir = shift; + my $master_pgdata = shift; + my $wals_archive_dir = catdir($tmp_dir, "master_wals_archive"); + my @wal_files = bsd_glob catfile($master_pgdata, "pg_wal", "0000000*"); + my $restore_command; + + remove_tree($wals_archive_dir); + make_path($wals_archive_dir) or die; + + # Move all old master WAL files to the archive. + # Old master should be stopped at this point. + foreach my $wal_file (@wal_files) + { + move($wal_file, "$wals_archive_dir") or die; + } + + if ($windows_os) + { + $restore_command = "copy $wals_archive_dir\\\%f \%p"; + } + else + { + $restore_command = "cp $wals_archive_dir/\%f \%p"; + } + + return $restore_command; +} + sub run_pg_rewind { my $test_mode = shift; @@ -249,6 +283,54 @@ sub run_pg_rewind ], 'pg_rewind remote'); } + elsif ($test_mode eq "archive") + { + + # Do rewind using a local pgdata as source and + # specified directory with target WALs archive. + my $restore_command = move_wals($tmp_folder, $master_pgdata); + + # Stop the new master and be ready to perform the rewind. + $node_standby->stop; + + command_ok( + [ + 'pg_rewind', + "--debug", + "--source-pgdata=$standby_pgdata", + "--target-pgdata=$master_pgdata", + "--no-sync", + "--restore-command=$restore_command" + ], + 'pg_rewind archive'); + } + elsif ($test_mode eq "archive_conf") + { + + # Do rewind using a local pgdata as source and + # specified directory with target WALs archive. + my $master_conf_path = catfile($master_pgdata, 'postgresql.conf'); + my $restore_command = move_wals($tmp_folder, $master_pgdata); + + # Stop the new master and be ready to perform the rewind. + $node_standby->stop; + + # Add restore_command to postgresql.conf of target cluster. + open(my $conf_fd, ">>", $master_conf_path) or die; + print $conf_fd "\nrestore_command='$restore_command'"; + close $conf_fd; + + command_ok( + [ + 'pg_rewind', + "--debug", + "--source-pgdata=$standby_pgdata", + "--target-pgdata=$master_pgdata", + "--no-sync", + "-r" + ], + 'pg_rewind archive_conf'); + } else { base-commit: 41531e42d34f4aca117d343b5e40f3f757dec5fe -- 2.17.1