From 21cd35e8850bc64c9d8d65e9863aa5c31b26c215 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 17 Apr 2024 15:56:33 -0400 Subject: [PATCH 1/2] Make PostgreSQL::Test::Cluster::init_from_backup handle tablespaces. This commit doesn't use this infrastructure for anything new, although it does adapt 010_pg_basebackup.pl to use it. However, a future commit will use this to improve test coverage for pg_combinebackup. --- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 21 +---- src/test/perl/PostgreSQL/Test/Cluster.pm | 93 ++++++++++++++++++-- 2 files changed, 90 insertions(+), 24 deletions(-) diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 63f7bd2735..31c369688e 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -407,25 +407,12 @@ SKIP: my $node2 = PostgreSQL::Test::Cluster->new('replica'); - # Recover main data directory - $node2->init_from_backup($node, 'tarbackup2', tar_program => $tar); - - # Recover tablespace into a new directory (not where it was!) - my $repTsDir = "$tempdir/tblspc1replica"; - my $realRepTsDir = "$real_sys_tempdir/tblspc1replica"; - mkdir $repTsDir; - PostgreSQL::Test::Utils::system_or_bail($tar, 'xf', $tblspc_tars[0], - '-C', $repTsDir); - - # Update tablespace map to point to new directory. - # XXX Ideally pg_basebackup would handle this. + # Recover the backup $tblspc_tars[0] =~ m|/([0-9]*)\.tar$|; my $tblspcoid = $1; - my $escapedRepTsDir = $realRepTsDir; - $escapedRepTsDir =~ s/\\/\\\\/g; - open my $mapfile, '>', $node2->data_dir . '/tablespace_map' or die $!; - print $mapfile "$tblspcoid $escapedRepTsDir\n"; - close $mapfile; + my $realRepTsDir = "$real_sys_tempdir/tblspc1replica"; + $node2->init_from_backup($node, 'tarbackup2', tar_program => $tar, + 'tablespace_map' => { $tblspcoid => $realRepTsDir }); $node2->start; my $result = $node2->safe_psql('postgres', 'SELECT * FROM test1'); diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm index 9b2879c145..52972a9746 100644 --- a/src/test/perl/PostgreSQL/Test/Cluster.pm +++ b/src/test/perl/PostgreSQL/Test/Cluster.pm @@ -777,7 +777,7 @@ sub backup_fs_cold =pod -=item $node->init_from_backup(root_node, backup_name) +=item $node->init_from_backup(root_node, backup_name, %params) Initialize a node from a backup, which may come from this node or a different node. root_node must be a PostgreSQL::Test::Cluster reference, backup_name the string name @@ -787,8 +787,13 @@ Does not start the node after initializing it. By default, the backup is assumed to be plain format. To restore from a tar-format backup, pass the name of the tar program to use in the -keyword parameter tar_program. Note that tablespace tar files aren't -handled here. +keyword parameter tar_program. + +If there are tablespace present in the backup, include tablespace_map as +a keyword parameter whose values is a hash. When tar_program is used, the +hash keys are tablespace OIDs; otherwise, they are the tablespace pathnames +used in the backup. In either case, the values are the tablespace pathnames +that should be used for the target cluster. To restore from an incremental backup, pass the parameter combine_with_prior as a reference to an array of prior backup names with which this backup @@ -843,12 +848,20 @@ sub init_from_backup } local %ENV = $self->_get_env(); - PostgreSQL::Test::Utils::system_or_bail('pg_combinebackup', '-d', - @prior_backup_path, $backup_path, '-o', $data_path); + my @combineargs = ('pg_combinebackup', '-d'); + if (exists $params{tablespace_map}) + { + while (my ($olddir, $newdir) = each %{$params{tablespace_map}}) + { + push @combineargs, "-T$olddir=$newdir"; + } + } + push @combineargs, @prior_backup_path, $backup_path, '-o', $data_path; + PostgreSQL::Test::Utils::system_or_bail(@combineargs); } elsif (defined $params{tar_program}) { - mkdir($data_path); + mkdir($data_path) || die "mkdir $data_path: $!"; PostgreSQL::Test::Utils::system_or_bail($params{tar_program}, 'xf', $backup_path . '/base.tar', '-C', $data_path); @@ -856,11 +869,77 @@ sub init_from_backup $params{tar_program}, 'xf', $backup_path . '/pg_wal.tar', '-C', $data_path . '/pg_wal'); + + # We need to generate a tablespace_map file. + open(my $tsmap, ">", "$data_path/tablespace_map") + || die "$data_path/tablespace_map: $!"; + + # Extract tarfiles and add tablespace_map entries + my @tstars = grep { /^\d+.tar/ } + PostgreSQL::Test::Utils::slurp_dir($backup_path); + for my $tstar (@tstars) + { + my $tsoid = $tstar; + $tsoid =~ s/\.tar$//; + + die "no tablespace mapping for $tstar" + if !exists $params{tablespace_map} || + !exists $params{tablespace_map}{$tsoid}; + my $newdir = $params{tablespace_map}{$tsoid}; + + mkdir($newdir) || die "mkdir $newdir: $!"; + PostgreSQL::Test::Utils::system_or_bail($params{tar_program}, 'xf', + $backup_path . '/' . $tstar, '-C', $newdir); + + my $escaped_newdir = $newdir; + $escaped_newdir =~ s/\\/\\\\/g; + print $tsmap "$tsoid $escaped_newdir\n"; + } + + # Close tablespace_map. + close($tsmap); } else { + my @tsoids; rmdir($data_path); - PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path); + + # Copy the main backup. Exclude tablespace links, but remember them. + PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path, + 'filterfn' => sub { + my ($path) = @_; + if ($path =~ /^pg_tblspc\/(\d+)$/ && -l "$backup_path/$path") + { + push @tsoids, $1; + return 0; + } + return 1; + }); + + # We need to generate a tablespace_map file. + open(my $tsmap, ">", "$data_path/tablespace_map") + || die "$data_path/tablespace_map: $!"; + + # Now use the list of tablespace links to copy each tablespace. + for my $tsoid (@tsoids) + { + my $olddir = readlink("$backup_path/pg_tblspc/$tsoid") + || die "readlink $backup_path/pg_tblspc/$tsoid: $!"; + + die "no tablespace mapping for $olddir" + if !exists $params{tablespace_map} || + !exists $params{tablespace_map}{$olddir}; + + my $newdir = $params{tablespace_map}{$olddir}; + PostgreSQL::Test::RecursiveCopy::copypath($olddir, $newdir); + + my $escaped_newdir = $newdir; + $escaped_newdir =~ s/\\/\\\\/g; + print $tsmap "$tsoid $escaped_newdir\n"; + } + + # Close tablespace_map. + close($tsmap); } chmod(0700, $data_path) or die $!; -- 2.39.3 (Apple Git-145)