On Tue, Dec 19, 2023 at 9:51 AM Michael Paquier <mich...@paquier.xyz> wrote: > > On Mon, Dec 18, 2023 at 08:48:09AM -0300, Euler Taveira wrote: > > It is cheaper. > > Agreed that this could just use a set of pg_logical_emit_message() > when jumping across N segments.
Thanks. I missed the point of using pg_logical_emit_message() over CREATE .. DROP TABLE to generate WAL. And, I agree that it's better and relatively cheaper in terms of amount of WAL generated. > Another thing that seems quite > important to me is to force a flush of WAL with the last segment > switch, and the new "flush" option of pg_logical_emit_message() can > be very handy for this purpose. I used pg_logical_emit_message() in non-transactional mode without needing an explicit WAL flush as the pg_switch_wal() does a WAL flush at the end [1]. Attached v4 patch. [1] /* * If this was an XLOG_SWITCH record, flush the record and the empty * padding space that fills the rest of the segment, and perform * end-of-segment actions (eg, notifying archiver). */ if (class == WALINSERT_SPECIAL_SWITCH) { TRACE_POSTGRESQL_WAL_SWITCH(); XLogFlush(EndPos); -- Bharath Rupireddy PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com
From b7fa7545eb983aaf92e3d7e99bdf76ef42b8e40e Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com> Date: Tue, 19 Dec 2023 05:49:20 +0000 Subject: [PATCH v4] Add a TAP test function to generate WAL This commit adds a perl function in Cluster.pm to generate WAL. Some TAP tests are now using their own way to generate WAL. Generalizing this functionality enables multiple TAP tests to reuse the functionality. --- src/test/perl/PostgreSQL/Test/Cluster.pm | 22 +++++++++ src/test/recovery/t/001_stream_rep.pl | 6 +-- src/test/recovery/t/019_replslot_limit.pl | 48 +++++-------------- .../t/035_standby_logical_decoding.pl | 7 +-- 4 files changed, 38 insertions(+), 45 deletions(-) diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm index a020377761..ad575ed6d6 100644 --- a/src/test/perl/PostgreSQL/Test/Cluster.pm +++ b/src/test/perl/PostgreSQL/Test/Cluster.pm @@ -3178,6 +3178,28 @@ sub create_logical_slot_on_standby =pod +=item $node->advance_wal($n) + +Advance WAL of given node by $n segments + +=cut + +sub advance_wal +{ + my ($self, $n) = @_; + + # Advance by $n segments (= (wal_segment_size * $n) bytes). + for (my $i = 0; $i < $n; $i++) + { + $self->safe_psql('postgres', qq{ + SELECT pg_logical_emit_message(false, '', 'foo'); + SELECT pg_switch_wal(); + }); + } +} + +=pod + =back =cut diff --git a/src/test/recovery/t/001_stream_rep.pl b/src/test/recovery/t/001_stream_rep.pl index 95f9b0d772..f0de921b4b 100644 --- a/src/test/recovery/t/001_stream_rep.pl +++ b/src/test/recovery/t/001_stream_rep.pl @@ -522,11 +522,7 @@ $node_primary->safe_psql('postgres', my $segment_removed = $node_primary->safe_psql('postgres', 'SELECT pg_walfile_name(pg_current_wal_lsn())'); chomp($segment_removed); -$node_primary->psql( - 'postgres', " - CREATE TABLE tab_phys_slot (a int); - INSERT INTO tab_phys_slot VALUES (generate_series(1,10)); - SELECT pg_switch_wal();"); +$node_primary->advance_wal(1); my $current_lsn = $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn();"); chomp($current_lsn); diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl index 7d94f15778..e4b75c6545 100644 --- a/src/test/recovery/t/019_replslot_limit.pl +++ b/src/test/recovery/t/019_replslot_limit.pl @@ -59,7 +59,7 @@ $result = $node_primary->safe_psql('postgres', is($result, "reserved|t", 'check the catching-up state'); # Advance WAL by five segments (= 5MB) on primary -advance_wal($node_primary, 1); +$node_primary->advance_wal(1); $node_primary->safe_psql('postgres', "CHECKPOINT;"); # The slot is always "safe" when fitting max_wal_size @@ -69,7 +69,7 @@ $result = $node_primary->safe_psql('postgres', is($result, "reserved|t", 'check that it is safe if WAL fits in max_wal_size'); -advance_wal($node_primary, 4); +$node_primary->advance_wal(4); $node_primary->safe_psql('postgres', "CHECKPOINT;"); # The slot is always "safe" when max_slot_wal_keep_size is not set @@ -100,7 +100,7 @@ $result = $node_primary->safe_psql('postgres', is($result, "reserved", 'check that max_slot_wal_keep_size is working'); # Advance WAL again then checkpoint, reducing remain by 2 MB. -advance_wal($node_primary, 2); +$node_primary->advance_wal(2); $node_primary->safe_psql('postgres', "CHECKPOINT;"); # The slot is still working @@ -118,7 +118,7 @@ $node_standby->stop; $result = $node_primary->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_size to '8MB'; SELECT pg_reload_conf();"); # Advance WAL again then checkpoint, reducing remain by 6 MB. -advance_wal($node_primary, 6); +$node_primary->advance_wal(6); $result = $node_primary->safe_psql('postgres', "SELECT wal_status as remain FROM pg_replication_slots WHERE slot_name = 'rep1'" ); @@ -134,7 +134,7 @@ $node_primary->wait_for_catchup($node_standby); $node_standby->stop; # Advance WAL again without checkpoint, reducing remain by 6 MB. -advance_wal($node_primary, 6); +$node_primary->advance_wal(6); # Slot gets into 'reserved' state $result = $node_primary->safe_psql('postgres', @@ -145,7 +145,7 @@ is($result, "extended", 'check that the slot state changes to "extended"'); $node_primary->safe_psql('postgres', "CHECKPOINT;"); # Advance WAL again without checkpoint; remain goes to 0. -advance_wal($node_primary, 1); +$node_primary->advance_wal(1); # Slot gets into 'unreserved' state and safe_wal_size is negative $result = $node_primary->safe_psql('postgres', @@ -174,7 +174,7 @@ $node_primary->safe_psql('postgres', # Advance WAL again. The slot loses the oldest segment by the next checkpoint my $logstart = -s $node_primary->logfile; -advance_wal($node_primary, 7); +$node_primary->advance_wal(7); # Now create another checkpoint and wait until the WARNING is issued $node_primary->safe_psql('postgres', @@ -275,18 +275,11 @@ $node_standby->init_from_backup($node_primary2, $backup_name, has_streaming => 1); $node_standby->append_conf('postgresql.conf', "primary_slot_name = 'rep1'"); $node_standby->start; -my @result = - split( - '\n', - $node_primary2->safe_psql( - 'postgres', - "CREATE TABLE tt(); - DROP TABLE tt; - SELECT pg_switch_wal(); - CHECKPOINT; - SELECT 'finished';", - timeout => $PostgreSQL::Test::Utils::timeout_default)); -is($result[1], 'finished', 'check if checkpoint command is not blocked'); +$node_primary2->advance_wal(1); +$result = $node_primary2->safe_psql('postgres', + "CHECKPOINT; SELECT 'finished';", + timeout => $PostgreSQL::Test::Utils::timeout_default); +is($result, 'finished', 'check if checkpoint command is not blocked'); $node_primary2->stop; $node_standby->stop; @@ -372,7 +365,7 @@ $logstart = -s $node_primary3->logfile; # freeze walsender and walreceiver. Slot will still be active, but walreceiver # won't get anything anymore. kill 'STOP', $senderpid, $receiverpid; -advance_wal($node_primary3, 2); +$node_primary3->advance_wal(2); my $msg_logged = 0; my $max_attempts = $PostgreSQL::Test::Utils::timeout_default; @@ -418,19 +411,4 @@ kill 'CONT', $receiverpid; $node_primary3->stop; $node_standby3->stop; -##################################### -# Advance WAL of $node by $n segments -sub advance_wal -{ - my ($node, $n) = @_; - - # Advance by $n segments (= (wal_segment_size * $n) bytes) on primary. - for (my $i = 0; $i < $n; $i++) - { - $node->safe_psql('postgres', - "CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();"); - } - return; -} - done_testing(); diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl index 9c34c0d36c..5d7c278d01 100644 --- a/src/test/recovery/t/035_standby_logical_decoding.pl +++ b/src/test/recovery/t/035_standby_logical_decoding.pl @@ -524,11 +524,8 @@ my $walfile_name = $node_primary->safe_psql('postgres', chomp($walfile_name); # Generate some activity and switch WAL file on the primary -$node_primary->safe_psql( - 'postgres', "create table retain_test(a int); - select pg_switch_wal(); - insert into retain_test values(1); - checkpoint;"); +$node_primary->advance_wal(1); +$node_primary->safe_psql('postgres', "checkpoint;"); # Wait for the standby to catch up $node_primary->wait_for_replay_catchup($node_standby); -- 2.34.1