From 8871cb932a047f04ca4a6c38d24dacd77b6fef6c Mon Sep 17 00:00:00 2001 From: Nisha Moond Date: Mon, 18 Nov 2024 16:54:50 +0530 Subject: [PATCH v50 2/2] Introduce inactive_timeout based replication slot invalidation Till now, postgres has the ability to invalidate inactive replication slots based on the amount of WAL (set via max_slot_wal_keep_size GUC) that will be needed for the slots in case they become active. However, choosing a default value for this GUC is a bit tricky. Because the amount of WAL a database generates, and the allocated storage per instance will vary greatly in production, making it difficult to pin down a one-size-fits-all value. It is often easy for users to set a timeout of say 1 or 2 or n days, after which all the inactive slots get invalidated. This commit introduces a GUC named replication_slot_inactive_timeout. When set, postgres invalidates slots (during non-shutdown checkpoints) that are inactive for longer than this amount of time. Note that the inactive timeout invalidation mechanism is not applicable for slots on the standby server that are being synced from the primary server (i.e., standby slots having 'synced' field 'true'). Synced slots are always considered to be inactive because they don't perform logical decoding to produce changes. --- doc/src/sgml/config.sgml | 35 ++++ doc/src/sgml/system-views.sgml | 12 +- src/backend/replication/logical/slotsync.c | 13 +- src/backend/replication/slot.c | 160 +++++++++++++-- src/backend/utils/misc/guc_tables.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/replication/slot.h | 22 ++ src/include/utils/guc_hooks.h | 2 + src/test/recovery/meson.build | 1 + src/test/recovery/t/050_invalidate_slots.pl | 190 ++++++++++++++++++ 10 files changed, 421 insertions(+), 27 deletions(-) create mode 100644 src/test/recovery/t/050_invalidate_slots.pl diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index a84e60c09b..25ca232a02 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4585,6 +4585,41 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows + + replication_slot_inactive_timeout (integer) + + replication_slot_inactive_timeout configuration parameter + + + + + Invalidate replication slots that are inactive for longer than this + amount of time. If this value is specified without units, + it is taken as seconds. A value of zero (which is default) disables + the inactive timeout invalidation mechanism. This parameter can only + be set in the postgresql.conf file or on the + server command line. + + + + Slot invalidation due to inactive timeout occurs during checkpoint. + The duration of slot inactivity is calculated using the slot's + pg_replication_slots.inactive_since + value. + + + + Note that the inactive timeout invalidation mechanism is not + applicable for slots on the standby server that are being synced + from the primary server (i.e., standby slots having + pg_replication_slots.synced + value true). + Synced slots are always considered to be inactive because they don't + perform logical decoding to produce changes. + + + + track_commit_timestamp (boolean) diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 61d28e701f..7eec92eac9 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2567,8 +2567,9 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The time since the slot has become inactive. - NULL if the slot is currently being used. - Note that for slots on the standby that are being synced from a + NULL if the slot is currently being used. Once the + slot is invalidated, this value will remain unchanged until we shutdown + the server. Note that for slots on the standby that are being synced from a primary server (whose synced field is true), the inactive_since indicates the last @@ -2618,6 +2619,13 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx perform logical decoding. It is set only for logical slots. + + + inactive_timeout means that the slot has remained + inactive beyond the duration specified by the + parameter. + + diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index 69a32422bf..4dc2811f10 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -1508,7 +1508,7 @@ ReplSlotSyncWorkerMain(char *startup_data, size_t startup_data_len) static void update_synced_slots_inactive_since(void) { - TimestampTz now = 0; + TimestampTz now; /* * We need to update inactive_since only when we are promoting standby to @@ -1523,6 +1523,9 @@ update_synced_slots_inactive_since(void) /* The slot sync worker or SQL function mustn't be running by now */ Assert((SlotSyncCtx->pid == InvalidPid) && !SlotSyncCtx->syncing); + /* Use same inactive_since time for all slots */ + now = GetCurrentTimestamp(); + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); for (int i = 0; i < max_replication_slots; i++) @@ -1537,13 +1540,7 @@ update_synced_slots_inactive_since(void) /* The slot must not be acquired by any process */ Assert(s->active_pid == 0); - /* Use the same inactive_since time for all the slots. */ - if (now == 0) - now = GetCurrentTimestamp(); - - SpinLockAcquire(&s->mutex); - s->inactive_since = now; - SpinLockRelease(&s->mutex); + ReplicationSlotSetInactiveSince(s, now, true); } } diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 0cbfd4fbaf..d65b8a6376 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -107,10 +107,11 @@ const char *const SlotInvalidationCauses[] = { [RS_INVAL_WAL_REMOVED] = "wal_removed", [RS_INVAL_HORIZON] = "rows_removed", [RS_INVAL_WAL_LEVEL] = "wal_level_insufficient", + [RS_INVAL_INACTIVE_TIMEOUT] = "inactive_timeout", }; /* Maximum number of invalidation causes */ -#define RS_INVAL_MAX_CAUSES RS_INVAL_WAL_LEVEL +#define RS_INVAL_MAX_CAUSES RS_INVAL_INACTIVE_TIMEOUT StaticAssertDecl(lengthof(SlotInvalidationCauses) == (RS_INVAL_MAX_CAUSES + 1), "array length mismatch"); @@ -140,6 +141,7 @@ ReplicationSlot *MyReplicationSlot = NULL; /* GUC variables */ int max_replication_slots = 10; /* the maximum number of replication * slots */ +int replication_slot_inactive_timeout_sec = 0; /* * This GUC lists streaming replication standby server slot names that @@ -644,6 +646,11 @@ retry: appendStringInfo(&err_detail, _("wal_level is insufficient for slot.")); break; + case RS_INVAL_INACTIVE_TIMEOUT: + appendStringInfo(&err_detail, _("inactivity exceeded the time limit set by \"%s\"."), + "replication_slot_inactive_timeout"); + break; + case RS_INVAL_NONE: pg_unreachable(); } @@ -745,16 +752,12 @@ ReplicationSlotRelease(void) */ SpinLockAcquire(&slot->mutex); slot->active_pid = 0; - slot->inactive_since = now; + ReplicationSlotSetInactiveSince(slot, now, false); SpinLockRelease(&slot->mutex); ConditionVariableBroadcast(&slot->active_cv); } else - { - SpinLockAcquire(&slot->mutex); - slot->inactive_since = now; - SpinLockRelease(&slot->mutex); - } + ReplicationSlotSetInactiveSince(slot, now, true); MyReplicationSlot = NULL; @@ -1550,7 +1553,8 @@ ReportSlotInvalidation(ReplicationSlotInvalidationCause cause, NameData slotname, XLogRecPtr restart_lsn, XLogRecPtr oldestLSN, - TransactionId snapshotConflictHorizon) + TransactionId snapshotConflictHorizon, + TimestampTz inactive_since) { StringInfoData err_detail; bool hint = false; @@ -1580,6 +1584,15 @@ ReportSlotInvalidation(ReplicationSlotInvalidationCause cause, case RS_INVAL_WAL_LEVEL: appendStringInfoString(&err_detail, _("Logical decoding on standby requires \"wal_level\" >= \"logical\" on the primary server.")); break; + + case RS_INVAL_INACTIVE_TIMEOUT: + Assert(inactive_since > 0); + appendStringInfo(&err_detail, + _("The slot has been inactive since %s, exceeding the time limit set by \"%s\"."), + timestamptz_to_str(inactive_since), + "replication_slot_inactive_timeout"); + break; + case RS_INVAL_NONE: pg_unreachable(); } @@ -1596,6 +1609,30 @@ ReportSlotInvalidation(ReplicationSlotInvalidationCause cause, pfree(err_detail.data); } +/* + * Is inactive timeout invalidation possible for this replication slot? + * + * Inactive timeout invalidation is allowed only when: + * + * 1. Inactive timeout is set + * 2. Slot is inactive + * 3. The slot is not being synced from the primary while the server + * is in recovery + * + * Note that the inactive timeout invalidation mechanism is not + * applicable for slots on the standby server that are being synced + * from the primary server (i.e., standby slots having 'synced' field 'true'). + * Synced slots are always considered to be inactive because they don't + * perform logical decoding to produce changes. + */ +static inline bool +IsSlotInactiveTimeoutPossible(ReplicationSlot *s) +{ + return (replication_slot_inactive_timeout_sec > 0 && + s->inactive_since > 0 && + !(RecoveryInProgress() && s->data.synced)); +} + /* * Helper for InvalidateObsoleteReplicationSlots * @@ -1623,6 +1660,7 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, TransactionId initial_catalog_effective_xmin = InvalidTransactionId; XLogRecPtr initial_restart_lsn = InvalidXLogRecPtr; ReplicationSlotInvalidationCause invalidation_cause_prev PG_USED_FOR_ASSERTS_ONLY = RS_INVAL_NONE; + TimestampTz inactive_since = 0; for (;;) { @@ -1630,6 +1668,7 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, NameData slotname; int active_pid = 0; ReplicationSlotInvalidationCause invalidation_cause = RS_INVAL_NONE; + TimestampTz now = 0; Assert(LWLockHeldByMeInMode(ReplicationSlotControlLock, LW_SHARED)); @@ -1640,6 +1679,16 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, break; } + if (cause == RS_INVAL_INACTIVE_TIMEOUT && + IsSlotInactiveTimeoutPossible(s)) + { + /* + * We get the current time beforehand to avoid system call while + * holding the spinlock. + */ + now = GetCurrentTimestamp(); + } + /* * Check if the slot needs to be invalidated. If it needs to be * invalidated, and is not currently acquired, acquire it and mark it @@ -1693,6 +1742,20 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, if (SlotIsLogical(s)) invalidation_cause = cause; break; + case RS_INVAL_INACTIVE_TIMEOUT: + + /* + * Check if the slot needs to be invalidated due to + * replication_slot_inactive_timeout GUC. + */ + if (now && + TimestampDifferenceExceeds(s->inactive_since, now, + replication_slot_inactive_timeout_sec * 1000)) + { + invalidation_cause = cause; + inactive_since = s->inactive_since; + } + break; case RS_INVAL_NONE: pg_unreachable(); } @@ -1718,11 +1781,13 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, active_pid = s->active_pid; /* - * If the slot can be acquired, do so and mark it invalidated - * immediately. Otherwise we'll signal the owning process, below, and - * retry. + * If the slot can be acquired, do so and mark it as invalidated. If + * the slot is already ours, mark it as invalidated. Otherwise, we'll + * signal the owning process below and retry. */ - if (active_pid == 0) + if (active_pid == 0 || + (MyReplicationSlot == s && + active_pid == MyProcPid)) { MyReplicationSlot = s; s->active_pid = MyProcPid; @@ -1777,7 +1842,8 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, { ReportSlotInvalidation(invalidation_cause, true, active_pid, slotname, restart_lsn, - oldestLSN, snapshotConflictHorizon); + oldestLSN, snapshotConflictHorizon, + inactive_since); if (MyBackendType == B_STARTUP) (void) SendProcSignal(active_pid, @@ -1823,7 +1889,8 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, ReportSlotInvalidation(invalidation_cause, false, active_pid, slotname, restart_lsn, - oldestLSN, snapshotConflictHorizon); + oldestLSN, snapshotConflictHorizon, + inactive_since); /* done with this slot for now */ break; @@ -1846,6 +1913,7 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, * - RS_INVAL_HORIZON: requires a snapshot <= the given horizon in the given * db; dboid may be InvalidOid for shared relations * - RS_INVAL_WAL_LEVEL: is logical + * - RS_INVAL_INACTIVE_TIMEOUT: inactive slot timeout has occurred * * NB - this runs as part of checkpoint, so avoid raising errors if possible. */ @@ -1898,7 +1966,8 @@ restart: } /* - * Flush all replication slots to disk. + * Flush all replication slots to disk. Also, invalidate obsolete slots during + * non-shutdown checkpoint. * * It is convenient to flush dirty replication slots at the time of checkpoint. * Additionally, in case of a shutdown checkpoint, we also identify the slots @@ -1956,6 +2025,38 @@ CheckPointReplicationSlots(bool is_shutdown) SaveSlotToPath(s, path, LOG); } LWLockRelease(ReplicationSlotAllocationLock); + + if (!is_shutdown) + { + elog(DEBUG1, "performing replication slot invalidation checks"); + + /* + * NB: We will make another pass over replication slots for + * invalidation checks to keep the code simple. Testing shows that + * there is no noticeable overhead (when compared with wal_removed + * invalidation) even if we were to do inactive_timeout invalidation + * of thousands of replication slots here. If it is ever proven that + * this assumption is wrong, we will have to perform the invalidation + * checks in the above for loop with the following changes: + * + * - Acquire ControlLock lock once before the loop. + * + * - Call InvalidatePossiblyObsoleteSlot for each slot. + * + * - Handle the cases in which ControlLock gets released just like + * InvalidateObsoleteReplicationSlots does. + * + * - Avoid saving slot info to disk two times for each invalidated + * slot. + * + * XXX: Should we move inactive_timeout invalidation check closer to + * wal_removed in CreateCheckPoint and CreateRestartPoint? + */ + InvalidateObsoleteReplicationSlots(RS_INVAL_INACTIVE_TIMEOUT, + 0, + InvalidOid, + InvalidTransactionId); + } } /* @@ -2250,6 +2351,7 @@ RestoreSlotFromDisk(const char *name) bool restored = false; int readBytes; pg_crc32c checksum; + TimestampTz now; /* no need to lock here, no concurrent access allowed yet */ @@ -2410,6 +2512,9 @@ RestoreSlotFromDisk(const char *name) NameStr(cp.slotdata.name)), errhint("Change \"wal_level\" to be \"replica\" or higher."))); + /* Use same inactive_since time for all slots */ + now = GetCurrentTimestamp(); + /* nothing can be active yet, don't lock anything */ for (i = 0; i < max_replication_slots; i++) { @@ -2440,9 +2545,11 @@ RestoreSlotFromDisk(const char *name) /* * Set the time since the slot has become inactive after loading the * slot from the disk into memory. Whoever acquires the slot i.e. - * makes the slot active will reset it. + * makes the slot active will reset it. Avoid calling + * ReplicationSlotSetInactiveSince() here, as it will not set the time + * for invalid slots. */ - slot->inactive_since = GetCurrentTimestamp(); + slot->inactive_since = now; restored = true; break; @@ -2845,3 +2952,22 @@ WaitForStandbyConfirmation(XLogRecPtr wait_for_lsn) ConditionVariableCancelSleep(); } + +/* + * GUC check_hook for replication_slot_inactive_timeout + * + * We don't allow the value of replication_slot_inactive_timeout other than 0 + * during the binary upgrade. + */ +bool +check_replication_slot_inactive_timeout(int *newval, void **extra, GucSource source) +{ + if (IsBinaryUpgrade && *newval != 0) + { + GUC_check_errdetail("The value of \"%s\" must be set to 0 during binary upgrade mode.", + "replication_slot_inactive_timeout"); + return false; + } + + return true; +} diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 8a67f01200..f5a6bf4c61 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3028,6 +3028,18 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"replication_slot_inactive_timeout", PGC_SIGHUP, REPLICATION_SENDING, + gettext_noop("Sets the amount of time a replication slot can remain inactive before " + "it will be invalidated."), + NULL, + GUC_UNIT_S + }, + &replication_slot_inactive_timeout_sec, + 0, 0, INT_MAX, + check_replication_slot_inactive_timeout, NULL, NULL + }, + { {"commit_delay", PGC_SUSET, WAL_SETTINGS, gettext_noop("Sets the delay in microseconds between transaction commit and " diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 39a3ac2312..7c6ae1baa2 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -336,6 +336,7 @@ #wal_sender_timeout = 60s # in milliseconds; 0 disables #track_commit_timestamp = off # collect timestamp of transaction commit # (change requires restart) +#replication_slot_inactive_timeout = 0 # in seconds; 0 disables # - Primary Server - diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 2c325fd942..98f858659c 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -56,6 +56,8 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_HORIZON, /* wal_level insufficient for slot */ RS_INVAL_WAL_LEVEL, + /* inactive slot timeout has occurred */ + RS_INVAL_INACTIVE_TIMEOUT, } ReplicationSlotInvalidationCause; extern PGDLLIMPORT const char *const SlotInvalidationCauses[]; @@ -224,6 +226,25 @@ typedef struct ReplicationSlotCtlData ReplicationSlot replication_slots[1]; } ReplicationSlotCtlData; +/* + * Set slot's inactive_since property unless it was previously invalidated. + */ +static inline void +ReplicationSlotSetInactiveSince(ReplicationSlot *s, TimestampTz now, + bool acquire_lock) +{ + if (s->data.invalidated != RS_INVAL_NONE) + return; + + if (acquire_lock) + SpinLockAcquire(&s->mutex); + + s->inactive_since = now; + + if (acquire_lock) + SpinLockRelease(&s->mutex); +} + /* * Pointers to shared memory */ @@ -233,6 +254,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; extern PGDLLIMPORT char *synchronized_standby_slots; +extern PGDLLIMPORT int replication_slot_inactive_timeout_sec; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 5813dba0a2..e36b6cfe21 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -174,5 +174,7 @@ extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); extern bool check_synchronized_standby_slots(char **newval, void **extra, GucSource source); extern void assign_synchronized_standby_slots(const char *newval, void *extra); +extern bool check_replication_slot_inactive_timeout(int *newval, void **extra, + GucSource source); #endif /* GUC_HOOKS_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index b1eb77b1ec..708a2a3798 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -51,6 +51,7 @@ tests += { 't/040_standby_failover_slots_sync.pl', 't/041_checkpoint_at_promote.pl', 't/042_low_level_backup.pl', + 't/050_invalidate_slots.pl', ], }, } diff --git a/src/test/recovery/t/050_invalidate_slots.pl b/src/test/recovery/t/050_invalidate_slots.pl new file mode 100644 index 0000000000..6a10cd0a54 --- /dev/null +++ b/src/test/recovery/t/050_invalidate_slots.pl @@ -0,0 +1,190 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +# Test for replication slots invalidation +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Utils; +use PostgreSQL::Test::Cluster; +use Test::More; +use Time::HiRes qw(usleep); + +# ============================================================================= +# Testcase start +# +# Test invalidation of streaming standby slot and logical failover slot on the +# primary due to inactive timeout. Also, test logical failover slot synced to +# the standby from the primary doesn't get invalidated on its own, but gets the +# invalidated state from the primary. + +# Initialize primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Avoid unpredictability +$primary->append_conf( + 'postgresql.conf', qq{ +checkpoint_timeout = 1h +autovacuum = off +}); +$primary->start; + +# Take backup +my $backup_name = 'my_backup'; +$primary->backup($backup_name); + +# Create standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup($primary, $backup_name, has_streaming => 1); + +my $connstr = $primary->connstr; +$standby1->append_conf( + 'postgresql.conf', qq( +hot_standby_feedback = on +primary_slot_name = 'sb_slot1' +primary_conninfo = '$connstr dbname=postgres' +)); + +# Create sync slot on the primary +$primary->psql('postgres', + q{SELECT pg_create_logical_replication_slot('sync_slot1', 'test_decoding', false, false, true);} +); + +# Create standby slot on the primary +$primary->safe_psql( + 'postgres', qq[ + SELECT pg_create_physical_replication_slot(slot_name := 'sb_slot1', immediately_reserve := true); +]); + +$standby1->start; + +# Wait until the standby has replayed enough data +$primary->wait_for_catchup($standby1); + +my $logstart = -s $standby1->logfile; + +# Set timeout GUC on the standby to verify that the next checkpoint will not +# invalidate synced slots. +my $inactive_timeout_1s = 1; +$standby1->safe_psql( + 'postgres', qq[ + ALTER SYSTEM SET replication_slot_inactive_timeout TO '${inactive_timeout_1s}s'; +]); +$standby1->reload; + +# Sync the primary slots to the standby +$standby1->safe_psql('postgres', "SELECT pg_sync_replication_slots();"); + +# Confirm that the logical failover slot is created on the standby +is( $standby1->safe_psql( + 'postgres', + q{SELECT count(*) = 1 FROM pg_replication_slots + WHERE slot_name = 'sync_slot1' AND synced + AND NOT temporary + AND invalidation_reason IS NULL;} + ), + "t", + 'logical slot sync_slot1 is synced to standby'); + +# Give enough time +sleep($inactive_timeout_1s + 1); + +# Despite inactive timeout being set, the synced slot won't get invalidated on +# its own on the standby. +$standby1->safe_psql('postgres', "CHECKPOINT"); +is( $standby1->safe_psql( + 'postgres', + q{SELECT count(*) = 1 FROM pg_replication_slots + WHERE slot_name = 'sync_slot1' + AND invalidation_reason IS NULL;} + ), + "t", + 'check that synced slot sync_slot1 has not been invalidated on standby'); + +$logstart = -s $primary->logfile; + +# Set timeout GUC so that the next checkpoint will invalidate inactive slots +$primary->safe_psql( + 'postgres', qq[ + ALTER SYSTEM SET replication_slot_inactive_timeout TO '${inactive_timeout_1s}s'; +]); +$primary->reload; + +# Wait for logical failover slot to become inactive on the primary. Note that +# nobody has acquired the slot yet, so it must get invalidated due to +# inactive timeout. +wait_for_slot_invalidation($primary, 'sync_slot1', $logstart, + $inactive_timeout_1s); + +# Re-sync the primary slots to the standby. Note that the primary slot was +# already invalidated (above) due to inactive timeout. The standby must just +# sync the invalidated state. +$standby1->safe_psql('postgres', "SELECT pg_sync_replication_slots();"); + +is( $standby1->safe_psql( + 'postgres', + q{SELECT count(*) = 1 FROM pg_replication_slots + WHERE slot_name = 'sync_slot1' + AND invalidation_reason = 'inactive_timeout';} + ), + "t", + 'check that invalidation of synced slot sync_slot1 is synced on standby'); + +# Make the standby slot on the primary inactive and check for invalidation +$standby1->stop; +wait_for_slot_invalidation($primary, 'sb_slot1', $logstart, + $inactive_timeout_1s); + +# Testcase end +# ============================================================================= + +# Wait for slot to first become inactive and then get invalidated +sub wait_for_slot_invalidation +{ + my ($node, $slot, $offset, $inactive_timeout_1s) = @_; + my $node_name = $node->name; + + trigger_slot_invalidation($node, $slot, $offset, $inactive_timeout_1s); + + # Check that an invalidated slot cannot be acquired + my ($result, $stdout, $stderr); + ($result, $stdout, $stderr) = $node->psql( + 'postgres', qq[ + SELECT pg_replication_slot_advance('$slot', '0/1'); + ]); + ok( $stderr =~ /can no longer get changes from replication slot "$slot"/, + "detected error upon trying to acquire invalidated slot $slot on node $node_name" + ) + or die + "could not detect error upon trying to acquire invalidated slot $slot on node $node_name"; +} + +# Trigger slot invalidation and confirm it in the server log +sub trigger_slot_invalidation +{ + my ($node, $slot, $offset, $inactive_timeout_1s) = @_; + my $node_name = $node->name; + my $invalidated = 0; + + # Give enough time to avoid multiple checkpoints + sleep($inactive_timeout_1s + 1); + + # Run a checkpoint + $node->safe_psql('postgres', "CHECKPOINT"); + + # The slot's invalidation should be logged + $node->wait_for_log(qr/invalidating obsolete replication slot \"$slot\"/, + $offset); + + # Check that the invalidation reason is 'inactive_timeout' + $node->poll_query_until( + 'postgres', qq[ + SELECT COUNT(slot_name) = 1 FROM pg_replication_slots + WHERE slot_name = '$slot' AND + invalidation_reason = 'inactive_timeout'; + ]) + or die + "Timed out while waiting for invalidation reason of slot $slot to be set on node $node_name"; +} + +done_testing(); -- 2.34.1