From 04c688ab4902bdacf4503b01800172eb9a638466 Mon Sep 17 00:00:00 2001 From: Nisha Moond Date: Mon, 23 Dec 2024 14:58:34 +0530 Subject: [PATCH v57 2/2] Introduce inactive_timeout based replication slot invalidation Till now, postgres has the ability to invalidate inactive replication slots based on the amount of WAL (set via max_slot_wal_keep_size GUC) that will be needed for the slots in case they become active. However, choosing a default value for this GUC is a bit tricky. Because the amount of WAL a database generates, and the allocated storage per instance will vary greatly in production, making it difficult to pin down a one-size-fits-all value. It is often easy for users to set a timeout of say 1 or 2 or n days, after which all the inactive slots get invalidated. This commit introduces a GUC named idle_replication_slot_timeout. When set, postgres invalidates slots (during non-shutdown checkpoints) that are idle for longer than this amount of time. Note that the idle timeout invalidation mechanism is not applicable for slots on the standby server that are being synced from the primary server (i.e., standby slots having 'synced' field 'true'). Synced slots are always considered to be inactive because they don't perform logical decoding to produce changes. --- doc/src/sgml/config.sgml | 39 ++++ doc/src/sgml/logical-replication.sgml | 5 + doc/src/sgml/system-views.sgml | 10 +- src/backend/replication/logical/slotsync.c | 4 +- src/backend/replication/slot.c | 158 +++++++++++++-- src/backend/utils/adt/timestamp.c | 19 ++ src/backend/utils/misc/guc_tables.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/bin/pg_basebackup/pg_createsubscriber.c | 4 + src/bin/pg_upgrade/server.c | 7 + src/include/replication/slot.h | 22 +++ src/include/utils/guc_hooks.h | 2 + src/include/utils/timestamp.h | 3 + src/test/recovery/meson.build | 1 + .../t/043_invalidate_inactive_slots.pl | 181 ++++++++++++++++++ 15 files changed, 452 insertions(+), 16 deletions(-) create mode 100644 src/test/recovery/t/043_invalidate_inactive_slots.pl diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index fbdd6ce574..8953c8fd4b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4593,6 +4593,45 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows + + idle_replication_slot_timeout (integer) + + idle_replication_slot_timeout configuration parameter + + + + + Invalidate replication slots that have remained idle longer than this + duration. If this value is specified without units, it is taken as + minutes. A value of zero disables the idle timeout invalidation mechanism. + The default is 24 hours. This parameter can only be set in the + postgresql.conf file or on the server command line. + + + + Slot invalidation due to idle timeout occurs during checkpoint. + Because checkpoints happen at checkpoint_timeout + intervals, there can be some lag between when the + idle_replication_slot_timeout was exceeded and when + the slot invalidation is triggered at the next checkpoint. + To avoid such lags, users can force a checkpoint to promptly invalidate + inactive slots. The duration of slot inactivity is calculated using the slot's + pg_replication_slots.inactive_since + value. + + + + Note that the idle timeout invalidation mechanism is not + applicable for slots on the standby server that are being synced + from the primary server (i.e., standby slots having + pg_replication_slots.synced + value true). + Synced slots are always considered to be inactive because they don't + perform logical decoding to produce changes. + + + + track_commit_timestamp (boolean) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 8290cd1a08..158ec18211 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -2163,6 +2163,11 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER plus some reserve for table synchronization. + + Logical replication slots are also affected by + idle_replication_slot_timeout. + + max_wal_senders should be set to at least the same as diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index a586156614..199d7248ee 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -2566,7 +2566,8 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The time when the slot became inactive. NULL if the - slot is currently being streamed. + slot is currently being streamed. If the slot becomes invalidated, + this value will remain unchanged until server shutdown. Note that for slots on the standby that are being synced from a primary server (whose synced field is true), the inactive_since @@ -2620,6 +2621,13 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx perform logical decoding. It is set only for logical slots. + + + idle_timeout means that the slot has remained + idle longer than the configured + duration. + + diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c index e3645aea53..9777c6a9cc 100644 --- a/src/backend/replication/logical/slotsync.c +++ b/src/backend/replication/logical/slotsync.c @@ -1540,9 +1540,7 @@ update_synced_slots_inactive_since(void) /* The slot must not be acquired by any process */ Assert(s->active_pid == 0); - SpinLockAcquire(&s->mutex); - s->inactive_since = now; - SpinLockRelease(&s->mutex); + ReplicationSlotSetInactiveSince(s, now, true); } } diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 2a99c1f053..601b71819d 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -107,10 +107,11 @@ const char *const SlotInvalidationCauses[] = { [RS_INVAL_WAL_REMOVED] = "wal_removed", [RS_INVAL_HORIZON] = "rows_removed", [RS_INVAL_WAL_LEVEL] = "wal_level_insufficient", + [RS_INVAL_IDLE_TIMEOUT] = "idle_timeout", }; /* Maximum number of invalidation causes */ -#define RS_INVAL_MAX_CAUSES RS_INVAL_WAL_LEVEL +#define RS_INVAL_MAX_CAUSES RS_INVAL_IDLE_TIMEOUT StaticAssertDecl(lengthof(SlotInvalidationCauses) == (RS_INVAL_MAX_CAUSES + 1), "array length mismatch"); @@ -141,6 +142,12 @@ ReplicationSlot *MyReplicationSlot = NULL; int max_replication_slots = 10; /* the maximum number of replication * slots */ +/* + * Invalidate replication slots that have remained idle longer than this + * duration; '0' disables it. + */ +int idle_replication_slot_timeout_min = HOURS_PER_DAY * MINS_PER_HOUR; + /* * This GUC lists streaming replication standby server slot names that * logical WAL sender processes will wait for. @@ -714,16 +721,12 @@ ReplicationSlotRelease(void) */ SpinLockAcquire(&slot->mutex); slot->active_pid = 0; - slot->inactive_since = now; + ReplicationSlotSetInactiveSince(slot, now, false); SpinLockRelease(&slot->mutex); ConditionVariableBroadcast(&slot->active_cv); } else - { - SpinLockAcquire(&slot->mutex); - slot->inactive_since = now; - SpinLockRelease(&slot->mutex); - } + ReplicationSlotSetInactiveSince(slot, now, true); MyReplicationSlot = NULL; @@ -1519,7 +1522,8 @@ ReportSlotInvalidation(ReplicationSlotInvalidationCause cause, NameData slotname, XLogRecPtr restart_lsn, XLogRecPtr oldestLSN, - TransactionId snapshotConflictHorizon) + TransactionId snapshotConflictHorizon, + TimestampTz inactive_since) { StringInfoData err_detail; bool hint = false; @@ -1549,6 +1553,16 @@ ReportSlotInvalidation(ReplicationSlotInvalidationCause cause, case RS_INVAL_WAL_LEVEL: appendStringInfoString(&err_detail, _("Logical decoding on standby requires \"wal_level\" >= \"logical\" on the primary server.")); break; + + case RS_INVAL_IDLE_TIMEOUT: + Assert(inactive_since > 0); + /* translator: second %s is a GUC variable name */ + appendStringInfo(&err_detail, + _("The slot has remained idle since %s, which is longer than the configured \"%s\" duration."), + timestamptz_to_str(inactive_since), + "idle_replication_slot_timeout"); + break; + case RS_INVAL_NONE: pg_unreachable(); } @@ -1565,6 +1579,30 @@ ReportSlotInvalidation(ReplicationSlotInvalidationCause cause, pfree(err_detail.data); } +/* + * Can invalidate an idle replication slot? + * + * Idle timeout invalidation is allowed only when: + * + * 1. Idle timeout is set + * 2. Slot is inactive + * 3. The slot is not being synced from the primary while the server + * is in recovery + * + * Note that the idle timeout invalidation mechanism is not + * applicable for slots on the standby server that are being synced + * from the primary server (i.e., standby slots having 'synced' field 'true'). + * Synced slots are always considered to be inactive because they don't + * perform logical decoding to produce changes. + */ +static inline bool +CanInvalidateIdleSlot(ReplicationSlot *s) +{ + return (idle_replication_slot_timeout_min > 0 && + s->inactive_since > 0 && + !(RecoveryInProgress() && s->data.synced)); +} + /* * Helper for InvalidateObsoleteReplicationSlots * @@ -1592,6 +1630,7 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, TransactionId initial_catalog_effective_xmin = InvalidTransactionId; XLogRecPtr initial_restart_lsn = InvalidXLogRecPtr; ReplicationSlotInvalidationCause invalidation_cause_prev PG_USED_FOR_ASSERTS_ONLY = RS_INVAL_NONE; + TimestampTz inactive_since = 0; for (;;) { @@ -1599,6 +1638,7 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, NameData slotname; int active_pid = 0; ReplicationSlotInvalidationCause invalidation_cause = RS_INVAL_NONE; + TimestampTz now = 0; Assert(LWLockHeldByMeInMode(ReplicationSlotControlLock, LW_SHARED)); @@ -1609,6 +1649,15 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, break; } + if (cause == RS_INVAL_IDLE_TIMEOUT) + { + /* + * We get the current time beforehand to avoid system call while + * holding the spinlock. + */ + now = GetCurrentTimestamp(); + } + /* * Check if the slot needs to be invalidated. If it needs to be * invalidated, and is not currently acquired, acquire it and mark it @@ -1662,6 +1711,21 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, if (SlotIsLogical(s)) invalidation_cause = cause; break; + case RS_INVAL_IDLE_TIMEOUT: + Assert(now > 0); + + /* + * Check if the slot needs to be invalidated due to + * idle_replication_slot_timeout GUC. + */ + if (CanInvalidateIdleSlot(s) && + TimestampDifferenceExceedsSeconds(s->inactive_since, now, + idle_replication_slot_timeout_min * SECS_PER_MINUTE)) + { + invalidation_cause = cause; + inactive_since = s->inactive_since; + } + break; case RS_INVAL_NONE: pg_unreachable(); } @@ -1747,7 +1811,8 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, { ReportSlotInvalidation(invalidation_cause, true, active_pid, slotname, restart_lsn, - oldestLSN, snapshotConflictHorizon); + oldestLSN, snapshotConflictHorizon, + inactive_since); if (MyBackendType == B_STARTUP) (void) SendProcSignal(active_pid, @@ -1793,7 +1858,8 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, ReportSlotInvalidation(invalidation_cause, false, active_pid, slotname, restart_lsn, - oldestLSN, snapshotConflictHorizon); + oldestLSN, snapshotConflictHorizon, + inactive_since); /* done with this slot for now */ break; @@ -1816,6 +1882,7 @@ InvalidatePossiblyObsoleteSlot(ReplicationSlotInvalidationCause cause, * - RS_INVAL_HORIZON: requires a snapshot <= the given horizon in the given * db; dboid may be InvalidOid for shared relations * - RS_INVAL_WAL_LEVEL: is logical + * - RS_INVAL_IDLE_TIMEOUT: idle slot timeout has occurred * * NB - this runs as part of checkpoint, so avoid raising errors if possible. */ @@ -1868,7 +1935,8 @@ restart: } /* - * Flush all replication slots to disk. + * Flush all replication slots to disk. Also, invalidate obsolete slots during + * non-shutdown checkpoint. * * It is convenient to flush dirty replication slots at the time of checkpoint. * Additionally, in case of a shutdown checkpoint, we also identify the slots @@ -1926,6 +1994,45 @@ CheckPointReplicationSlots(bool is_shutdown) SaveSlotToPath(s, path, LOG); } LWLockRelease(ReplicationSlotAllocationLock); + + if (!is_shutdown) + { + elog(DEBUG1, "performing replication slot invalidation checks"); + + /* + * NB: We will make another pass over replication slots for + * invalidation checks to keep the code simple. Testing shows that + * there is no noticeable overhead (when compared with wal_removed + * invalidation) even if we were to do idle_timeout invalidation of + * thousands of replication slots here. If it is ever proven that this + * assumption is wrong, we will have to perform the invalidation + * checks in the above for loop with the following changes: + * + * - Acquire ControlLock lock once before the loop. + * + * - Call InvalidatePossiblyObsoleteSlot for each slot. + * + * - Handle the cases in which ControlLock gets released just like + * InvalidateObsoleteReplicationSlots does. + * + * - Avoid saving slot info to disk two times for each invalidated + * slot. + * + * XXX: Should we move idle_timeout invalidation check closer to + * wal_removed in CreateCheckPoint and CreateRestartPoint? + * + * XXX: Slot invalidation due to 'idle_timeout' applies only to + * released slots, and is based on the 'idle_replication_slot_timeout' + * GUC. Active slots currently in use for replication are excluded to + * prevent accidental invalidation. Slots where communication between + * the publisher and subscriber is down are also excluded, as they are + * managed by the 'wal_sender_timeout'. + */ + InvalidateObsoleteReplicationSlots(RS_INVAL_IDLE_TIMEOUT, + 0, + InvalidOid, + InvalidTransactionId); + } } /* @@ -2414,7 +2521,9 @@ RestoreSlotFromDisk(const char *name) /* * Set the time since the slot has become inactive after loading the * slot from the disk into memory. Whoever acquires the slot i.e. - * makes the slot active will reset it. + * makes the slot active will reset it. Avoid calling + * ReplicationSlotSetInactiveSince() here, as it will not set the time + * for invalid slots. */ slot->inactive_since = now; @@ -2836,6 +2945,12 @@ RaiseSlotInvalidationError(ReplicationSlot *slot) "wal_level"); break; + case RS_INVAL_IDLE_TIMEOUT: + /* translator: %s is a GUC variable name */ + appendStringInfo(&err_detail, _("This slot has been invalidated because it has remained idle longer than the configured \"%s\" duration."), + "idle_replication_slot_timeout"); + break; + case RS_INVAL_NONE: pg_unreachable(); } @@ -2846,3 +2961,22 @@ RaiseSlotInvalidationError(ReplicationSlot *slot) NameStr(slot->data.name)), errdetail_internal("%s", err_detail.data)); } + +/* + * GUC check_hook for idle_replication_slot_timeout + * + * The idle_replication_slot_timeout must be disabled (set to 0) + * during the binary upgrade. + */ +bool +check_idle_replication_slot_timeout(int *newval, void **extra, GucSource source) +{ + if (IsBinaryUpgrade && *newval != 0) + { + GUC_check_errdetail("The value of \"%s\" must be set to 0 during binary upgrade mode.", + "idle_replication_slot_timeout"); + return false; + } + + return true; +} diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c index 18d7d8a108..e1d96b0e29 100644 --- a/src/backend/utils/adt/timestamp.c +++ b/src/backend/utils/adt/timestamp.c @@ -1786,6 +1786,25 @@ TimestampDifferenceExceeds(TimestampTz start_time, return (diff >= msec * INT64CONST(1000)); } +/* + * Check if the difference between two timestamps is >= a given + * threshold (expressed in seconds). + */ +bool +TimestampDifferenceExceedsSeconds(TimestampTz start_time, + TimestampTz stop_time, + int threshold_sec) +{ + long secs; + int usecs; + + /* Calculate the difference in seconds */ + TimestampDifference(start_time, stop_time, &secs, &usecs); + + /* Return if the difference meets or exceeds the threshold */ + return (secs >= threshold_sec); +} + /* * Convert a time_t to TimestampTz. * diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 8cf1afbad2..fe13a2c646 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3047,6 +3047,18 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"idle_replication_slot_timeout", PGC_SIGHUP, REPLICATION_SENDING, + gettext_noop("Sets the duration a replication slot can remain idle before " + "it is invalidated."), + NULL, + GUC_UNIT_MIN + }, + &idle_replication_slot_timeout_min, + HOURS_PER_DAY * MINS_PER_HOUR, 0, INT_MAX / SECS_PER_MINUTE, + check_idle_replication_slot_timeout, NULL, NULL + }, + { {"commit_delay", PGC_SUSET, WAL_SETTINGS, gettext_noop("Sets the delay in microseconds between transaction commit and " diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index a2ac7575ca..720c50e966 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -337,6 +337,7 @@ #wal_sender_timeout = 60s # in milliseconds; 0 disables #track_commit_timestamp = off # collect timestamp of transaction commit # (change requires restart) +#idle_replication_slot_timeout = 1d # in minutes; 0 disables # - Primary Server - diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c index e96370a9ec..a9ac00f44f 100644 --- a/src/bin/pg_basebackup/pg_createsubscriber.c +++ b/src/bin/pg_basebackup/pg_createsubscriber.c @@ -1438,6 +1438,10 @@ start_standby_server(const struct CreateSubscriberOptions *opt, bool restricted_ appendPQExpBuffer(pg_ctl_cmd, "\"%s\" start -D ", pg_ctl_path); appendShellString(pg_ctl_cmd, subscriber_dir); appendPQExpBuffer(pg_ctl_cmd, " -s -o \"-c sync_replication_slots=off\""); + + /* Prevent unintended slot invalidation */ + appendPQExpBuffer(pg_ctl_cmd, " -o \"-c idle_replication_slot_timeout=0\""); + if (restricted_access) { appendPQExpBuffer(pg_ctl_cmd, " -o \"-p %s\"", opt->sub_port); diff --git a/src/bin/pg_upgrade/server.c b/src/bin/pg_upgrade/server.c index 91bcb4dbc7..93940825d5 100644 --- a/src/bin/pg_upgrade/server.c +++ b/src/bin/pg_upgrade/server.c @@ -252,6 +252,13 @@ start_postmaster(ClusterInfo *cluster, bool report_and_exit_on_error) if (GET_MAJOR_VERSION(cluster->major_version) >= 1700) appendPQExpBufferStr(&pgoptions, " -c max_slot_wal_keep_size=-1"); + /* + * Use idle_replication_slot_timeout=0 to prevent slot invalidation due to + * idle_timeout by checkpointer process during upgrade. + */ + if (GET_MAJOR_VERSION(cluster->major_version) >= 1800) + appendPQExpBufferStr(&pgoptions, " -c idle_replication_slot_timeout=0"); + /* * Use -b to disable autovacuum and logical replication launcher * (effective in PG17 or later for the latter). diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index f5f2d22163..b54970f22f 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -56,6 +56,8 @@ typedef enum ReplicationSlotInvalidationCause RS_INVAL_HORIZON, /* wal_level insufficient for slot */ RS_INVAL_WAL_LEVEL, + /* idle slot timeout has occurred */ + RS_INVAL_IDLE_TIMEOUT, } ReplicationSlotInvalidationCause; extern PGDLLIMPORT const char *const SlotInvalidationCauses[]; @@ -228,6 +230,25 @@ typedef struct ReplicationSlotCtlData ReplicationSlot replication_slots[1]; } ReplicationSlotCtlData; +/* + * Set slot's inactive_since property unless it was previously invalidated. + */ +static inline void +ReplicationSlotSetInactiveSince(ReplicationSlot *s, TimestampTz now, + bool acquire_lock) +{ + if (s->data.invalidated != RS_INVAL_NONE) + return; + + if (acquire_lock) + SpinLockAcquire(&s->mutex); + + s->inactive_since = now; + + if (acquire_lock) + SpinLockRelease(&s->mutex); +} + /* * Pointers to shared memory */ @@ -237,6 +258,7 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot; /* GUCs */ extern PGDLLIMPORT int max_replication_slots; extern PGDLLIMPORT char *synchronized_standby_slots; +extern PGDLLIMPORT int idle_replication_slot_timeout_min; /* shmem initialization functions */ extern Size ReplicationSlotsShmemSize(void); diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 5813dba0a2..d7a7dffab5 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -174,5 +174,7 @@ extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); extern bool check_synchronized_standby_slots(char **newval, void **extra, GucSource source); extern void assign_synchronized_standby_slots(const char *newval, void *extra); +extern bool check_idle_replication_slot_timeout(int *newval, void **extra, + GucSource source); #endif /* GUC_HOOKS_H */ diff --git a/src/include/utils/timestamp.h b/src/include/utils/timestamp.h index a6ce03ed46..7f3d0001ef 100644 --- a/src/include/utils/timestamp.h +++ b/src/include/utils/timestamp.h @@ -143,5 +143,8 @@ extern int date2isoyear(int year, int mon, int mday); extern int date2isoyearday(int year, int mon, int mday); extern bool TimestampTimestampTzRequiresRewrite(void); +extern bool TimestampDifferenceExceedsSeconds(TimestampTz start_time, + TimestampTz stop_time, + int threshold_sec); #endif /* TIMESTAMP_H */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index b1eb77b1ec..3b8f45c93e 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -51,6 +51,7 @@ tests += { 't/040_standby_failover_slots_sync.pl', 't/041_checkpoint_at_promote.pl', 't/042_low_level_backup.pl', + 't/043_invalidate_inactive_slots.pl', ], }, } diff --git a/src/test/recovery/t/043_invalidate_inactive_slots.pl b/src/test/recovery/t/043_invalidate_inactive_slots.pl new file mode 100644 index 0000000000..fd227bfc65 --- /dev/null +++ b/src/test/recovery/t/043_invalidate_inactive_slots.pl @@ -0,0 +1,181 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +# Test for replication slots invalidation +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Utils; +use PostgreSQL::Test::Cluster; +use Test::More; + +# ============================================================================= +# Testcase start +# +# Test invalidation of streaming standby slot and logical failover slot on the +# primary due to idle timeout. Also, test logical failover slot synced to +# the standby from the primary doesn't get invalidated on its own, but gets the +# invalidated state from the primary. + +# Initialize primary +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 'logical'); + +# Avoid unpredictability +$primary->append_conf( + 'postgresql.conf', qq{ +checkpoint_timeout = 1h +}); +$primary->start; + +# Take backup +my $backup_name = 'my_backup'; +$primary->backup($backup_name); + +# Create sync slot on the primary +$primary->psql('postgres', + q{SELECT pg_create_logical_replication_slot('sync_slot1', 'test_decoding', false, false, true);} +); + +# Create standby slot on the primary +$primary->safe_psql( + 'postgres', qq[ + SELECT pg_create_physical_replication_slot(slot_name := 'sb_slot1', immediately_reserve := true); +]); + +# Create standby +my $standby1 = PostgreSQL::Test::Cluster->new('standby1'); +$standby1->init_from_backup($primary, $backup_name, has_streaming => 1); + +my $connstr = $primary->connstr; +$standby1->append_conf( + 'postgresql.conf', qq( +hot_standby_feedback = on +primary_slot_name = 'sb_slot1' +idle_replication_slot_timeout = 1 +primary_conninfo = '$connstr dbname=postgres' +)); +$standby1->start; + +# Wait until the standby has replayed enough data +$primary->wait_for_catchup($standby1); + +# Set timeout GUC on the standby to verify that the next checkpoint will not +# invalidate synced slots. +my $idle_timeout_1min = 1; + +# Sync the primary slots to the standby +$standby1->safe_psql('postgres', "SELECT pg_sync_replication_slots();"); + +# Confirm that the logical failover slot is created on the standby +is( $standby1->safe_psql( + 'postgres', + q{SELECT count(*) = 1 FROM pg_replication_slots + WHERE slot_name = 'sync_slot1' AND synced + AND NOT temporary + AND invalidation_reason IS NULL;} + ), + 't', + 'logical slot sync_slot1 is synced to standby'); + +# Give enough time for inactive_since to exceed the timeout +sleep($idle_timeout_1min * 60 + 10); + +# On standby, synced slots are not invalidated by the idle timeout +# until the invalidation state is propagated from the primary. +$standby1->safe_psql('postgres', "CHECKPOINT"); +is( $standby1->safe_psql( + 'postgres', + q{SELECT count(*) = 1 FROM pg_replication_slots + WHERE slot_name = 'sync_slot1' + AND invalidation_reason IS NULL;} + ), + 't', + 'check that synced slot sync_slot1 has not been invalidated on standby'); + +my $logstart = -s $primary->logfile; + +# Set timeout GUC so that the next checkpoint will invalidate inactive slots +$primary->safe_psql( + 'postgres', qq[ + ALTER SYSTEM SET idle_replication_slot_timeout TO '${idle_timeout_1min}min'; +]); +$primary->reload; + +# Wait for logical failover slot to become inactive on the primary. Note that +# nobody has acquired the slot yet, so it must get invalidated due to +# idle timeout. +wait_for_slot_invalidation($primary, 'sync_slot1', $logstart, + $idle_timeout_1min); + +# Re-sync the primary slots to the standby. Note that the primary slot was +# already invalidated (above) due to idle timeout. The standby must just +# sync the invalidated state. +$standby1->safe_psql('postgres', "SELECT pg_sync_replication_slots();"); + +is( $standby1->safe_psql( + 'postgres', + q{SELECT count(*) = 1 FROM pg_replication_slots + WHERE slot_name = 'sync_slot1' + AND invalidation_reason = 'idle_timeout';} + ), + "t", + 'check that invalidation of synced slot sync_slot1 is synced on standby'); + +# Make the standby slot on the primary inactive and check for invalidation +$standby1->stop; +wait_for_slot_invalidation($primary, 'sb_slot1', $logstart, + $idle_timeout_1min); + +# Testcase end +# ============================================================================= + +# Wait for slot to first become idle and then get invalidated +sub wait_for_slot_invalidation +{ + my ($node, $slot, $offset, $idle_timeout_min) = @_; + my $node_name = $node->name; + + trigger_slot_invalidation($node, $slot, $offset, $idle_timeout_min); + + # Check that an invalidated slot cannot be acquired + my ($result, $stdout, $stderr); + ($result, $stdout, $stderr) = $node->psql( + 'postgres', qq[ + SELECT pg_replication_slot_advance('$slot', '0/1'); + ]); + ok( $stderr =~ /can no longer get changes from replication slot "$slot"/, + "detected error upon trying to acquire invalidated slot $slot on node $node_name" + ) + or die + "could not detect error upon trying to acquire invalidated slot $slot on node $node_name"; +} + +# Trigger slot invalidation and confirm it in the server log +sub trigger_slot_invalidation +{ + my ($node, $slot, $offset, $idle_timeout_min) = @_; + my $node_name = $node->name; + my $invalidated = 0; + + # Give enough time for inactive_since to exceed the timeout + sleep($idle_timeout_min * 60 + 10); + + # Run a checkpoint + $node->safe_psql('postgres', "CHECKPOINT"); + + # The slot's invalidation should be logged + $node->wait_for_log(qr/invalidating obsolete replication slot \"$slot\"/, + $offset); + + # Check that the invalidation reason is 'idle_timeout' + $node->poll_query_until( + 'postgres', qq[ + SELECT COUNT(slot_name) = 1 FROM pg_replication_slots + WHERE slot_name = '$slot' AND + invalidation_reason = 'idle_timeout'; + ]) + or die + "Timed out while waiting for invalidation reason of slot $slot to be set on node $node_name"; +} + +done_testing(); -- 2.34.1