From b9e897487cf5a03b4bd0c53e4050f07fea96181a Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <brupireddy@microsoft.com>
Date: Thu, 14 Oct 2021 10:53:40 +0000
Subject: [PATCH v1] add retry mechanism with a GUC before failing the standby
 with recovery ended before configured recovery fatal error

---
 src/backend/access/transam/xlog.c | 73 +++++++++++++++++++++++++++++++
 src/backend/utils/misc/guc.c      | 12 +++++
 src/include/access/xlog.h         |  1 +
 3 files changed, 86 insertions(+)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ffc9fa5bf1..9b62a68870 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -108,6 +108,7 @@ int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
 int			max_slot_wal_keep_size_mb = -1;
+int			recovery_target_retry_timeout = 30;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -968,6 +969,7 @@ static void WALInsertLockAcquire(void);
 static void WALInsertLockAcquireExclusive(void);
 static void WALInsertLockRelease(void);
 static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
+static bool WaitForRecoveryTarget(void);
 
 /*
  * Insert an XLOG record represented by an already-constructed chain of data
@@ -7385,6 +7387,34 @@ StartupXLOG(void)
 
 				/* Else, try to fetch the next WAL record */
 				record = ReadRecord(xlogreader, LOG, false);
+
+				if (ArchiveRecoveryRequested &&
+					recoveryTarget != RECOVERY_TARGET_UNSET &&
+					record == NULL && !reachedRecoveryTarget)
+				{
+					bool retry_timeout;
+
+					while (record == NULL)
+					{
+						if (recovery_target_retry_timeout > 0)
+						{
+							retry_timeout = WaitForRecoveryTarget();
+
+							/* try to fetch the next WAL record */
+							record = ReadRecord(xlogreader, LOG, false);
+
+							/* yay! we've got a record */
+							if (record)
+								break;
+
+							/* we've waited enough, no luck, so let's exit now. */
+							if (retry_timeout)
+								break;
+						}
+						else
+							break;
+					}
+				}
 			} while (record != NULL);
 
 			/*
@@ -7957,6 +7987,49 @@ StartupXLOG(void)
 		RequestCheckpoint(CHECKPOINT_FORCE);
 }
 
+static bool
+WaitForRecoveryTarget(void)
+{
+	static long	total_wait_time_msec = 0;
+	long	wait_time_msec = 1000;
+
+	/* calculate total wait time in first call */
+	if (total_wait_time_msec == 0)
+		total_wait_time_msec = recovery_target_retry_timeout * 1000;
+
+	/* quick exit without arming the latch if it's already past time */
+	if (total_wait_time_msec <= 0)
+	{
+		elog(LOG, "timeout occurred while waiting for recovery target, waited %d seconds",
+			 recovery_target_retry_timeout);
+		return true;
+	}
+
+	ResetLatch(&XLogCtl->recoveryWakeupLatch);
+
+	/* might change the trigger file's location */
+	HandleStartupProcInterrupts();
+
+	elog(LOG, "waiting for %ld milliseconds before recovery target retry", wait_time_msec);
+
+	(void) WaitLatch(&XLogCtl->recoveryWakeupLatch,
+					 WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					 wait_time_msec,
+					 WAIT_EVENT_RECOVERY_APPLY_DELAY);
+
+	total_wait_time_msec -= wait_time_msec;
+
+	/* exit if waited enough */
+	if (total_wait_time_msec <= 0)
+	{
+		elog(LOG, "timeout occurred while waiting for recovery target, waited %d seconds",
+			 recovery_target_retry_timeout);
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * Checks if recovery has reached a consistent state. When consistency is
  * reached and we have a valid starting standby snapshot, tell postmaster
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index ce58018635..f67f454ed2 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3110,6 +3110,18 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"recovery_target_retry_timeout", PGC_SIGHUP, REPLICATION_STANDBY,
+			gettext_noop("Sets the time to wait before retrying to reach recovery target "
+						 "after a failed attempt."),
+			NULL,
+			GUC_UNIT_S
+		},
+		&recovery_target_retry_timeout,
+		30, 10, 86400,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the size of write ahead log segments."),
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 1570860f95..96db8f7f48 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -131,6 +131,7 @@ extern int	recovery_min_apply_delay;
 extern char *PrimaryConnInfo;
 extern char *PrimarySlotName;
 extern bool wal_receiver_create_temp_slot;
+extern int  recovery_target_retry_timeout;
 
 /* indirectly set via GUC system */
 extern TransactionId recoveryTargetXid;
-- 
2.25.1