On Tue, Jan 12, 2021 at 11:30:13PM +1300, Thomas Munro wrote: > I haven't heard any user complaints, and I'd personally be happy with > a fix on master only.
I have been looking again at that, and the rebased version that Andres has provided would take care of that. Any thoughts? -- Michael
From 0a8fb2599283b9703466b79f0fd490b3a565ff62 Mon Sep 17 00:00:00 2001 From: Michael Paquier <mich...@paquier.xyz> Date: Wed, 13 Jan 2021 16:23:57 +0900 Subject: [PATCH] Fix O(recycled_segments^2) stat() calls while recycling WAL files. Author: Michael Paquier <michael.paqu...@gmail.com> Reviewed-By: Andres Freund <and...@anarazel.de> Discussion: https://postgr.es/m/CAB7nPqTB3VcKSSrW2Qj59tYYR2H4+n=5pzbdwou+x9iqvnm...@mail.gmail.com Backpatch: --- src/backend/access/transam/xlog.c | 60 +++++++++++++++---------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b18257c198..1bc6c9683d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -930,7 +930,7 @@ static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); static void RemoveTempXlogFiles(void); static void RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr lastredoptr, XLogRecPtr endptr); -static void RemoveXlogFile(const char *segname, XLogRecPtr lastredoptr, XLogRecPtr endptr); +static void RemoveXlogFile(const char *segname, XLogSegNo *endlogSegNo, XLogSegNo recycleSegNo); static void UpdateLastRemovedPtr(char *filename); static void ValidateXLOGDirectoryStructure(void); static void CleanupBackupHistory(void); @@ -4055,6 +4055,12 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr lastredoptr, XLogRecPtr endptr) DIR *xldir; struct dirent *xlde; char lastoff[MAXFNAMELEN]; + XLogSegNo endlogSegNo; + XLogSegNo recycleSegNo; + + /* Initialize info about where to try to recycle to */ + XLByteToSeg(endptr, endlogSegNo, wal_segment_size); + recycleSegNo = XLOGfileslop(lastredoptr); /* * Construct a filename of the last segment to be kept. The timeline ID @@ -4093,7 +4099,7 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr lastredoptr, XLogRecPtr endptr) /* Update the last removed location in shared memory first */ UpdateLastRemovedPtr(xlde->d_name); - RemoveXlogFile(xlde->d_name, lastredoptr, endptr); + RemoveXlogFile(xlde->d_name, &endlogSegNo, recycleSegNo); } } } @@ -4123,13 +4129,21 @@ RemoveNonParentXlogFiles(XLogRecPtr switchpoint, TimeLineID newTLI) struct dirent *xlde; char switchseg[MAXFNAMELEN]; XLogSegNo endLogSegNo; + XLogSegNo switchLogSegNo; + XLogSegNo recycleSegNo; - XLByteToPrevSeg(switchpoint, endLogSegNo, wal_segment_size); + /* + * Initialize info about where to begin the work. This will recycle, + * somewhat arbitrarily, 10 future segments. + */ + XLByteToPrevSeg(switchpoint, switchLogSegNo, wal_segment_size); + XLByteToSeg(switchpoint, endLogSegNo, wal_segment_size); + recycleSegNo = endLogSegNo + 10; /* * Construct a filename of the last segment to be kept. */ - XLogFileName(switchseg, newTLI, endLogSegNo, wal_segment_size); + XLogFileName(switchseg, newTLI, switchLogSegNo, wal_segment_size); elog(DEBUG2, "attempting to remove WAL segments newer than log file %s", switchseg); @@ -4157,7 +4171,7 @@ RemoveNonParentXlogFiles(XLogRecPtr switchpoint, TimeLineID newTLI) * - but seems safer to let them be archived and removed later. */ if (!XLogArchiveIsReady(xlde->d_name)) - RemoveXlogFile(xlde->d_name, InvalidXLogRecPtr, switchpoint); + RemoveXlogFile(xlde->d_name, &endLogSegNo, recycleSegNo); } } @@ -4167,36 +4181,22 @@ RemoveNonParentXlogFiles(XLogRecPtr switchpoint, TimeLineID newTLI) /* * Recycle or remove a log file that's no longer needed. * - * endptr is current (or recent) end of xlog, and lastredoptr is the - * redo pointer of the last checkpoint. These are used to determine - * whether we want to recycle rather than delete no-longer-wanted log files. - * If lastredoptr is not known, pass invalid, and the function will recycle, - * somewhat arbitrarily, 10 future segments. + * segname is the name of the segment to recycle or remove. endlogSegNo + * is the segment number of the current (or recent) end of WAL. recycleSegNo + * is the segment number to recycle up to. + * + * endlogSegNo gets incremented if the segment is recycled so as it is not + * checked again with future callers of this function. */ static void -RemoveXlogFile(const char *segname, XLogRecPtr lastredoptr, XLogRecPtr endptr) +RemoveXlogFile(const char *segname, XLogSegNo *endlogSegNo, + XLogSegNo recycleSegNo) { char path[MAXPGPATH]; #ifdef WIN32 char newpath[MAXPGPATH]; #endif struct stat statbuf; - XLogSegNo endlogSegNo; - XLogSegNo recycleSegNo; - - if (wal_recycle) - { - /* - * Initialize info about where to try to recycle to. - */ - XLByteToSeg(endptr, endlogSegNo, wal_segment_size); - if (lastredoptr == InvalidXLogRecPtr) - recycleSegNo = endlogSegNo + 10; - else - recycleSegNo = XLOGfileslop(lastredoptr); - } - else - recycleSegNo = 0; /* keep compiler quiet */ snprintf(path, MAXPGPATH, XLOGDIR "/%s", segname); @@ -4206,9 +4206,9 @@ RemoveXlogFile(const char *segname, XLogRecPtr lastredoptr, XLogRecPtr endptr) * symbolic links pointing to a separate archive directory. */ if (wal_recycle && - endlogSegNo <= recycleSegNo && + *endlogSegNo <= recycleSegNo && lstat(path, &statbuf) == 0 && S_ISREG(statbuf.st_mode) && - InstallXLogFileSegment(&endlogSegNo, path, + InstallXLogFileSegment(endlogSegNo, path, true, recycleSegNo, true)) { ereport(DEBUG2, @@ -4216,7 +4216,7 @@ RemoveXlogFile(const char *segname, XLogRecPtr lastredoptr, XLogRecPtr endptr) segname))); CheckpointStats.ckpt_segs_recycled++; /* Needn't recheck that slot on future iterations */ - endlogSegNo++; + (*endlogSegNo)++; } else { -- 2.30.0
signature.asc
Description: PGP signature
