From c2e24db75eb6d1f1e647173081964971a8454c9b Mon Sep 17 00:00:00 2001 From: Matthias van de Meent Date: Mon, 20 Jun 2022 10:21:22 +0200 Subject: [PATCH v8] Add protections in xlog record APIs against large numbers and overflows. Before this, it was possible for an extension to create malicious WAL records that were too large to replay; or that would overflow the xl_tot_len field, causing potential corruption in WAL record IO ops. Emitting invalid records was also possible through pg_logical_emit_message(), which allowed you to emit arbitrary amounts of data up to 2GB, much higher than the replay limit of approximately 1GB. This patch adds a limit to the size of an XLogRecord (1020MiB), checks against overflows, and ensures that the reader infrastructure can read any max-sized XLogRecords, such that the wal-generating backend will fail when it attempts to insert unreadable records, instead of that insertion succeeding but breaking any replication streams. --- src/backend/access/transam/xloginsert.c | 77 ++++++++++++++++++++++--- src/include/access/xloginsert.h | 4 +- src/include/access/xlogrecord.h | 13 +++++ 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index f3c29fa909..4b343c1c12 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -141,7 +141,18 @@ static XLogRecData *XLogRecordAssemble(RmgrId rmid, uint8 info, bool *topxid_included); static bool XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, char *dest, uint16 *dlen); +static inline void XLogErrorDataLimitExceeded(); +/* + * Error due to exceeding the maximum size of a WAL record, or registering + * more datas than are being accounted for by the XLog infrastructure. + */ +static inline void +XLogErrorDataLimitExceeded() +{ + elog(ERROR, "too much WAL data"); +} + /* * Begin constructing a WAL record. This must be called before the * XLogRegister* functions and XLogInsert(). @@ -348,14 +359,29 @@ XLogRegisterBlock(uint8 block_id, RelFileLocator *rlocator, ForkNumber forknum, * XLogRecGetData(). */ void -XLogRegisterData(char *data, int len) +XLogRegisterData(char *data, uint32 len) { XLogRecData *rdata; - Assert(begininsert_called); + Assert(begininsert_called && XLogRecordLengthIsValid(len)); + + /* + * Check against max_rdatas; and ensure we don't fill a record with + * more data than can be replayed. Records are allocated in one chunk + * with some overhead, so ensure XLogRecordLengthIsValid() for that + * size of record. + * + * Additionally, check that we don't accidentally overflow the + * intermediate sum value on 32-bit systems by ensuring that the + * sum of the two inputs is no less than one of the inputs. + */ + if (num_rdatas >= max_rdatas || +#if SIZEOF_SIZE_T == 4 + mainrdata_len + len < len || +#endif + !XLogRecordLengthIsValid((size_t) mainrdata_len + (size_t) len)) + XLogErrorDataLimitExceeded(); - if (num_rdatas >= max_rdatas) - elog(ERROR, "too much WAL data"); rdata = &rdatas[num_rdatas++]; rdata->data = data; @@ -386,12 +412,12 @@ XLogRegisterData(char *data, int len) * limited) */ void -XLogRegisterBufData(uint8 block_id, char *data, int len) +XLogRegisterBufData(uint8 block_id, char *data, uint32 len) { registered_buffer *regbuf; XLogRecData *rdata; - Assert(begininsert_called); + Assert(begininsert_called && len <= UINT16_MAX); /* find the registered buffer struct */ regbuf = ®istered_buffers[block_id]; @@ -399,8 +425,16 @@ XLogRegisterBufData(uint8 block_id, char *data, int len) elog(ERROR, "no block with id %d registered with WAL insertion", block_id); - if (num_rdatas >= max_rdatas) - elog(ERROR, "too much WAL data"); + /* + * Check against max_rdatas; and ensure we don't register more data per + * buffer than can be handled by the physical data format; + * i.e. that regbuf->rdata_len does not grow beyond what + * XLogRecordBlockHeader->data_length can hold. + */ + if (num_rdatas >= max_rdatas || + regbuf->rdata_len + len > UINT16_MAX) + XLogErrorDataLimitExceeded(); + rdata = &rdatas[num_rdatas++]; rdata->data = data; @@ -519,6 +553,12 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, XLogRecPtr *fpw_lsn, int *num_fpi, bool *topxid_included) { XLogRecData *rdt; + /* + * Overflow of total_len is not normally possible, considering that + * this value will be at most 33 RegBufDatas (at UINT16_MAX each) + * plus one MaxXLogRecordSize, which together are still an order of + * magnitude smaller than UINT32_MAX. + */ uint32 total_len = 0; int block_id; pg_crc32c rdata_crc; @@ -527,6 +567,9 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, XLogRecord *rechdr; char *scratch = hdr_scratch; + /* ensure that any assembled record can be decoded */ + Assert(AllocSizeIsValid(DecodeXLogRecordRequiredSpace(MaxXLogRecordSize))); + /* * Note: this function can be called multiple times for the same record. * All the modifications we do to the rdata chains below must handle that. @@ -756,12 +799,18 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, if (needs_data) { + /* + * When copying to XLogRecordBlockHeader, the length is narrowed + * to an uint16. We double-check that that is still correct. + */ + Assert(regbuf->rdata_len <= UINT16_MAX); + /* * Link the caller-supplied rdata chain for this buffer to the * overall list. */ bkpb.fork_flags |= BKPBLOCK_HAS_DATA; - bkpb.data_length = regbuf->rdata_len; + bkpb.data_length = (uint16) regbuf->rdata_len; total_len += regbuf->rdata_len; rdt_datas_last->next = regbuf->rdata_head; @@ -858,6 +907,16 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, for (rdt = hdr_rdt.next; rdt != NULL; rdt = rdt->next) COMP_CRC32C(rdata_crc, rdt->data, rdt->len); + /* + * Ensure that the XLogRecord is not too large. + * + * XLogReader machinery is only able to handle records up to a certain + * size (ignoring machine resource limitations), so make sure we will + * not emit records larger than those sizes we advertise we support. + */ + if (!XLogRecordLengthIsValid(total_len)) + XLogErrorDataLimitExceeded(); + /* * Fill in the fields in the record header. Prev-link is filled in later, * once we know where in the WAL the record will be inserted. The CRC does diff --git a/src/include/access/xloginsert.h b/src/include/access/xloginsert.h index c04f77b173..aed4643d1c 100644 --- a/src/include/access/xloginsert.h +++ b/src/include/access/xloginsert.h @@ -43,12 +43,12 @@ extern void XLogBeginInsert(void); extern void XLogSetRecordFlags(uint8 flags); extern XLogRecPtr XLogInsert(RmgrId rmid, uint8 info); extern void XLogEnsureRecordSpace(int max_block_id, int ndatas); -extern void XLogRegisterData(char *data, int len); +extern void XLogRegisterData(char *data, uint32 len); extern void XLogRegisterBuffer(uint8 block_id, Buffer buffer, uint8 flags); extern void XLogRegisterBlock(uint8 block_id, RelFileLocator *rlocator, ForkNumber forknum, BlockNumber blknum, char *page, uint8 flags); -extern void XLogRegisterBufData(uint8 block_id, char *data, int len); +extern void XLogRegisterBufData(uint8 block_id, char *data, uint32 len); extern void XLogResetInsertion(void); extern bool XLogCheckBufferNeedsBackup(Buffer buffer); diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h index 835151ec92..065bc63a79 100644 --- a/src/include/access/xlogrecord.h +++ b/src/include/access/xlogrecord.h @@ -54,6 +54,19 @@ typedef struct XLogRecord #define SizeOfXLogRecord (offsetof(XLogRecord, xl_crc) + sizeof(pg_crc32c)) +/* + * XLogReader needs to allocate all the data of an xlog record in a single + * chunk. This means that a single XLogRecord cannot exceed MaxAllocSize + * in length if we ignore any allocation overhead of the XLogReader. + * + * To accommodate some overhead, hhis MaxXLogRecordSize value allows for + * 4MB -1b of allocation overhead in anything that allocates xlog record + * data, which should be enough for effectively all purposes. + */ +#define MaxXLogRecordSize (1020 * 1024 * 1024) + +#define XLogRecordLengthIsValid(len) ((len) >= 0 && (len) < MaxXLogRecordSize) + /* * The high 4 bits in xl_info may be used freely by rmgr. The * XLR_SPECIAL_REL_UPDATE and XLR_CHECK_CONSISTENCY bits can be passed by -- 2.30.2