From f38e8ee17cf2b6e413ae6c2ffd8400c26457180d Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Thu, 24 Mar 2022 09:23:15 +0000 Subject: [PATCH v14] Refactor pg_waldump code This patch puts some generic chunks of pg_waldump's code into separate reusable functions in xlogreader.h/.c. --- src/backend/access/transam/xlogreader.c | 202 ++++++++++++++++++++++++ src/bin/pg_waldump/pg_waldump.c | 197 ++--------------------- src/include/access/xlogreader.h | 28 ++++ 3 files changed, 240 insertions(+), 187 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index e437c42992..5909ec180e 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1950,6 +1950,208 @@ XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, return true; } +/* + * Calculate the size of a record, split into !FPI and FPI parts. + */ +void +XLogRecGetLen(XLogReaderState *record, uint32 *rec_len, + uint32 *fpi_len) +{ + int block_id; + + /* + * Calculate the amount of FPI data in the record. + * + * XXX: We peek into xlogreader's private decoded backup blocks for the + * bimg_len indicating the length of FPI data. + */ + *fpi_len = 0; + for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + { + if (XLogRecHasBlockImage(record, block_id)) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + } + + /* + * Calculate the length of the record as the total length - the length of + * all the block images. + */ + *rec_len = XLogRecGetTotalLen(record) - *fpi_len; +} + +/* + * Returns a string giving information about all the blocks in an + * XLogRecord. + */ +void +XLogRecGetBlockRefInfo(XLogReaderState *record, char *delimiter, + uint32 *fpi_len, bool detailed_format, + StringInfo buf) +{ + int block_id; + + Assert(record != NULL); + + if (detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); + + for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + { + RelFileNode rnode = {InvalidOid, InvalidOid, InvalidOid}; + ForkNumber forknum = InvalidForkNumber; + BlockNumber blk = InvalidBlockNumber; + + if (!XLogRecHasBlockRef(record, block_id)) + continue; + + XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); + + if (detailed_format) + { + /* Get block references in detailed format. */ + + appendStringInfo(buf, + "\tblkref #%d: rel %u/%u/%u fork %s blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + forkNames[forknum], + blk); + + if (XLogRecHasBlockImage(record, block_id)) + { + uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info; + + /* Calculate the amount of FPI data in the record. */ + if (fpi_len) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + + if (BKPIMAGE_COMPRESSED(bimg_info)) + { + const char *method; + + if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0) + method = "pglz"; + else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) + method = "lz4"; + else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) + method = "zstd"; + else + method = "unknown"; + + appendStringInfo(buf, + " (FPW%s); hole: offset: %u, length: %u, " + "compression saved: %u, method: %s", + XLogRecBlockImageApply(record, block_id) ? + "" : " for WAL verification", + XLogRecGetBlock(record, block_id)->hole_offset, + XLogRecGetBlock(record, block_id)->hole_length, + BLCKSZ - + XLogRecGetBlock(record, block_id)->hole_length - + XLogRecGetBlock(record, block_id)->bimg_len, + method); + } + else + { + appendStringInfo(buf, + " (FPW%s); hole: offset: %u, length: %u", + XLogRecBlockImageApply(record, block_id) ? + "" : " for WAL verification", + XLogRecGetBlock(record, block_id)->hole_offset, + XLogRecGetBlock(record, block_id)->hole_length); + } + } + } + else + { + /* Get block references in short format. */ + + if (forknum != MAIN_FORKNUM) + { + appendStringInfo(buf, + ", blkref #%d: rel %u/%u/%u fork %s blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + forkNames[forknum], + blk); + } + else + { + appendStringInfo(buf, + ", blkref #%d: rel %u/%u/%u blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + blk); + } + + if (XLogRecHasBlockImage(record, block_id)) + { + /* Calculate the amount of FPI data in the record. */ + if (fpi_len) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + + if (XLogRecBlockImageApply(record, block_id)) + appendStringInfo(buf, " FPW"); + else + appendStringInfo(buf, " FPW for WAL verification"); + } + } + + if (detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); + } + + if (!detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); +} + +/* + * Store per-rmgr and per-record statistics for a given record. + */ +void +XLogRecStoreStats(XLogStats *stats, XLogReaderState *record) +{ + RmgrId rmid; + uint8 recid; + uint32 rec_len; + uint32 fpi_len; + + Assert(stats != NULL && record != NULL); + + stats->count++; + + rmid = XLogRecGetRmid(record); + + XLogRecGetLen(record, &rec_len, &fpi_len); + + /* Update per-rmgr statistics */ + + stats->rmgr_stats[rmid].count++; + stats->rmgr_stats[rmid].rec_len += rec_len; + stats->rmgr_stats[rmid].fpi_len += fpi_len; + + /* + * Update per-record statistics, where the record is identified by a + * combination of the RmgrId and the four bits of the xl_info field that + * are the rmgr's domain (resulting in sixteen possible entries per + * RmgrId). + */ + + recid = XLogRecGetInfo(record) >> 4; + + /* + * XACT records need to be handled differently. Those records use the + * first bit of those four bits for an optional flag variable and the + * following three bits for the opcode. We filter opcode out of xl_info + * and use it as the identifier of the record. + */ + if (rmid == RM_XACT_ID) + recid &= 0x07; + + stats->record_stats[rmid][recid].count++; + stats->record_stats[rmid][recid].rec_len += rec_len; + stats->record_stats[rmid][recid].fpi_len += fpi_len; +} + /* * Returns the data associated with a block reference, or NULL if there is * no data (e.g. because a full-page image was taken instead). The returned diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 92238f30c9..179cd617e4 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -66,23 +66,6 @@ typedef struct XLogDumpConfig bool filter_by_fpw; } XLogDumpConfig; -typedef struct Stats -{ - uint64 count; - uint64 rec_len; - uint64 fpi_len; -} Stats; - -#define MAX_XLINFO_TYPES 16 - -typedef struct XLogDumpStats -{ - uint64 count; - XLogRecPtr startptr; - XLogRecPtr endptr; - Stats rmgr_stats[RM_NEXT_ID]; - Stats record_stats[RM_NEXT_ID][MAX_XLINFO_TYPES]; -} XLogDumpStats; #define fatal_error(...) do { pg_log_fatal(__VA_ARGS__); exit(EXIT_FAILURE); } while(0) @@ -453,81 +436,6 @@ XLogRecordHasFPW(XLogReaderState *record) return false; } -/* - * Calculate the size of a record, split into !FPI and FPI parts. - */ -static void -XLogDumpRecordLen(XLogReaderState *record, uint32 *rec_len, uint32 *fpi_len) -{ - int block_id; - - /* - * Calculate the amount of FPI data in the record. - * - * XXX: We peek into xlogreader's private decoded backup blocks for the - * bimg_len indicating the length of FPI data. - */ - *fpi_len = 0; - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (XLogRecHasBlockImage(record, block_id)) - *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; - } - - /* - * Calculate the length of the record as the total length - the length of - * all the block images. - */ - *rec_len = XLogRecGetTotalLen(record) - *fpi_len; -} - -/* - * Store per-rmgr and per-record statistics for a given record. - */ -static void -XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats, - XLogReaderState *record) -{ - RmgrId rmid; - uint8 recid; - uint32 rec_len; - uint32 fpi_len; - - stats->count++; - - rmid = XLogRecGetRmid(record); - - XLogDumpRecordLen(record, &rec_len, &fpi_len); - - /* Update per-rmgr statistics */ - - stats->rmgr_stats[rmid].count++; - stats->rmgr_stats[rmid].rec_len += rec_len; - stats->rmgr_stats[rmid].fpi_len += fpi_len; - - /* - * Update per-record statistics, where the record is identified by a - * combination of the RmgrId and the four bits of the xl_info field that - * are the rmgr's domain (resulting in sixteen possible entries per - * RmgrId). - */ - - recid = XLogRecGetInfo(record) >> 4; - - /* - * XACT records need to be handled differently. Those records use the - * first bit of those four bits for an optional flag variable and the - * following three bits for the opcode. We filter opcode out of xl_info - * and use it as the identifier of the record. - */ - if (rmid == RM_XACT_ID) - recid &= 0x07; - - stats->record_stats[rmid][recid].count++; - stats->record_stats[rmid][recid].rec_len += rec_len; - stats->record_stats[rmid][recid].fpi_len += fpi_len; -} - /* * Print a record to stdout */ @@ -538,15 +446,12 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) const RmgrDescData *desc = &RmgrDescTable[XLogRecGetRmid(record)]; uint32 rec_len; uint32 fpi_len; - RelFileNode rnode; - ForkNumber forknum; - BlockNumber blk; - int block_id; uint8 info = XLogRecGetInfo(record); XLogRecPtr xl_prev = XLogRecGetPrev(record); StringInfoData s; + char delim = {'\n'}; - XLogDumpRecordLen(record, &rec_len, &fpi_len); + XLogRecGetLen(record, &rec_len, &fpi_len); printf("rmgr: %-11s len (rec/tot): %6u/%6u, tx: %10u, lsn: %X/%08X, prev %X/%08X, ", desc->rm_name, @@ -564,93 +469,11 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) initStringInfo(&s); desc->rm_desc(&s, record); printf("%s", s.data); - pfree(s.data); - - if (!config->bkp_details) - { - /* print block references (short format) */ - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (!XLogRecHasBlockRef(record, block_id)) - continue; - - XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); - if (forknum != MAIN_FORKNUM) - printf(", blkref #%d: rel %u/%u/%u fork %s blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - forkNames[forknum], - blk); - else - printf(", blkref #%d: rel %u/%u/%u blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - blk); - if (XLogRecHasBlockImage(record, block_id)) - { - if (XLogRecBlockImageApply(record, block_id)) - printf(" FPW"); - else - printf(" FPW for WAL verification"); - } - } - putchar('\n'); - } - else - { - /* print block references (detailed format) */ - putchar('\n'); - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (!XLogRecHasBlockRef(record, block_id)) - continue; - - XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); - printf("\tblkref #%d: rel %u/%u/%u fork %s blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - forkNames[forknum], - blk); - if (XLogRecHasBlockImage(record, block_id)) - { - uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info; - if (BKPIMAGE_COMPRESSED(bimg_info)) - { - const char *method; - - if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0) - method = "pglz"; - else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) - method = "lz4"; - else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) - method = "zstd"; - else - method = "unknown"; - - printf(" (FPW%s); hole: offset: %u, length: %u, " - "compression saved: %u, method: %s", - XLogRecBlockImageApply(record, block_id) ? - "" : " for WAL verification", - XLogRecGetBlock(record, block_id)->hole_offset, - XLogRecGetBlock(record, block_id)->hole_length, - BLCKSZ - - XLogRecGetBlock(record, block_id)->hole_length - - XLogRecGetBlock(record, block_id)->bimg_len, - method); - } - else - { - printf(" (FPW%s); hole: offset: %u, length: %u", - XLogRecBlockImageApply(record, block_id) ? - "" : " for WAL verification", - XLogRecGetBlock(record, block_id)->hole_offset, - XLogRecGetBlock(record, block_id)->hole_length); - } - } - putchar('\n'); - } - } + resetStringInfo(&s); + XLogRecGetBlockRefInfo(record, &delim, NULL, config->bkp_details, &s); + printf("%s", s.data); + pfree(s.data); } /* @@ -698,7 +521,7 @@ XLogDumpStatsRow(const char *name, * Display summary statistics about the records seen so far. */ static void -XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats) +XLogDumpDisplayStats(XLogDumpConfig *config, XLogStats *stats) { int ri, rj; @@ -859,7 +682,7 @@ main(int argc, char **argv) XLogReaderState *xlogreader_state; XLogDumpPrivate private; XLogDumpConfig config; - XLogDumpStats stats; + XLogStats stats; XLogRecord *record; XLogRecPtr first_record; char *waldir = NULL; @@ -913,7 +736,7 @@ main(int argc, char **argv) memset(&private, 0, sizeof(XLogDumpPrivate)); memset(&config, 0, sizeof(XLogDumpConfig)); - memset(&stats, 0, sizeof(XLogDumpStats)); + memset(&stats, 0, sizeof(XLogStats)); private.timeline = 1; private.startptr = InvalidXLogRecPtr; @@ -1295,7 +1118,7 @@ main(int argc, char **argv) { if (config.stats == true) { - XLogDumpCountRecord(&config, &stats, xlogreader_state); + XLogRecStoreStats(&stats, xlogreader_state); stats.endptr = xlogreader_state->EndRecPtr; } else diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index f4388cc9be..4021ae05df 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -39,6 +39,7 @@ #endif #include "access/xlogrecord.h" +#include "lib/stringinfo.h" /* WALOpenSegment represents a WAL segment being read. */ typedef struct WALOpenSegment @@ -314,6 +315,26 @@ struct XLogReaderState bool nonblocking; }; +#define MAX_XLINFO_TYPES 16 + +typedef struct XLogRecStats +{ + uint64 count; + uint64 rec_len; + uint64 fpi_len; +} XLogRecStats; + +typedef struct XLogStats +{ + uint64 count; +#ifdef FRONTEND + XLogRecPtr startptr; + XLogRecPtr endptr; +#endif + XLogRecStats rmgr_stats[RM_NEXT_ID]; + XLogRecStats record_stats[RM_NEXT_ID][MAX_XLINFO_TYPES]; +} XLogStats; + /* * Check if XLogNextRecord() has any more queued records or an error to return. */ @@ -430,5 +451,12 @@ extern char *XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size * extern bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum); +extern void XLogRecGetLen(XLogReaderState *record, uint32 *rec_len, + uint32 *fpi_len); +extern void XLogRecGetBlockRefInfo(XLogReaderState *record, + char *delimiter, uint32 *fpi_len, + bool detailed_format, + StringInfo blk_ref); +extern void XLogRecStoreStats(XLogStats *stats, XLogReaderState *record); #endif /* XLOGREADER_H */ -- 2.25.1