From 79e5f8d77140f26e1c27924aef1cbb14a89eb5d6 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Wed, 23 Mar 2022 15:49:03 +0000 Subject: [PATCH v13] Refactor pg_waldump code This patch puts some generic chunks of pg_waldump's code into separate reusable functions in xlogreader.h/.c. --- src/backend/access/transam/xlogreader.c | 197 +++++++++++++++++++++ src/bin/pg_waldump/pg_waldump.c | 217 ++---------------------- src/common/relpath.c | 17 ++ src/include/access/xlogreader.h | 49 +++++- src/include/common/relpath.h | 1 + 5 files changed, 280 insertions(+), 201 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index e437c42992..cf31df7e24 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1950,6 +1950,203 @@ XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, return true; } +/* + * Calculate the size of a record, split into !FPI and FPI parts. + */ +void +XLogRecGetLen(XLogReaderState *record, uint32 *rec_len, + uint32 *fpi_len) +{ + int block_id; + + /* + * Calculate the amount of FPI data in the record. + * + * XXX: We peek into xlogreader's private decoded backup blocks for the + * bimg_len indicating the length of FPI data. + */ + *fpi_len = 0; + for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + { + if (XLogRecHasBlockImage(record, block_id)) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + } + + /* + * Calculate the length of the record as the total length - the length of + * all the block images. + */ + *rec_len = XLogRecGetTotalLen(record) - *fpi_len; +} + +/* + * Returns a string giving information about all the blocks in an + * XLogRecord. + */ +void +XLogRecGetBlockRefInfo(XLogReaderState *record, char *delimiter, + uint32 *fpi_len, bool detailed_format, + StringInfo buf) +{ + RelFileNode rnode; + ForkNumber forknum; + BlockNumber blk; + int block_id; + + if (detailed_format && delimiter) + appendStringInfoChar(buf, '\n'); + + for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + { + if (!XLogRecHasBlockRef(record, block_id)) + continue; + + XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); + + if (detailed_format) + { + /* Get block references in detailed format. */ + + appendStringInfo(buf, + "\tblkref #%d: rel %u/%u/%u fork %s blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + get_forkname(forknum), + blk); + + if (XLogRecHasBlockImage(record, block_id)) + { + uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info; + + /* Calculate the amount of FPI data in the record. */ + if (fpi_len) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + + if (BKPIMAGE_COMPRESSED(bimg_info)) + { + const char *method; + + if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0) + method = "pglz"; + else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) + method = "lz4"; + else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) + method = "zstd"; + else + method = "unknown"; + + appendStringInfo(buf, + " (FPW%s); hole: offset: %u, length: %u, " + "compression saved: %u, method: %s", + XLogRecBlockImageApply(record, block_id) ? + "" : " for WAL verification", + XLogRecGetBlock(record, block_id)->hole_offset, + XLogRecGetBlock(record, block_id)->hole_length, + BLCKSZ - + XLogRecGetBlock(record, block_id)->hole_length - + XLogRecGetBlock(record, block_id)->bimg_len, + method); + } + else + { + appendStringInfo(buf, + " (FPW%s); hole: offset: %u, length: %u", + XLogRecBlockImageApply(record, block_id) ? + "" : " for WAL verification", + XLogRecGetBlock(record, block_id)->hole_offset, + XLogRecGetBlock(record, block_id)->hole_length); + } + } + } + else + { + /* Get block references in short format. */ + + if (forknum != MAIN_FORKNUM) + { + appendStringInfo(buf, + ", blkref #%d: rel %u/%u/%u fork %s blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + get_forkname(forknum), + blk); + } + else + { + appendStringInfo(buf, + ", blkref #%d: rel %u/%u/%u blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + blk); + } + + if (XLogRecHasBlockImage(record, block_id)) + { + /* Calculate the amount of FPI data in the record. */ + if (fpi_len) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + + if (XLogRecBlockImageApply(record, block_id)) + appendStringInfo(buf, " FPW"); + else + appendStringInfo(buf, " FPW for WAL verification"); + } + } + + if (detailed_format && delimiter) + appendStringInfoChar(buf, '\n'); + } + + if (!detailed_format && delimiter) + appendStringInfoChar(buf, '\n'); +} + +/* + * Store per-rmgr and per-record statistics for a given record. + */ +void +XLogRecStoreStats(XLogStats *stats, XLogReaderState *record) +{ + RmgrId rmid; + uint8 recid; + uint32 rec_len; + uint32 fpi_len; + + stats->count++; + + rmid = XLogRecGetRmid(record); + + XLogRecGetLen(record, &rec_len, &fpi_len); + + /* Update per-rmgr statistics */ + + stats->rmgr_stats[rmid].count++; + stats->rmgr_stats[rmid].rec_len += rec_len; + stats->rmgr_stats[rmid].fpi_len += fpi_len; + + /* + * Update per-record statistics, where the record is identified by a + * combination of the RmgrId and the four bits of the xl_info field that + * are the rmgr's domain (resulting in sixteen possible entries per + * RmgrId). + */ + + recid = XLogRecGetInfo(record) >> 4; + + /* + * XACT records need to be handled differently. Those records use the + * first bit of those four bits for an optional flag variable and the + * following three bits for the opcode. We filter opcode out of xl_info + * and use it as the identifier of the record. + */ + if (rmid == RM_XACT_ID) + recid &= 0x07; + + stats->record_stats[rmid][recid].count++; + stats->record_stats[rmid][recid].rec_len += rec_len; + stats->record_stats[rmid][recid].fpi_len += fpi_len; +} + /* * Returns the data associated with a block reference, or NULL if there is * no data (e.g. because a full-page image was taken instead). The returned diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index fc081adfb8..03bc22c0f9 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -57,23 +57,6 @@ typedef struct XLogDumpConfig bool filter_by_xid_enabled; } XLogDumpConfig; -typedef struct Stats -{ - uint64 count; - uint64 rec_len; - uint64 fpi_len; -} Stats; - -#define MAX_XLINFO_TYPES 16 - -typedef struct XLogDumpStats -{ - uint64 count; - XLogRecPtr startptr; - XLogRecPtr endptr; - Stats rmgr_stats[RM_NEXT_ID]; - Stats record_stats[RM_NEXT_ID][MAX_XLINFO_TYPES]; -} XLogDumpStats; #define fatal_error(...) do { pg_log_fatal(__VA_ARGS__); exit(EXIT_FAILURE); } while(0) @@ -391,81 +374,6 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, return count; } -/* - * Calculate the size of a record, split into !FPI and FPI parts. - */ -static void -XLogDumpRecordLen(XLogReaderState *record, uint32 *rec_len, uint32 *fpi_len) -{ - int block_id; - - /* - * Calculate the amount of FPI data in the record. - * - * XXX: We peek into xlogreader's private decoded backup blocks for the - * bimg_len indicating the length of FPI data. - */ - *fpi_len = 0; - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (XLogRecHasBlockImage(record, block_id)) - *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; - } - - /* - * Calculate the length of the record as the total length - the length of - * all the block images. - */ - *rec_len = XLogRecGetTotalLen(record) - *fpi_len; -} - -/* - * Store per-rmgr and per-record statistics for a given record. - */ -static void -XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats, - XLogReaderState *record) -{ - RmgrId rmid; - uint8 recid; - uint32 rec_len; - uint32 fpi_len; - - stats->count++; - - rmid = XLogRecGetRmid(record); - - XLogDumpRecordLen(record, &rec_len, &fpi_len); - - /* Update per-rmgr statistics */ - - stats->rmgr_stats[rmid].count++; - stats->rmgr_stats[rmid].rec_len += rec_len; - stats->rmgr_stats[rmid].fpi_len += fpi_len; - - /* - * Update per-record statistics, where the record is identified by a - * combination of the RmgrId and the four bits of the xl_info field that - * are the rmgr's domain (resulting in sixteen possible entries per - * RmgrId). - */ - - recid = XLogRecGetInfo(record) >> 4; - - /* - * XACT records need to be handled differently. Those records use the - * first bit of those four bits for an optional flag variable and the - * following three bits for the opcode. We filter opcode out of xl_info - * and use it as the identifier of the record. - */ - if (rmid == RM_XACT_ID) - recid &= 0x07; - - stats->record_stats[rmid][recid].count++; - stats->record_stats[rmid][recid].rec_len += rec_len; - stats->record_stats[rmid][recid].fpi_len += fpi_len; -} - /* * Print a record to stdout */ @@ -476,15 +384,13 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) const RmgrDescData *desc = &RmgrDescTable[XLogRecGetRmid(record)]; uint32 rec_len; uint32 fpi_len; - RelFileNode rnode; - ForkNumber forknum; - BlockNumber blk; - int block_id; uint8 info = XLogRecGetInfo(record); XLogRecPtr xl_prev = XLogRecGetPrev(record); StringInfoData s; + StringInfoData blk_ref; + char delim = {'\n'}; - XLogDumpRecordLen(record, &rec_len, &fpi_len); + XLogRecGetLen(record, &rec_len, &fpi_len); printf("rmgr: %-11s len (rec/tot): %6u/%6u, tx: %10u, lsn: %X/%08X, prev %X/%08X, ", desc->rm_name, @@ -504,91 +410,12 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) printf("%s", s.data); pfree(s.data); - if (!config->bkp_details) - { - /* print block references (short format) */ - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (!XLogRecHasBlockRef(record, block_id)) - continue; + initStringInfo(&blk_ref); + XLogRecGetBlockRefInfo(record, &delim, NULL, config->bkp_details, + &blk_ref); - XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); - if (forknum != MAIN_FORKNUM) - printf(", blkref #%d: rel %u/%u/%u fork %s blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - forkNames[forknum], - blk); - else - printf(", blkref #%d: rel %u/%u/%u blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - blk); - if (XLogRecHasBlockImage(record, block_id)) - { - if (XLogRecBlockImageApply(record, block_id)) - printf(" FPW"); - else - printf(" FPW for WAL verification"); - } - } - putchar('\n'); - } - else - { - /* print block references (detailed format) */ - putchar('\n'); - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (!XLogRecHasBlockRef(record, block_id)) - continue; - - XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); - printf("\tblkref #%d: rel %u/%u/%u fork %s blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - forkNames[forknum], - blk); - if (XLogRecHasBlockImage(record, block_id)) - { - uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info; - - if (BKPIMAGE_COMPRESSED(bimg_info)) - { - const char *method; - - if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0) - method = "pglz"; - else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) - method = "lz4"; - else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) - method = "zstd"; - else - method = "unknown"; - - printf(" (FPW%s); hole: offset: %u, length: %u, " - "compression saved: %u, method: %s", - XLogRecBlockImageApply(record, block_id) ? - "" : " for WAL verification", - XLogRecGetBlock(record, block_id)->hole_offset, - XLogRecGetBlock(record, block_id)->hole_length, - BLCKSZ - - XLogRecGetBlock(record, block_id)->hole_length - - XLogRecGetBlock(record, block_id)->bimg_len, - method); - } - else - { - printf(" (FPW%s); hole: offset: %u, length: %u", - XLogRecBlockImageApply(record, block_id) ? - "" : " for WAL verification", - XLogRecGetBlock(record, block_id)->hole_offset, - XLogRecGetBlock(record, block_id)->hole_length); - } - } - putchar('\n'); - } - } + printf("%s", blk_ref.data); + pfree(blk_ref.data); } /* @@ -606,21 +433,11 @@ XLogDumpStatsRow(const char *name, fpi_len_pct, tot_len_pct; - n_pct = 0; - if (total_count != 0) - n_pct = 100 * (double) n / total_count; - - rec_len_pct = 0; - if (total_rec_len != 0) - rec_len_pct = 100 * (double) rec_len / total_rec_len; - - fpi_len_pct = 0; - if (total_fpi_len != 0) - fpi_len_pct = 100 * (double) fpi_len / total_fpi_len; - - tot_len_pct = 0; - if (total_len != 0) - tot_len_pct = 100 * (double) tot_len / total_len; + XLOG_GET_STATS_PERCENTAGE(n_pct, rec_len_pct, fpi_len_pct, + tot_len_pct, total_count, + rec_len, total_rec_len, + fpi_len, total_fpi_len, + tot_len, total_len); printf("%-27s " "%20" INT64_MODIFIER "u (%6.02f) " @@ -636,7 +453,7 @@ XLogDumpStatsRow(const char *name, * Display summary statistics about the records seen so far. */ static void -XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats) +XLogDumpDisplayStats(XLogDumpConfig *config, XLogStats *stats) { int ri, rj; @@ -792,7 +609,7 @@ main(int argc, char **argv) XLogReaderState *xlogreader_state; XLogDumpPrivate private; XLogDumpConfig config; - XLogDumpStats stats; + XLogStats stats; XLogRecord *record; XLogRecPtr first_record; char *waldir = NULL; @@ -842,7 +659,7 @@ main(int argc, char **argv) memset(&private, 0, sizeof(XLogDumpPrivate)); memset(&config, 0, sizeof(XLogDumpConfig)); - memset(&stats, 0, sizeof(XLogDumpStats)); + memset(&stats, 0, sizeof(XLogStats)); private.timeline = 1; private.startptr = InvalidXLogRecPtr; @@ -1153,7 +970,7 @@ main(int argc, char **argv) { if (config.stats == true) { - XLogDumpCountRecord(&config, &stats, xlogreader_state); + XLogRecStoreStats(&stats, xlogreader_state); stats.endptr = xlogreader_state->EndRecPtr; } else diff --git a/src/common/relpath.c b/src/common/relpath.c index 636c96efd3..00bf1f37d5 100644 --- a/src/common/relpath.c +++ b/src/common/relpath.c @@ -39,6 +39,23 @@ const char *const forkNames[] = { StaticAssertDecl(lengthof(forkNames) == (MAX_FORKNUM + 1), "array length mismatch"); +/* + * get_forkname - return fork name given fork number + * + * This function is defined with "extern PGDLLIMPORT ..." in the core here so + * that the loadable modules can access it. + */ +const char *const +get_forkname(ForkNumber num) +{ + /* + * As this function gets called by external modules, let's ensure that the + * fork number passed in is valid. + */ + Assert(num > InvalidForkNumber && num <= MAX_FORKNUM); + + return forkNames[num]; +} /* * forkname_to_number - look up fork number by name diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index f4388cc9be..c9acbfe051 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -39,6 +39,7 @@ #endif #include "access/xlogrecord.h" +#include "lib/stringinfo.h" /* WALOpenSegment represents a WAL segment being read. */ typedef struct WALOpenSegment @@ -314,6 +315,46 @@ struct XLogReaderState bool nonblocking; }; +#define MAX_XLINFO_TYPES 16 + +#define XLOG_GET_STATS_PERCENTAGE(n_pct, rec_len_pct, fpi_len_pct, \ + tot_len_pct, total_count, \ + rec_len, total_rec_len, \ + fpi_len, total_fpi_len, \ + tot_len, total_len) \ +do { \ + n_pct = 0; \ + if (total_count != 0) \ + n_pct = 100 * (double) n / total_count; \ + rec_len_pct = 0; \ + if (total_rec_len != 0) \ + rec_len_pct = 100 * (double) rec_len / total_rec_len; \ + fpi_len_pct = 0; \ + if (total_fpi_len != 0) \ + fpi_len_pct = 100 * (double) fpi_len / total_fpi_len; \ + tot_len_pct = 0; \ + if (total_len != 0) \ + tot_len_pct = 100 * (double) tot_len / total_len; \ +} while(0) + +typedef struct XLogRecStats +{ + uint64 count; + uint64 rec_len; + uint64 fpi_len; +} XLogRecStats; + +typedef struct XLogStats +{ + uint64 count; +#ifdef FRONTEND + XLogRecPtr startptr; + XLogRecPtr endptr; +#endif + XLogRecStats rmgr_stats[RM_NEXT_ID]; + XLogRecStats record_stats[RM_NEXT_ID][MAX_XLINFO_TYPES]; +} XLogStats; + /* * Check if XLogNextRecord() has any more queued records or an error to return. */ @@ -430,5 +471,11 @@ extern char *XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size * extern bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum); - +extern void XLogRecGetLen(XLogReaderState *record, uint32 *rec_len, + uint32 *fpi_len); +extern void XLogRecGetBlockRefInfo(XLogReaderState *record, + char *delimiter, uint32 *fpi_len, + bool detailed_format, + StringInfo blk_ref); +extern void XLogRecStoreStats(XLogStats *stats, XLogReaderState *record); #endif /* XLOGREADER_H */ diff --git a/src/include/common/relpath.h b/src/include/common/relpath.h index a4b5dc853b..072d381d5f 100644 --- a/src/include/common/relpath.h +++ b/src/include/common/relpath.h @@ -58,6 +58,7 @@ typedef enum ForkNumber extern const char *const forkNames[]; +extern PGDLLIMPORT const char *const get_forkname(ForkNumber num); extern ForkNumber forkname_to_number(const char *forkName); extern int forkname_chars(const char *str, ForkNumber *fork); -- 2.25.1