From a221feb6942e4721d4b15cbfdeed68e3fed4d342 Mon Sep 17 00:00:00 2001 From: Zhong ShiHao Date: Sun, 13 Sep 2026 22:01:31 -0400 Subject: [PATCH] Report pruning statistics in pg_stat_all_tables On access pruning has no counters today. When a scan wants to prune a page but cannot get a cleanup lock on it, because another session holds a pin, it gives up without leaving any trace. The dead tuples stay on the page until vacuum gets to them, and nothing in the system tells you that this happened. This matters more than it used to. On access pruning can now set pages all visible and update the free space map, so losing it costs more than a few dead tuples. It also loses an index only scan and pushes work onto vacuum. Add four columns to pg_stat_all_tables. prune_onaccess counts pages that a scan pruned. prune_onaccess_missed counts prune attempts that were dropped because the page was pinned by someone else. pages_all_visible_onaccess counts pages that on access pruning marked all visible in the visibility map. vacuum_missed_dead_pages counts pages that vacuum could not clean up for the same reason. Vacuum already counted these, but only printed the number in its log output, so there was no way to track it over time. The three new scan side counters go into the per relation pending stats, so the scan path does not touch shared memory. The prune path already reported reclaimed dead tuples to pgstat, so this adds no new work there. Also add an isolation test. It pins a page with a held cursor, makes the page worth pruning from another session, and shows that the same query prunes the page only once the pin is gone. --- doc/src/sgml/monitoring.sgml | 46 +++++++++++++ src/backend/access/heap/pruneheap.c | 15 ++++- src/backend/access/heap/vacuumlazy.c | 1 + src/backend/catalog/system_views.sql | 4 ++ src/backend/utils/activity/pgstat_relation.c | 55 +++++++++++++++- src/backend/utils/adt/pgstatfuncs.c | 12 ++++ src/include/catalog/pg_proc.dat | 20 ++++++ src/include/pgstat.h | 17 ++++- .../isolation/expected/prune-onaccess-pin.out | 65 +++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + .../isolation/specs/prune-onaccess-pin.spec | 59 +++++++++++++++++ src/test/regress/expected/rules.out | 12 ++++ 12 files changed, 303 insertions(+), 4 deletions(-) create mode 100644 src/test/isolation/expected/prune-onaccess-pin.out create mode 100644 src/test/isolation/specs/prune-onaccess-pin.spec diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 62dadf3e86c..6bf80569170 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4669,6 +4669,52 @@ description | Waiting for a newly initialized WAL file to reach durable storage + + + prune_onaccess bigint + + + Number of pages of this table pruned opportunistically, that is, during + a scan rather than by VACUUM + + + + + + prune_onaccess_missed bigint + + + Number of times opportunistic pruning of a page of this table was + abandoned because the page could not be cleanup-locked, that is, + another session held a pin on it. A high value relative to + prune_onaccess means dead tuples are being + left for VACUUM that a scan could have removed. + + + + + + pages_all_visible_onaccess bigint + + + Number of pages of this table newly marked all-visible in the + visibility map by opportunistic pruning + + + + + + vacuum_missed_dead_pages bigint + + + Cumulative number of pages of this table that a VACUUM + could not clean up because it failed to acquire a cleanup lock on them, + that is, another session held a pin. This is the + VACUUM counterpart of + prune_onaccess_missed + + + last_vacuum timestamp with time zone diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 54c69b69346..781a05af24f 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -333,9 +333,20 @@ heap_page_prune_opt(Relation relation, Buffer buffer, Buffer *vmbuffer, */ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer); - /* OK, try to get exclusive buffer lock */ + /* + * OK, try to get exclusive buffer lock. + * + * Failing to get the cleanup lock means somebody else holds a pin on + * the page, so we give up on pruning it. Count that: a workload that + * keeps buffers pinned for longer, direct I/O for example, can + * silently lose most of its opportunistic pruning this way, and the + * resulting bloat is otherwise hard to attribute. + */ if (!ConditionalLockBufferForCleanup(buffer)) + { + pgstat_count_prune_onaccess_missed(relation); return; + } /* * Now that we have buffer lock, get accurate information about the @@ -368,6 +379,8 @@ heap_page_prune_opt(Relation relation, Buffer buffer, Buffer *vmbuffer, heap_page_prune_and_freeze(¶ms, &presult, &dummy_off_loc, NULL, NULL); + pgstat_count_prune_onaccess(relation, presult.newly_all_visible); + /* * Report the number of tuples reclaimed to pgstats. This is * presult.ndeleted minus the number of newly-LP_DEAD-set items. diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 063ef2208de..52adcb20337 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -988,6 +988,7 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, Max(vacrel->new_live_tuples, 0), vacrel->recently_dead_tuples + vacrel->missed_dead_tuples, + vacrel->missed_dead_pages, starttime); pgstat_progress_end_command(); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ad340887f54..a4b9f4100ca 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -735,6 +735,10 @@ CREATE VIEW pg_stat_all_tables AS pg_stat_get_dead_tuples(C.oid) AS n_dead_tup, pg_stat_get_mod_since_analyze(C.oid) AS n_mod_since_analyze, pg_stat_get_ins_since_vacuum(C.oid) AS n_ins_since_vacuum, + pg_stat_get_prune_onaccess(C.oid) AS prune_onaccess, + pg_stat_get_prune_onaccess_missed(C.oid) AS prune_onaccess_missed, + pg_stat_get_pages_all_visible_onaccess(C.oid) AS pages_all_visible_onaccess, + pg_stat_get_vacuum_missed_dead_pages(C.oid) AS vacuum_missed_dead_pages, pg_stat_get_last_vacuum_time(C.oid) as last_vacuum, pg_stat_get_last_autovacuum_time(C.oid) as last_autovacuum, pg_stat_get_last_analyze_time(C.oid) as last_analyze, diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 5c70543ba88..4b329135e20 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -261,7 +261,8 @@ pgstat_drop_relation(Relation rel) */ void pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, - PgStat_Counter deadtuples, TimestampTz starttime) + PgStat_Counter deadtuples, + PgStat_Counter missed_dead_pages, TimestampTz starttime) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; @@ -287,6 +288,14 @@ pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, tabentry->live_tuples = livetuples; tabentry->dead_tuples = deadtuples; + /* + * Pages this VACUUM could not clean up because it failed to acquire a + * cleanup lock on them. Like prune_onaccess_missed, this accumulates: + * it measures how much cleanup work concurrent buffer pins are + * deferring, rather than being a property of the last VACUUM. + */ + tabentry->vacuum_missed_dead_pages += missed_dead_pages; + /* * It is quite possible that a non-aggressive VACUUM ended up skipping * various pages, however, we'll zero the insert counter here regardless. @@ -523,6 +532,46 @@ pgstat_update_heap_dead_tuples(Relation rel, int delta) } } +/* + * Count an on-access prune of one heap page. + * + * newly_all_visible is true if this prune also marked the page all-visible + * in the visibility map. + */ +void +pgstat_count_prune_onaccess(Relation rel, bool newly_all_visible) +{ + if (pgstat_should_count_relation(rel)) + { + PgStat_RelationStatus *pgstat_info = rel->pgstat_info; + + Assert(pgstat_info->kind == PGSTAT_KIND_RELATION); + + pgstat_info->tab.counts.prune_onaccess++; + if (newly_all_visible) + pgstat_info->tab.counts.pages_all_visible_onaccess++; + } +} + +/* + * Count an on-access prune that was abandoned because the buffer cleanup + * lock could not be acquired, i.e. some other backend held a pin on the + * page. A page counted here keeps its dead tuples until some later prune + * or VACUUM gets to it. + */ +void +pgstat_count_prune_onaccess_missed(Relation rel) +{ + if (pgstat_should_count_relation(rel)) + { + PgStat_RelationStatus *pgstat_info = rel->pgstat_info; + + Assert(pgstat_info->kind == PGSTAT_KIND_RELATION); + + pgstat_info->tab.counts.prune_onaccess_missed++; + } +} + /* * Support function for the SQL-callable pgstat* functions. Returns * the collected statistics for one table or NULL. NULL doesn't mean @@ -931,6 +980,10 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) tabentry->tuples_deleted += lstats->tab.counts_xact.tuples_deleted; tabentry->tuples_hot_updated += lstats->tab.counts_xact.tuples_hot_updated; tabentry->tuples_newpage_updated += lstats->tab.counts_xact.tuples_newpage_updated; + tabentry->prune_onaccess += lstats->tab.counts.prune_onaccess; + tabentry->prune_onaccess_missed += lstats->tab.counts.prune_onaccess_missed; + tabentry->pages_all_visible_onaccess += + lstats->tab.counts.pages_all_visible_onaccess; /* * If table was truncated/dropped, first reset the live/dead counters. diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 64b6f60516c..08667a32f5d 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -81,6 +81,18 @@ PG_STAT_GET_RELENTRY_INT64(live_tuples) /* pg_stat_get_mod_since_analyze */ PG_STAT_GET_RELENTRY_INT64(mod_since_analyze) +/* pg_stat_get_pages_all_visible_onaccess */ +PG_STAT_GET_RELENTRY_INT64(pages_all_visible_onaccess) + +/* pg_stat_get_prune_onaccess */ +PG_STAT_GET_RELENTRY_INT64(prune_onaccess) + +/* pg_stat_get_prune_onaccess_missed */ +PG_STAT_GET_RELENTRY_INT64(prune_onaccess_missed) + +/* pg_stat_get_vacuum_missed_dead_pages */ +PG_STAT_GET_RELENTRY_INT64(vacuum_missed_dead_pages) + /* pg_stat_get_numscans */ PG_STAT_GET_RELENTRY_INT64(numscans) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f46427258e3..f9c83da8bde 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5593,6 +5593,26 @@ proname => 'pg_stat_get_tuples_newpage_updated', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', prosrc => 'pg_stat_get_tuples_newpage_updated' }, +{ oid => '8093', + descr => 'statistics: number of on-access prunes of this table', + proname => 'pg_stat_get_prune_onaccess', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_prune_onaccess' }, +{ oid => '8094', + descr => 'statistics: number of on-access prunes skipped because the page was pinned', + proname => 'pg_stat_get_prune_onaccess_missed', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_prune_onaccess_missed' }, +{ oid => '8095', + descr => 'statistics: number of pages set all-visible by on-access pruning', + proname => 'pg_stat_get_pages_all_visible_onaccess', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_pages_all_visible_onaccess' }, +{ oid => '8096', + descr => 'statistics: number of pages VACUUM could not clean up because they were pinned', + proname => 'pg_stat_get_vacuum_missed_dead_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_vacuum_missed_dead_pages' }, { oid => '2878', descr => 'statistics: number of live tuples', proname => 'pg_stat_get_live_tuples', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 187d82c96fe..43908c63a48 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -146,9 +146,14 @@ typedef struct PgStat_TableCounts PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + + /* on-access (opportunistic) pruning */ + PgStat_Counter prune_onaccess; + PgStat_Counter prune_onaccess_missed; + PgStat_Counter pages_all_visible_onaccess; } PgStat_TableCounts; -StaticAssertDecl(sizeof(PgStat_TableCounts) == 5 * sizeof(PgStat_Counter), +StaticAssertDecl(sizeof(PgStat_TableCounts) == 8 * sizeof(PgStat_Counter), "PgStat_TableCounts has no padding"); /* ---------- @@ -275,7 +280,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCBD +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCBE typedef struct PgStat_ArchiverStats { @@ -519,6 +524,11 @@ typedef struct PgStat_StatTabEntry PgStat_Counter tuples_hot_updated; PgStat_Counter tuples_newpage_updated; + PgStat_Counter prune_onaccess; + PgStat_Counter prune_onaccess_missed; + PgStat_Counter pages_all_visible_onaccess; + PgStat_Counter vacuum_missed_dead_pages; + PgStat_Counter live_tuples; PgStat_Counter dead_tuples; PgStat_Counter mod_since_analyze; @@ -782,6 +792,7 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, + PgStat_Counter missed_dead_pages, TimestampTz starttime); extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, @@ -866,6 +877,8 @@ extern void pgstat_count_heap_update(Relation rel, bool hot, bool newpage); extern void pgstat_count_heap_delete(Relation rel); extern void pgstat_count_truncate(Relation rel); extern void pgstat_update_heap_dead_tuples(Relation rel, int delta); +extern void pgstat_count_prune_onaccess(Relation rel, bool newly_all_visible); +extern void pgstat_count_prune_onaccess_missed(Relation rel); extern void pgstat_twophase_postcommit(FullTransactionId fxid, uint16 info, void *recdata, uint32 len); diff --git a/src/test/isolation/expected/prune-onaccess-pin.out b/src/test/isolation/expected/prune-onaccess-pin.out new file mode 100644 index 00000000000..e56f6b5cb58 --- /dev/null +++ b/src/test/isolation/expected/prune-onaccess-pin.out @@ -0,0 +1,65 @@ +Parsed test spec with 3 sessions + +starting permutation: p_begin p_declare p_fetch f_fill r_scan r_ff r_stats p_commit r_scan r_ff r_stats +pg_stat_force_next_flush +------------------------ + +(1 row) + +step p_begin: BEGIN; +step p_declare: DECLARE c CURSOR FOR SELECT id FROM prunetest; +step p_fetch: FETCH 1 FROM c; +id +-- + 4 +(1 row) + +step f_fill: + DO $$ + BEGIN + WHILE (SELECT pg_relation_size('prunetest')) <= + current_setting('block_size')::int LOOP + INSERT INTO prunetest + SELECT g, repeat('z', 200) FROM generate_series(100, 110) g; + END LOOP; + END $$; + +step r_scan: SELECT count(*) > 0 AS scanned FROM prunetest; +scanned +------- +t +(1 row) + +step r_ff: SELECT pg_stat_force_next_flush(); +pg_stat_force_next_flush +------------------------ + +(1 row) + +step r_stats: SELECT prune_onaccess, prune_onaccess_missed + FROM pg_stat_all_tables WHERE relname = 'prunetest'; +prune_onaccess|prune_onaccess_missed +--------------+--------------------- + 0| 1 +(1 row) + +step p_commit: COMMIT; +step r_scan: SELECT count(*) > 0 AS scanned FROM prunetest; +scanned +------- +t +(1 row) + +step r_ff: SELECT pg_stat_force_next_flush(); +pg_stat_force_next_flush +------------------------ + +(1 row) + +step r_stats: SELECT prune_onaccess, prune_onaccess_missed + FROM pg_stat_all_tables WHERE relname = 'prunetest'; +prune_onaccess|prune_onaccess_missed +--------------+--------------------- + 1| 1 +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index fc45d504d2b..e8eb25d66a6 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -131,3 +131,4 @@ test: ddl-dependency-locking test: tablespace-dependency-locking test: pub-concurrent-drop test: drop-owned-grant +test: prune-onaccess-pin diff --git a/src/test/isolation/specs/prune-onaccess-pin.spec b/src/test/isolation/specs/prune-onaccess-pin.spec new file mode 100644 index 00000000000..7ab71b34c3a --- /dev/null +++ b/src/test/isolation/specs/prune-onaccess-pin.spec @@ -0,0 +1,59 @@ +# Opportunistic pruning lost to a concurrent buffer pin +# +# On-access pruning gives up silently when it cannot get a cleanup lock on +# the page, i.e. when some other session holds a pin on it. This test shows +# that prune_onaccess_missed counts exactly that, and that the very same scan +# prunes the page once the pin is gone. +# +# The page must not be prunable while the pin is being taken (otherwise the +# pinning scan prunes it itself), and must be prunable afterwards. Hence the +# setup leaves plenty of free space on the page and "filler" fills it up. + +setup +{ + CREATE TABLE prunetest (id int, v text) WITH (fillfactor = 100); + INSERT INTO prunetest SELECT g, repeat('x', 200) FROM generate_series(1, 20) g; + -- leaves an old pd_prune_xid behind, but not enough dead space to make a + -- scan want to prune the page yet + UPDATE prunetest SET v = repeat('y', 200) WHERE id <= 3; + SELECT pg_stat_force_next_flush(); +} + +teardown +{ + DROP TABLE prunetest; +} + +session pinner +setup { SET stats_fetch_consistency = 'none'; } +step p_begin { BEGIN; } +step p_declare { DECLARE c CURSOR FOR SELECT id FROM prunetest; } +# leaves the scan positioned on, and holding a pin on, the first page +step p_fetch { FETCH 1 FROM c; } +step p_commit { COMMIT; } + +session filler +# fill the first page until the relation has to extend, so that a scan will +# want to prune it. Written this way to not depend on the block size. +step f_fill +{ + DO $$ + BEGIN + WHILE (SELECT pg_relation_size('prunetest')) <= + current_setting('block_size')::int LOOP + INSERT INTO prunetest + SELECT g, repeat('z', 200) FROM generate_series(100, 110) g; + END LOOP; + END $$; +} + +session reader +setup { SET stats_fetch_consistency = 'none'; } +step r_scan { SELECT count(*) > 0 AS scanned FROM prunetest; } +step r_ff { SELECT pg_stat_force_next_flush(); } +step r_stats { SELECT prune_onaccess, prune_onaccess_missed + FROM pg_stat_all_tables WHERE relname = 'prunetest'; } + +# The first scan runs into the pin and gives up; the second, identical scan +# runs after the pin is released and prunes the page. +permutation p_begin p_declare p_fetch f_fill r_scan r_ff r_stats p_commit r_scan r_ff r_stats diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 4a8cc759d7b..e55d8cd88ec 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1834,6 +1834,10 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_dead_tuples(c.oid) AS n_dead_tup, pg_stat_get_mod_since_analyze(c.oid) AS n_mod_since_analyze, pg_stat_get_ins_since_vacuum(c.oid) AS n_ins_since_vacuum, + pg_stat_get_prune_onaccess(c.oid) AS prune_onaccess, + pg_stat_get_prune_onaccess_missed(c.oid) AS prune_onaccess_missed, + pg_stat_get_pages_all_visible_onaccess(c.oid) AS pages_all_visible_onaccess, + pg_stat_get_vacuum_missed_dead_pages(c.oid) AS vacuum_missed_dead_pages, pg_stat_get_last_vacuum_time(c.oid) AS last_vacuum, pg_stat_get_last_autovacuum_time(c.oid) AS last_autovacuum, pg_stat_get_last_analyze_time(c.oid) AS last_analyze, @@ -2357,6 +2361,10 @@ pg_stat_sys_tables| SELECT relid, n_dead_tup, n_mod_since_analyze, n_ins_since_vacuum, + prune_onaccess, + prune_onaccess_missed, + pages_all_visible_onaccess, + vacuum_missed_dead_pages, last_vacuum, last_autovacuum, last_analyze, @@ -2412,6 +2420,10 @@ pg_stat_user_tables| SELECT relid, n_dead_tup, n_mod_since_analyze, n_ins_since_vacuum, + prune_onaccess, + prune_onaccess_missed, + pages_all_visible_onaccess, + vacuum_missed_dead_pages, last_vacuum, last_autovacuum, last_analyze, base-commit: 6547a2f6cccf8cfbfdd91a0b278bb65f0b00bdd2 -- 2.37.1 (Apple Git-137.1)