From f2b5b75f5063702cbc3c64efdc1e7ef3cf1acdb4 Mon Sep 17 00:00:00 2001 From: Melanie Plageman Date: Mon, 4 Jul 2022 15:44:17 -0400 Subject: [PATCH v22 3/3] Add system view tracking IO ops per backend type Add pg_stat_buffers, a system view which tracks the number of IO operations (allocs, writes, fsyncs, and extends) done through each IO path (e.g. shared buffers, local buffers, unbuffered IO) by each type of backend. Some of these should always be zero. For example, checkpointer does not use a BufferAccessStrategy (currently), so the "strategy" IO Path for checkpointer will be 0 for all IO operations (alloc, write, fsync, and extend). All possible combinations of IOPath and IOOp are enumerated in the view but not all are populated or even possible at this point. View stats are fetched from statistics incremented when a backend performs an IO operation and maintained by the cumulative statistics subsystem. Each row of the view is stats for a particular backend type for a particular IO Path (e.g. shared buffer accesses by checkpointer) and each column in the view is the total number of IO operations done (e.g. writes). So a cell in the view would be, for example, the number of shared buffers written by checkpointer since the last stats reset. Note that some of the cells in the view are redundant with fields in pg_stat_bgwriter (e.g. buffers_backend), however these have been kept in pg_stat_bgwriter for backwards compatibility. Deriving the redundant pg_stat_bgwriter stats from the IO operations stats structures was also problematic due to the separate reset targets for 'bgwriter' and 'buffers'. Suggested by Andres Freund Author: Melanie Plageman Reviewed-by: Justin Pryzby Discussion: https://www.postgresql.org/message-id/flat/20200124195226.lth52iydq2n2uilq%40alap3.anarazel.de --- doc/src/sgml/monitoring.sgml | 106 ++++++++++++++++++++++++++- src/backend/catalog/system_views.sql | 11 +++ src/backend/utils/adt/pgstatfuncs.c | 66 +++++++++++++++++ src/include/catalog/pg_proc.dat | 9 +++ src/test/regress/expected/rules.out | 8 ++ src/test/regress/expected/stats.out | 52 +++++++++++++ src/test/regress/sql/stats.sql | 14 ++++ 7 files changed, 265 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 602f0a827a..ee276b44ca 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -448,6 +448,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_bufferspg_stat_buffers + A row for each IO path for each backend type showing + statistics about backend IO operations. See + + pg_stat_buffers for details. + + + pg_stat_walpg_stat_wal One row only, showing statistics about WAL activity. See @@ -3595,7 +3604,102 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i stats_reset timestamp with time zone - Time at which these statistics were last reset + Time at which these statistics were last reset. + + + + + + + + + + <structname>pg_stat_buffers</structname> + + + pg_stat_buffers + + + + The pg_stat_buffers view has a row for each backend + type for each possible IO path containing global data for the cluster for + that backend and IO path. + + + + <structname>pg_stat_buffers</structname> View + + + + + Column Type + + + Description + + + + + + + backend_type text + + + Type of backend (e.g. background worker, autovacuum worker). + + + + + + io_path text + + + IO path taken (e.g. shared buffers, direct). + + + + + + alloc bigint + + + Number of buffers allocated. + + + + + + extend bigint + + + Number of buffers extended. + + + + + + fsync bigint + + + Number of buffers fsynced. + + + + + + write bigint + + + Number of buffers written. + + + + + + stats_reset timestamp with time zone + + + Time at which these statistics were last reset. diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index fedaed533b..b756480eab 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1115,6 +1115,17 @@ CREATE VIEW pg_stat_bgwriter AS pg_stat_get_buf_alloc() AS buffers_alloc, pg_stat_get_bgwriter_stat_reset_time() AS stats_reset; +CREATE VIEW pg_stat_buffers AS +SELECT + b.backend_type, + b.io_path, + b.alloc, + b.extend, + b.fsync, + b.write, + b.stats_reset +FROM pg_stat_get_buffers() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 47ceba78e6..c94d98de62 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1739,6 +1739,72 @@ pg_stat_get_buf_alloc(PG_FUNCTION_ARGS) PG_RETURN_INT64(pgstat_fetch_stat_bgwriter()->buf_alloc); } +/* +* When adding a new column to the pg_stat_buffers view, add a new enum +* value here above BUFFERS_NUM_COLUMNS. +*/ +enum +{ + BUFFERS_COLUMN_BACKEND_TYPE, + BUFFERS_COLUMN_IO_PATH, + BUFFERS_COLUMN_ALLOCS, + BUFFERS_COLUMN_EXTENDS, + BUFFERS_COLUMN_FSYNCS, + BUFFERS_COLUMN_WRITES, + BUFFERS_COLUMN_RESET_TIME, + BUFFERS_NUM_COLUMNS, +}; + +Datum +pg_stat_get_buffers(PG_FUNCTION_ARGS) +{ + PgStat_IOPathOps *io_path_ops; + ReturnSetInfo *rsinfo; + Datum reset_time; + + SetSingleFuncCall(fcinfo, 0); + io_path_ops = pgstat_fetch_backend_io_path_ops(); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + /* + * Currently it is not permitted to reset IO operation stats for individual + * IO Paths or individual BackendTypes. All IO Operation statistics are + * reset together. As such, it is easiest to reuse the first reset timestamp + * available. + */ + reset_time = TimestampTzGetDatum(io_path_ops->stat_reset_timestamp); + + for (int i = 0; i < BACKEND_NUM_TYPES; i++) + { + PgStat_IOOpCounters *counters = io_path_ops->data; + Datum backend_type_desc = + CStringGetTextDatum(GetBackendTypeDesc(idx_get_backend_type(i))); + /* const char *log_name = GetBackendTypeDesc(idx_get_backend_type(i)); */ + + for (int j = 0; j < IOPATH_NUM_TYPES; j++) + { + Datum values[BUFFERS_NUM_COLUMNS]; + bool nulls[BUFFERS_NUM_COLUMNS]; + memset(values, 0, sizeof(values)); + memset(nulls, 0, sizeof(nulls)); + + values[BUFFERS_COLUMN_BACKEND_TYPE] = backend_type_desc; + values[BUFFERS_COLUMN_IO_PATH] = CStringGetTextDatum(pgstat_io_path_desc(j)); + values[BUFFERS_COLUMN_RESET_TIME] = TimestampTzGetDatum(reset_time); + values[BUFFERS_COLUMN_ALLOCS] = Int64GetDatum(counters->allocs); + values[BUFFERS_COLUMN_EXTENDS] = Int64GetDatum(counters->extends); + values[BUFFERS_COLUMN_FSYNCS] = Int64GetDatum(counters->fsyncs); + values[BUFFERS_COLUMN_WRITES] = Int64GetDatum(counters->writes); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); + counters++; + } + io_path_ops++; + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2e41f4d9e8..462bcbbe15 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5646,6 +5646,15 @@ proname => 'pg_stat_get_buf_alloc', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => '', prosrc => 'pg_stat_get_buf_alloc' }, +{ oid => '8459', descr => 'statistics: counts of all IO operations done through all IO paths by each type of backend.', + proname => 'pg_stat_get_buffers', provolatile => 's', proisstrict => 'f', + prorows => '52', proretset => 't', + proparallel => 'r', prorettype => 'record', proargtypes => '', + proallargtypes => '{text,text,int8,int8,int8,int8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o}', + proargnames => '{backend_type,io_path,alloc,extend,fsync,write,stats_reset}', + prosrc => 'pg_stat_get_buffers' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 7ec3d2688f..4f266fea8e 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1822,6 +1822,14 @@ pg_stat_bgwriter| SELECT pg_stat_get_bgwriter_timed_checkpoints() AS checkpoints pg_stat_get_buf_fsync_backend() AS buffers_backend_fsync, pg_stat_get_buf_alloc() AS buffers_alloc, pg_stat_get_bgwriter_stat_reset_time() AS stats_reset; +pg_stat_buffers| SELECT b.backend_type, + b.io_path, + b.alloc, + b.extend, + b.fsync, + b.write, + b.stats_reset + FROM pg_stat_get_buffers() b(backend_type, io_path, alloc, extend, fsync, write, stats_reset); pg_stat_database| SELECT d.oid AS datid, d.datname, CASE diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 5b0ebf090f..350171f2be 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -554,4 +554,56 @@ SELECT pg_stat_get_live_tuples(:drop_stats_test_subxact_oid); DROP TABLE trunc_stats_test, trunc_stats_test1, trunc_stats_test2, trunc_stats_test3, trunc_stats_test4; DROP TABLE prevstats; +SELECT pg_stat_reset_shared('buffers'); + pg_stat_reset_shared +---------------------- + +(1 row) + +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT write = 0 FROM pg_stat_buffers WHERE io_path = 'Shared' and backend_type = 'checkpointer'; + ?column? +---------- + t +(1 row) + +CREATE TABLE test_buffer_stats(a int, b int); +INSERT INTO test_buffer_stats SELECT i, i FROM generate_series(1,1000)i; +CHECKPOINT; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT write != 0 FROM pg_stat_buffers WHERE io_path = 'Shared' and backend_type = 'checkpointer'; + ?column? +---------- + t +(1 row) + +SELECT pg_stat_reset_shared('buffers'); + pg_stat_reset_shared +---------------------- + +(1 row) + +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT write = 0 FROM pg_stat_buffers WHERE io_path = 'Shared' and backend_type = 'checkpointer'; + ?column? +---------- + t +(1 row) + +DROP TABLE test_buffer_stats; -- End of Stats Test diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 3f3cf8fb56..11c69ca423 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -285,4 +285,18 @@ SELECT pg_stat_get_live_tuples(:drop_stats_test_subxact_oid); DROP TABLE trunc_stats_test, trunc_stats_test1, trunc_stats_test2, trunc_stats_test3, trunc_stats_test4; DROP TABLE prevstats; + +SELECT pg_stat_reset_shared('buffers'); +SELECT pg_stat_force_next_flush(); +SELECT write = 0 FROM pg_stat_buffers WHERE io_path = 'Shared' and backend_type = 'checkpointer'; +CREATE TABLE test_buffer_stats(a int, b int); +INSERT INTO test_buffer_stats SELECT i, i FROM generate_series(1,1000)i; +CHECKPOINT; +SELECT pg_stat_force_next_flush(); +SELECT write != 0 FROM pg_stat_buffers WHERE io_path = 'Shared' and backend_type = 'checkpointer'; +SELECT pg_stat_reset_shared('buffers'); +SELECT pg_stat_force_next_flush(); +SELECT write = 0 FROM pg_stat_buffers WHERE io_path = 'Shared' and backend_type = 'checkpointer'; +DROP TABLE test_buffer_stats; + -- End of Stats Test -- 2.37.0