Code rebased to current master.
Updated to incorporate additional recommendations from the the list
- add units to variables in view
- remove 'backend_' prefix from variables/functions
- update documentation
- add functional test for allocated_bytes
- refactor allocation reporting to reduce number of functions and
branches/reduce performance hit
- zero allocated bytes after fork to avoid double counting postmaster
allocations
--
Reid Thompson
Senior Software Engineer
Crunchy Data, Inc.
reid.thomp...@crunchydata.com
www.crunchydata.com
From 3d772d8620faba4bd4e091d6618c63557fbf6749 Mon Sep 17 00:00:00 2001
From: Reid Thompson <jreidthomp...@nc.rr.com>
Date: Thu, 11 Aug 2022 12:01:25 -0400
Subject: [PATCH] Add tracking of backend memory allocated to pg_stat_activity
This new field displays the current bytes of memory allocated to the
backend process. It is updated as memory for the process is
malloc'd/free'd. Memory allocated to items on the freelist is included in
the displayed value. Dynamic shared memory allocations are included
only in the value displayed for the backend that created them, they are
not included in the value for backends that are attached to them to
avoid double counting. On occasion, orphaned memory segments may be
cleaned up on postmaster startup. This may result in decreasing the sum
without a prior increment. We limit the floor of backend_mem_allocated
to zero. Updated pg_stat_activity documentation for the new column.
---
doc/src/sgml/monitoring.sgml | 15 ++++
src/backend/catalog/system_views.sql | 1 +
src/backend/postmaster/autovacuum.c | 6 ++
src/backend/postmaster/postmaster.c | 13 ++++
src/backend/postmaster/syslogger.c | 3 +
src/backend/storage/ipc/dsm_impl.c | 81 +++++++++++++++++++++
src/backend/utils/activity/backend_status.c | 45 ++++++++++++
src/backend/utils/adt/pgstatfuncs.c | 4 +-
src/backend/utils/mmgr/aset.c | 17 +++++
src/backend/utils/mmgr/generation.c | 15 ++++
src/backend/utils/mmgr/slab.c | 21 ++++++
src/include/catalog/pg_proc.dat | 6 +-
src/include/utils/backend_status.h | 59 ++++++++++++++-
src/test/regress/expected/rules.out | 9 ++-
src/test/regress/expected/stats.out | 11 +++
src/test/regress/sql/stats.sql | 3 +
16 files changed, 300 insertions(+), 9 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 5579b8b9e0..ffe7d2566c 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -947,6 +947,21 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
</para></entry>
</row>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>allocated_bytes</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Memory currently allocated to this backend in bytes. This is the balance
+ of bytes allocated and freed by this backend. Dynamic shared memory
+ allocations are included only in the value displayed for the backend that
+ created them, they are not included in the value for backends that are
+ attached to them to avoid double counting. Use <function>pg_size_pretty</function>
+ described in <xref linkend="functions-admin-dbsize"/> to make this value
+ more easily readable.
+ </para></entry>
+ </row>
+
<row>
<entry role="catalog_table_entry"><para role="column_definition">
<structfield>query</structfield> <type>text</type>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..9ea8f78c95 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -864,6 +864,7 @@ CREATE VIEW pg_stat_activity AS
S.state,
S.backend_xid,
s.backend_xmin,
+ S.allocated_bytes,
S.query_id,
S.query,
S.backend_type
diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c
index 601834d4b4..f54606104d 100644
--- a/src/backend/postmaster/autovacuum.c
+++ b/src/backend/postmaster/autovacuum.c
@@ -407,6 +407,9 @@ StartAutoVacLauncher(void)
#ifndef EXEC_BACKEND
case 0:
+ /* Zero allocated bytes to avoid double counting parent allocation */
+ pgstat_zero_my_allocated_bytes();
+
/* in postmaster child ... */
InitPostmasterChild();
@@ -1485,6 +1488,9 @@ StartAutoVacWorker(void)
#ifndef EXEC_BACKEND
case 0:
+ /* Zero allocated bytes to avoid double counting parent allocation */
+ pgstat_zero_my_allocated_bytes();
+
/* in postmaster child ... */
InitPostmasterChild();
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index a8a246921f..89a6caec78 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -4102,6 +4102,9 @@ BackendStartup(Port *port)
{
free(bn);
+ /* Zero allocated bytes to avoid double counting parent allocation */
+ pgstat_zero_my_allocated_bytes();
+
/* Detangle from postmaster */
InitPostmasterChild();
@@ -5307,6 +5310,11 @@ StartChildProcess(AuxProcType type)
MemoryContextDelete(PostmasterContext);
PostmasterContext = NULL;
+ /* Zero allocated bytes to avoid double counting parent allocation.
+ * Needs to be after the MemoryContextDelete(PostmasterContext) above.
+ */
+ pgstat_zero_my_allocated_bytes();
+
AuxiliaryProcessMain(type); /* does not return */
}
#endif /* EXEC_BACKEND */
@@ -5700,6 +5708,11 @@ do_start_bgworker(RegisteredBgWorker *rw)
MemoryContextDelete(PostmasterContext);
PostmasterContext = NULL;
+ /* Zero allocated bytes to avoid double counting parent allocation.
+ * Needs to be after the MemoryContextDelete(PostmasterContext) above.
+ */
+ pgstat_zero_my_allocated_bytes();
+
StartBackgroundWorker();
exit(1); /* should not get here */
diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c
index d6d02e3c63..4cbc59cda5 100644
--- a/src/backend/postmaster/syslogger.c
+++ b/src/backend/postmaster/syslogger.c
@@ -679,6 +679,9 @@ SysLogger_Start(void)
#ifndef EXEC_BACKEND
case 0:
+ /* Zero allocated bytes to avoid double counting parent allocation */
+ pgstat_zero_my_allocated_bytes();
+
/* in postmaster child ... */
InitPostmasterChild();
diff --git a/src/backend/storage/ipc/dsm_impl.c b/src/backend/storage/ipc/dsm_impl.c
index 6ddd46a4e7..65d59fc43e 100644
--- a/src/backend/storage/ipc/dsm_impl.c
+++ b/src/backend/storage/ipc/dsm_impl.c
@@ -66,6 +66,7 @@
#include "postmaster/postmaster.h"
#include "storage/dsm_impl.h"
#include "storage/fd.h"
+#include "utils/backend_status.h"
#include "utils/guc.h"
#include "utils/memutils.h"
@@ -232,6 +233,14 @@ dsm_impl_posix(dsm_op op, dsm_handle handle, Size request_size,
name)));
return false;
}
+
+ /*
+ * Detach and destroy pass through here, only decrease the memory
+ * shown allocated in pg_stat_activity when the creator destroys the
+ * allocation.
+ */
+ if (op == DSM_OP_DESTROY)
+ pgstat_report_allocated_bytes(*mapped_size, DECREASE);
*mapped_address = NULL;
*mapped_size = 0;
if (op == DSM_OP_DESTROY && shm_unlink(name) != 0)
@@ -332,6 +341,36 @@ dsm_impl_posix(dsm_op op, dsm_handle handle, Size request_size,
name)));
return false;
}
+
+ /*
+ * Attach and create pass through here, only update backend memory
+ * allocated in pg_stat_activity for the creator process.
+ */
+ if (op == DSM_OP_CREATE)
+ {
+ /*
+ * Posix creation calls dsm_impl_posix_resize implying that resizing
+ * occurs or may be added in the future. As implemented
+ * dsm_impl_posix_resize utilizes fallocate or truncate, passing the
+ * whole new size as input, growing the allocation as needed (only
+ * truncate supports shrinking). We update by replacing the old
+ * allocation with the new.
+ */
+#if defined(HAVE_POSIX_FALLOCATE) && defined(__linux__)
+ /*
+ * posix_fallocate does not shrink allocations, adjust only on
+ * allocation increase.
+ */
+ if (request_size > *mapped_size)
+ {
+ pgstat_report_allocated_bytes(*mapped_size, DECREASE);
+ pgstat_report_allocated_bytes(request_size, INCREASE);
+ }
+#else
+ pgstat_report_allocated_bytes(*mapped_size, DECREASE);
+ pgstat_report_allocated_bytes(request_size, INCREASE);
+#endif
+ }
*mapped_address = address;
*mapped_size = request_size;
close(fd);
@@ -537,6 +576,14 @@ dsm_impl_sysv(dsm_op op, dsm_handle handle, Size request_size,
name)));
return false;
}
+
+ /*
+ * Detach and destroy pass through here, only decrease the memory
+ * shown allocated in pg_stat_activity when the creator destroys the
+ * allocation.
+ */
+ if (op == DSM_OP_DESTROY)
+ pgstat_report_allocated_bytes(*mapped_size, DECREASE);
*mapped_address = NULL;
*mapped_size = 0;
if (op == DSM_OP_DESTROY && shmctl(ident, IPC_RMID, NULL) < 0)
@@ -584,6 +631,13 @@ dsm_impl_sysv(dsm_op op, dsm_handle handle, Size request_size,
name)));
return false;
}
+
+ /*
+ * Attach and create pass through here, only update backend memory
+ * allocated in pg_stat_activity for the creator process.
+ */
+ if (op == DSM_OP_CREATE)
+ pgstat_report_allocated_bytes(request_size, INCREASE);
*mapped_address = address;
*mapped_size = request_size;
@@ -652,6 +706,13 @@ dsm_impl_windows(dsm_op op, dsm_handle handle, Size request_size,
return false;
}
+ /*
+ * Detach and destroy pass through here, only decrease the memory
+ * shown allocated in pg_stat_activity when the creator destroys the
+ * allocation.
+ */
+ if (op == DSM_OP_DESTROY)
+ pgstat_report_allocated_bytes(*mapped_size, DECREASE);
*impl_private = NULL;
*mapped_address = NULL;
*mapped_size = 0;
@@ -768,6 +829,12 @@ dsm_impl_windows(dsm_op op, dsm_handle handle, Size request_size,
return false;
}
+ /*
+ * Attach and create pass through here, only update backend memory
+ * allocated in pg_stat_activity for the creator process.
+ */
+ if (op == DSM_OP_CREATE)
+ pgstat_report_allocated_bytes(info.RegionSize, INCREASE);
*mapped_address = address;
*mapped_size = info.RegionSize;
*impl_private = hmap;
@@ -812,6 +879,13 @@ dsm_impl_mmap(dsm_op op, dsm_handle handle, Size request_size,
name)));
return false;
}
+
+ /*
+ * Detach and destroy pass through here, only decrease the memory
+ * shown allocated in pg_stat_activity when the creator destroys the
+ * allocation.
+ */
+ pgstat_report_allocated_bytes(*mapped_size, DECREASE);
*mapped_address = NULL;
*mapped_size = 0;
if (op == DSM_OP_DESTROY && unlink(name) != 0)
@@ -933,6 +1007,13 @@ dsm_impl_mmap(dsm_op op, dsm_handle handle, Size request_size,
name)));
return false;
}
+
+ /*
+ * Attach and create pass through here, only update backend memory
+ * allocated in pg_stat_activity for the creator process.
+ */
+ if (op == DSM_OP_CREATE)
+ pgstat_report_allocated_bytes(request_size, INCREASE);
*mapped_address = address;
*mapped_size = request_size;
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index 1146a6c33c..3785e8af53 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -49,6 +49,9 @@ int pgstat_track_activity_query_size = 1024;
/* exposed so that backend_progress.c can access it */
PgBackendStatus *MyBEEntry = NULL;
+/* Memory allocated to this backend prior to pgstats initialization */
+uint64 local_my_allocated_bytes = 0;
+uint64 *my_allocated_bytes = &local_my_allocated_bytes;
static PgBackendStatus *BackendStatusArray = NULL;
static char *BackendAppnameBuffer = NULL;
@@ -400,6 +403,15 @@ pgstat_bestart(void)
lbeentry.st_progress_command_target = InvalidOid;
lbeentry.st_query_id = UINT64CONST(0);
+ /* Alter allocation reporting from local_my_allocated_bytes to shared memory */
+ pgstat_set_allocated_bytes_storage(&MyBEEntry->allocated_bytes);
+
+ /* Populate sum of memory allocated prior to pgstats initialization to pgstats
+ * and zero the local variable.
+ */
+ lbeentry.allocated_bytes += local_my_allocated_bytes;
+ local_my_allocated_bytes = 0;
+
/*
* we don't zero st_progress_param here to save cycles; nobody should
* examine it until st_progress_command has been set to something other
@@ -459,6 +471,11 @@ pgstat_beshutdown_hook(int code, Datum arg)
{
volatile PgBackendStatus *beentry = MyBEEntry;
+ /*
+ * Stop reporting memory allocation changes to &MyBEEntry->allocated_bytes
+ */
+ pgstat_reset_allocated_bytes_storage();
+
/*
* Clear my status entry, following the protocol of bumping st_changecount
* before and after. We use a volatile pointer here to ensure the
@@ -1191,3 +1208,31 @@ pgstat_clip_activity(const char *raw_activity)
return activity;
}
+
+/*
+ * Configure bytes allocated reporting to report allocated bytes to
+ * *allocated_bytes. *allocated_bytes needs to be valid until
+ * pgstat_set_allocated_bytes_storage() is called.
+ *
+ * Expected to be called during backend startup (in pgstat_bestart), to point
+ * my_allocated_bytes into shared memory.
+ */
+void
+pgstat_set_allocated_bytes_storage(uint64 *new_allocated_bytes)
+{
+ my_allocated_bytes = new_allocated_bytes;
+ *new_allocated_bytes = local_my_allocated_bytes;
+}
+
+/*
+ * Reset allocated bytes storage location.
+ *
+ * Expected to be called during backend shutdown, before the location set up
+ * by pgstat_set_allocated_bytes_storage() becomes invalid.
+ */
+void
+pgstat_reset_allocated_bytes_storage(void)
+{
+ my_allocated_bytes = &local_my_allocated_bytes;
+}
+
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index ae3365d917..170d60df8b 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -559,7 +559,7 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
Datum
pg_stat_get_activity(PG_FUNCTION_ARGS)
{
-#define PG_STAT_GET_ACTIVITY_COLS 30
+#define PG_STAT_GET_ACTIVITY_COLS 31
int num_backends = pgstat_fetch_stat_numbackends();
int curr_backend;
int pid = PG_ARGISNULL(0) ? -1 : PG_GETARG_INT32(0);
@@ -615,6 +615,8 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
else
nulls[16] = true;
+ values[30] = UInt64GetDatum(beentry->allocated_bytes);
+
/* Values only available to role member or pg_read_all_stats */
if (HAS_PGSTAT_PERMISSIONS(beentry->st_userid))
{
diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c
index b6a8bbcd59..b202e115b6 100644
--- a/src/backend/utils/mmgr/aset.c
+++ b/src/backend/utils/mmgr/aset.c
@@ -47,6 +47,7 @@
#include "postgres.h"
#include "port/pg_bitutils.h"
+#include "utils/backend_status.h"
#include "utils/memdebug.h"
#include "utils/memutils.h"
#include "utils/memutils_memorychunk.h"
@@ -521,6 +522,7 @@ AllocSetContextCreateInternal(MemoryContext parent,
name);
((MemoryContext) set)->mem_allocated = firstBlockSize;
+ pgstat_report_allocated_bytes(firstBlockSize, INCREASE);
return (MemoryContext) set;
}
@@ -543,6 +545,7 @@ AllocSetReset(MemoryContext context)
AllocSet set = (AllocSet) context;
AllocBlock block;
Size keepersize PG_USED_FOR_ASSERTS_ONLY;
+ uint64 deallocation = 0;
Assert(AllocSetIsValid(set));
@@ -585,6 +588,7 @@ AllocSetReset(MemoryContext context)
{
/* Normal case, release the block */
context->mem_allocated -= block->endptr - ((char *) block);
+ deallocation += block->endptr - ((char *) block);
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
@@ -595,6 +599,7 @@ AllocSetReset(MemoryContext context)
}
Assert(context->mem_allocated == keepersize);
+ pgstat_report_allocated_bytes(deallocation, DECREASE);
/* Reset block size allocation sequence, too */
set->nextBlockSize = set->initBlockSize;
@@ -613,6 +618,7 @@ AllocSetDelete(MemoryContext context)
AllocSet set = (AllocSet) context;
AllocBlock block = set->blocks;
Size keepersize PG_USED_FOR_ASSERTS_ONLY;
+ uint64 deallocation = 0;
Assert(AllocSetIsValid(set));
@@ -651,11 +657,13 @@ AllocSetDelete(MemoryContext context)
freelist->first_free = (AllocSetContext *) oldset->header.nextchild;
freelist->num_free--;
+ deallocation += oldset->header.mem_allocated;
/* All that remains is to free the header/initial block */
free(oldset);
}
Assert(freelist->num_free == 0);
+ pgstat_report_allocated_bytes(deallocation, DECREASE);
}
/* Now add the just-deleted context to the freelist. */
@@ -672,7 +680,10 @@ AllocSetDelete(MemoryContext context)
AllocBlock next = block->next;
if (block != set->keeper)
+ {
context->mem_allocated -= block->endptr - ((char *) block);
+ deallocation += block->endptr - ((char *) block);
+ }
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
@@ -685,6 +696,7 @@ AllocSetDelete(MemoryContext context)
}
Assert(context->mem_allocated == keepersize);
+ pgstat_report_allocated_bytes(deallocation + context->mem_allocated, DECREASE);
/* Finally, free the context header, including the keeper block */
free(set);
@@ -734,6 +746,7 @@ AllocSetAlloc(MemoryContext context, Size size)
return NULL;
context->mem_allocated += blksize;
+ pgstat_report_allocated_bytes(blksize, INCREASE);
block->aset = set;
block->freeptr = block->endptr = ((char *) block) + blksize;
@@ -944,6 +957,7 @@ AllocSetAlloc(MemoryContext context, Size size)
return NULL;
context->mem_allocated += blksize;
+ pgstat_report_allocated_bytes(blksize, INCREASE);
block->aset = set;
block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
@@ -1043,6 +1057,7 @@ AllocSetFree(void *pointer)
block->next->prev = block->prev;
set->header.mem_allocated -= block->endptr - ((char *) block);
+ pgstat_report_allocated_bytes(block->endptr - ((char *) block), DECREASE);
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
@@ -1173,7 +1188,9 @@ AllocSetRealloc(void *pointer, Size size)
/* updated separately, not to underflow when (oldblksize > blksize) */
set->header.mem_allocated -= oldblksize;
+ pgstat_report_allocated_bytes(oldblksize, DECREASE);
set->header.mem_allocated += blksize;
+ pgstat_report_allocated_bytes(blksize, INCREASE);
block->freeptr = block->endptr = ((char *) block) + blksize;
diff --git a/src/backend/utils/mmgr/generation.c b/src/backend/utils/mmgr/generation.c
index b432a92be3..459eb985d6 100644
--- a/src/backend/utils/mmgr/generation.c
+++ b/src/backend/utils/mmgr/generation.c
@@ -37,6 +37,7 @@
#include "lib/ilist.h"
#include "port/pg_bitutils.h"
+#include "utils/backend_status.h"
#include "utils/memdebug.h"
#include "utils/memutils.h"
#include "utils/memutils_memorychunk.h"
@@ -267,6 +268,7 @@ GenerationContextCreate(MemoryContext parent,
name);
((MemoryContext) set)->mem_allocated = firstBlockSize;
+ pgstat_report_allocated_bytes(firstBlockSize, INCREASE);
return (MemoryContext) set;
}
@@ -283,6 +285,7 @@ GenerationReset(MemoryContext context)
{
GenerationContext *set = (GenerationContext *) context;
dlist_mutable_iter miter;
+ uint64 deallocation = 0;
Assert(GenerationIsValid(set));
@@ -305,9 +308,14 @@ GenerationReset(MemoryContext context)
if (block == set->keeper)
GenerationBlockMarkEmpty(block);
else
+ {
+ deallocation += block->blksize;
GenerationBlockFree(set, block);
+ }
}
+ pgstat_report_allocated_bytes(deallocation, DECREASE);
+
/* set it so new allocations to make use of the keeper block */
set->block = set->keeper;
@@ -328,6 +336,9 @@ GenerationDelete(MemoryContext context)
{
/* Reset to release all releasable GenerationBlocks */
GenerationReset(context);
+
+ pgstat_report_allocated_bytes(context->mem_allocated, DECREASE);
+
/* And free the context header and keeper block */
free(context);
}
@@ -374,6 +385,7 @@ GenerationAlloc(MemoryContext context, Size size)
return NULL;
context->mem_allocated += blksize;
+ pgstat_report_allocated_bytes(blksize, INCREASE);
/* block with a single (used) chunk */
block->context = set;
@@ -477,6 +489,7 @@ GenerationAlloc(MemoryContext context, Size size)
return NULL;
context->mem_allocated += blksize;
+ pgstat_report_allocated_bytes(blksize, INCREASE);
/* initialize the new block */
GenerationBlockInit(set, block, blksize);
@@ -726,6 +739,8 @@ GenerationFree(void *pointer)
dlist_delete(&block->node);
set->header.mem_allocated -= block->blksize;
+ pgstat_report_allocated_bytes(block->blksize, DECREASE);
+
free(block);
}
diff --git a/src/backend/utils/mmgr/slab.c b/src/backend/utils/mmgr/slab.c
index 6df0839b6a..f38256f6f3 100644
--- a/src/backend/utils/mmgr/slab.c
+++ b/src/backend/utils/mmgr/slab.c
@@ -53,6 +53,7 @@
#include "postgres.h"
#include "lib/ilist.h"
+#include "utils/backend_status.h"
#include "utils/memdebug.h"
#include "utils/memutils.h"
#include "utils/memutils_memorychunk.h"
@@ -238,6 +239,12 @@ SlabContextCreate(MemoryContext parent,
parent,
name);
+ /*
+ * If SlabContextCreate is updated to add headerSize to
+ * context->mem_allocated, then update here and SlabDelete appropriately
+ */
+ pgstat_report_allocated_bytes(headerSize, INCREASE);
+
return (MemoryContext) slab;
}
@@ -253,6 +260,7 @@ SlabReset(MemoryContext context)
{
SlabContext *slab = (SlabContext *) context;
int i;
+ uint64 deallocation = 0;
Assert(SlabIsValid(slab));
@@ -278,9 +286,11 @@ SlabReset(MemoryContext context)
free(block);
slab->nblocks--;
context->mem_allocated -= slab->blockSize;
+ deallocation += slab->blockSize;
}
}
+ pgstat_report_allocated_bytes(deallocation, DECREASE);
slab->minFreeChunks = 0;
Assert(slab->nblocks == 0);
@@ -294,8 +304,17 @@ SlabReset(MemoryContext context)
void
SlabDelete(MemoryContext context)
{
+ /*
+ * Until header allocation is included in context->mem_allocated, cast to
+ * slab and decrement the headerSize
+ */
+ SlabContext *slab = castNode(SlabContext, context);
+
/* Reset to release all the SlabBlocks */
SlabReset(context);
+
+ pgstat_report_allocated_bytes(slab->headerSize, DECREASE);
+
/* And free the context header */
free(context);
}
@@ -364,6 +383,7 @@ SlabAlloc(MemoryContext context, Size size)
slab->minFreeChunks = slab->chunksPerBlock;
slab->nblocks += 1;
context->mem_allocated += slab->blockSize;
+ pgstat_report_allocated_bytes(slab->blockSize, INCREASE);
}
/* grab the block from the freelist (even the new block is there) */
@@ -537,6 +557,7 @@ SlabFree(void *pointer)
free(block);
slab->nblocks--;
slab->header.mem_allocated -= slab->blockSize;
+ pgstat_report_allocated_bytes(slab->blockSize, DECREASE);
}
else
dlist_push_head(&slab->freelist[block->nfree], &block->node);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index f9301b2627..1bf02758d4 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5373,9 +5373,9 @@
proname => 'pg_stat_get_activity', prorows => '100', proisstrict => 'f',
proretset => 't', provolatile => 's', proparallel => 'r',
prorettype => 'record', proargtypes => 'int4',
- proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,int4,int8}',
- proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,leader_pid,query_id}',
+ proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,int4,int8,int8}',
+ proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
+ proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,leader_pid,query_id,allocated_bytes}',
prosrc => 'pg_stat_get_activity' },
{ oid => '3318',
descr => 'statistics: information about progress of backends running maintenance command',
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index b582b46e9f..e5aa90b101 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -15,6 +15,7 @@
#include "miscadmin.h" /* for BackendType */
#include "storage/backendid.h"
#include "utils/backend_progress.h"
+#include "common/int.h"
/* ----------
@@ -32,6 +33,13 @@ typedef enum BackendState
STATE_DISABLED
} BackendState;
+/* Enum helper for reporting memory allocated bytes */
+enum allocation_direction
+{
+ DECREASE = -1,
+ IGNORE,
+ INCREASE,
+};
/* ----------
* Shared-memory data structures
@@ -169,6 +177,9 @@ typedef struct PgBackendStatus
/* query identifier, optionally computed using post_parse_analyze_hook */
uint64 st_query_id;
+
+ /* Current memory allocated to this backend */
+ uint64 allocated_bytes;
} PgBackendStatus;
@@ -282,6 +293,7 @@ extern PGDLLIMPORT int pgstat_track_activity_query_size;
* ----------
*/
extern PGDLLIMPORT PgBackendStatus *MyBEEntry;
+extern PGDLLIMPORT uint64 *my_allocated_bytes;
/* ----------
@@ -313,7 +325,8 @@ extern const char *pgstat_get_backend_current_activity(int pid, bool checkUser);
extern const char *pgstat_get_crashed_backend_activity(int pid, char *buffer,
int buflen);
extern uint64 pgstat_get_my_query_id(void);
-
+extern void pgstat_set_allocated_bytes_storage(uint64 *allocated_bytes);
+extern void pgstat_reset_allocated_bytes_storage(void);
/* ----------
* Support functions for the SQL-callable functions to
@@ -325,5 +338,49 @@ extern PgBackendStatus *pgstat_fetch_stat_beentry(BackendId beid);
extern LocalPgBackendStatus *pgstat_fetch_stat_local_beentry(int beid);
extern char *pgstat_clip_activity(const char *raw_activity);
+/* ----------
+ * pgstat_report_allocated_bytes() -
+ *
+ * Called to report change in memory allocated for this backend.
+ *
+ * my_allocated_bytes initially points to local memory, making it safe to call
+ * this before pgstats has been initialized. allocation_direction is a
+ * positive/negative multiplier enum defined above.
+ * ----------
+ */
+static inline void
+pgstat_report_allocated_bytes(uint64 allocated_bytes, int allocation_direction)
+{
+ uint64 temp;
+
+ /* Avoid *my_allocated_bytes unsigned integer overflow on DECREASE */
+ if (allocation_direction == DECREASE &&
+ pg_sub_u64_overflow(*my_allocated_bytes, allocated_bytes, &temp))
+ {
+ *my_allocated_bytes = 0;
+ ereport(LOG,
+ errmsg("Backend %d deallocated %ld bytes, exceeding the %ld bytes it is currently reporting allocated. Setting reported to 0.",
+ MyProcPid, allocated_bytes, *my_allocated_bytes));
+ }
+ else
+ *my_allocated_bytes += (allocated_bytes) * allocation_direction;
+
+ return;
+}
+
+/* ---------
+ * pgstat_zero_my_allocated_bytes() -
+ *
+ * Called to zero out local allocated bytes variable after fork to avoid double
+ * counting allocations.
+ * ---------
+ */
+static inline void
+pgstat_zero_my_allocated_bytes(void)
+{
+ *my_allocated_bytes = 0;
+
+ return;
+}
#endif /* BACKEND_STATUS_H */
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 37c1c86473..5263294ad4 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1756,10 +1756,11 @@ pg_stat_activity| SELECT s.datid,
s.state,
s.backend_xid,
s.backend_xmin,
+ s.allocated_bytes,
s.query_id,
s.query,
s.backend_type
- FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id)
+ FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id, allocated_bytes)
LEFT JOIN pg_database d ON ((s.datid = d.oid)))
LEFT JOIN pg_authid u ON ((s.usesysid = u.oid)));
pg_stat_all_indexes| SELECT c.oid AS relid,
@@ -1874,7 +1875,7 @@ pg_stat_gssapi| SELECT s.pid,
s.gss_auth AS gss_authenticated,
s.gss_princ AS principal,
s.gss_enc AS encrypted
- FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id)
+ FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id, allocated_bytes)
WHERE (s.client_port IS NOT NULL);
pg_stat_progress_analyze| SELECT s.pid,
s.datid,
@@ -2055,7 +2056,7 @@ pg_stat_replication| SELECT s.pid,
w.sync_priority,
w.sync_state,
w.reply_time
- FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id)
+ FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id, allocated_bytes)
JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid)))
LEFT JOIN pg_authid u ON ((s.usesysid = u.oid)));
pg_stat_replication_slots| SELECT s.slot_name,
@@ -2089,7 +2090,7 @@ pg_stat_ssl| SELECT s.pid,
s.ssl_client_dn AS client_dn,
s.ssl_client_serial AS client_serial,
s.ssl_issuer_dn AS issuer_dn
- FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id)
+ FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid, query_id, allocated_bytes)
WHERE (s.client_port IS NOT NULL);
pg_stat_subscription| SELECT su.oid AS subid,
su.subname,
diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out
index 1d84407a03..ab7e95c367 100644
--- a/src/test/regress/expected/stats.out
+++ b/src/test/regress/expected/stats.out
@@ -1126,4 +1126,15 @@ SELECT pg_stat_get_subscription_stats(NULL);
(1 row)
+-- ensure that allocated_bytes exist for backends
+SELECT allocated_bytes > 0 AS result FROM pg_stat_activity WHERE backend_type
+IN ('checkpointer', 'background writer', 'walwriter', 'autovacuum launcher');
+ result
+--------
+ t
+ t
+ t
+ t
+(4 rows)
+
-- End of Stats Test
diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql
index b4d6753c71..2f0b1cc9d8 100644
--- a/src/test/regress/sql/stats.sql
+++ b/src/test/regress/sql/stats.sql
@@ -535,5 +535,8 @@ SET enable_seqscan TO on;
SELECT pg_stat_get_replication_slot(NULL);
SELECT pg_stat_get_subscription_stats(NULL);
+-- ensure that allocated_bytes exist for backends
+SELECT allocated_bytes > 0 AS result FROM pg_stat_activity WHERE backend_type
+IN ('checkpointer', 'background writer', 'walwriter', 'autovacuum launcher');
-- End of Stats Test
--
2.25.1
