Changeset: 9e984cfd6561 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/9e984cfd6561 Modified Files: ChangeLog clients/Tests/exports.stable.out gdk/gdk.h gdk/gdk_align.c gdk/gdk_analytic_func.c gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_bbp.c gdk/gdk_cand.c gdk/gdk_group.c gdk/gdk_hash.c gdk/gdk_heap.c gdk/gdk_join.c gdk/gdk_orderidx.c gdk/gdk_private.h gdk/gdk_rsort.c gdk/gdk_sample.c gdk/gdk_sketch.c gdk/gdk_strimps.c gdk/gdk_unique.c sql/backends/monet5/copy_convert.c sql/backends/monet5/copy_misc.c sql/storage/bat/bat_storage.c testing/Mtest.py.in tools/mserver/mserver5.1.in tools/mserver/mserver5.c Branch: default Log Message:
Introduced TEMPFARM (settable with --dbtemp) role for heaps. A TEMPFARM is a database farm which is only used for temporary heaps, i.e. heaps that are created and destroyed in the same function. The Heap structure now has a field `role` to hold the role of the heap (usually the same as the batRole field in the BAT structure). Using this, we can now better keep track of memory usage by client processes. Fixed some incorrect uses of the farmid value on the way. diffs (truncated from 814 to 300 lines): diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,16 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Wed Oct 21 2026 Sjoerd Mullender <[email protected]> +- There is a new type of database farm, like the --dbextra farm, for + temporary heaps. Temporary heaps are heaps which are created and + destroyed in the same function, so they never need to be saved to disk. + By using the new --dbtemp option in mserver5, it is possible to choose + a separate directory in case the heap needs to be memory mapped, or by + specifying `in-memory` it is possible to direct the server to never + use a file-backed memory map (in this case, there should be enough + memory for these heaps). + * Tue Oct 6 2026 Sjoerd Mullender <[email protected]> - The function sys.local_timezone() has been renamed to sys.system_timezone() since it returns the time zone of the server. diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -235,6 +235,7 @@ void BBPrelinquishbats(void); int BBPrename(BAT *b, const char *nme); int BBPretain(bat b); gdk_return BBPsave(BAT *b); +int BBPselectfarm(role_t role, int type, enum heaptype hptype); void BBPtmlock(void); void BBPtmunlock(void); int BBPunfix(bat b); diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -1443,6 +1443,7 @@ typedef enum { PERSISTENT = 0, TRANSIENT, SYSTRANS, + TEMPFARM, } role_t; /* Heap storage modes */ @@ -1464,7 +1465,8 @@ typedef struct { #endif ATOMIC_TYPE refs; /* reference count for this heap */ - bte farmid; /* id of farm where heap is located */ + int8_t farmid; /* id of farm where heap is located */ + role_t role; /* role of this heap */ bool cleanhash; /* string heaps must clean hash */ bool dirty; /* specific heap dirty marker */ bool remove; /* remove storage file when freeing */ @@ -3892,6 +3894,15 @@ gdk_export int BBPreadBBPline(FILE *fp, /* global calls */ gdk_export gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror); +enum heaptype { + offheap, + varheap, + hashheap, + orderidxheap, + strimpheap, + dataheap +}; +gdk_export int BBPselectfarm(role_t role, int type, enum heaptype hptype); /* update interface */ gdk_export gdk_return BBPsave(BAT *b); diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c --- a/gdk/gdk_align.c +++ b/gdk/gdk_align.c @@ -247,6 +247,7 @@ BATmaterialize(BAT *b, BUN cap) *tail = (Heap) { .farmid = BBPselectfarm(b->batRole, TYPE_oid, offheap), + .role = b->batRole, .parentid = b->batCacheid, .dirty = true, .refs = ATOMIC_VAR_INIT(1), diff --git a/gdk/gdk_analytic_func.c b/gdk/gdk_analytic_func.c --- a/gdk/gdk_analytic_func.c +++ b/gdk/gdk_analytic_func.c @@ -22,6 +22,7 @@ GDKinitialize_segment_tree(void) return NULL; *h = (Heap) { .farmid = BBPselectfarm(TRANSIENT, TYPE_bte, dataheap), + .role = TRANSIENT, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c --- a/gdk/gdk_bat.c +++ b/gdk/gdk_bat.c @@ -131,6 +131,7 @@ BATcreatedesc(oid hseq, int tt, bool hea } *h = (Heap) { .farmid = BBPselectfarm(role, tt, offheap), + .role = role, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; @@ -142,6 +143,7 @@ BATcreatedesc(oid hseq, int tt, bool hea } *vh = (Heap) { .farmid = BBPselectfarm(role, tt, varheap), + .role = role, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; @@ -506,6 +508,7 @@ BATclear(BAT *b, bool force) } *th = (Heap) { .farmid = b->tvheap->farmid, + .role = b->tvheap->role, .parentid = b->tvheap->parentid, .dirty = true, .hasfile = b->tvheap->hasfile, @@ -2465,9 +2468,13 @@ BATassertProps(BAT *b) assert(isview1 || b->ttype == TYPE_void || BBPfarms[b->theap->farmid].roles & (1 << b->batRole)); + assert(b->ttype == TYPE_void || + BBPfarms[b->theap->farmid].roles & (1 << b->theap->role)); assert(isview2 || b->tvheap == NULL || (BBPfarms[b->tvheap->farmid].roles & (1 << b->batRole))); + assert(b->tvheap == NULL || + (BBPfarms[b->tvheap->farmid].roles & (1 << b->tvheap->role))); cmpf = ATOMcompare(b->ttype); eqf = ATOMequal(b->ttype); @@ -2721,9 +2728,11 @@ BATassertProps(BAT *b) .heapbckt.parentid = b->batCacheid, .heaplink.parentid = b->batCacheid, .heaplink.farmid = BBPselectfarm( - TRANSIENT, b->ttype, hashheap), + TEMPFARM, b->ttype, hashheap), + .heaplink.role = TEMPFARM, .heapbckt.farmid = BBPselectfarm( - TRANSIENT, b->ttype, hashheap), + TEMPFARM, b->ttype, hashheap), + .heapbckt.role = TEMPFARM, }; if (ATOMsize(b->ttype) == 1) mask = (BUN) 1 << 8; diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -32,6 +32,7 @@ unshare_varsized_heap(BAT *b) *h = (Heap) { .parentid = b->batCacheid, .farmid = BBPselectfarm(b->batRole, b->ttype, varheap), + .role = b->batRole, .refs = ATOMIC_VAR_INIT(1), }; strtconcat(h->filename, sizeof(h->filename), @@ -550,6 +551,7 @@ append_varsized_bat(BAT *b, BATiter *ni, *h = (Heap) { .parentid = b->batCacheid, .farmid = BBPselectfarm(b->batRole, b->ttype, varheap), + .role = b->batRole, .refs = ATOMIC_VAR_INIT(1), }; strtconcat(h->filename, sizeof(h->filename), diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -448,6 +448,7 @@ vheapinit(BAT *b, const char *buf, unsig .dirty = false, .parentid = b->batCacheid, .farmid = BBPselectfarm(PERSISTENT, b->ttype, varheap), + .role = PERSISTENT, .hasfile = free > 0, }; strtconcat(b->tvheap->filename, sizeof(b->tvheap->filename), @@ -587,6 +588,7 @@ heapinit(BAT *b, const char *buf, settailname(b->theap, filename, t, width); b->theap->storage = STORE_INVALID; b->theap->farmid = BBPselectfarm(PERSISTENT, b->ttype, offheap); + b->theap->role = PERSISTENT, b->theap->dirty = false; b->theap->parentid = b->batCacheid; if (minpos < b->batCount) @@ -1102,6 +1104,9 @@ BBPaddfarm(const char *dirname, uint32_t GDKerror("bad rolemask\n"); return GDK_FAIL; } + /* transient and systrans must be same farm */ + if (rolemask & (1 << TRANSIENT)) + rolemask |= 1 << SYSTRANS; if (strcmp(dirname, "in-memory") == 0 || /* backward compatibility: */ strcmp(dirname, ":memory:") == 0) { dirname = NULL; @@ -1158,6 +1163,10 @@ BBPaddfarm(const char *dirname, uint32_t } return GDK_SUCCEED; } + if (BBPfarms[i].roles & rolemask) { + GDKerror("role added multiple times"); + return GDK_FAIL; + } } if (logerror) GDKerror("too many farms\n"); @@ -1168,6 +1177,7 @@ gdk_return BBPchkfarms(void) { const char *dir = NULL; + const char *tdir = NULL; uint32_t rolemask = 0; if ((BBPfarms[0].roles & 1) == 0) { GDKerror("Must call BBPaddfarms at least once for persistent data\n"); @@ -1177,17 +1187,30 @@ BBPchkfarms(void) if (BBPfarms[i].roles != 0) { dir = BBPfarms[i].dirname; rolemask |= BBPfarms[i].roles; + if (BBPfarms[i].roles & (1U << TRANSIENT)) + tdir = BBPfarms[i].dirname; } } if (dir == NULL) dir = "in-memory"; + if (tdir == NULL) + tdir = "in-memory"; if ((rolemask & (1U << TRANSIENT)) == 0) { gdk_return rc = BBPaddfarm(dir, 1U << TRANSIENT, true); if (rc != GDK_SUCCEED) return rc; + tdir = dir; } if ((rolemask & (1U << SYSTRANS)) == 0) { - gdk_return rc = BBPaddfarm(dir, 1U << SYSTRANS, true); + int farmid = BBPselectfarm(TRANSIENT, TYPE_int, offheap); + if (farmid < 0) { + GDKerror("transient farm not found"); + return GDK_FAIL; + } + BBPfarms[farmid].roles |= 1 << SYSTRANS; + } + if ((rolemask & (1U << TEMPFARM)) == 0) { + gdk_return rc = BBPaddfarm(tdir, 1U << TEMPFARM, true); if (rc != GDK_SUCCEED) return rc; } @@ -2476,9 +2499,11 @@ new_bbpentry(FILE *fp, bat i, BUN size, assert(bi->b->batRole == PERSISTENT); assert(0 <= bi->h->farmid && bi->h->farmid < MAXFARMS); assert(BBPfarms[bi->h->farmid].roles & (1U << PERSISTENT)); + assert(bi->h->role == PERSISTENT); if (bi->vh) { assert(0 <= bi->vh->farmid && bi->vh->farmid < MAXFARMS); assert(BBPfarms[bi->vh->farmid].roles & (1U << PERSISTENT)); + assert(bi->vh->role == PERSISTENT); } assert(size <= bi->count || size == BUN_NONE); assert(BBP_options(i) == NULL || strpbrk(BBP_options(i), "\r\n") == NULL); diff --git a/gdk/gdk_cand.c b/gdk/gdk_cand.c --- a/gdk/gdk_cand.c +++ b/gdk/gdk_cand.c @@ -1342,6 +1342,7 @@ BATnegcands(oid tseq, BUN nr, BAT *odels } *dels = (Heap) { .farmid = BBPselectfarm(bn->batRole, bn->ttype, varheap), + .role = bn->batRole, .parentid = bn->batCacheid, .dirty = true, .refs = ATOMIC_VAR_INIT(1), @@ -1406,6 +1407,7 @@ BATmaskedcands(oid hseq, BUN nr, BAT *ma } *msks = (Heap) { .farmid = BBPselectfarm(bn->batRole, bn->ttype, varheap), + .role = bn->batRole, .parentid = bn->batCacheid, .dirty = true, .refs = ATOMIC_VAR_INIT(1), @@ -1522,6 +1524,7 @@ BATunmask(BAT *b) } *dels = (Heap) { .farmid = BBPselectfarm(TRANSIENT, TYPE_void, varheap), + .role = TRANSIENT, .parentid = bn->batCacheid, .dirty = true, .refs = ATOMIC_VAR_INIT(1), diff --git a/gdk/gdk_group.c b/gdk/gdk_group.c --- a/gdk/gdk_group.c +++ b/gdk/gdk_group.c @@ -1154,8 +1154,10 @@ BATgroup_internal(BAT **groups, BAT **ex * tells us which power of two */ bits = 8 * SIZEOF_OID - ctz(nbucket); hsh = (Hash) { - .heaplink.farmid = BBPselectfarm(TRANSIENT, bi.type, hashheap), - .heapbckt.farmid = BBPselectfarm(TRANSIENT, bi.type, hashheap), + .heaplink.farmid = BBPselectfarm(TEMPFARM, bi.type, hashheap), + .heaplink.role = TEMPFARM, + .heapbckt.farmid = BBPselectfarm(TEMPFARM, bi.type, hashheap), + .heapbckt.role = TEMPFARM, }; hs = &hsh; if (hs->heaplink.farmid < 0 || hs->heapbckt.farmid < 0) { diff --git a/gdk/gdk_hash.c b/gdk/gdk_hash.c _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
