Changeset: 124995815925 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/124995815925 Branch: default Log Message:
Merge with Dec2023 branch. diffs (truncated from 506 to 300 lines): diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -220,6 +220,7 @@ void BBPcold(bat i); int BBPfix(bat b); unsigned BBPheader(FILE *fp, int *lineno, bat *bbpsize, lng *logno, lng *transid, bool allow_hge_upgrade); bat BBPindex(const char *nme); +gdk_return BBPjson_upgrade(json_storage_conversion); void BBPkeepref(BAT *b) __attribute__((__nonnull__(1))); bat BBPlimit; void BBPlock(void); diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -755,7 +755,8 @@ typedef struct { #define GDKLIBRARY_TAILN 061043U /* first in Jul2021: str offset heaps names don't take width into account */ #define GDKLIBRARY_HASHASH 061044U /* first in Jul2021: hashash bit in string heaps */ #define GDKLIBRARY_HSIZE 061045U /* first in Jan2022: heap "size" values */ -#define GDKLIBRARY 061046U /* first in Sep2022 */ +#define GDKLIBRARY_JSON 061046U /* first in Sep2022: json storage changes*/ +#define GDKLIBRARY 061047U /* first in Dec2023 */ /* The batRestricted field indicates whether a BAT is readonly. * we have modes: BAT_WRITE = all permitted diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -139,7 +139,7 @@ insert_string_bat(BAT *b, BATiter *ni, s } MT_lock_set(&b->theaplock); - if (HEAPgrow(&b->tvheap, toff + ni->vh->size, force) != GDK_SUCCEED) { + if (HEAPgrow(&b->tvheap, toff + ni->vhfree, force) != GDK_SUCCEED) { MT_lock_unset(&b->theaplock); return GDK_FAIL; } diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -1002,6 +1002,7 @@ BBPheader(FILE *fp, int *lineno, bat *bb return 0; } if (bbpversion != GDKLIBRARY && + bbpversion != GDKLIBRARY_JSON && bbpversion != GDKLIBRARY_HSIZE && bbpversion != GDKLIBRARY_HASHASH && bbpversion != GDKLIBRARY_TAILN && @@ -1474,6 +1475,244 @@ movestrbats(void) } #endif +#ifdef GDKLIBRARY_JSON +static gdk_return +jsonupgradebat(BAT *b, json_storage_conversion fixJSONStorage) +{ + const char *nme = BBP_physical(b->batCacheid); + char *srcdir = GDKfilepath(NOFARM, BATDIR, nme, NULL); + + if (srcdir == NULL) { + TRC_CRITICAL(GDK, "GDKfilepath failed\n"); + return GDK_FAIL; + } + + char *s; + if ((s = strrchr(srcdir, DIR_SEP)) != NULL) + *s = 0; + const char *bnme; + if ((bnme = strrchr(nme, DIR_SEP)) != NULL) { + bnme++; + } else { + bnme = nme; + } + + long_str filename; + snprintf(filename, sizeof(filename), "BACKUP%c%s", DIR_SEP, bnme); + + /* A json column should not normally have any index structures */ + HASHdestroy(b); + IMPSdestroy(b); + OIDXdestroy(b); + PROPdestroy(b); + STRMPdestroy(b); + RTREEdestroy(b); + + /* backup the current heaps */ + if (GDKmove(b->theap->farmid, srcdir, bnme, "tail", + BAKDIR, bnme, "tail", false) != GDK_SUCCEED) { + GDKfree(srcdir); + TRC_CRITICAL(GDK, "cannot make backup of %s.tail\n", nme); + return GDK_FAIL; + } + if (GDKmove(b->theap->farmid, srcdir, bnme, "theap", + BAKDIR, bnme, "theap", true) != GDK_SUCCEED) { + GDKfree(srcdir); + TRC_CRITICAL(GDK, "cannot make backup of %s.theap\n", nme); + return GDK_FAIL; + } + + /* load the old heaps */ + Heap h1 = *b->theap; + h1.base = NULL; + h1.dirty = false; + strconcat_len(h1.filename, sizeof(h1.filename), filename, ".tail", NULL); + if (HEAPload(&h1, filename, "tail", false) != GDK_SUCCEED) { + GDKfree(srcdir); + TRC_CRITICAL(GDK, "loading old tail heap " + "for BAT %d failed\n", b->batCacheid); + return GDK_FAIL; + } + + Heap vh1 = *b->tvheap; + vh1.base = NULL; + vh1.dirty = false; + strconcat_len(vh1.filename, sizeof(vh1.filename), filename, ".theap", NULL); + if (HEAPload(&vh1, filename, "theap", false) != GDK_SUCCEED) { + GDKfree(srcdir); + HEAPfree(&h1, false); + TRC_CRITICAL(GDK, "loading old string heap " + "for BAT %d failed\n", b->batCacheid); + return GDK_FAIL; + } + + /* create the new heaps */ + Heap *h2 = GDKmalloc(sizeof(Heap)); + Heap *vh2 = GDKmalloc(sizeof(Heap)); + if (h2 == NULL || vh2 == NULL) { + GDKfree(h2); + GDKfree(vh2); + GDKfree(srcdir); + HEAPfree(&h1, false); + HEAPfree(&vh1, false); + TRC_CRITICAL(GDK, "allocating new heaps " + "for BAT %d failed\n", b->batCacheid); + return GDK_FAIL; + } + *h2 = *b->theap; + h2->base = NULL; + if (HEAPalloc(h2, b->batCapacity, b->twidth) != GDK_SUCCEED) { + GDKfree(h2); + GDKfree(vh2); + GDKfree(srcdir); + HEAPfree(&h1, false); + HEAPfree(&vh1, false); + TRC_CRITICAL(GDK, "allocating new tail heap " + "for BAT %d failed\n", b->batCacheid); + return GDK_FAIL; + + } + h2->dirty = true; + h2->free = h1.free; + + *vh2 = *b->tvheap; + strconcat_len(vh2->filename, sizeof(vh2->filename), nme, ".theap", NULL); + strHeap(vh2, b->batCapacity); + if (vh2->base == NULL) { + GDKfree(srcdir); + HEAPfree(&h1, false); + HEAPfree(&vh1, false); + HEAPfree(h2, false); + GDKfree(h2); + GDKfree(vh2); + TRC_CRITICAL(GDK, "allocating new string heap " + "for BAT %d failed\n", b->batCacheid); + return GDK_FAIL; + } + vh2->dirty = true; + ATOMIC_INIT(&h2->refs, 1); + ATOMIC_INIT(&vh2->refs, 1); + Heap *ovh = b->tvheap; + b->tvheap = vh2; + vh2 = NULL; + + for (BUN i = 0; i < b->batCount; i++) { + var_t o = ((var_t *) h1.base)[i]; + const char *s = vh1.base + o; + char *ns; + if (fixJSONStorage(&ns, &s) != GDK_SUCCEED) { + GDKfree(srcdir); + HEAPfree(&h1, false); + HEAPfree(&vh1, false); + HEAPdecref(h2, false); + HEAPdecref(b->tvheap, false); + b->tvheap = ovh; + TRC_CRITICAL(GDK, "converting value " + "in BAT %d failed\n", b->batCacheid); + return GDK_FAIL; + } + var_t no = strPut(b, &o, ns); + GDKfree(ns); + if (no == 0) { + GDKfree(srcdir); + HEAPfree(&h1, false); + HEAPfree(&vh1, false); + HEAPdecref(h2, false); + HEAPdecref(b->tvheap, false); + b->tvheap = ovh; + TRC_CRITICAL(GDK, "storing new value " + "in BAT %d failed\n", b->batCacheid); + return GDK_FAIL; + + } + ((var_t *)h2->base)[i] = no; + } + + /* cleanup */ + HEAPfree(&h1, false); + HEAPfree(&vh1, false); + if (HEAPsave(h2, nme, BATtailname(b), true, h2->free, NULL) != + GDK_SUCCEED) { + HEAPdecref(h2, false); + HEAPdecref(b->tvheap, false); + b->tvheap = ovh; + GDKfree(srcdir); + TRC_CRITICAL(GDK, "saving heap failed\n"); + return GDK_FAIL; + } + + if (HEAPsave(b->tvheap, nme, "theap", true, b->tvheap->free, + &b->theaplock) != GDK_SUCCEED) { + HEAPfree(b->tvheap, false); + b->tvheap = ovh; + GDKfree(srcdir); + TRC_CRITICAL(GDK, "saving string failed\n"); + return GDK_FAIL; + } + + HEAPdecref(b->theap, false); + b->theap = h2; + HEAPfree(h2, false); + HEAPdecref(ovh, false); + HEAPfree(b->tvheap, false); + GDKfree(srcdir); + + return GDK_SUCCEED; +} + +gdk_return +BBPjson_upgrade(json_storage_conversion fixJSONStorage) +{ + bat bid; + BAT *b; + int JSON_type = ATOMindex("json"); + bat *upd = GDKmalloc(sizeof(bat) * ATOMIC_GET(&BBPsize)); + BUN nupd = 0; + + if (upd == NULL) { + TRC_CRITICAL(GDK, "could not create bat\n"); + return GDK_FAIL; + } + upd[nupd++] = 0; /* first entry unused */ + + BBPlock(); + + for (bid = 1; bid < (bat) ATOMIC_GET(&BBPsize); bid++) { + if ((b = BBP_desc(bid)) == NULL) { + /* not a valid BAT */ + continue; + } + + if (b->ttype < 0) { + const char *nme; + + nme = ATOMunknown_name(b->ttype); + if (strcmp(nme, "json") != 0) + continue; + } else if (b->ttype != JSON_type) { + continue; + } + fprintf(stderr, "Upgrading json bat %d\n", bid); + if (jsonupgradebat(b, fixJSONStorage) != GDK_SUCCEED) { + BBPunlock(); + GDKfree(upd); + return GDK_FAIL; + } + upd[nupd++] = bid; + } + BBPunlock(); + if (nupd > 1 && + TMsubcommit_list(upd, NULL, nupd, -1, -1) != GDK_SUCCEED) { + TRC_CRITICAL(GDK, "failed to commit changes\n"); + GDKfree(upd); + return GDK_FAIL; + } + GDKfree(upd); + GDKunlink(0, BATDIR, "jsonupgradeneeded", NULL); + return GDK_SUCCEED; +} +#endif + static bool BBPtrim(bool aggressive) { @@ -1815,6 +2054,38 @@ BBPinit(bool allow_hge_upgrade) return res; #endif +#ifdef GDKLIBRARY_JSON + if (bbpversion <= GDKLIBRARY_JSON) { + char *jsonupgradestr; + if (GDKinmemory(0)) { _______________________________________________ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org