Changeset: c82f0e5b818c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c82f0e5b818c Modified Files: gdk/ChangeLog gdk/gdk_bbp.mx Branch: default Log Message:
Implemented automatic conversion from 32-bit OIDs to 64-bit OIDs. Columns of type OID and the offset columns of variable sized types get rewritten, as do string heaps (which includes the URL type). Data heaps of other variabled sized types do not get rewritten. All known variable sized types (blob and wkb (geom)) write the data heap using the HEAP_* functions which are not dependent on the size of the OID type (they do depend on the size of size_t, but that didn't change). diffs (truncated from 448 to 300 lines): diff --git a/gdk/ChangeLog b/gdk/ChangeLog --- a/gdk/ChangeLog +++ b/gdk/ChangeLog @@ -1,3 +1,7 @@ # ChangeLog file for MonetDB # This file is updated with Maddlog +* Wed May 11 2011 Sjoerd Mullender <[email protected]> +- Implemented automatic conversion of a 64-bit database with 32-bit OIDs + to one with 64-bit OIDs. + diff --git a/gdk/gdk_bbp.mx b/gdk/gdk_bbp.mx --- a/gdk/gdk_bbp.mx +++ b/gdk/gdk_bbp.mx @@ -493,15 +493,264 @@ static int BBPrecover_subdir(void); static int BBPdiskscan(const char *); +#if SIZEOF_SIZE_T == 8 && SIZEOF_OID == 8 +/* Convert 32-bit OIDs to 64 bits. + * This function must be called at the end of BBPinit(), just before + * "normal processing" starts. All errors that happen during the + * conversion are fatal. This function is "safe" in the sense that + * when it is interrupted, recovery will just start over. No + * permanent changes are made to the database until all string heaps + * have been converted, and then the changes are made atomically. */ + +static void +fixoidheapcolumn(BAT *b, const char *nme, const char *filename, + const char *headtail, const char *htheap) +{ + bat bid = ABS(b->batCacheid); + Heap h1, h2; + int *old; + oid *new; + BUN i; + char *s; + unsigned short w; + + if (b->H->type == TYPE_str) { + h1 = b->H->heap; + h1.filename = NULL; + h1.base = NULL; + h1.dirty = 0; + h1.parentid = 0; + h2 = *b->H->vheap; + h2.filename = NULL; + h2.base = NULL; + h2.dirty = 0; + h2.parentid = 0; + + /* load old string heap */ + if (HEAPload(&h1, filename, headtail, 0) < 0) + GDKfatal("fixoidheap: loading old %s heap " + "for BAT %d failed\n", headtail, bid); + if (HEAPload(&h2, filename, htheap, 0) < 0) + GDKfatal("fixoidheap: loading old string heap " + "for BAT %d failed\n", bid); + + /* create new string heap */ + if (HEAPdelete(&b->H->heap, nme, headtail)) + GDKfatal("fixoidheap: deleting %s heap " + "for BAT %d failed\n", headtail, bid); + b->H->heap.filename = GDKmalloc(strlen(nme) + 12); + if (b->H->heap.filename == NULL) + GDKfatal("fixoidheap: GDKmalloc failed\n"); + GDKfilepath(b->H->heap.filename, NULL, nme, headtail); + w = b->H->width; /* remember old width */ + b->H->width = 1; + b->H->shift = 0; + if (HEAPalloc(&b->H->heap, b->U->capacity, SIZEOF_OID) < 0) + GDKfatal("fixoidheap: allocating new %s heap " + "for BAT %d failed\n", headtail, bid); + + b->H->heap.dirty = TRUE; + if (HEAPdelete(b->H->vheap, nme, htheap)) + GDKfatal("fixoidheap: deleting string heap " + "for BAT %d failed\n", bid); + b->H->vheap->filename = GDKmalloc(strlen(nme) + 12); + if (b->H->vheap->filename == NULL) + GDKfatal("fixoidheap: GDKmalloc failed\n"); + GDKfilepath(b->H->vheap->filename, NULL, nme, htheap); + if (ATOMheap(TYPE_str, b->H->vheap, b->U->capacity)) + GDKfatal("fixoidheap: initializing new string " + "heap for BAT %d failed\n", bid); + b->H->vheap->parentid = bid; + + /* do the conversion */ + b->H->heap.dirty = TRUE; + b->H->vheap->dirty = TRUE; + for (i = 0; i < b->U->count; i++) { + /* s = h2.base + VarHeapVal(h1.base, i, w); */ + switch (w) { + case 1: + s = h2.base + (((var_t) ((unsigned char *) h1.base)[i] + ((GDK_STRHASHTABLE * sizeof(unsigned short)) >> 3)) << 3); + break; + case 2: + s = h2.base + (((var_t) ((unsigned short *) h1.base)[i] + ((GDK_STRHASHTABLE * sizeof(unsigned short)) >> 3)) << 3); + break; + case 4: + s = h2.base + ((var_t) ((unsigned int *) h1.base)[i] << 3); + break; + default: + assert(0); + } + b->H->heap.free += b->H->width; + Hputvalue(b, Hloc(b, i), s, 0); + } + HEAPfree(&h1); + HEAPfree(&h2); + } else if (b->H->type == TYPE_oid || + (b->H->type != TYPE_void && b->H->varsized)) { + h1 = b->H->heap; + h1.filename = NULL; + h1.base = NULL; + h1.dirty = 0; + h1.parentid = 0; + + /* load old heap */ + if (HEAPload(&h1, filename, headtail, 0) < 0) + GDKfatal("fixoidheap: loading old %s heap " + "for BAT %d failed\n", headtail, bid); + + /* create new heap */ + if (HEAPdelete(&b->H->heap, nme, headtail)) + GDKfatal("fixoidheap: deleting %s heap " + "for BAT %d failed\n", headtail, bid); + b->H->heap.filename = GDKmalloc(strlen(nme) + 12); + if (b->H->heap.filename == NULL) + GDKfatal("fixoidheap: GDKmalloc failed\n"); + GDKfilepath(b->H->heap.filename, NULL, nme, headtail); + b->H->width = SIZEOF_OID; + b->H->shift = 3; + assert(b->H->width == (1 << b->H->shift)); + if (HEAPalloc(&b->H->heap, b->U->capacity, SIZEOF_OID) < 0) + GDKfatal("fixoidheap: allocating new %s heap " + "for BAT %d failed\n", headtail, bid); + + b->H->heap.dirty = TRUE; + old = (int *) h1.base + b->U->first; + new = (oid *) b->H->heap.base + b->U->first; + if (b->H->varsized) + for (i = 0; i < b->U->count; i++) + new[i] = (oid) old[i] << 3; + else + for (i = 0; i < b->U->count; i++) + new[i] = old[i] == int_nil ? oid_nil : (oid) old[i]; + b->H->heap.free = h1.free << 1; + HEAPfree(&h1); + } + return; + + bunins_failed: + GDKfatal("fixoidheap: memory allocation failed\n"); +} + +static void +fixoidheap(void) +{ + bat bid; + BATstore *bs; + BAT *b; + str nme, bnme; + long_str filename; + int ht, tt; + + fprintf(stderr, + "# upgrading database from 32 bit OIDs to 64 bit OIDs\n"); + fflush(stderr); + + for (bid = 1; bid < BBPsize; bid++) { + if ((bs = BBP[bid].cache) == NULL) + continue; /* not a valid BAT */ + /* OID and (non-void) varsized columns have to be rewritten */ + if (bs->H.type != TYPE_oid && + (bs->H.type == TYPE_void || !bs->H.varsized) && + bs->T.type != TYPE_oid && + (bs->T.type == TYPE_void || !bs->T.varsized)) + continue; /* nothing to do for this BAT */ + if ((ht = bs->H.type) < 0) { + /* as yet unknown head column type */ + nme = ATOMunknown_name(ht); + if (strcmp(nme, "url") == 0) + bs->H.type = TYPE_str; + else if (strcmp(nme, "sqlblob") == 0 || + strcmp(nme, "wkb") == 0) + bs->H.type = TYPE_int; + else if (bs->H.varsized) + GDKfatal("fixoidheap: unrecognized column " + "type %s for BAT %d\n", nme, bid); + else if (bs->H.width == 1) + bs->H.type = TYPE_bte; + else if (bs->H.width == 2) + bs->H.type = TYPE_sht; + else if (bs->H.width == 4) + bs->H.type = TYPE_int; + else + bs->H.type = TYPE_lng; + } + if ((tt = bs->T.type) < 0) { + nme = ATOMunknown_name(tt); + if (strcmp(nme, "url") == 0) + bs->T.type = TYPE_str; + else if (strcmp(nme, "sqlblob") == 0 || + strcmp(nme, "wkb") == 0) + bs->T.type = TYPE_int; + else if (bs->H.varsized) + GDKfatal("fixoidheap: unrecognized column " + "type %s for BAT %d\n", nme, bid); + else if (bs->H.width == 1) + bs->H.type = TYPE_bte; + else if (bs->H.width == 2) + bs->H.type = TYPE_sht; + else if (bs->H.width == 4) + bs->H.type = TYPE_int; + else + bs->H.type = TYPE_lng; + } + BBPfix(bid); + b = BBPdescriptor(bid); + if (b == NULL) + GDKfatal("fixoidheap: BBPdescriptor(%d) failed\n", bid); + + nme = BBP_physical(bid); + if ((bnme = strrchr(nme, DIR_SEP)) == NULL) + bnme = nme; + else + bnme++; + sprintf(filename, "BACKUP%c%s", DIR_SEP, bnme); + + /* make backup copy */ + if (b->H->type == TYPE_oid || + (b->H->varsized && b->H->type != TYPE_void)) { + assert(b->H->type != TYPE_oid || b->H->width == 4); + b->H->heap.dirty = TRUE; + if (b->H->type == TYPE_str) + b->H->vheap->dirty = TRUE; + } + if (b->T->type == TYPE_oid || + (b->T->varsized && b->T->type != TYPE_void)) { + assert(b->T->type != TYPE_oid || b->T->width == 4); + b->T->heap.dirty = TRUE; + if (b->T->type == TYPE_str) + b->T->vheap->dirty = TRUE; + } + if (BBPsave(b)) + GDKfatal("fixoidheap: creating backup for BAT %d failed\n", bid); + fixoidheapcolumn(b, nme, filename, "head", "hheap"); + fixoidheapcolumn(BATmirror(b), nme, filename, "tail", "theap"); + + if (ht < 0) + bs->H.type = ht; + if (tt < 0) + bs->T.type = tt; + + BBPunfix(bid); + BBP_unload_inc(bid, "fixoidheap"); + if (BBPfree(b, "fixoidheap")) + GDKfatal("fixoidheap: BBPfree failed\n"); + b = NULL; + } + + /* make permanent */ + if (TMcommit()) + GDKfatal("fixoidheap: commit failed\n"); +} +#endif + #if GDK_VARSHIFT != 0 /* Convert the old-style string heap to the new style. - This function must be called at the end of BBPinit(), just before - "normal processing" starts. All errors that happen during the - conversion are fatal. This function is "safe" in the sense that - when it is interrupted, recovery will just start over. No - permanent changes are made to the database until all string heaps - have been converted, and then the changes are made atomically. -*/ + * This function must be called at the end of BBPinit(), just before + * "normal processing" starts. All errors that happen during the + * conversion are fatal. This function is "safe" in the sense that + * when it is interrupted, recovery will just start over. No + * permanent changes are made to the database until all string heaps + * have been converted, and then the changes are made atomically. */ static void fixstrheap(void) @@ -517,6 +766,9 @@ str s; var_t loc; + fprintf(stderr, "# upgrading string heaps to new storage scheme\n"); + fflush(stderr); + for (bid = 1; bid < BBPsize; bid++) { if ((bs = BBP[bid].cache) == NULL) continue; /* not a valid BAT */ @@ -605,12 +857,14 @@ The default src=0 is to read the full BBPdir file. @c static BATstore * _______________________________________________ Checkin-list mailing list [email protected] http://mail.monetdb.org/mailman/listinfo/checkin-list
