Changeset: 757ca2e33646 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=757ca2e33646 Modified Files: gdk/gdk_bbp.c gdk/gdk_imprints.c Branch: Oct2014 Log Message:
Implemented persistent imprints. Imprints are saved to disk when they are created, and used when found when imprints are requested. The start of the imprints heap contains some information with which it can be determined whether the imprint is valid. Saving is done in two stages: first the heap is written, and when that is successful, the first word of the file is overwritten with an indicator that the heap was written successfully. diffs (152 lines): diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -3777,6 +3777,12 @@ BBPdiskscan(const char *parent) } else if (strncmp(p + 1, "thash", 5) == 0) { BAT *b = getdesc(bid); delete = (b == NULL || !b->T->hash); + } else if (strncmp(p + 1, "himprints", 9) == 0) { + BAT *b = getdesc(bid); + delete = b == NULL; + } else if (strncmp(p + 1, "timprints", 9) == 0) { + BAT *b = getdesc(bid); + delete = b == NULL; } else if (strncmp(p + 1, "priv", 4) != 0 && strncmp(p + 1, "new", 3) != 0 && strncmp(p + 1, "head", 4) != 0 && strncmp(p + 1, "tail", 4) != 0) { ok = FALSE; } diff --git a/gdk/gdk_imprints.c b/gdk/gdk_imprints.c --- a/gdk/gdk_imprints.c +++ b/gdk/gdk_imprints.c @@ -587,6 +587,7 @@ BATimprints(BAT *b) BUN cnt; str nme = BBP_physical(b->batCacheid); size_t pages; + int fd; ALGODEBUG fprintf(stderr, "#BATimprints(b=%s#" BUNFMT ") %s: " "created imprints\n", BATgetId(b), @@ -599,6 +600,54 @@ BATimprints(BAT *b) "BATimprints"); return NULL; } + imprints->imprints = GDKzalloc(sizeof(Heap)); + if (imprints->imprints == NULL || + (imprints->imprints->filename = + GDKmalloc(strlen(nme) + 12)) == NULL) { + GDKfree(imprints->imprints); + GDKfree(imprints); + GDKerror("#BATimprints: memory allocation error.\n"); + BBPunfix(smp->batCacheid); + MT_lock_unset(&GDKimprintsLock(abs(b->batCacheid)), + "BATimprints"); + return NULL; + } + sprintf(imprints->imprints->filename, "%s.%cimprints", nme, + b->batCacheid > 0 ? 't' : 'h'); + pages = (((size_t) BATcount(b) * b->T->width) + IMPS_PAGE - 1) / IMPS_PAGE; + imprints->imprints->farmid = BBPselectfarm(PERSISTENT, b->ttype, + imprintsheap); + if ((fd = GDKfdlocate(imprints->imprints->farmid, nme, "rb", + b->batCacheid > 0 ? "timprints" : "himprints")) >= 0) { + size_t hdata[4]; + struct stat st; + if (read(fd, hdata, sizeof(hdata)) == sizeof(hdata) && + hdata[0] & ((size_t) 1 << 16) && + hdata[3] == (size_t) BATcount(b) && + fstat(fd, &st) == 0 && + st.st_size >= (off_t) (imprints->imprints->size = imprints->imprints->free = 64 * b->T->width + + pages * ((bte) hdata[0] / 8) + + hdata[2] * sizeof(cchdc_t) + + sizeof(uint64_t) /* padding for alignment */ + + 4 * SIZEOF_SIZE_T) && + HEAPload(imprints->imprints, nme, b->batCacheid > 0 ? "timprints" : "himprints", 0) >= 0) { + /* usable */ + imprints->bits = (bte) (hdata[0] & 0xFF); + imprints->impcnt = (BUN) hdata[1]; + imprints->dictcnt = (BUN) hdata[2]; + imprints->bins = imprints->imprints->base + 4 * SIZEOF_SIZE_T; + imprints->imps = (char *) imprints->bins + 64 * b->T->width; + imprints->dict = (void *) ((size_t) ((char *) imprints->imps + pages * (imprints->bits / 8) + sizeof(uint64_t)) & ~(sizeof(uint64_t) - 1)); + b->T->imprints = imprints; + close(fd); + goto do_return; + } + close(fd); + /* file exists, but can't be used: delete it */ + GDKunlink(imprints->imprints->farmid, BATDIR, nme, + b->batCacheid > 0 ? "timprints" : "himprints"); + } + #define SMP_SIZE 2048 s = BATsample(b, SMP_SIZE); if (s == NULL) { @@ -643,24 +692,6 @@ BATimprints(BAT *b) if (cnt < 8) imprints->bits = 8; - /* bins of histogram */ - imprints->imprints = GDKzalloc(sizeof(Heap)); - if (imprints->imprints == NULL || - (imprints->imprints->filename = - GDKmalloc(strlen(nme) + 12)) == NULL) { - GDKfree(imprints->imprints); - GDKfree(imprints); - GDKerror("#BATimprints: memory allocation error.\n"); - BBPunfix(smp->batCacheid); - MT_lock_unset(&GDKimprintsLock(abs(b->batCacheid)), - "BATimprints"); - return NULL; - } - sprintf(imprints->imprints->filename, "%s.%cimprints", nme, - b->batCacheid > 0 ? 't' : 'h'); - pages = (((size_t) BATcount(b) * b->T->width) + IMPS_PAGE - 1) / IMPS_PAGE; - imprints->imprints->farmid = BBPselectfarm(TRANSIENT, b->ttype, - imprintsheap); /* The heap we create here consists of three parts: * bins, max 64 entries with bin boundaries, domain of b; * imps, max one entry per "page", entry is "bits" wide; @@ -729,13 +760,30 @@ BATimprints(BAT *b) } assert(imprints->impcnt <= pages); assert(imprints->dictcnt <= pages); + imprints->imprints->free = (size_t) ((char *) ((cchdc_t *) imprints->dict + imprints->dictcnt) - imprints->imprints->base); /* add info to heap for when they become persistent */ ((size_t *) imprints->imprints->base)[0] = (size_t) imprints->bits; ((size_t *) imprints->imprints->base)[1] = (size_t) imprints->impcnt; ((size_t *) imprints->imprints->base)[2] = (size_t) imprints->dictcnt; ((size_t *) imprints->imprints->base)[3] = (size_t) BATcount(b); + if (HEAPsave(imprints->imprints, nme, b->batCacheid > 0 ? "timprints" : "himprints") == 0 && + (fd = GDKfdlocate(imprints->imprints->farmid, nme, "rb+", + b->batCacheid > 0 ? "timprints" : "himprints")) >= 0) { + ((size_t *) imprints->imprints->base)[0] |= (size_t) 1 << 16; + write(fd, imprints->imprints->base, sizeof(size_t)); +#if defined(NATIVE_WIN32) + _commit(fd); +#elif defined(HAVE_FDATASYNC) + fdatasync(fd); +#elif defined(HAVE_FSYNC) + fsync(fd); +#endif + close(fd); + } b->T->imprints = imprints; } + + do_return: MT_lock_unset(&GDKimprintsLock(abs(b->batCacheid)), "BATimprints"); if (o != NULL) { @@ -825,11 +873,8 @@ IMPSremove(BAT *b) imprints = b->T->imprints; b->T->imprints = NULL; - if (imprints->imprints->storage != STORE_MEM) - HEAPdelete(imprints->imprints, BBP_physical(b->batCacheid), - b->batCacheid > 0 ? "timprints" : "himprints"); - else - HEAPfree(imprints->imprints); + HEAPdelete(imprints->imprints, BBP_physical(b->batCacheid), + b->batCacheid > 0 ? "timprints" : "himprints"); GDKfree(imprints->imprints); GDKfree(imprints); _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list