Changeset: bb73d36663f1 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bb73d36663f1 Modified Files: clients/Tests/exports.stable.out common/utils/muuid.c gdk/gdk.h gdk/gdk_atoms.c gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_heap.c gdk/gdk_imprints.c gdk/gdk_logger.c gdk/gdk_posix.c gdk/gdk_posix.h gdk/gdk_private.h gdk/gdk_setop.c gdk/gdk_storage.c gdk/gdk_utils.c monetdb5/mal/mal_parser.c monetdb5/modules/kernel/status.c monetdb5/modules/mal/bbp.c monetdb5/modules/mal/mat.c monetdb5/modules/mal/remote.c monetdb5/modules/mal/tablet.c sql/backends/monet5/sql_result.c sql/backends/monet5/vaults/fits.c sql/benchmarks/tpch/Tests/16-explain.stable.out sql/benchmarks/tpch/Tests/16-plan.stable.out sql/benchmarks/tpch/fileleak/Tests/delete_all.stable.out sql/benchmarks/tpch/fileleak/Tests/leaks.stable.out sql/server/rel_optimizer.c sql/storage/bat/bat_storage.c sql/storage/restrict/restrict_storage.c sql/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.out Branch: default Log Message:
Merge with Jan2014 branch. diffs (truncated from 1373 to 300 lines): diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -253,9 +253,7 @@ BAT *BUNinplace(BAT *b, BUN p, const voi BAT *BUNins(BAT *b, const void *left, const void *right, bit force); BUN BUNlocate(BAT *b, const void *left, const void *right); BAT *BUNreplace(BAT *b, const void *left, const void *right, bit force); -size_t GDK_mem_bigsize; size_t GDK_mem_maxsize; -int GDK_mem_pagebits; size_t GDK_vm_maxsize; int GDK_vm_trim; int GDKatomcnt; @@ -292,7 +290,7 @@ ssize_t GDKstrFromStr(unsigned char *dst str GDKstrdup(const char *s); int GDKsyserror(_In_z_ _Printf_format_string_ const char *format, ...) __attribute__((__format__(__printf__, 1, 2))); ThreadRec GDKthreads[THREADS]; -int GDKupgradevarheap(COLrec *c, var_t v, int copyall); +int GDKupgradevarheap(COLrec *c, var_t v, int copyall, int mayshare); lng GDKusec(void); BAT *GDKval; const char *GDKversion(void); @@ -305,7 +303,7 @@ void HEAP_free(Heap *heap, var_t block); void HEAP_initialize(Heap *heap, size_t nbytes, size_t nprivate, int alignment); var_t HEAP_malloc(Heap *heap, size_t nbytes); int HEAPcopy(Heap *dst, Heap *src); -int HEAPextend(Heap *h, size_t size); +int HEAPextend(Heap *h, size_t size, int mayshare); int HEAPfree(Heap *h); size_t HEAPmemsize(Heap *h); size_t HEAPvmsize(Heap *h); @@ -319,7 +317,6 @@ void MT_exiting_thread(void); MT_Id MT_getpid(void); size_t MT_getrss(void); void MT_global_exit(int status) __attribute__((__noreturn__)); -char *MT_heapbase; void MT_init(void); int MT_join_thread(MT_Id t); int MT_kill_thread(MT_Id t); diff --git a/common/utils/muuid.c b/common/utils/muuid.c --- a/common/utils/muuid.c +++ b/common/utils/muuid.c @@ -47,22 +47,16 @@ generateUUID(void) uuid_unparse(uuid, out); #else /* try to do some pseudo interesting stuff, and stash it in the - * format of an UUID to at least return some uniform answer */ + * format of a UUID to at least return some uniform answer */ char out[37]; - char *p = out; /* generate something like this: * cefa7a9c-1dd2-11b2-8350-880020adbeef ("%08x-%04x-%04x-%04x-%012x") */ - p += snprintf(p, 5, "%04x", rand() % 65536); - p += snprintf(p, 6, "%04x-", rand() % 65536); - p += snprintf(p, 6, "%04x-", rand() % 65536); - p += snprintf(p, 6, "%04x-", rand() % 65536); - p += snprintf(p, 6, "%04x-", rand() % 65536); - p += snprintf(p, 5, "%04x", rand() % 65536); - p += snprintf(p, 5, "%04x", rand() % 65536); - p += snprintf(p, 5, "%04x", rand() % 65536); + snprintf(out, sizeof(out), "%04x%04x-%04x-%04x-%04x-%04x%04x%04x", + rand() % 65536, rand() % 65536, + rand() % 65536, rand() % 65536, + rand() % 65536, rand() % 65536, + rand() % 65536, rand() % 65536); #endif - return(strdup(out)); + return strdup(out); } - -/* vim:set ts=4 sw=4 noexpandtab: */ diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -990,7 +990,7 @@ typedef int (*GDKfcn) (); * HEAPfree (Heap *h); * @item int * @tab - * HEAPextend (Heap *h, size_t size); + * HEAPextend (Heap *h, size_t size, int mayshare); * @item int * @tab * HEAPload (Heap *h, str nme,ext, int trunc); @@ -1013,7 +1013,7 @@ typedef int (*GDKfcn) (); * isolate you from the different ways heaps can be accessed. */ gdk_export int HEAPfree(Heap *h); -gdk_export int HEAPextend(Heap *h, size_t size); +gdk_export int HEAPextend(Heap *h, size_t size, int mayshare); gdk_export int HEAPcopy(Heap *dst, Heap *src); gdk_export size_t HEAPvmsize(Heap *h); gdk_export size_t HEAPmemsize(Heap *h); @@ -1187,7 +1187,7 @@ gdk_export bte ATOMelmshift(int sz); if ((b)->HT->width < SIZEOF_VAR_T && \ ((b)->HT->width <= 2 ? _d - GDK_VAROFFSET : _d) >= ((size_t) 1 << (8 * (b)->HT->width))) { \ /* doesn't fit in current heap, upgrade it */ \ - if (GDKupgradevarheap((b)->HT, _d, (copyall)) == GDK_FAIL) \ + if (GDKupgradevarheap((b)->HT, _d, (copyall), (b)->batRestricted == BAT_READ) == GDK_FAIL) \ goto bunins_failed; \ } \ _ptr = (p); \ @@ -1235,7 +1235,7 @@ gdk_export bte ATOMelmshift(int sz); if ((b)->HT->width < SIZEOF_VAR_T && \ ((b)->HT->width <= 2 ? _d - GDK_VAROFFSET : _d) >= ((size_t) 1 << (8 * (b)->HT->width))) { \ /* doesn't fit in current heap, upgrade it */ \ - if (GDKupgradevarheap((b)->HT, _d, 0) == GDK_FAIL) \ + if (GDKupgradevarheap((b)->HT, _d, 0, (b)->batRestricted == BAT_READ) == GDK_FAIL) \ goto bunins_failed; \ } \ _ptr = (p); \ @@ -1297,7 +1297,7 @@ gdk_export bte ATOMelmshift(int sz); #define bunfastins_check(b, p, h, t) bunfastins(b, h, t) -gdk_export int GDKupgradevarheap(COLrec *c, var_t v, int copyall); +gdk_export int GDKupgradevarheap(COLrec *c, var_t v, int copyall, int mayshare); gdk_export BAT *BUNfastins(BAT *b, const void *left, const void *right); gdk_export BAT *BUNins(BAT *b, const void *left, const void *right, bit force); gdk_export BAT *BUNappend(BAT *b, const void *right, bit force); @@ -1590,12 +1590,6 @@ gdk_export gdk_return BATgroup(BAT **gro * for the expected usage pattern of the memory in a heap. */ -gdk_export int GDK_mem_pagebits; /* page size for non-linear mmaps */ - -#define REMAP_PAGE_BITS GDK_mem_pagebits -#define REMAP_PAGE_MAXBITS (REMAP_PAGE_BITS+3) -#define REMAP_PAGE_MAXSIZE ((size_t) 1 << REMAP_PAGE_MAXBITS) /* max page bytesize of unary BUN heap (8-byte atom) */ - /* Buffer management advice for heaps */ #define BUF_NORMAL 0 /* No further special treatment */ #define BUF_RANDOM 1 /* Expect random page references */ @@ -2189,7 +2183,6 @@ gdk_export BAT *BATimprints(BAT *b); /* we prefer to use vm_alloc routines on size > GDKmmap */ gdk_export void *GDKmmap(const char *path, int mode, size_t len); -gdk_export size_t GDK_mem_bigsize; /* size after which we use anonymous VM rather than malloc */ gdk_export size_t GDK_mem_maxsize; /* max allowed size of committed memory */ gdk_export size_t GDK_vm_maxsize; /* max allowed size of reserved vm */ gdk_export int GDK_vm_trim; /* allow trimming */ diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c --- a/gdk/gdk_atoms.c +++ b/gdk/gdk_atoms.c @@ -942,7 +942,7 @@ strHeap(Heap *d, size_t cap) cap = MAX(cap, BATTINY); size = GDK_STRHASHTABLE * sizeof(stridx_t) + MIN(GDK_ELIMLIMIT, cap * GDK_VARALIGN); - if (HEAPalloc(d, size, 1) >= 0) { + if (HEAPalloc(d, size, 1) == 0) { d->free = GDK_STRHASHTABLE * sizeof(stridx_t); memset(d->base, 0, d->free); d->hashash = 1; /* new string heaps get the hash value (and length) stored */ @@ -1119,7 +1119,7 @@ strPut(Heap *h, var_t *dst, const char * return 0; } HEAPDEBUG fprintf(stderr, "#HEAPextend in strPut %s " SZFMT " " SZFMT "\n", h->filename, h->size, newsize); - if (HEAPextend(h, newsize) < 0) { + if (HEAPextend(h, newsize, TRUE) < 0) { return 0; } #ifndef NDEBUG diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c --- a/gdk/gdk_bat.c +++ b/gdk/gdk_bat.c @@ -364,7 +364,7 @@ BATattach(int tt, const char *heapfile) } bn->batRestricted = BAT_READ; bn->T->heap.size = (size_t) st.st_size; - bn->T->heap.newstorage = bn->T->heap.storage = (bn->T->heap.size < REMAP_PAGE_MAXSIZE) ? STORE_MEM : STORE_MMAP; + bn->T->heap.newstorage = bn->T->heap.storage = (bn->T->heap.size < GDK_mmap_minsize) ? STORE_MEM : STORE_MMAP; if (HEAPload(&bn->T->heap, BBP_physical(bn->batCacheid), "tail", TRUE) < 0) { HEAPfree(&bn->T->heap); GDKfree(bs); @@ -479,12 +479,14 @@ BATextend(BAT *b, BUN newcap) hheap_size *= Hsize(b); if (b->H->heap.base && GDKdebug & HEAPMASK) fprintf(stderr, "#HEAPextend in BATextend %s " SZFMT " " SZFMT "\n", b->H->heap.filename, b->H->heap.size, hheap_size); - if (b->H->heap.base && HEAPextend(&b->H->heap, hheap_size) < 0) + if (b->H->heap.base && + HEAPextend(&b->H->heap, hheap_size, b->batRestricted == BAT_READ) < 0) return NULL; theap_size *= Tsize(b); if (b->T->heap.base && GDKdebug & HEAPMASK) fprintf(stderr, "#HEAPextend in BATextend %s " SZFMT " " SZFMT "\n", b->T->heap.filename, b->T->heap.size, theap_size); - if (b->T->heap.base && HEAPextend(&b->T->heap, theap_size) < 0) + if (b->T->heap.base && + HEAPextend(&b->T->heap, theap_size, b->batRestricted == BAT_READ) < 0) return NULL; HASHdestroy(b); IMPSdestroy(b); @@ -837,13 +839,13 @@ BATcopy(BAT *b, int ht, int tt, int writ if (bn->hvarsized && bn->htype) { bn->H->shift = b->H->shift; bn->H->width = b->H->width; - if (HEAPextend(&bn->H->heap, BATcapacity(bn) << bn->H->shift) < 0) + if (HEAPextend(&bn->H->heap, BATcapacity(bn) << bn->H->shift, TRUE) < 0) goto bunins_failed; } if (bn->tvarsized && bn->ttype) { bn->T->shift = b->T->shift; bn->T->width = b->T->width; - if (HEAPextend(&bn->T->heap, BATcapacity(bn) << bn->T->shift) < 0) + if (HEAPextend(&bn->T->heap, BATcapacity(bn) << bn->T->shift, TRUE) < 0) goto bunins_failed; } diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -63,7 +63,7 @@ * using BATins (BATdel). */ static BAT * -insert_string_bat(BAT *b, BAT *n, int append) +insert_string_bat(BAT *b, BAT *n, int append, int force) { BATiter ni; /* iterator */ int tt; /* tail type */ @@ -110,7 +110,9 @@ insert_string_bat(BAT *b, BAT *n, int ap /* make sure we get alignment right */ toff = (toff + GDK_VARALIGN - 1) & ~(GDK_VARALIGN - 1); assert(((toff >> GDK_VARSHIFT) << GDK_VARSHIFT) == toff); - if (HEAPextend(b->T->vheap, toff + n->T->vheap->size) < 0) { + /* if in "force" mode, the heap may be shared when + * memory mapped */ + if (HEAPextend(b->T->vheap, toff + n->T->vheap->size, force) < 0) { toff = ~(size_t) 0; goto bunins_failed; } @@ -121,7 +123,7 @@ insert_string_bat(BAT *b, BAT *n, int ap if (b->T->width < SIZEOF_VAR_T && ((size_t) 1 << 8 * b->T->width) < (b->T->width <= 2 ? (b->T->vheap->size >> GDK_VARSHIFT) - GDK_VAROFFSET : (b->T->vheap->size >> GDK_VARSHIFT))) { /* offsets aren't going to fit */ - if (GDKupgradevarheap(b->T, (var_t) (b->T->vheap->size >> GDK_VARSHIFT), 0) == GDK_FAIL) { + if (GDKupgradevarheap(b->T, (var_t) (b->T->vheap->size >> GDK_VARSHIFT), 0, force) == GDK_FAIL) { toff = ~(size_t) 0; goto bunins_failed; } @@ -398,7 +400,7 @@ BATins(BAT *b, BAT *n, bit force) !GDK_ELIMDOUBLES(n->T->vheap) && b->T->vheap->hashash == n->T->vheap->hashash && VIEWtparent(n) == 0)) { - b = insert_string_bat(b, n, 0); + b = insert_string_bat(b, n, 0, force); } else if (b->htype == TYPE_void) { if (!ATOMvarsized(b->ttype) && BATatoms[b->ttype].atomFix == NULL && @@ -600,7 +602,7 @@ BATappend(BAT *b, BAT *n, bit force) !GDK_ELIMDOUBLES(n->T->vheap) && b->T->vheap->hashash == n->T->vheap->hashash && VIEWtparent(n) == 0) { - b = insert_string_bat(b, n, 1); + b = insert_string_bat(b, n, 1, force); if (b == NULL) return NULL; } else if (b->htype == TYPE_void) { diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c --- a/gdk/gdk_heap.c +++ b/gdk/gdk_heap.c @@ -60,19 +60,16 @@ #include "gdk_private.h" static void * -HEAPcreatefile(size_t *maxsz, char *fn, storage_t mode) +HEAPcreatefile(size_t *maxsz, const char *fn) { - size_t size = *maxsz; void *base = NULL; int fd; - size = (*maxsz + (size_t) 0xFFFF) & ~ (size_t) 0xFFFF; /* round up to 64k */ + /* round up to mulitple of GDK_mmap_pagesize */ fd = GDKfdlocate(fn, "wb", NULL); if (fd >= 0) { close(fd); - base = GDKload(fn, NULL, size, size, mode); - if (base) - *maxsz = size; + base = GDKload(fn, NULL, *maxsz, maxsz, STORE_MMAP); } return base; } @@ -118,18 +115,6 @@ HEAPalloc(Heap *h, size_t nitems, size_t if (itemsize && nitems > (h->size / itemsize)) return -1; - if (h->filename) { - GDKfilepath(nme, BATDIR, h->filename, NULL); - /* if we're going to use mmap anyway (size >= - * GDK_mem_bigsize -- see GDKmallocmax), and the file - * we want to use already exists and is large enough - * for the size we want, force non-anonymous mmap */ - if (h->size >= GDK_mem_bigsize && - stat(nme, &st) == 0 && st.st_size >= (off_t) h->size) { - minsize = GDK_mem_bigsize; /* force mmap */ - } - } - if (h->filename == NULL || h->size < minsize) { _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list