Changeset: c9ad3649c580 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/c9ad3649c580 Modified Files: gdk/gdk_heap.c Branch: default Log Message:
Dynamically lower boundary when we start memory mapping heaps. If a to-be-allocated heap is larger than 1/64th of the available memory (total memory minus what we've already allocated), then use memory map. diffs (67 lines): diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c --- a/gdk/gdk_heap.c +++ b/gdk/gdk_heap.c @@ -152,9 +152,11 @@ HEAPalloc(Heap *h, size_t nitems, size_t h->free = 0; h->cleanhash = false; + size_t allocated; if (GDKinmemory(h->farmid) || - (GDKmem_cursize() + h->size < GDK_mem_maxsize && - h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient))) { + ((allocated = GDKmem_cursize()) + h->size < GDK_mem_maxsize && + h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient) && + h->size < ((GDK_mem_maxsize - allocated) >> 6))) { h->storage = STORE_MEM; h->base = GDKmalloc(h->size); TRC_DEBUG(HEAP, "%s %zu %p\n", h->filename, h->size, h->base); @@ -162,6 +164,8 @@ HEAPalloc(Heap *h, size_t nitems, size_t if (!GDKinmemory(h->farmid) && h->base == NULL) { char *nme; + if (h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient)) + fprintf(stderr, "#%s: emergency mmap %11zu %11zu %11zu\n", __func__, h->size, (size_t) GDKmem_cursize(), (size_t) GDKvm_cursize()); nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL); if (nme == NULL) return GDK_FAIL; @@ -244,8 +248,12 @@ HEAPextend(Heap *h, size_t size, bool ma /* extend a malloced heap, possibly switching over to * file-mapped storage */ Heap bak = *h; - bool exceeds_swap = size + GDKmem_cursize() >= GDK_mem_maxsize; - bool must_mmap = !GDKinmemory(h->farmid) && (exceeds_swap || h->newstorage != STORE_MEM || size >= (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient)); + size_t allocated; + bool must_mmap = (!GDKinmemory(h->farmid) && + (h->newstorage != STORE_MEM || + (allocated = GDKmem_cursize()) + size >= GDK_mem_maxsize || + size >= (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient) || + size >= ((GDK_mem_maxsize - allocated) >> 6))); h->size = size; @@ -266,6 +274,8 @@ HEAPextend(Heap *h, size_t size, bool ma /* too big: convert it to a disk-based temporary heap */ bool existing = false; + if (size < (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient)) + fprintf(stderr, "#%s: emergency mmap %11zu %11zu %11zu\n", __func__, h->size, (size_t) GDKmem_cursize(), (size_t) GDKvm_cursize()); assert(h->storage == STORE_MEM); assert(ext != NULL); /* if the heap file already exists, we want to switch @@ -750,8 +760,14 @@ HEAPload_intern(Heap *h, const char *nme char *srcpath, *dstpath, *tmp; int t0; - if (h->storage == STORE_INVALID || h->newstorage == STORE_INVALID) - h->storage = h->newstorage = h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient) ? STORE_MEM : STORE_MMAP; + if (h->storage == STORE_INVALID || h->newstorage == STORE_INVALID) { + size_t allocated; + h->storage = h->newstorage = h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient) && + (allocated = GDKmem_cursize()) < GDK_mem_maxsize && + h->size < ((GDK_mem_maxsize - allocated) >> 6) ? STORE_MEM : STORE_MMAP; + if (h->storage == STORE_MMAP && h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient)) + fprintf(stderr, "#%s: emergency mmap %11zu %11zu\n", __func__, h->size, (size_t) GDKmem_cursize()); + } minsize = (h->size + GDK_mmap_pagesize - 1) & ~(GDK_mmap_pagesize - 1); if (h->storage != STORE_MEM && minsize != h->size) _______________________________________________ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org