Changeset: c9ad3649c580 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/c9ad3649c580
Modified Files:
gdk/gdk_heap.c
Branch: default
Log Message:
Dynamically lower boundary when we start memory mapping heaps.
If a to-be-allocated heap is larger than 1/64th of the available memory
(total memory minus what we've already allocated), then use memory map.
diffs (67 lines):
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -152,9 +152,11 @@ HEAPalloc(Heap *h, size_t nitems, size_t
h->free = 0;
h->cleanhash = false;
+ size_t allocated;
if (GDKinmemory(h->farmid) ||
- (GDKmem_cursize() + h->size < GDK_mem_maxsize &&
- h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent :
GDK_mmap_minsize_transient))) {
+ ((allocated = GDKmem_cursize()) + h->size < GDK_mem_maxsize &&
+ h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent :
GDK_mmap_minsize_transient) &&
+ h->size < ((GDK_mem_maxsize - allocated) >> 6))) {
h->storage = STORE_MEM;
h->base = GDKmalloc(h->size);
TRC_DEBUG(HEAP, "%s %zu %p\n", h->filename, h->size, h->base);
@@ -162,6 +164,8 @@ HEAPalloc(Heap *h, size_t nitems, size_t
if (!GDKinmemory(h->farmid) && h->base == NULL) {
char *nme;
+ if (h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent :
GDK_mmap_minsize_transient))
+ fprintf(stderr, "#%s: emergency mmap %11zu %11zu
%11zu\n", __func__, h->size, (size_t) GDKmem_cursize(), (size_t)
GDKvm_cursize());
nme = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
if (nme == NULL)
return GDK_FAIL;
@@ -244,8 +248,12 @@ HEAPextend(Heap *h, size_t size, bool ma
/* extend a malloced heap, possibly switching over to
* file-mapped storage */
Heap bak = *h;
- bool exceeds_swap = size + GDKmem_cursize() >= GDK_mem_maxsize;
- bool must_mmap = !GDKinmemory(h->farmid) && (exceeds_swap ||
h->newstorage != STORE_MEM || size >= (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient));
+ size_t allocated;
+ bool must_mmap = (!GDKinmemory(h->farmid) &&
+ (h->newstorage != STORE_MEM ||
+ (allocated = GDKmem_cursize()) + size >=
GDK_mem_maxsize ||
+ size >= (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient) ||
+ size >= ((GDK_mem_maxsize - allocated) >>
6)));
h->size = size;
@@ -266,6 +274,8 @@ HEAPextend(Heap *h, size_t size, bool ma
/* too big: convert it to a disk-based temporary heap */
bool existing = false;
+ if (size < (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient))
+ fprintf(stderr, "#%s: emergency mmap %11zu
%11zu %11zu\n", __func__, h->size, (size_t) GDKmem_cursize(), (size_t)
GDKvm_cursize());
assert(h->storage == STORE_MEM);
assert(ext != NULL);
/* if the heap file already exists, we want to switch
@@ -750,8 +760,14 @@ HEAPload_intern(Heap *h, const char *nme
char *srcpath, *dstpath, *tmp;
int t0;
- if (h->storage == STORE_INVALID || h->newstorage == STORE_INVALID)
- h->storage = h->newstorage = h->size < (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient) ? STORE_MEM :
STORE_MMAP;
+ if (h->storage == STORE_INVALID || h->newstorage == STORE_INVALID) {
+ size_t allocated;
+ h->storage = h->newstorage = h->size < (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient) &&
+ (allocated = GDKmem_cursize()) < GDK_mem_maxsize &&
+ h->size < ((GDK_mem_maxsize - allocated) >> 6) ?
STORE_MEM : STORE_MMAP;
+ if (h->storage == STORE_MMAP && h->size < (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient))
+ fprintf(stderr, "#%s: emergency mmap %11zu %11zu\n",
__func__, h->size, (size_t) GDKmem_cursize());
+ }
minsize = (h->size + GDK_mmap_pagesize - 1) & ~(GDK_mmap_pagesize - 1);
if (h->storage != STORE_MEM && minsize != h->size)
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
