sorry, I was helping Stefan de Konink, but he had a read-only version of the MonetDB sources.
> -----Original Message----- > From: Stefan Manegold [mailto:[email protected]] > Sent: Friday, April 03, 2009 1:18 PM > To: [email protected]; Peter Boncz > Subject: Re: [Monetdb-checkins] MonetDB/src/gdk gdk.mx, , 1.278, 1.279 > gdk_bat.mx, , 1.213, 1.214 gdk_posix.mx, , 1.168, 1.169 gdk_relop.mx, , > 1.161, 1.162 gdk_utils.mx, , 1.241, 1.242 > > Peter, > > these changes lead to > > $ Mserver > # MonetDB Server v4.29.0 > # based on GDK v1.29.0 > # Copyright (c) 1993-July 2008, CWI. All rights reserved. > # Copyright (c) August 2008-2009, MonetDB B.V.. All rights reserved. > # Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs; dynamically > linked. > # Visit http://monetdb.cwi.nl/ for further information. > !ERROR: CMDfind: value not found. > !ERROR: CMDfind: operation failed. > !ERROR: interpret_params: int(param 1): evaluation error. > MonetDB> > > In fact, Sjoerd fixed this yesterday in > > =================================================================== > 2009/04/02 - sjoerd: MonetDB/src/gdk/gdk_utils.mx,1.241 > Reintroduced monet_pid, gdk_mem_pagebits, and gdk_mem_bigsize into the > MonetDB environment, but not through advertizing them as options. > This should fix bug 2726337. > =================================================================== > > Your checkin apparently removed this fix, again. > > Why? > > Stefan > > > ps: page 14 of > http://homepages.cwi.nl/~manegold/Presentations/MonetDBdevTTT.pdf > says > "Do never copy/overwrite files checked-out from CVS!" > > > On Fri, Apr 03, 2009 at 09:14:55AM +0000, Peter Boncz wrote: > > Update of /cvsroot/monetdb/MonetDB/src/gdk > > In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv31767/gdk > > > > Modified Files: > > gdk.mx gdk_bat.mx gdk_posix.mx gdk_relop.mx gdk_utils.mx > > Log Message: > > various tweaks to the memory mangement > > > > - tune the GDK_mmap_minsize to vary between full RAM size and 128MB > > (lowest when memory pressure is extreme) > > > > - in fetchjoin, when the optimization of string heap copying is applied > > (instead of creating a new string heap and inserting into it), try > > to share the heaps in virtual memory (rather than copying it) > > This is possible with the logical view mechanism. > > > > now fetchjoin, batcopy and remap use this VM heap sharing. > > > > - disable vmalloc on Linux -- well actually on platforms > > that do not have posix_fadvise. A bit of a hack. > > > > > > > > U gdk.mx > > Index: gdk.mx > > =================================================================== > > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk.mx,v > > retrieving revision 1.278 > > retrieving revision 1.279 > > diff -u -d -r1.278 -r1.279 > > --- gdk.mx 13 Mar 2009 17:58:12 -0000 1.278 > > +++ gdk.mx 3 Apr 2009 09:14:51 -0000 1.279 > > @@ -1048,6 +1048,7 @@ > > gdk_export int HEAPextend(Heap *h, size_t size); > > gdk_export int HEAPload(Heap *h, str nme, str ext, int trunc); > > gdk_export int HEAPsave(Heap *h, str nme, str ext); > > +gdk_export int HEAPshare(Heap *dst, Heap *src, int copy_on_write); > > gdk_export int HEAPcopy(Heap *dst, Heap *src); > > gdk_export int HEAPwarm(Heap *h); > > gdk_export size_t HEAPsubcopy(Heap *dst, Heap *src, size_t size, size_t > offset); > > > > U gdk_posix.mx > > Index: gdk_posix.mx > > =================================================================== > > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v > > retrieving revision 1.168 > > retrieving revision 1.169 > > diff -u -d -r1.168 -r1.169 > > --- gdk_posix.mx 1 Apr 2009 12:37:27 -0000 1.168 > > +++ gdk_posix.mx 3 Apr 2009 09:14:52 -0000 1.169 > > @@ -824,6 +824,13 @@ > > } > > } > > } > > + /* could not save anything -- reset the save_tile to the start > to try all heaps again */ > > + for (i = MT_mmap_next(MT_mmap_cur); i != MT_mmap_cur; i = > MT_mmap_next(i)) { > > + if (MT_mmap_tab[i].fd >= 0 && > > + MT_mmap_tab[i].writable && > > + (MT_mmap_tab[i].pincnt == 0 || MT_mmap_tab[i].len > > target)) > > + MT_mmap_tab[i].save_tile = 1; > > + } > > MT_mmap_cur = i; > > } > > (void) pthread_mutex_unlock(&MT_mmap_lock); > > > > U gdk_relop.mx > > Index: gdk_relop.mx > > =================================================================== > > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_relop.mx,v > > retrieving revision 1.161 > > retrieving revision 1.162 > > diff -u -d -r1.161 -r1.162 > > --- gdk_relop.mx 30 Mar 2009 07:30:13 -0000 1.161 > > +++ gdk_relop.mx 3 Apr 2009 09:14:52 -0000 1.162 > > @@ -972,7 +972,13 @@ > > bn->theap->filename = (str) > > GDKmalloc(strlen(nme) + > 12); > > GDKfilepath(bn->theap->filename, NULL, nme, > "theap"); > > } > > - if (HEAPcopy(bn->theap, r->theap) < 0) { > > + if (r->batRestricted == BAT_READ && HEAPshare(bn->theap, > r->theap, 0) == 0) { > > + int parent = r->T->parentid; > > + if (!parent) parent = -r->batCacheid; > > + bn->P->lview = TRUE; > > + bn->T->parentid = parent; > > + BBPshare(parent); > > + } else if (HEAPcopy(bn->theap, r->theap) < 0) { > > BBPreclaim(bn); > > goto ready; > > } > > > > U gdk_utils.mx > > Index: gdk_utils.mx > > =================================================================== > > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_utils.mx,v > > retrieving revision 1.241 > > retrieving revision 1.242 > > diff -u -d -r1.241 -r1.242 > > --- gdk_utils.mx 2 Apr 2009 08:44:03 -0000 1.241 > > +++ gdk_utils.mx 3 Apr 2009 09:14:53 -0000 1.242 > > @@ -1053,6 +1053,7 @@ > > #endif > > } > > size = (size + 7) & ~7; /* round up to a multiple of eight */ > > +#ifndef HAVE_POSIX_FADVISE > > if (size > GDK_mem_bigsize) { > > size_t newsize = size + sizeof(size_t) + sizeof(size_t); > > size_t newmax = MAX(*maxsize, newsize); > > @@ -1067,6 +1068,7 @@ > > *maxsize = newmax - (sizeof(size_t) + sizeof(size_t)); > > return (void *) s; > > } > > +#endif > > CHKMEM(size, 0); > > GDKmalloc_prefixsize(s, size); > > if (s == NULL) { > > @@ -1203,6 +1205,7 @@ > > } > > } > > #ifndef GDK_MEM_MISALIGN > > +#ifndef HAVE_POSIX_FADVISE > > else if (size <= GDK_mem_bigsize) { > > void *oldblk = blk; > > size_t newsize = size + 8; > > @@ -1236,6 +1239,7 @@ > > return blk; > > } > > } > > +#endif > > /* alloc© due to failed realloc (may be very big heap that needs > vm) */ > > #else > > /* alloc© because we cannot trust realloc due to misalignment */ > > @@ -1446,7 +1450,9 @@ > > } > > } > > > > -#ifndef WIN32 > > +#ifdef HAVE_POSIX_FADVISE > > +static str highload_name[] = { "idle", "low", "medium", "high", > "extreme" }; > > + > > static int membits = 0; > > static void > > GDKvmtrim(void *limit) > > @@ -1458,21 +1464,25 @@ > > /* compute rounded down log2 of memsize */ > > size_t s = GDK_mem_maxsize_max; > > while((s >>= 1) > 0) membits++; > > + membits = (MAX(0,membits-27))/4; > > } > > - > > do { > > stream *fp = NULL; > > - > > MT_sleep_ms(highload ? 500 : 5000); > > MEMDEBUG { > > fp = GDKout; > > - THRprintf(fp, "#GDKvmtrim(" LLFMT ")\n", (long long) > *(size_t *) limit); > > + THRprintf(fp, "#GDKvmtrim(load=%s, rsstarget=" SZFMT ", > GDK_mmap_minsize=" SZFMT ")\n", > > + highload_name[highload], *(size_t *) limit, > GDK_mmap_minsize); > > } > > highload = MT_mmap_trim(*(size_t *) limit, fp); > > if (highload) { /* memcrunch distress varies from 1 to 4 */ > > /* rss is above 0.75mem_maxsize!! start using mmap-on- > tempfile */ > > if (!bak_mmap_minsize) bak_mmap_minsize = > GDK_mmap_minsize; > > - GDK_mmap_minsize = ((size_t) 1) << (membits- > (highload+highload)); /* HEAPalloc/extend > [256-16]MB uses it */ > > + /* minsize = memsize iff highload = 0 > > + * = ..in between.. iff highload = 1-3 > > + * = 128MB iff highload = 4 > > + */ > > + GDK_mmap_minsize = ((size_t) 1) << (27 + membits*(4- > highload)); > > } else if (bak_mmap_minsize) { > > GDK_mmap_minsize = bak_mmap_minsize; /* revert to > > default > setting */ > > bak_mmap_minsize = 0; > > @@ -1490,7 +1500,6 @@ > > char *p; > > opt *n; > > int i, j, nlen = 0; > > - char buf[16]; > > > > /* some sanity checks (should also find if symbols are not defined) > */ > > assert(sizeof(char) == SIZEOF_CHAR); > > @@ -1606,20 +1615,8 @@ > > } > > if (GDKgetenv_isyes("gdk_embedded") || GDKgetenv_isyes("embedded")) > > GDKembedded = 1; > > - if (GDKgetenv("gdk_mem_pagebits") == NULL) { > > - snprintf(buf, sizeof(buf), "%d", GDK_mem_pagebits); > > - GDKsetenv("gdk_mem_pagebits", buf); > > - } > > - if (GDKgetenv("gdk_mem_bigsize") == NULL) { > > - snprintf(buf, sizeof(buf), SZFMT, GDK_mem_bigsize); > > - GDKsetenv("gdk_mem_bigsize", buf); > > - } > > - if (GDKgetenv("monet_pid") == NULL) { > > - snprintf(buf, sizeof(buf), "%d", (int) getpid()); > > - GDKsetenv("monet_pid", buf); > > - } > > > > -#ifndef WIN32 > > +#ifdef HAVE_POSIX_FADVISE > > if (!GDKembedded && GDK_vm_trim) { > > MT_Id t; > > > > > > U gdk_bat.mx > > Index: gdk_bat.mx > > =================================================================== > > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_bat.mx,v > > retrieving revision 1.213 > > retrieving revision 1.214 > > diff -u -d -r1.213 -r1.214 > > --- gdk_bat.mx 30 Mar 2009 07:30:07 -0000 1.213 > > +++ gdk_bat.mx 3 Apr 2009 09:14:52 -0000 1.214 > > @@ -670,11 +670,9 @@ > > as rcopy(). > > @{ > > @c > > -static int > > -heapcopy(Heap *dst, Heap *src, int *remap) > > -{ > > - if (*remap && src->storage == STORE_MMAP) { > > - /* use copy-on-write mmap for isolatable copy */ > > +int HEAPshare(Heap *dst, Heap *src, int copy_on_write) { > > + /* use copy-on-write mmap for isolatable copy */ > > + if (src->storage == STORE_MMAP) { > > *dst = *src; > > dst->base = (char *) -1; > > dst->filename = GDKstrdup(src->filename); > > @@ -682,16 +680,25 @@ > > char path[PATHLENGTH]; > > > > GDKfilepath(path, BATDIR, dst->filename, NULL); > > - dst->base = GDKmmap(path, MMAP_READ | MMAP_SEQUENTIAL | > MMAP_COPY, 0, dst->maxsize); > > + dst->base = GDKmmap(path, MMAP_READ | MMAP_SEQUENTIAL | > (copy_on_write?MMAP_COPY:0), 0, dst->maxsize); > > if (dst->base != (char *) -1) { > > dst->copied = 1; > > - dst->newstorage = dst->storage = STORE_PRIV; > > + dst->newstorage = dst->storage = > copy_on_write?STORE_PRIV:STORE_MMAP; > > return 0; > > } > > GDKfree(dst->filename); > > dst->filename = NULL; > > } > > } > > + return -1; > > +} > > + > > +static int > > +heapcopy(Heap *dst, Heap *src, int *remap) > > +{ > > + if (*remap && HEAPshare(dst, src, *remap) == 0) { > > + return 0; /* use a shared memory heap */ > > + } > > *remap = 0; > > return HEAPcopy(dst, src); > > } > > @@ -2766,7 +2773,7 @@ > > int b0, b1, b2 = 0, b3 = 0; > > > > /* copy-on-write isolated bats that change mode should be made > independent */ > > - if (b->P->lview && batunshare(b, "BATsetaccess") < 0) > > + if (b->P->lview && newmode != BAT_READ && batunshare(b, > "BATsetaccess") < 0) > > return NULL; > > > > if (b->batSharecnt && newmode != BAT_READ) { > > > > > > ------------------------------------------------------------------------- > ----- > > _______________________________________________ > > Monetdb-checkins mailing list > > [email protected] > > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins > > -- > | Dr. Stefan Manegold | mailto:[email protected] | > | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | > | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | > | The Netherlands | Fax : +31 (20) 592-4312 | ------------------------------------------------------------------------------ _______________________________________________ Monetdb-developers mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/monetdb-developers
