On Sat, Oct 25, 2008 at 07:30:50AM +0000, Martin Kersten wrote:
> Update of /cvsroot/monetdb/MonetDB/src/gdk
> In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv30373
>
> Modified Files:
> gdk_posix.mx
> Log Message:
> The memory map table maintained for the vmtrim
> should be aligned to the number of mapped files.
> In more recent applications, such as the SkyServer
> this may be around 2000 BATs easily.
> For the time being the limit of 100 is increased to 4096
>
> The remainder are additions of '#' in error messages
> to prepare for problems in the API.
just an idea (without background knowledge):
shouldn't we introduce a new symbol/chracter in the protocol to indicate
warnings and informative message?
'!' indicates errors (obviously never to be ignored)
'#' indicates (usually to be ignored) comments
'?' to be added to indicate warnings (and "informative messages)?
'?' to be added for ... ??
Stefan
>
>
>
> U gdk_posix.mx
> Index: gdk_posix.mx
> ===================================================================
> RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v
> retrieving revision 1.157
> retrieving revision 1.158
> diff -u -d -r1.157 -r1.158
> --- gdk_posix.mx 10 Oct 2008 08:50:35 -0000 1.157
> +++ gdk_posix.mx 25 Oct 2008 07:30:48 -0000 1.158
> @@ -389,7 +389,7 @@
> * hope the VM paging algorithm behaves better than Linux which just runs off
> * the cliff and if MonetDB does not prevent RSS from being too high, enters
> coma.
> *
> - * We will only eb able to sensibly test this on Windows64. On Windows32,
> mmap sizes
> + * We will only be able to sensibly test this on Windows64. On Windows32,
> mmap sizes
> * do not significantly exceed RAM sizes so MonetDB swapping actually will
> not happen
> * (of course, you've got this nasty problem of VM fragemntation and failing
> mmaps instead).
> *
> @@ -488,6 +488,10 @@
> * to the cursors (i.e. pointers to b->batBuns->free or the cursors
> * in radix-cluster), which an enhanced version of this thread might
> * take into account.
> + *
> + * [Kersten] The memory map table should be aligned to the number of
> + * mapped files. In more recent applications, such as the SkyServer
> + * this may be around 2000 BATs easily.
> */
>
> #ifdef HAVE_PTHREAD_H
> @@ -519,7 +523,7 @@
> #endif
>
> #define MT_MMAP_TILE (1<<27)
> -#define MT_MMAP_BUFSIZE 100
> +#define MT_MMAP_BUFSIZE 4096
> MT_mmap_t MT_mmap_tab[MT_MMAP_BUFSIZE];
> int MT_mmap_cur = -1, MT_mmap_first = -1, MT_mmap_free = 0;
>
> @@ -598,7 +602,7 @@
> if (MT_mmap_cur == -1)
> MT_mmap_cur = i;
> #ifdef MMAP_DEBUG
> - stream_printf(GDKerr, "MT_mmap_new: %s fd=%d\n", path, fd);
> + stream_printf(GDKerr, "#MT_mmap_new: %s fd=%d\n", path, fd);
> #endif
> strncpy(MT_mmap_tab[i].path, path, 128);
> MT_mmap_tab[i].base = base;
> @@ -642,13 +646,13 @@
> /* tell the OS quite clearly that you want to
> drop this */
> ret = posix_fadvise(MT_mmap_tab[victim].fd,
> 0LL, MT_mmap_tab[victim].len, POSIX_FADV_DONTNEED);
> #ifdef MMAP_DEBUG
> - stream_printf(GDKerr, "MT_mmap_del:
> posix_fadvise(%s,fd=%d,%uMB,POSIX_FADV_DONTNEED) = %d\n",
> MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, (unsigned int)
> (MT_mmap_tab[victim].len >> 20), ret);
> + stream_printf(GDKerr, "#MT_mmap_del:
> posix_fadvise(%s,fd=%d,%uMB,POSIX_FADV_DONTNEED) = %d\n",
> MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, (unsigned int)
> (MT_mmap_tab[victim].len >> 20), ret);
> #endif
> }
> #endif
> ret = close(MT_mmap_tab[victim].fd);
> #ifdef MMAP_DEBUG
> - stream_printf(GDKerr, "MT_mmap_del: close(%s fd=%d) =
> %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, ret);
> + stream_printf(GDKerr, "#MT_mmap_del: close(%s fd=%d) =
> %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, ret);
> #endif
> MT_mmap_tab[victim].next = MT_mmap_free;
> MT_mmap_empty(victim);
> @@ -674,7 +678,7 @@
> if (MT_mmap_tab[i].fd >= 0) {
> ret = posix_fadvise(MT_mmap_tab[i].fd, 0, len,
> advice);
> #ifdef MMAP_DEBUG
> - stream_printf(GDKerr, "MT_fadvise:
> posix_fadvise(%s,fd=%d,%uMB,%d) = %d\n", MT_mmap_tab[i].path,
> MT_mmap_tab[i].fd, (unsigned int) (len >> 20), advice, ret);
> + stream_printf(GDKerr, "#MT_fadvise:
> posix_fadvise(%s,fd=%d,%uMB,%d) = %d\n", MT_mmap_tab[i].path,
> MT_mmap_tab[i].fd, (unsigned int) (len >> 20), advice, ret);
> #endif
> }
> }
> @@ -697,7 +701,7 @@
> int ret = posix_madvise(MT_mmap_tab[i].base + off, MT_MMAP_TILE,
> POSIX_MADV_DONTNEED);
>
> if (err) {
> - stream_printf(err, "MT_mmap_unload_tile:
> posix_madvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n",
> + stream_printf(err, "#MT_mmap_unload_tile:
> posix_madvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n",
> MT_mmap_tab[i].path, (unsigned int) (off >> 20),
> (unsigned int) (MT_MMAP_TILE >> 20),
> MT_mmap_tab[i].fd, ret);
> }
> @@ -706,7 +710,7 @@
> /* tell the OS quite clearly that you want to drop this */
> ret = posix_fadvise(MT_mmap_tab[i].fd, off, MT_MMAP_TILE,
> POSIX_FADV_DONTNEED);
> if (err) {
> - stream_printf(err, "MT_mmap_unload_tile:
> posix_fadvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n",
> + stream_printf(err, "#MT_mmap_unload_tile:
> posix_fadvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n",
> MT_mmap_tab[i].path, (unsigned int) (off
> >> 20),
> (unsigned int) (MT_MMAP_TILE >> 20),
> MT_mmap_tab[i].fd, ret);
> }
> @@ -721,7 +725,7 @@
>
> /* save to disk an 128MB tile, and observe how long this takes */
> if (err) {
> - stream_printf(err, "MT_mmap_save_tile:
> msync(%s,off=%uM,%u,SYNC)...\n",
> + stream_printf(err, "#MT_mmap_save_tile:
> msync(%s,off=%uM,%u,SYNC)...\n",
> MT_mmap_tab[i].path, (unsigned int) (tile >> 20),
> (unsigned int) (MT_MMAP_TILE >> 20));
> }
> @@ -729,7 +733,7 @@
> ret = MT_msync(MT_mmap_tab[i].base, tile, MT_MMAP_TILE, MMAP_SYNC);
> t = GDKms() - t;
> if (err) {
> - stream_printf(err, "MT_mmap_save_tile:
> msync(%s,tile=%uM,%uM,SYNC) = %d (%dms)\n",
> + stream_printf(err, "#MT_mmap_save_tile:
> msync(%s,tile=%uM,%uM,SYNC) = %d (%dms)\n",
> MT_mmap_tab[i].path, (unsigned int) (tile >> 20),
> (unsigned int) (MT_MMAP_TILE >> 20), ret, t);
> }
> @@ -768,7 +772,7 @@
>
> (void) pthread_mutex_lock(&MT_mmap_lock);
> if (err) {
> - stream_printf(err, "MT_mmap_trim(%u MB): rss = %u MB\n",
> (unsigned int) ((target) >> 20), (unsigned int) (rss >> 20));
> + stream_printf(err, "#MT_mmap_trim(%u MB): rss = %u MB\n",
> (unsigned int) ((target) >> 20), (unsigned int) (rss >> 20));
> }
> if (rss > target) {
> /* try to selectively unload pages from the writable regions */
> @@ -1085,7 +1089,7 @@
> int ret = munmap(p, len);
>
> #ifdef MMAP_DEBUG
> - stream_printf(GDKerr, "munmap(" LLFMT "," LLFMT ",%d) = %d\n", (long
> long) p, (long long) len, ret);
> + stream_printf(GDKerr, "#munmap(" LLFMT "," LLFMT ",%d) = %d\n", (long
> long) p, (long long) len, ret);
> #endif
> MT_mmap_del(p, len);
> return ret;
> @@ -1097,7 +1101,7 @@
> int ret = msync(((char*) p)+off, len, (mode & MMAP_SYNC) ? MS_SYNC :
> ((mode & MMAP_ASYNC) ? MS_ASYNC : MS_INVALIDATE));
>
> #ifdef MMAP_DEBUG
> - stream_printf(GDKerr, "msync(" LLFMT "," LLFMT ",%s) = %d\n", (long
> long) p, (long long) len, (mode & MMAP_SYNC) ? "MS_SYNC" : ((mode &
> MMAP_ASYNC) ? "MS_ASYNC" : "MS_INVALIDATE"), ret);
> + stream_printf(GDKerr, "#msync(" LLFMT "," LLFMT ",%s) = %d\n", (long
> long) p, (long long) len, (mode & MMAP_SYNC) ? "MS_SYNC" : ((mode &
> MMAP_ASYNC) ? "MS_ASYNC" : "MS_INVALIDATE"), ret);
> #endif
> if (ret < 0)
> return errno;
> @@ -1110,7 +1114,7 @@
> int ret = posix_madvise(p, len, advise);
>
> #ifdef MMAP_DEBUG
> - stream_printf(GDKerr, "posix_madvise(" PTRFMT "," SZFMT ",%d) = %d\n",
> PTRFMTCAST p, len, advise, ret);
> + stream_printf(GDKerr, "#posix_madvise(" PTRFMT "," SZFMT ",%d) = %d\n",
> PTRFMTCAST p, len, advise, ret);
> #endif
> if (MT_fadvise(p, len, advise))
> ret = -1;
> @@ -1468,7 +1472,7 @@
> }
> if (heapstatus == _HEAPBADPTR || heapstatus == _HEAPBADBEGIN ||
> heapstatus == _HEAPBADNODE) {
>
> - stream_printf(GDKerr, "mallinfo(): heap is corrupt.");
> + stream_printf(GDKerr, "#mallinfo(): heap is corrupt.");
> }
> _heapmin();
> return _ret;
> @@ -1604,7 +1608,7 @@
> return;
>
> if (!FindClose(dir->find_file_handle))
> - stream_printf(GDKerr, "rewinddir(): FindClose() failed\n");
> + stream_printf(GDKerr, "#rewinddir(): FindClose() failed\n");
>
> mask = malloc(strlen(dir->dir_name) + 3);
> sprintf(mask, "%s\\*", dir->dir_name);
> @@ -2116,7 +2120,7 @@
> p = (void *) VirtualAlloc(a, size, MEM_COMMIT | mode, PAGE_READWRITE);
> (void) pthread_mutex_unlock(&MT_mmap_lock);
> if (p == NULL) {
> - stream_printf(GDKerr, "VirtualAlloc(" PTRFMT "," SZFMT
> ",MEM_COMMIT,PAGE_READWRITE): failed\n", PTRFMTCAST a, size);
> + stream_printf(GDKerr, "#VirtualAlloc(" PTRFMT "," SZFMT
> ",MEM_COMMIT,PAGE_READWRITE): failed\n", PTRFMTCAST a, size);
> }
> return p;
> }
> @@ -2126,9 +2130,9 @@
> MT_vmfree(void *p, size_t size)
> {
> if (VirtualFree(p, size, MEM_DECOMMIT) == 0)
> - stream_printf(GDKerr, "VirtualFree(" PTRFMT "," SZFMT
> ",MEM_DECOMMIT): failed\n", PTRFMTCAST p, size);
> + stream_printf(GDKerr, "#VirtualFree(" PTRFMT "," SZFMT
> ",MEM_DECOMMIT): failed\n", PTRFMTCAST p, size);
> if (VirtualFree(p, 0, MEM_RELEASE) == 0)
> - stream_printf(GDKerr, "VirtualFree(" PTRFMT ",0,MEM_RELEASE):
> failed\n", PTRFMTCAST p);
> + stream_printf(GDKerr, "#VirtualFree(" PTRFMT ",0,MEM_RELEASE):
> failed\n", PTRFMTCAST p);
> }
>
> void *
> @@ -2149,7 +2153,7 @@
> size_t ret = VirtualFree(p + newsize, oldsize - newsize,
> MEM_DECOMMIT);
>
> if (ret == 0)
> - stream_printf(GDKerr, "VirtualFree(" PTRFMT "," SSZFMT
> ",MEM_DECOMMIT): failed\n", PTRFMTCAST(p + newsize), (ssize_t) (oldsize -
> newsize));
> + stream_printf(GDKerr, "#VirtualFree(" PTRFMT "," SSZFMT
> ",MEM_DECOMMIT): failed\n", PTRFMTCAST(p + newsize), (ssize_t) (oldsize -
> newsize));
> } else if (oldsize < newsize) {
> (void) pthread_mutex_lock(&MT_mmap_lock);
> a = (char *) VirtualAlloc(p, newsize, MEM_COMMIT,
> PAGE_READWRITE);
>
>
> -------------------------------------------------------------------------
> This SF.Net email is sponsored by the Moblin Your Move Developer's challenge
> Build the coolest Linux based applications with Moblin SDK & win great prizes
> Grand prize is a trip for two to an Open Source event anywhere in the world
> http://moblin-contest.org/redirect.php?banner_id=100&url=/
> _______________________________________________
> Monetdb-checkins mailing list
> [EMAIL PROTECTED]
> https://lists.sourceforge.net/lists/listinfo/monetdb-checkins
>
>
--
| Dr. Stefan Manegold | mailto:[EMAIL PROTECTED] |
| CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ |
| 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 |
| The Netherlands | Fax : +31 (20) 592-4312 |
-------------------------------------------------------------------------
This SF.Net email is sponsored by the Moblin Your Move Developer's challenge
Build the coolest Linux based applications with Moblin SDK & win great prizes
Grand prize is a trip for two to an Open Source event anywhere in the world
http://moblin-contest.org/redirect.php?banner_id=100&url=/
_______________________________________________
Monetdb-developers mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/monetdb-developers
