Changeset: d1b0bad1529f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d1b0bad1529f Added Files: monetdb5/modules/mal/Tests/mosaic_prefix.stable.err monetdb5/modules/mal/Tests/mosaic_prefix.stable.out Modified Files: monetdb5/modules/mal/mosaic_prefix.c Branch: mosaic Log Message:
Prefix compression for lng type. diffs (truncated from 521 to 300 lines): diff --git a/monetdb5/modules/mal/Tests/mosaic_prefix.stable.err b/monetdb5/modules/mal/Tests/mosaic_prefix.stable.err new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/mosaic_prefix.stable.err @@ -0,0 +1,30 @@ +stderr of test 'mosaic_prefix` in directory 'monetdb5/modules/mal` itself: + + +# 12:42:06 > +# 12:42:06 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=32744" "--set" "mapi_usock=/var/tmp/mtest-6218/.s.monetdb.32744" "--set" "monet_prompt=" "--forcemito" "--set" "mal_listing=2" "--dbpath=/export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mal" "mosaic_prefix.mal" +# 12:42:06 > + +# builtin opt gdk_dbpath = /export/scratch1/mk/mosaic//Linux/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 50000 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 32744 +# cmdline opt mapi_usock = /var/tmp/mtest-6218/.s.monetdb.32744 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mal +# cmdline opt gdk_debug = 536870922 + +# 12:42:06 > +# 12:42:06 > "Done." +# 12:42:06 > + diff --git a/monetdb5/modules/mal/Tests/mosaic_prefix.stable.out b/monetdb5/modules/mal/Tests/mosaic_prefix.stable.out new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/mosaic_prefix.stable.out @@ -0,0 +1,118 @@ +stdout of test 'mosaic_prefix` in directory 'monetdb5/modules/mal` itself: + + +# 12:42:06 > +# 12:42:06 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=32744" "--set" "mapi_usock=/var/tmp/mtest-6218/.s.monetdb.32744" "--set" "monet_prompt=" "--forcemito" "--set" "mal_listing=2" "--dbpath=/export/scratch1/mk/mosaic//Linux/var/MonetDB/mTests_monetdb5_modules_mal" "mosaic_prefix.mal" +# 12:42:06 > + +# MonetDB 5 server v11.20.0 +# This is an unreleased version +# Serving database 'mTests_monetdb5_modules_mal', using 8 threads +# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 15.590 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://vienna.ins.cwi.nl:32744/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-6218/.s.monetdb.32744 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded +function user.main():void; + b := bat.new(:oid,:int); + bat.append(b,50); + bat.append(b,51); + bat.append(b,51); + bat.append(b,51); + bat.append(b,51); + bat.append(b,51); + bat.append(b,51); + bat.append(b,60); + bat.append(b,67); + bat.append(b,67); + bat.append(b,67); + bat.append(b,67); + bat.append(b,67); + bat.append(b,67); + bat.append(b,67); + bat.append(b,255); + bat.append(b,256); + bat.append(b,256); + bat.append(b,256); + bat.append(b,256); + bat.append(b,256); + bat.append(b,255); + bat.append(b,255); + bat.append(b,255); + bat.append(b,255); + bat.append(b,255); + io.print(b); + x := mosaic.compress(b,"prefix"); +#mosaic.dump(x); + z := mosaic.decompress(x); + io.print(z); +end main; +#--------------------------# +# h t # name +# void int # type +#--------------------------# +[ 0@0, 50 ] +[ 1@0, 51 ] +[ 2@0, 51 ] +[ 3@0, 51 ] +[ 4@0, 51 ] +[ 5@0, 51 ] +[ 6@0, 51 ] +[ 7@0, 60 ] +[ 8@0, 67 ] +[ 9@0, 67 ] +[ 10@0, 67 ] +[ 11@0, 67 ] +[ 12@0, 67 ] +[ 13@0, 67 ] +[ 14@0, 67 ] +[ 15@0, 255 ] +[ 16@0, 256 ] +[ 17@0, 256 ] +[ 18@0, 256 ] +[ 19@0, 256 ] +[ 20@0, 256 ] +[ 21@0, 255 ] +[ 22@0, 255 ] +[ 23@0, 255 ] +[ 24@0, 255 ] +[ 25@0, 255 ] +#--------------------------# +# h t # name +# void int # type +#--------------------------# +[ 0@0, 50 ] +[ 1@0, 51 ] +[ 2@0, 51 ] +[ 3@0, 51 ] +[ 4@0, 51 ] +[ 5@0, 51 ] +[ 6@0, 51 ] +[ 7@0, 60 ] +[ 8@0, 67 ] +[ 9@0, 67 ] +[ 10@0, 67 ] +[ 11@0, 67 ] +[ 12@0, 67 ] +[ 13@0, 67 ] +[ 14@0, 67 ] +[ 15@0, 255 ] +[ 16@0, 256 ] +[ 17@0, 256 ] +[ 18@0, 256 ] +[ 19@0, 256 ] +[ 20@0, 256 ] +[ 21@0, 255 ] +[ 22@0, 255 ] +[ 23@0, 255 ] +[ 24@0, 255 ] +[ 25@0, 255 ] + +# 12:42:06 > +# 12:42:06 > "Done." +# 12:42:06 > + diff --git a/monetdb5/modules/mal/mosaic_prefix.c b/monetdb5/modules/mal/mosaic_prefix.c --- a/monetdb5/modules/mal/mosaic_prefix.c +++ b/monetdb5/modules/mal/mosaic_prefix.c @@ -109,6 +109,16 @@ MOSadvance_prefix(Client cntxt, MOStask task->blk = (MosaicBlk) (((char*) dst) + wordaligned(bytes, int)); //mnstr_printf(cntxt->fdout,"advance mask width %d bytes %d %d \n",bits,bytes,(int)wordaligned(bytes,int)); } + break; + case 8: + { lng *dst = (lng*) (((char*) task->blk) + MosaicBlkSize); + lng mask = *dst++; + lng val = *dst++; + bits = val & (~mask); + bytes = (MOSgetCnt(task->blk) * bits)/8 + ((MOSgetCnt(task->blk) %8) != 0); + task->blk = (MosaicBlk) (((char*) dst) + wordaligned(bytes, int)); + //mnstr_printf(cntxt->fdout,"advance mask width %d bytes %d %d \n",bits,bytes,(int)wordaligned(bytes,int)); + } } } @@ -149,6 +159,13 @@ MOSestimate_prefix(Client cntxt, MOStask switch(size){ case 1: { bte *v = (bte*) task->src, *w= v+1, val= *v,val2= *w, mask; + // search first non-identical value + for(i = 0;i < task->elm;i++, w++) + if( *v != *w ){ + val2 = *w; + break; + } + w = v+1; Prefix(bits, mask, val, val2, 8); val = *v & mask; for(i = 0; i < task->elm; w++, i++){ @@ -164,6 +181,13 @@ MOSestimate_prefix(Client cntxt, MOStask break; case 2: { sht *v = (sht*) task->src, *w= v+1, val= *v,val2= *w, mask; + // search first non-identical value + for(i = 0;i < task->elm;i++, w++) + if( *v != *w ){ + val2 = *w; + break; + } + w = v+1; Prefix(bits, mask, val, val2, 16); val = *v & mask; for(i = 0; i < task->elm; w++, i++){ @@ -179,6 +203,35 @@ MOSestimate_prefix(Client cntxt, MOStask break; case 4: { int *v = (int*) task->src, *w= v+1, val= *v,val2= *w, mask; + // search first non-identical value + for(i = 0;i < task->elm;i++, w++) + if( *v != *w ){ + val2 = *w; + break; + } + w = v+1; + Prefix(bits, mask, val, val2, 32); + val = *v & mask; + for(i = 0; i < task->elm; w++, i++){ + if ( val != (*w & mask) ) + break; + } + if ( i > MOSlimit() ) i = MOSlimit(); + bits = i * (32 -bits); + store = bits/8 + ((bits % 8) >0); + store = MosaicBlkSize + 2 * sizeof(int) + wordaligned( store,int); + factor = ( (flt)i * sizeof(int))/ store; + } + break; + case 8: + { lng *v = (lng*) task->src, *w= v+1, val= *v,val2= *w, mask; + // search first non-identical value + for(i = 0;i < task->elm;i++, w++) + if( *v != *w ){ + val2 = *w; + break; + } + w = v+1; Prefix(bits, mask, val, val2, 32); val = *v & mask; for(i = 0; i < task->elm; w++, i++){ @@ -192,69 +245,164 @@ MOSestimate_prefix(Client cntxt, MOStask factor = ( (flt)i * sizeof(int))/ store; } } -#ifdef _DEBUG_MOSAIC_ - mnstr_printf(cntxt->fdout,"#estimate rle "BUNFMT" elm %4.3f factor\n",i,factor); -#endif - return factor; -} + #ifdef _DEBUG_MOSAIC_ + mnstr_printf(cntxt->fdout,"#estimate rle "BUNFMT" elm %4.3f factor\n",i,factor); + #endif + return factor; + } -#define compress() \ -{ cell = (i * rbits)/32;\ - lshift= 32 -((i * rbits) % 32) ;\ - if ( lshift > rbits){\ - base[cell]= base[cell] | (m << (lshift-rbits));\ - }else{ \ - rshift= 32 - ((i+1) * rbits) % 32;\ - base[cell]= base[cell] | (m >> (rbits-lshift));\ - base[cell+1]= 0 | (m << rshift);\ - }\ -} + #define compress() \ + { cell = (i * rbits)/32;\ + lshift= 32 -((i * rbits) % 32) ;\ + if ( lshift > rbits){\ + base[cell]= base[cell] | (m << (lshift-rbits));\ + }else{ \ + rshift= 32 - ((i+1) * rbits) % 32;\ + base[cell]= base[cell] | (m >> (rbits-lshift));\ + base[cell+1]= 0 | (m << rshift);\ + }\ + } -void -MOScompress_prefix(Client cntxt, MOStask task) -{ - BUN i ; - unsigned int *base; - unsigned int cell, bits, rbits, lshift, rshift; - int size; - MosaicBlk blk = task->blk; + void + MOScompress_prefix(Client cntxt, MOStask task) + { + BUN i ; + unsigned int *base; + unsigned int cell, bits, rbits, lshift, rshift; + int size; + MosaicBlk blk = task->blk; - (void) cntxt; - MOSsetTag(blk, MOSAIC_PREFIX); + (void) cntxt; _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
