Changeset: 521416a9ac2e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=521416a9ac2e Added Files: monetdb5/modules/mal/Tests/manifold.mal monetdb5/modules/mal/Tests/manifold.stable.err monetdb5/modules/mal/Tests/manifold.stable.out monetdb5/modules/mal/manifold.c monetdb5/modules/mal/manifold.h monetdb5/modules/mal/manifold.mal Modified Files: clients/Tests/MAL-signatures.stable.out clients/Tests/exports.stable.out monetdb5/modules/mal/Makefile.ag monetdb5/modules/mal/Tests/All monetdb5/modules/mal/mal_init.mal monetdb5/optimizer/opt_multiplex.c monetdb5/optimizer/opt_prelude.c monetdb5/optimizer/opt_prelude.h monetdb5/optimizer/opt_remap.c monetdb5/optimizer/opt_remap.h monetdb5/optimizer/optimizer.mal Branch: default Log Message:
Manifold implementation A multiplex operation leads to a MAL loop when the underlying BAT version is not available. This is quite expensive and interferes with the dataflow optimizer. The manifold function partly rectifies this situation by simulating the BAT loop, provided the underlying MAL command returns a single BAT and the argument types are fixed width. Simulation on mmath.sin() with 1M elm shows the following: vectorise: 52ms manifold : 70ms multiplex:1469ms diffs (truncated from 747 to 300 lines): diff --git a/clients/Tests/MAL-signatures.stable.out b/clients/Tests/MAL-signatures.stable.out --- a/clients/Tests/MAL-signatures.stable.out +++ b/clients/Tests/MAL-signatures.stable.out @@ -39939,10 +39939,6 @@ command lsst.xmatch(l:bat[:oid,:lng],r:b address LSSTxmatch; comment Return the HtmID pairs that lie within the same triangle at level depth -pattern mal.multiplex(a:any...):any -address OPTremapMultiplex; -pattern mal.multiplex(a:any...):any... -address OPTremapMultiplex; command mapreduce.getCloud(nme:str):bat[:oid,:str] address MRgetCloud; comment Localize the elements of a named cloud @@ -40251,6 +40247,10 @@ pattern mat.slice(first:wrd,last:wrd,b:b address MATpackSlice; comment Materialize a sliced MAT into a BAT +pattern mal.manifold(mod:str,fcn:str,a:any...):bat[:oid,:any] +address MANIFOLDevaluate; +pattern mal.multiplex(mod:str,fcn:str,a:any...):any... +address OPTremapMultiplex; command mbr.#cmp():int address mbrCOMP; command mbr.#fromstr():mbr diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -1532,6 +1532,8 @@ str MALpipeline(Client c); str MALreader(Client c); void MALresourceFairness(lng usec); str MALstartDataflow(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); +str MANIFOLDevaluate(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); +MALfcn MANIFOLDtypecheck(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); str MANUALcompletion(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); str MANUALcreate0(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); str MANUALcreate1(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); @@ -2021,6 +2023,9 @@ str QOTshowPlan(Client cntxt, MalBlkPtr void QOTstatisticsExit(void); void QOTupdateStatistics(str nme, int prop, lng val); QueryQueue QRYqueue; +str RAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); +str RAPIpostlude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); +str RAPIprelude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); str RECYCLEappendSQL(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p); str RECYCLEcolumn(Client cntxt, str sch, str tbl, str col); str RECYCLEdeleteSQL(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p); @@ -2056,6 +2061,7 @@ str RUNcostPrediction(Client cntxt, MalB str RUNisolation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p); str RUNpickResult(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p); str RUNvolumeCost(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p); +char *R_HomeDir(); str SABAOTHdeserialise(sabdb **ret, str *sabdb); str SABAOTHfreeStatus(sabdb **ret); str SABAOTHgetDBfarm(str *ret); diff --git a/monetdb5/modules/mal/Makefile.ag b/monetdb5/modules/mal/Makefile.ag --- a/monetdb5/modules/mal/Makefile.ag +++ b/monetdb5/modules/mal/Makefile.ag @@ -47,6 +47,7 @@ lib_mal = { mat.c mat.h \ mdb.c mdb.h \ mkey.c mkey.h \ + manifold.c manifold.h \ pcre.c \ pqueue.c pqueue.h \ profiler.c profiler.h \ @@ -72,7 +73,7 @@ headers_mal = { DIR = libdir/monetdb5 SOURCES = language.mal constraints.mal mal_init.mal box.mal bbp.mal \ profiler.mal const.mal batExtensions.mal \ - inspect.mal manual.mal mal_io.mal pqueue.mal mkey.mal \ + inspect.mal manual.mal mal_io.mal pqueue.mal mkey.mal manifold.mal \ iterator.mal clients.mal \ factories.mal groupby.mal mdb.mal pcre.mal mat.mal \ urlbox.mal transaction.mal \ @@ -84,7 +85,7 @@ headers_mal = { } EXTRA_DIST = batExtensions.mal iterator.mal constraints.mal \ - groupby.mal mal_init.mal manual.mal mkey.mal pcre.mal \ + groupby.mal mal_init.mal manual.mal mkey.mal manifold.mal pcre.mal \ profiler.mal recycle.mal remote.mal sabaoth.mal trader.mal \ transaction.mal txtsim.mal tablet.mal tablet.h sample.mal json_util.mal \ mal_mapi.mal mat.mal tokenizer.mal pqueue.mal calc.mal \ diff --git a/monetdb5/modules/mal/Tests/All b/monetdb5/modules/mal/Tests/All --- a/monetdb5/modules/mal/Tests/All +++ b/monetdb5/modules/mal/Tests/All @@ -69,6 +69,8 @@ cluster00 tokenizer00 zorder +manifold + #HAVE_RAPTOR?rdf # might show different output if openssl is compiled without full sha2 diff --git a/monetdb5/modules/mal/Tests/manifold.mal b/monetdb5/modules/mal/Tests/manifold.mal new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/manifold.mal @@ -0,0 +1,48 @@ +# the multiplex code blocks. + +t0:= alarm.usec(); +b:= bat.new(:oid,:dbl); + +INT_MAX := 2147483647; +dbgmsk_restore := mdb.getDebug(); +dbgmsk_unset := 8+8388608; +dbgmsk_keep := calc.xor(INT_MAX,dbgmsk_unset); +dbgmsk_set := calc.and(dbgmsk_restore,dbgmsk_keep); +mdb.setDebug(dbgmsk_set); + +r:= mmath.srand(0); +barrier i:= 0:lng; + k:= mmath.rand(); + l:= calc.dbl(k); + bat.append(b,l); + redo i:= iterator.next(1:lng,1000000:lng); +exit i; + +mdb.setDebug(dbgmsk_restore); +mdb.setDebug(dbgmsk_set); + +t0:= alarm.usec(); +z:bat[:oid,:dbl]:= mal.manifold("mmath","sin",b); +t1:= alarm.usec(); +d0:= t1-t0; +io.printf("#mal.manifold(sin) %d\n",d0); +s:= aggr.sum(z); +io.printf("#sum %f\n",s); + + +t0:= alarm.usec(); +x:bat[:oid,:dbl]:= mal.multiplex("mmath","sin",b); +t1:= alarm.usec(); +d0:= t1-t0; +io.printf("#mal.multiplex(sin) %d\n",d0); +s:= aggr.sum(x); +io.printf("#sum %f\n",s); + + +t0:= alarm.usec(); +x:= batmmath.sin(b); +t1:= alarm.usec(); +d0:= t1-t0; +io.printf("#mmath.sin() %d\n",d0); +s:= aggr.sum(x); +io.printf("#sum %f\n",s); diff --git a/monetdb5/modules/mal/Tests/manifold.stable.err b/monetdb5/modules/mal/Tests/manifold.stable.err new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/manifold.stable.err @@ -0,0 +1,29 @@ +stderr of test 'manifold` in directory 'monetdb5/modules/mal` itself: + + +# 23:08:15 > +# 23:08:15 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=34659" "--set" "mapi_usock=/var/tmp/mtest-26669/.s.monetdb.34659" "--set" "monet_prompt=" "--forcemito" "--set" "mal_listing=2" "--dbpath=/export/scratch1/mk/current//Linux/var/MonetDB/mTests_monetdb5_modules_mal" "manifold.mal" +# 23:08:15 > + +# builtin opt gdk_dbpath = /export/scratch1/mk/current//Linux/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = yes +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 50000 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 34659 +# cmdline opt mapi_usock = /var/tmp/mtest-26669/.s.monetdb.34659 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /export/scratch1/mk/current//Linux/var/MonetDB/mTests_monetdb5_modules_mal + +# 23:08:16 > +# 23:08:16 > "Done." +# 23:08:16 > + diff --git a/monetdb5/modules/mal/Tests/manifold.stable.out b/monetdb5/modules/mal/Tests/manifold.stable.out new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/manifold.stable.out @@ -0,0 +1,76 @@ +stdout of test 'manifold` in directory 'monetdb5/modules/mal` itself: + + +# 23:08:15 > +# 23:08:15 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=34659" "--set" "mapi_usock=/var/tmp/mtest-26669/.s.monetdb.34659" "--set" "monet_prompt=" "--forcemito" "--set" "mal_listing=2" "--dbpath=/export/scratch1/mk/current//Linux/var/MonetDB/mTests_monetdb5_modules_mal" "manifold.mal" +# 23:08:15 > + +# MonetDB 5 server v11.18.0 +# This is an unreleased version +# Serving database 'mTests_monetdb5_modules_mal', using 8 threads +# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically linked +# Found 15.591 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2013 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://vienna.ins.cwi.nl:34659/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-26669/.s.monetdb.34659 +# MonetDB/GIS module loaded +# MonetDB/JAQL module loaded +# MonetDB/SQL module loaded +#WARNING To speedup mmath.sin a bulk operator implementation is needed +function user.main():void; +# the multiplex code blocks. + t0 := alarm.usec(); + b := bat.new(:oid,:dbl); + INT_MAX := 2147483647; + dbgmsk_restore := mdb.getDebug(); + dbgmsk_unset := calc.+(8,8388608); + dbgmsk_keep := calc.xor(INT_MAX,dbgmsk_unset); + dbgmsk_set := calc.and(dbgmsk_restore,dbgmsk_keep); + mdb.setDebug(dbgmsk_set); + r := mmath.srand(0); +barrier i := 0:lng; + k := mmath.rand(); + l := calc.dbl(k); + bat.append(b,l); + redo i := iterator.next(1:lng,1000000:lng); +exit i; + mdb.setDebug(dbgmsk_restore); + mdb.setDebug(dbgmsk_set); + t0 := alarm.usec(); + z:bat[:oid,:dbl] := mal.manifold("mmath","sin",b); + t1 := alarm.usec(); + d0 := calc.-(t1,t0); + io.printf("#mal.manifold(sin) %d\n",d0); + s := aggr.sum(z); + io.printf("#sum %f\n",s); + t0 := alarm.usec(); + X_43 := bat.reverse(b); + X_45 := bat.new(nil:oid,nil:dbl); +barrier (X_49,X_50) := iterator.new(b); + X_52 := mmath.sin(X_50); + X_53 := algebra.fetch(X_43,X_49); + bat.insert(X_45,X_53,X_52); + redo (X_49,X_50) := iterator.next(b); +exit (X_49,X_50); + x:bat[:oid,:dbl] := X_45; + t1 := alarm.usec(); + d0 := calc.-(t1,t0); + io.printf("#mal.multiplex(sin) %d\n",d0); + s := aggr.sum(x); + io.printf("#sum %f\n",s); + t0 := alarm.usec(); + x:bat[:oid,:dbl] := batmmath.sin(b); + t1 := alarm.usec(); + d0 := calc.-(t1,t0); + io.printf("#mmath.sin() %d\n",d0); + s := aggr.sum(x); + io.printf("#sum %f\n",s); +end main; + + +# 23:08:16 > +# 23:08:16 > "Done." +# 23:08:16 > + diff --git a/monetdb5/modules/mal/mal_init.mal b/monetdb5/modules/mal/mal_init.mal --- a/monetdb5/modules/mal/mal_init.mal +++ b/monetdb5/modules/mal/mal_init.mal @@ -85,6 +85,7 @@ include bbp; include const; include box; include mal_io; +include manifold; include factories; include recycle; include remote; diff --git a/monetdb5/modules/mal/manifold.c b/monetdb5/modules/mal/manifold.c new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/manifold.c @@ -0,0 +1,270 @@ +/* + * The contents of this file are subject to the MonetDB Public License + * Version 1.1 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * http://www.monetdb.org/Legal/MonetDBLicense + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + * License for the specific language governing rights and limitations + * under the License. + * + * The Original Code is the MonetDB Database System. + * + * The Initial Developer of the Original Code is CWI. + * Portions created by CWI are Copyright (C) 1997-July 2008 CWI. + * Copyright August 2008-2013 MonetDB B.V. + * All Rights Reserved. + */ + +/* _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list