MonetDB: default - Fix compiler "variable may be used uninitiali...
Changeset: 3d370112d4b8 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3d370112d4b8 Modified Files: monetdb5/optimizer/opt_reorder.c Branch: default Log Message: Fix compiler "variable may be used uninitialized" error diffs (12 lines): diff --git a/monetdb5/optimizer/opt_reorder.c b/monetdb5/optimizer/opt_reorder.c --- a/monetdb5/optimizer/opt_reorder.c +++ b/monetdb5/optimizer/opt_reorder.c @@ -206,7 +206,7 @@ OPTbreadthfirst(Client cntxt, MalBlkPtr static int OPTpostponeAppends(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { - int i,j,k=0, actions =0, last; + int i,j,k=0, actions =0, last=-1; InstrPtr *old, *appends; int limit; (void) cntxt; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Fix preprocessor compilation issue
Changeset: 37403127e328 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=37403127e328 Modified Files: sql/backends/monet5/vaults/bam/bam_wrapper.c Branch: default Log Message: Fix preprocessor compilation issue diffs (12 lines): diff --git a/sql/backends/monet5/vaults/bam/bam_wrapper.c b/sql/backends/monet5/vaults/bam/bam_wrapper.c --- a/sql/backends/monet5/vaults/bam/bam_wrapper.c +++ b/sql/backends/monet5/vaults/bam/bam_wrapper.c @@ -23,7 +23,7 @@ #ifdef HAVE_SAMTOOLS_KSTRING_H #include -#elif HAVE_BAM_KSTRING_H +#elif defined (HAVE_BAM_KSTRING_H) #include #else /* Ubuntu doesn't distribute samtools/kstring.h, so we need our own ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: profiler - Flush stdout after every JSON object
Changeset: 7f1d8af451e8 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7f1d8af451e8 Modified Files: clients/mapiclient/stethoscope.c Branch: profiler Log Message: Flush stdout after every JSON object diffs (18 lines): diff --git a/clients/mapiclient/stethoscope.c b/clients/mapiclient/stethoscope.c --- a/clients/mapiclient/stethoscope.c +++ b/clients/mapiclient/stethoscope.c @@ -342,10 +342,12 @@ main(int argc, char **argv) buffer[len + n] = 0; response = buffer; if(json) { - if(trace != NULL) + if(trace != NULL) { fprintf(trace, "%s", response); - else + } else { printf("%s", response); + fflush(stdout); + } } while ((e = strchr(response, '\n')) != NULL) { *e = 0; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: profiler - Changes in stethoscope:
Changeset: 997c7811bf34 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=997c7811bf34 Modified Files: clients/mapiclient/stethoscope.c Branch: profiler Log Message: Changes in stethoscope: - Always either write to stdout or to specified file - Add json flag that, when set, makes stethoscope output json - Fix help messages diffs (255 lines): diff --git a/clients/mapiclient/stethoscope.c b/clients/mapiclient/stethoscope.c --- a/clients/mapiclient/stethoscope.c +++ b/clients/mapiclient/stethoscope.c @@ -69,10 +69,12 @@ static stream *conn = NULL; static char hostname[128]; -static char *basefilename = "stethoscope"; +static char *filename = NULL; static int beat = 50; +static int json = 0; static Mapi dbh; static MapiHdl hdl = NULL; +static FILE *trace = NULL; /* * Tuple level reformatting @@ -80,51 +82,56 @@ static MapiHdl hdl = NULL; static void renderEvent(EventRecord *ev){ + FILE *s; + if(trace != NULL) + s = trace; + else + s = stdout; if( ev->eventnr ==0 && ev->version){ - printf("[ "); - printf("0, "); - printf("0, "); - printf("\"\", " ); - printf("0, "); - printf("\"system\", "); - printf("0, "); - printf("0, "); - printf("0, "); - printf("0, "); - printf("0, "); - printf("0, "); - printf("\""); - printf("version:%s, release:%s, threads:%s, memory:%s, host:%s, oid:%d, package:%s ", + fprintf(s, "[ "); + fprintf(s, "0, "); + fprintf(s, "0, "); + fprintf(s, "\"\", " ); + fprintf(s, "0, "); + fprintf(s, "\"system\", "); + fprintf(s, "0, "); + fprintf(s, "0, "); + fprintf(s, "0, "); + fprintf(s, "0, "); + fprintf(s, "0, "); + fprintf(s, "0, "); + fprintf(s, "\""); + fprintf(s, "version:%s, release:%s, threads:%s, memory:%s, host:%s, oid:%d, package:%s ", ev->version, ev->release, ev->threads, ev->memory, ev->host, ev->oid, ev->package); - printf("\" ]\n"); + fprintf(s, "\" ]\n"); return ; } if( ev->eventnr < 0) return; - printf("[ "); - printf(LLFMT", ", ev->eventnr); - printf("\"%s\", ", ev->clk); + fprintf(s, "[ "); + fprintf(s, LLFMT", ", ev->eventnr); + fprintf(s, "\"%s\", ", ev->clk); if( ev->function && *ev->function) - printf("\"%s[%d]%d\", ", ev->function, ev->pc, ev->tag); + fprintf(s, "\"%s[%d]%d\", ", ev->function, ev->pc, ev->tag); else - printf("\"\", "); - printf("%d, ", ev->thread); + fprintf(s, "\"\", "); + fprintf(s, "%d, ", ev->thread); switch(ev->state){ - case MDB_START: printf("\"start\", "); break; - case MDB_DONE: printf("\"done \", "); break; - case MDB_WAIT: printf("\"wait \", "); break; - case MDB_PING: printf("\"ping \", "); break; - case MDB_SYSTEM: printf("\"system\","); + case MDB_START: fprintf(s, "\"start\", "); break; + case MDB_DONE: fprintf(s, "\"done \", "); break; + case MDB_WAIT: fprintf(s, "\"wait \", "); break; + case MDB_PING: fprintf(s, "\"ping \", "); break; + case MDB_SYSTEM: fprintf(s, "\"system\","); } - printf(LLFMT", ", ev->ticks); - printf(LLFMT", ", ev->rss); - printf(LLFMT", ", ev->size); - printf(LLFMT", ", ev->inblock); - printf(LLFMT", ", ev->oublock); - printf(LLFMT", ", ev->majflt); - printf(LLFMT", ", ev->swaps); - printf(LLFMT", ", ev->csw); - printf("\"%s\" ]\n", ev->stmt); + fprintf(s, LLFMT", ", ev->ticks); + fprintf(s, LLFMT", ", ev->rss); + fprintf(s, LLFMT", ", ev->size); + fprintf(s, LLFMT", ", ev->inblock); + fprintf(s, LLFMT", ", ev->oublock); + fprintf(s, LLFMT", ", ev->majflt); + fprintf(s, LLFMT", ", ev->swaps); + fprintf(s, LLFMT", ", ev->csw); + fprintf(s, "\"%s\" ]\n", ev->stmt); } static void @@ -133,10 +140,13 @@ usageStethoscope(void) fprintf(stderr, "stethoscope [options] \n"); fprintf(stderr, " -d | --dbname=\n"); fprintf(stderr, " -u | --user=\n"); +fprintf(stderr, " -P | --password=\n"); fprintf(stderr, " -p | --port=\n"); fprintf(stderr, " -h | --host=\n"); +fprintf(stderr, " -j | --json\n"); fprintf(stderr, " -o | --output=\n"); fprintf(stderr, " -b | --beat= in milliseconds
MonetDB: profiler - Enable beautified statements
Changeset: 795a98661f1b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=795a98661f1b Modified Files: monetdb5/mal/mal_profiler.c Branch: profiler Log Message: Enable beautified statements diffs (21 lines): diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c --- a/monetdb5/mal/mal_profiler.c +++ b/monetdb5/mal/mal_profiler.c @@ -212,7 +212,7 @@ renderProfilerEvent(MalBlkPtr mb, MalStk //GDKfree(stmt); // ship the beautified version as well -/* + stmt = shortStmtRendering(mb, stk, pci); stmtq = mal_quote(stmt, strlen(stmt)); if (stmtq != NULL) { @@ -220,7 +220,7 @@ renderProfilerEvent(MalBlkPtr mb, MalStk GDKfree(stmtq); } GDKfree(stmt); -*/ + // collect the prerequisite pre-requisite statements prereq[0]='['; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: profiler - Change help message in stable.err
Changeset: 03dbc027b047 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=03dbc027b047 Modified Files: clients/mapiclient/Tests/stethoscope--help.stable.err Branch: profiler Log Message: Change help message in stable.err diffs (41 lines): diff --git a/clients/mapiclient/Tests/stethoscope--help.stable.err b/clients/mapiclient/Tests/stethoscope--help.stable.err --- a/clients/mapiclient/Tests/stethoscope--help.stable.err +++ b/clients/mapiclient/Tests/stethoscope--help.stable.err @@ -1,25 +1,26 @@ stderr of test 'stethoscope--help` in directory 'clients/mapiclient` itself: -# 11:43:47 > -# 11:43:47 > Mtimeout -timeout 120 ./stethoscope--help stethoscope--help -# 11:43:47 > +# 17:08:24 > +# 17:08:24 > "./stethoscope--help" "stethoscope--help" +# 17:08:24 > -# 10:36:09 > -# 10:36:09 > Mtimeout -timeout 60 stethoscope --help -# 10:36:09 > +# 17:08:24 > +# 17:08:24 > Mtimeout -timeout 60 stethoscope --help +# 17:08:24 > stethoscope [options] -d | --dbname= -u | --user= + -P | --password= -p | --port= -h | --host= + -j | --json -o | --output= -b | --beat= in milliseconds (default 50) - -? | --help + -D | --debug -? | --help - -# 19:24:05 > -# 19:24:05 > "Done." - +# 17:08:24 > +# 17:08:24 > "Done." +# 17:08:24 > ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - NodeJS client source code is now hosted on gi...
Changeset: 938ce957c0eb for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=938ce957c0eb Removed Files: clients/nodejs/Tests/nodetest.js clients/nodejs/monetdb/README.md clients/nodejs/monetdb/mapiclient.js clients/nodejs/monetdb/package.json Branch: default Log Message: NodeJS client source code is now hosted on github (https://github.com/MonetDB/monetdb-nodejs) and can be found on npm as well (https://www.npmjs.com/package/monetdb) diffs (truncated from 843 to 300 lines): diff --git a/clients/nodejs/Tests/nodetest.js b/clients/nodejs/Tests/nodetest.js deleted file mode 100644 --- a/clients/nodejs/Tests/nodetest.js +++ /dev/null @@ -1,147 +0,0 @@ -var monetdb = require('../monetdb'); -var assert = require('assert'); - -var dbport = parseInt(process.argv[2]); -var dbname = process.argv[3]; - -/* lets first check some failing connection attempts */ -monetdb.connect({host:'veryinvalidhostnamethathopefullyresolvesnowhere'}, function(err) { - assert(err); -}); - -monetdb.connect({dbname:'nonexist', port:dbport}, function(err) { - assert(err); -}); - -var failedconn = monetdb.connect({dbname:dbname, user:'nonexist', port:dbport}, function(err) { - assert(err); -}); -/* try to query on a failed connection, we need this callback */ -failedconn.query('SELECT 1',function(err,res) { - assert(err); -}); - -/* now actually connect */ -var conn = monetdb.connect({dbname:dbname, port:dbport, debug: false}, function(err) { - assert.equal(null, err); - assert.equal(conn.env.gdk_dbname, dbname); -}); - - -/* some querying, call chaining */ -conn.query('start transaction'). - query('create table foo(a int, b float, c clob)'). - query("insert into foo values (42,4.2,'42'),(43,4.3,'43'),(44,4.4,'44'),(45,4.5,'45')"); - -conn.query('select * from foo', function(err, res) { - assert.equal(null, err); - - assert.equal('table', res.type); - assert.equal(4, res.rows); - assert.equal(3, res.cols); - - assert.equal(3, res.structure.length); - - assert.equal('a', res.structure[0].column); - assert.equal('int', res.structure[0].type); - - assert.equal(4, res.data.length); - - assert.equal(42, res.data[0][res.structure[0].index]); - assert.equal(4.3, res.data[1][res.col['b']]); - assert.equal('44', res.data[2][2]); -}); - -/* we can also just put the queries in one .query call */ -conn.query('delete from foo; drop table foo; rollback'); - -/* query that will stress multi-block operations */ -function rep(str,n) { - ret = ''; - for (var i = 0; i< n; i++) { - ret += str; - } - return ret; -} - -var longstr = rep('ABCDEFGHIJKLMNOP', 10); -conn.query("SELECT '" + longstr + "'", function(err, res) { - assert.equal(null, err); - assert.equal(longstr,res.data[0][0]); -}); - -/* failing query */ -conn.query('MEHR BIER', function(err, res) { - assert(err); -}); - -/* prepared statements */ -conn.query('MEHR BIER', - ['connections', 0, false], function(err, res) { - assert(err); -}); - - -/* fire-and-forget query with parameters */ -conn.query('SELECT id from tables where name=? and type=? and temporary=?', - ['connections', 0, 0], function(err, res) { - assert.equal(null, err); - assert(res.rows > 0); -}); - -/* Try the log callback functionality */ -var nr_log_callbacks = 0; -conn.log_callback = function(message, error, result) { - assert(message); // message must contain something - ++nr_log_callbacks; -} -conn.query('SELECT id FROM tables WHERE name=? AND type=? AND temporary=?', - ['connections', 0, 0], function(err, res) { - assert.equal(null, err); - assert(nr_log_callbacks > 0); - conn.log_callback = null; -}); - -/* some quoting fun, jesus */ -conn.query("SELECT 'asdf','\"', '\\\"', '\"', '\\''", function(err, res) { - assert.equal(null, err); - assert.equal('\\asdf', res.data[0][0]); - assert.equal('"', res.data[0][1]); - assert.equal('\"', res.data[0][2]); - assert.equal('\\"', res.data[0][3]); - assert.equal("'", res.data[0][4]); -}); - -/* Retrieving escaped characters from the database */ -conn.query('start transaction'). - query('create table foo(a string)'). - query("insert into foo values ('\t\n\r\n\tlalala\t\n\r')"); - -conn.query("select * from foo", function(err, res) { - assert.equal(null, err); - assert.equal(1, res.rows); - assert.equal('\t\n\r\n\tlalala\t\n\r', res.data[0][0]); -}); - -conn.query('delete from foo; drop table foo; rollback'); - - -/* prepared statements can also be re-used */ -conn.prepare('SELECT id from tables where name=? and type=? and temporary=?', function(err, res){ - assert.equal(null, err); - - /* parameters can also be given as array */ -
MonetDB: default - Update npm version
Changeset: 6b31db1e14d5 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6b31db1e14d5 Modified Files: clients/nodejs/monetdb/package.json Branch: default Log Message: Update npm version diffs (11 lines): diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.2.7, + version: 0.2.8, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Also support dates in prepared statement exec...
Changeset: 41c633c2aefe for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=41c633c2aefe Modified Files: clients/nodejs/monetdb/mapiclient.js Branch: default Log Message: Also support dates in prepared statement execution diffs (18 lines): diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -147,10 +147,10 @@ MonetDBConnection.prototype.prepare = fu } var colData = resp.data[resp.rows-bindparams.length+paramIndex]; if(colData) { - if(colData[0] == timestamp) { - s = timestamp +s; - } else if(colData[0] == timestamptz) { - s = timestamptz +s + var toCheck = ['timestamp', 'timestamptz', 'date']; + var i = toCheck.indexOf(colData[0]); + if(i = 0) { + s = toCheck[i] + ' ' + s; } } return s; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Extend the prepared statement handling to sup...
Changeset: 28883be78b39 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=28883be78b39 Modified Files: clients/nodejs/monetdb/mapiclient.js clients/nodejs/monetdb/package.json Branch: default Log Message: Extend the prepared statement handling to support json and timestamptz diffs (38 lines): diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -137,13 +137,21 @@ MonetDBConnection.prototype.prepare = fu /* escape single quotes except if they are already escaped */ s = ' + param.replace(/([^\\])'/g,$1\\') + '; break + case 'object': + case 'array': + s = json ' + JSON.stringify(param).replace(/([^\\])'/g,$1\\') + '; + break; default: s = param; break; } var colData = resp.data[resp.rows-bindparams.length+paramIndex]; - if(colData colData[0] == timestamp) { - s = timestamp +s; + if(colData) { + if(colData[0] == timestamp) { + s = timestamp +s; + } else if(colData[0] == timestamptz) { + s = timestamptz +s + } } return s; }).join(', '); diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.2.6, + version: 0.2.7, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Properly reconstruct escaped characters \t, \...
Changeset: 8df3a20c7058 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8df3a20c7058 Modified Files: clients/nodejs/Tests/nodetest.js clients/nodejs/monetdb/mapiclient.js clients/nodejs/monetdb/package.json Branch: default Log Message: Properly reconstruct escaped characters \t, \n, and \r, when retrieved from the database diffs (52 lines): diff --git a/clients/nodejs/Tests/nodetest.js b/clients/nodejs/Tests/nodetest.js --- a/clients/nodejs/Tests/nodetest.js +++ b/clients/nodejs/Tests/nodetest.js @@ -112,6 +112,20 @@ conn.query(SELECT 'asdf','\', '\\\ assert.equal(', res.data[0][4]); }); +/* Retrieving escaped characters from the database */ +conn.query('start transaction'). + query('create table foo(a string)'). + query(insert into foo values ('\t\n\r\n\tlalala\t\n\r')); + +conn.query(select * from foo, function(err, res) { + assert.equal(null, err); + assert.equal(1, res.rows); + assert.equal('\t\n\r\n\tlalala\t\n\r', res.data[0][0]); +}); + +conn.query('delete from foo; drop table foo; rollback'); + + /* prepared statements can also be re-used */ conn.prepare('SELECT id from tables where name=? and type=? and temporary=?', function(err, res){ assert.equal(null, err); diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -386,7 +386,12 @@ exports.parsetuples = _parsetuples = fu break; case 'ESCAPED': state = 'INQUOTES'; - curtok += chr; +switch(chr) { +case 't': curtok += '\t'; break; +case 'n': curtok += '\n'; break; +case 'r': curtok += '\r'; break; +default: curtok += chr; +} break; case 'INQUOTES': if (chr == '') { diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.2.5, + version: 0.2.6, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merge with default
Changeset: de97b2973b94 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=de97b2973b94 Added Files: clients/mapiclient/Tests/tachograph--help clients/mapiclient/Tests/tachograph--help.bat clients/mapiclient/Tests/tachograph--help.stable.err clients/mapiclient/Tests/tachograph--help.stable.out monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows.single sql/backends/monet5/vaults/74_netcdf.mal sql/backends/monet5/vaults/74_netcdf.sql sql/backends/monet5/vaults/netcdf.c sql/backends/monet5/vaults/netcdf.mal sql/backends/monet5/vaults/netcdf_vault.h sql/test/mergetables/Tests/doubletable.stable.err sql/test/mergetables/Tests/doubletable.stable.out sql/test/mergetables/Tests/forex1.stable.err sql/test/mergetables/Tests/forex1.stable.out sql/test/mergetables/Tests/mergedropcrash.sql sql/test/mergetables/Tests/mergedropcrash.stable.err sql/test/mergetables/Tests/mergedropcrash.stable.out sql/test/remote/Tests/ssbm.stable.out.int128 Removed Files: clients/Tests/SQL-dump.stable.out.oid32 clients/Tests/SQL-dump.stable.out.oid32.int128 Modified Files: .hgtags MonetDB.spec NT/installer32/MonetDB-ODBC-Installer.vdproj NT/installer32/MonetDB5-Geom-Module.vdproj NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB-ODBC-Installer.vdproj NT/installer64/MonetDB5-Geom-Module.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj NT/monetdb_config.h.in NT/rules.msc buildtools/ChangeLog-Archive buildtools/ChangeLog.Oct2014 buildtools/autogen/autogen/am.py clients/Tests/All clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/SQL-dump.stable.out clients/Tests/SQL-dump.stable.out.int128 clients/Tests/exports.stable.out clients/mapiclient/Tests/All clients/mapiclient/Tests/tomograph--help.stable.err clients/mapiclient/eventparser.c clients/mapiclient/tachograph.c clients/mapilib/mapi.rc clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc clients/python2/setup.py clients/python3/setup.py configure.ag debian/changelog gdk/ChangeLog-Archive gdk/ChangeLog.Oct2014 gdk/gdk_aggr.c gdk/gdk_align.c gdk/gdk_atomic.h gdk/gdk_calc.c gdk/gdk_imprints.c gdk/gdk_join.c gdk/gdk_logger.c gdk/gdk_private.h gdk/gdk_select.c gdk/gdk_system.c gdk/gdk_system.h gdk/gdk_utils.c gdk/libbat.rc geom/monetdb5/geom.mal java/Makefile.ag java/build.properties java/pom.xml java/release.txt libversions monetdb5/mal/Tests/performanceTests/performanceLog monetdb5/mal/Tests/performanceTests/tst901a.mal monetdb5/mal/Tests/performanceTests/tst901b.mal monetdb5/mal/mal_dataflow.c monetdb5/mal/mal_parser.c monetdb5/modules/atoms/color.mal monetdb5/modules/atoms/inet.mal monetdb5/modules/atoms/json.mal monetdb5/modules/atoms/mtime.mal monetdb5/modules/atoms/url.mal monetdb5/modules/atoms/uuid.mal monetdb5/modules/atoms/xml.mal monetdb5/tools/libmonetdb5.rc sql/backends/monet5/sql.c sql/backends/monet5/sql.mal sql/backends/monet5/sql_upgrades.c sql/backends/monet5/vaults/Makefile.ag sql/benchmarks/tpch/Tests/01-explain.stable.out.32bit sql/benchmarks/tpch/Tests/04-explain.stable.out.32bit sql/benchmarks/tpch/Tests/05-explain.stable.out.32bit sql/benchmarks/tpch/Tests/16-explain.stable.out.32bit sql/benchmarks/tpch/Tests/20-explain.stable.out.32bit sql/benchmarks/tpch/Tests/21-explain.stable.out.32bit sql/benchmarks/tpch/Tests/22-explain.stable.out.32bit sql/jdbc/tests/Tests/All sql/jdbc/tests/Tests/Test_Dobjects.stable.out sql/scripts/75_storagemodel.sql sql/server/rel_optimizer.c sql/server/rel_schema.c sql/storage/bat/bat_storage.c sql/storage/bat/bat_table.c sql/storage/store.c sql/test/BugTracker-2013/Tests/between.Bug-3259.stable.out.int128.single sql/test/BugTracker-2013/Tests/between.Bug-3259.stable.out.single sql/test/BugTracker-2014/Tests/nil_2dec_lng.Bug-3592.stable.out.single sql/test/BugTracker-2014/Tests/typecast_error.Bug-3622.sql sql/test/BugTracker-2014/Tests/typecast_error.Bug-3622.stable.out sql/test/BugTracker-2015/Tests/readonly.Bug-3709.stable.err sql/test/Tests/All sql/test/Tests/systemfunctions.stable.out sql/test/Tests/systemfunctions.stable.out.int128
MonetDB: bamloader - Merge with default
Changeset: c3b0b8e5956b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c3b0b8e5956b Added Files: clients/R/build-for-cran.sh clients/Tests/MAL-signatures.malC clients/Tests/MAL-signatures.stable.err clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/SQL-dump.sql clients/Tests/SQL-dump.stable.err clients/Tests/SQL-dump.stable.out clients/Tests/SQL-dump.stable.out.int128 clients/Tests/SQL-dump.stable.out.oid32 clients/Tests/SQL-dump.stable.out.oid32.int128 debian/monetdb5-server-hugeint.install debian/monetdb5-sql-hugeint.install monetdb5/mal/Tests/tst062.stable.err monetdb5/mal/Tests/tst062.stable.out monetdb5/modules/mal/Tests/remote07.malC monetdb5/optimizer/Tests/joinchain.mal monetdb5/optimizer/Tests/joinchain.stable.err monetdb5/optimizer/Tests/joinchain.stable.out sql/benchmarks/ATIS/Tests/select_group.stable.out.int128 sql/benchmarks/tpch/15a.sql sql/benchmarks/tpch/LOCKED/Tests/17.stable.out.int128 sql/benchmarks/tpch/Tests/05-explain.stable.out.32bit sql/benchmarks/tpch/Tests/17.stable.out.int128 sql/benchmarks/tpch/Tests/22.stable.out.int128 sql/benchmarks/tpch/c.sql-primary sql/benchmarks/tpch/c.sql-primary-foreign sql/scripts/41_md5sum.sql sql/test/BugTracker-2009/Tests/case_evaluates_all_branches.SF-2893484.stable.out.int128 sql/test/BugTracker-2010/Tests/crash-with-huge-query.Bug-2647.stable.out.int128 sql/test/BugTracker-2011/Tests/subquery_in_from_clause.Bug-2812.stable.out.int128 sql/test/BugTracker-2012/Tests/aggregate_incorrect_precision_scale.Bug-3182.stable.out.int128 sql/test/BugTracker-2012/Tests/currenttime.Bug-2781.SQL.py sql/test/BugTracker-2015/Tests/alias.Bug-3704.sql sql/test/BugTracker-2015/Tests/alias.Bug-3704.stable.err sql/test/BugTracker-2015/Tests/alias.Bug-3704.stable.out sql/test/BugTracker-2015/Tests/assertHead.Bug-3706.sql sql/test/BugTracker-2015/Tests/assertHead.Bug-3706.stable.err sql/test/BugTracker-2015/Tests/assertHead.Bug-3706.stable.out sql/test/BugTracker-2015/Tests/case_aggr.Bug-3705.sql sql/test/BugTracker-2015/Tests/case_aggr.Bug-3705.stable.err sql/test/BugTracker-2015/Tests/case_aggr.Bug-3705.stable.out sql/test/BugTracker-2015/Tests/find_fk.Bug-3690.sql sql/test/BugTracker-2015/Tests/find_fk.Bug-3690.stable.err sql/test/BugTracker-2015/Tests/find_fk.Bug-3690.stable.out sql/test/BugTracker-2015/Tests/float-space-parse.Bug-3691.sql sql/test/BugTracker-2015/Tests/float-space-parse.Bug-3691.stable.err sql/test/BugTracker-2015/Tests/float-space-parse.Bug-3691.stable.out sql/test/BugTracker-2015/Tests/ifthenelse.Bug-3699.sql sql/test/BugTracker-2015/Tests/ifthenelse.Bug-3699.stable.err sql/test/BugTracker-2015/Tests/ifthenelse.Bug-3699.stable.out sql/test/BugTracker-2015/Tests/incorrect_result_type.Bug-3687.sql sql/test/BugTracker-2015/Tests/incorrect_result_type.Bug-3687.stable.err sql/test/BugTracker-2015/Tests/incorrect_result_type.Bug-3687.stable.out sql/test/BugTracker-2015/Tests/incorrect_result_type.Bug-3687.stable.out.int128 sql/test/BugTracker-2015/Tests/inf-nan-handling.Bug-3696.sql sql/test/BugTracker-2015/Tests/inf-nan-handling.Bug-3696.stable.err sql/test/BugTracker-2015/Tests/inf-nan-handling.Bug-3696.stable.out sql/test/BugTracker-2015/Tests/insert_into_merge_table_crash.Bug-3703.sql sql/test/BugTracker-2015/Tests/insert_into_merge_table_crash.Bug-3703.stable.err sql/test/BugTracker-2015/Tests/insert_into_merge_table_crash.Bug-3703.stable.out sql/test/BugTracker-2015/Tests/is_null_limit.Bug-3684.sql sql/test/BugTracker-2015/Tests/is_null_limit.Bug-3684.stable.err sql/test/BugTracker-2015/Tests/is_null_limit.Bug-3684.stable.out sql/test/BugTracker-2015/Tests/operator-priority.Bug-3686.sql sql/test/BugTracker-2015/Tests/operator-priority.Bug-3686.stable.err sql/test/BugTracker-2015/Tests/operator-priority.Bug-3686.stable.out sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693.sql sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693.stable.err sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693.stable.out sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693_2.sql sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693_2.stable.err sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693_2.stable.out sql/test/BugTracker-2015/Tests/readonly.Bug-3709.sql sql/test/BugTracker-2015/Tests/readonly.Bug-3709.stable.err
MonetDB: bamloader - Merge with default
Changeset: 04e4a6a7dac6 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=04e4a6a7dac6 Modified Files: clients/nodejs/monetdb/mapiclient.js clients/nodejs/monetdb/package.json clients/odbc/driver/ODBCStmt.h clients/odbc/driver/SQLExtendedFetch.c clients/odbc/driver/SQLFetch.c clients/odbc/driver/SQLFetchScroll.c configure.ag gdk/gdk_atomic.h gdk/gdk_system.c gdk/gdk_system.h gdk/gdk_utils.c monetdb5/mal/mal_instruction.c monetdb5/optimizer/Tests/manifold2.stable.out.single sql/backends/monet5/UDF/Tests/udf-fuse.stable.out sql/backends/monet5/UDF/Tests/udf-reverse.stable.out sql/backends/monet5/rest/Tests/jsonstore00.stable.out sql/backends/monet5/sql_scenario.c sql/test/BugTracker-2010/Tests/LIMIT_OFFSET_big-endian.Bug-2622.stable.out sql/test/BugTracker-2010/Tests/group-by_ordered_column.Bug-2564.stable.out sql/test/BugTracker/Tests/insert_not_exists.SF-1380287.stable.out sql/test/Skyserver/Tests/Skyserver.stable.out sql/test/Tests/decimal2.stable.out sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_MapiClient.stable.out testing/Mtest.py.in Branch: bamloader Log Message: Merge with default diffs (truncated from 838 to 300 lines): diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -122,26 +122,31 @@ MonetDBConnection.prototype.prepare = fu thizz.query(query, function(error, resp) { if (!error) { var execfun = function(bindparams, ecallback) { - var quoted = bindparams.map(function(param) { + var quoted = bindparams.map(function(param, paramIndex) { if(param === null) { return NULL; } var type = typeof param; + var s; switch(type) { case 'boolean': case 'number': - return '' + param; + s = '' + param; break case 'string': /* escape single quotes except if they are already escaped */ - return ' + param.replace(/([^\\])'/g,$1\\') + '; + s = ' + param.replace(/([^\\])'/g,$1\\') + '; break default: - return param; + s = param; break; } + var colData = resp.data[resp.rows-bindparams.length+paramIndex]; + if(colData colData[0] == timestamp) { + s = timestamp +s; + } + return s; }).join(', '); - var execquery = 'EXEC ' + resp.queryid + '(' + quoted + ')'; thizz.query(execquery, ecallback); } diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.2.4, + version: 0.2.5, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, diff --git a/clients/odbc/driver/ODBCStmt.h b/clients/odbc/driver/ODBCStmt.h --- a/clients/odbc/driver/ODBCStmt.h +++ b/clients/odbc/driver/ODBCStmt.h @@ -199,9 +199,9 @@ SQLRETURN MNDBColAttribute(ODBCStmt *stm SQLRETURN MNDBExecDirect(ODBCStmt *stmt, SQLCHAR *szSqlStr, SQLINTEGER nSqlStr); SQLRETURN MNDBExecute(ODBCStmt *stmt); -SQLRETURN MNDBFetch(ODBCStmt *stmt); +SQLRETURN MNDBFetch(ODBCStmt *stmt, SQLUSMALLINT *RowStatusArray); SQLRETURN MNDBFetchScroll(ODBCStmt *stmt, SQLSMALLINT nOrientation, - SQLLEN nOffset); + SQLLEN nOffset, SQLUSMALLINT *RowStatusArray); SQLRETURN MNDBFreeStmt(ODBCStmt *stmt, SQLUSMALLINT option);
MonetDB: default - MonetDB/NodeJS: Update package version
Changeset: ec75240aad7f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec75240aad7f Modified Files: clients/nodejs/monetdb/package.json Branch: default Log Message: MonetDB/NodeJS: Update package version diffs (11 lines): diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.2.4, + version: 0.2.5, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merge with default
Changeset: c66c91916d41 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c66c91916d41 Added Files: clients/R/Makefile.ag clients/R/MonetDB.R/Makefile.ag clients/R/MonetDB.R/src/Makefile.ag clients/R/MonetDB.R/src/mapisplit-r.c clients/R/MonetDB.R/src/mapisplit.h clients/R/MonetDB.R/src/profiler-r.c clients/R/MonetDB.R/src/profiler.h clients/mapiclient/mapisplit-dummy.c clients/mapiclient/profiler-dummy.c monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.single monetdb5/optimizer/Tests/manifold2.stable.out.single sql/backends/monet5/Tests/rapi12.sql sql/backends/monet5/Tests/rapi12.stable.err sql/backends/monet5/Tests/rapi12.stable.out sql/backends/monet5/generator/90_generator_hge.mal sql/backends/monet5/generator/90_generator_hge.sql sql/test/bugs/Tests/apply_with_union_assertion.sql sql/test/bugs/Tests/apply_with_union_assertion.stable.err sql/test/bugs/Tests/apply_with_union_assertion.stable.out sql/test/copy/Tests/format_date.sql sql/test/copy/Tests/format_date.stable.err sql/test/copy/Tests/format_date.stable.out sql/test/mergetables/Tests/crash0.stable.err sql/test/mergetables/Tests/crash0.stable.out sql/test/mergetables/Tests/mergedrop.stable.err sql/test/mergetables/Tests/mergedrop.stable.out sql/test/pg_regress/Tests/int4.stable.out.int128 sql/test/pg_regress/Tests/int8.stable.err.int128 sql/test/pg_regress/Tests/int8.stable.out.int128 sql/test/pg_regress/Tests/numeric_big.stable.err.int128 sql/test/pg_regress/Tests/strings.stable.err.int128 Removed Files: monetdb5/mal/Tests/LICENSE monetdb5/modules/atoms/Tests/LICENSE monetdb5/modules/mal/Tests/LICENSE monetdb5/scheduler/Tests/LICENSE monetdb5/tests/BugsViaSourgeforce/Tests/All monetdb5/tests/gdkTests/Tests/casts.stable.out.Windows monetdb5/tools/Tests/LICENSE sql/backends/monet5/generator/89_generator_hge.mal sql/backends/monet5/generator/89_generator_hge.sql sql/backends/monet5/vaults/Tests/All sql/test/BugTracker-2009/Tests/overflow.SF-2853458.stable.err.Windows sql/test/Dump/Tests/dump.stable.out.Windows Modified Files: COPYING HowToStart.rst Makefile.ag MonetDB.spec NT/Makefile NT/installer32/cv64-32.sh NT/installer64/cv32-64.sh NT/installer64/fixmsi.py NT/monetdb_config.h.in NT/rules.msc NT/updatecodes.py NT/wincompile.py NT/winconfig.py README bootstrap buildtools/ChangeLog buildtools/Makefile.ag buildtools/autogen/autogen.py buildtools/autogen/autogen/am.py buildtools/autogen/autogen/codegen.py buildtools/autogen/autogen/msc.py buildtools/autogen/autogen/var.py buildtools/autogen/setup.py buildtools/conf/Maddlog buildtools/conf/Makefile.ag buildtools/conf/fixlicense.py buildtools/conf/license.rtf buildtools/conf/rules.mk buildtools/conf/website.html buildtools/doc/HowToRelease.rst buildtools/doc/windowsbuild.rst clients/Makefile.ag clients/NT/Makefile.ag clients/NT/mclient.bat clients/NT/msqldump.bat clients/NT/stethoscope.bat clients/R/MonetDB.R/DESCRIPTION clients/R/MonetDB.R/NAMESPACE clients/R/MonetDB.R/R/control.R clients/R/MonetDB.R/R/dbi.R clients/R/MonetDB.R/R/dplyr.R clients/R/MonetDB.R/man/control.Rd clients/R/MonetDB.R/man/src_monetdb.Rd clients/R/MonetDB.R/src/Makevars.win clients/R/MonetDB.R/src/mapisplit.c clients/R/MonetDB.R/src/profiler.c clients/R/Tests/dbi.R clients/R/Tests/dplyr.R clients/R/Tests/dplyr.stable.out clients/R/Tests/survey.R clients/Tests/MAL-signatures_all.stable.out clients/Tests/MAL-signatures_all.stable.out.int128 clients/Tests/MAL-signatures_fits_geom.stable.out clients/Tests/MAL-signatures_fits_geom.stable.out.int128 clients/Tests/MAL-signatures_geom.stable.out clients/Tests/MAL-signatures_geom.stable.out.int128 clients/Tests/MAL-signatures_none.stable.out clients/Tests/MAL-signatures_none.stable.out.int128 clients/Tests/SQL-dump_all.stable.out clients/Tests/SQL-dump_all.stable.out.int128 clients/Tests/SQL-dump_all.stable.out.oid32 clients/Tests/SQL-dump_all.stable.out.oid32.int128 clients/Tests/SQL-dump_geom.stable.out clients/Tests/SQL-dump_geom.stable.out.32bit clients/Tests/SQL-dump_geom.stable.out.64bit.oid32 clients/Tests/SQL-dump_geom.stable.out.int128
MonetDB: bamloader - Do not track pyc file
Changeset: f6f83b8f23f3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f6f83b8f23f3 Removed Files: sql/backends/monet5/bam/Tests/bam.pyc Branch: bamloader Log Message: Do not track pyc file diffs (7 lines): diff --git a/sql/backends/monet5/bam/Tests/bam.pyc b/sql/backends/monet5/bam/Tests/bam.pyc deleted file mode 100644 index 920c4a9a9889e1cc40f6d04cb102a0f9b57a828a..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch literal 0 Hc$@O1 ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Add timestamp prefix to timestamp arguments i...
Changeset: f84672e52d8f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f84672e52d8f Modified Files: clients/nodejs/monetdb/mapiclient.js Branch: default Log Message: Add timestamp prefix to timestamp arguments in the execution of prepared statements diffs (40 lines): diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -122,26 +122,31 @@ MonetDBConnection.prototype.prepare = fu thizz.query(query, function(error, resp) { if (!error) { var execfun = function(bindparams, ecallback) { - var quoted = bindparams.map(function(param) { + var quoted = bindparams.map(function(param, paramIndex) { if(param === null) { return NULL; } var type = typeof param; + var s; switch(type) { case 'boolean': case 'number': - return '' + param; + s = '' + param; break case 'string': /* escape single quotes except if they are already escaped */ - return ' + param.replace(/([^\\])'/g,$1\\') + '; + s = ' + param.replace(/([^\\])'/g,$1\\') + '; break default: - return param; + s = param; break; } + var colData = resp.data[resp.rows-bindparams.length+paramIndex]; + if(colData colData[0] == timestamp) { + s = timestamp +s; + } + return s; }).join(', '); - var execquery = 'EXEC ' + resp.queryid + '(' + quoted + ')'; thizz.query(execquery, ecallback); } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merged with default
Changeset: b847a55bc9f2 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b847a55bc9f2 Added Files: clients/R/MonetDB.R/src/Makevars.win clients/R/MonetDB.R/src/profiler.c clients/mapiclient/eventparser.c clients/mapiclient/eventparser.h clients/mapiclient/tachograph.c monetdb5/mal/Tests/tst880.stable.err monetdb5/mal/Tests/tst880.stable.out sql/test/BugTracker-2015/Tests/inet-funcs.Bug-3645.sql sql/test/BugTracker-2015/Tests/inet-funcs.Bug-3645.stable.err sql/test/BugTracker-2015/Tests/inet-funcs.Bug-3645.stable.out sql/test/BugTracker-2015/Tests/reserved_keywords.Bug-3613.sql sql/test/BugTracker-2015/Tests/reserved_keywords.Bug-3613.stable.err sql/test/BugTracker-2015/Tests/reserved_keywords.Bug-3613.stable.out Removed Files: monetdb5/extras/rdf/30_rdf.mal monetdb5/extras/rdf/Makefile.ag monetdb5/extras/rdf/rdf.h monetdb5/extras/rdf/rdf_shredder.c monetdb5/extras/rdf/rdfalgebra.c monetdb5/extras/rdf/rdfalgebra.mal monetdb5/mal/Tests/saveload.mal monetdb5/mal/Tests/saveload.stable.err monetdb5/mal/Tests/saveload.stable.out monetdb5/mal/Tests/tst090.mal monetdb5/mal/Tests/tst260.mal monetdb5/mal/Tests/tst260.stable.err monetdb5/mal/Tests/tst260.stable.out monetdb5/mal/Tests/tst612.mal monetdb5/mal/Tests/tst612.stable.err monetdb5/mal/Tests/tst612.stable.out monetdb5/mal/Tests/tst805.mal monetdb5/mal/Tests/tst805.stable.err monetdb5/mal/Tests/tst805.stable.out monetdb5/mal/Tests/tst867.mal monetdb5/mal/Tests/tst867.stable.err monetdb5/mal/Tests/tst867.stable.out monetdb5/mal/Tests/tst950.mal monetdb5/modules/kernel/Tests/vacuum.mal monetdb5/modules/kernel/Tests/vacuum.stable.err monetdb5/modules/kernel/Tests/vacuum.stable.out monetdb5/modules/kernel/counters.c monetdb5/modules/kernel/counters.h monetdb5/modules/kernel/counters.mal monetdb5/modules/mal/Tests/cluster00.mal monetdb5/modules/mal/Tests/cluster00.stable.err monetdb5/modules/mal/Tests/cluster00.stable.out monetdb5/modules/mal/trader.c monetdb5/modules/mal/trader.h monetdb5/modules/mal/trader.mal monetdb5/optimizer/Tests/ESexample.mal monetdb5/optimizer/Tests/ESexample.stable.err monetdb5/optimizer/Tests/ESexample.stable.out monetdb5/optimizer/Tests/SRexample.mal monetdb5/optimizer/Tests/SRexample.stable.err monetdb5/optimizer/Tests/SRexample.stable.out monetdb5/optimizer/Tests/emptyset00.mal monetdb5/optimizer/Tests/emptyset00.stable.err monetdb5/optimizer/Tests/emptyset00.stable.out monetdb5/optimizer/Tests/emptyset01.mal monetdb5/optimizer/Tests/emptyset01.stable.err monetdb5/optimizer/Tests/emptyset01.stable.out monetdb5/optimizer/Tests/groups00.mal monetdb5/optimizer/Tests/groups00.stable.err monetdb5/optimizer/Tests/groups00.stable.out monetdb5/optimizer/Tests/octopus00.mal monetdb5/optimizer/Tests/octopus01.mal monetdb5/optimizer/Tests/rangePush00.mal monetdb5/optimizer/Tests/rangePush00.stable.err monetdb5/optimizer/Tests/rangePush00.stable.out monetdb5/optimizer/Tests/rangePush01.mal monetdb5/optimizer/Tests/rangePush01.stable.err monetdb5/optimizer/Tests/rangePush01.stable.out monetdb5/optimizer/Tests/tst4820.mal monetdb5/optimizer/Tests/tst4820.stable.err monetdb5/optimizer/Tests/tst4820.stable.out monetdb5/optimizer/opt_centipede.c monetdb5/optimizer/opt_centipede.h monetdb5/optimizer/opt_cluster.c monetdb5/optimizer/opt_cluster.h monetdb5/optimizer/opt_emptySet.c monetdb5/optimizer/opt_emptySet.h monetdb5/optimizer/opt_groups.c monetdb5/optimizer/opt_groups.h monetdb5/optimizer/opt_mapreduce.c monetdb5/optimizer/opt_mapreduce.h monetdb5/optimizer/opt_octopus.c monetdb5/optimizer/opt_octopus.h monetdb5/optimizer/opt_pushranges.c monetdb5/optimizer/opt_pushranges.h monetdb5/optimizer/opt_strengthReduction.c monetdb5/optimizer/opt_strengthReduction.h monetdb5/scheduler/run_octopus.c monetdb5/scheduler/run_octopus.h monetdb5/scheduler/run_octopus.mal monetdb5/tests/BugDay_2005-10-06_4.9.3/Tests/return_types_of_head_and_tail.SF-701641.stable.out.int128 monetdb5/tests/BugReports/Tests/no.050.mal monetdb5/tests/BugReports/Tests/no.050.stable.err monetdb5/tests/BugReports/Tests/no.050.stable.out monetdb5/tests/BugReports/Tests/no.051.mal monetdb5/tests/BugReports/Tests/no.051.stable.err monetdb5/tests/BugReports/Tests/no.051.stable.out
MonetDB: bamloader - Fixed test
Changeset: d96643f0932f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d96643f0932f Modified Files: sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out Branch: bamloader Log Message: Fixed test diffs (12 lines): diff --git a/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out b/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out --- a/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out +++ b/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out @@ -84,7 +84,7 @@ function user.main():void; bat.append(is,17922987); res3:bat[:oid,:str] := mal.manifold(bam,seq_char,is,sc_seqs,sc_poss,sc_cgrs); io.print(res3); -end main; +end user.main; [ false ] #--# # ht # name ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Replace BBPreleaseref with BBPunfix
Changeset: a81783156d70 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a81783156d70 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Replace BBPreleaseref with BBPunfix diffs (81 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -312,7 +312,7 @@ bam_flag_bat(bat * ret, bat * bid, str * /* allocate result BAT */ output = BATnew(TYPE_void, TYPE_bit, BATcount(input), TRANSIENT); if (output == NULL) { - BBPreleaseref(input-batCacheid); + BBPunfix(input-batCacheid); throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL); } @@ -328,7 +328,7 @@ bam_flag_bat(bat * ret, bat * bid, str * finish_props(); /* release input BAT-descriptor */ - BBPreleaseref(input-batCacheid); + BBPunfix(input-batCacheid); BBPkeepref((*ret = output-batCacheid)); @@ -348,7 +348,7 @@ bam_flag_bat(bat * ret, bat * bid, str * /* allocate result BAT */ \ output = BATnew(TYPE_void, TYPE_str, BATcount(input), TRANSIENT); \ if (output == NULL) { \ - BBPreleaseref(input-batCacheid); \ + BBPunfix(input-batCacheid); \ throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); \ } \ BATseqbase(output, input-hseqbase); \ @@ -360,8 +360,8 @@ bam_flag_bat(bat * ret, bat * bid, str * str r, msg; \ \ if ((msg = transform_fn(r, t)) != MAL_SUCCEED) { \ - BBPreleaseref(input-batCacheid); \ - BBPreleaseref(output-batCacheid); \ + BBPunfix(input-batCacheid); \ + BBPunfix(output-batCacheid); \ return msg; \ } \ BUNappend(output, (ptr) r, FALSE); \ @@ -369,7 +369,7 @@ bam_flag_bat(bat * ret, bat * bid, str * } \ \ /* release input BAT-descriptor */ \ - BBPreleaseref(input-batCacheid); \ + BBPunfix(input-batCacheid); \ \ BBPkeepref((*ret = output-batCacheid)); \ \ @@ -416,7 +416,7 @@ seq_length_bat(bat * ret, bat * bid) BATloop(input, p, q) { cur_in = (str) BUNtail(li, p); if ((msg = seq_length(cur_out, cur_in)) != MAL_SUCCEED) { - BBPreleaseref(output-batCacheid); + BBPunfix(output-batCacheid); return msg; } update_props(int); @@ -426,7 +426,7 @@ seq_length_bat(bat * ret, bat * bid) finish_props(); /* release input BAT-descriptor */ - BBPreleaseref(input-batCacheid); + BBPunfix(input-batCacheid); BBPkeepref((*ret = output-batCacheid)); @@ -493,9 +493,9 @@ seq_char_bat(bat * ret, int * ref_pos, b cleanup: /* release input BAT-descriptors */ - if(seqs) BBPreleaseref(seqs-batCacheid); - if(poss) BBPreleaseref(poss-batCacheid); - if(cigars) BBPreleaseref(cigars-batCacheid); + if(seqs) BBPunfix(seqs-batCacheid); + if(poss) BBPunfix(poss-batCacheid); + if(cigars) BBPunfix(cigars-batCacheid); if(result) BBPkeepref((*ret = result-batCacheid)); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Updated readme and version number
Changeset: 57cf0e1e0c2e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=57cf0e1e0c2e Modified Files: clients/nodejs/monetdb/README.md clients/nodejs/monetdb/package.json Branch: default Log Message: Updated readme and version number diffs (29 lines): diff --git a/clients/nodejs/monetdb/README.md b/clients/nodejs/monetdb/README.md --- a/clients/nodejs/monetdb/README.md +++ b/clients/nodejs/monetdb/README.md @@ -93,6 +93,14 @@ Empty the query queue and then close the +# a name=log/aQuery logging +Every time a query result is returned from the database, the callback function *conn.log_callback*, which defaults to null, will be executed right before the query callback function is executed (conn refers to a connection object returned by the [connect function](#connect)). If you have some general behavior that you want to be executed every time a query finishes, you can set *conn.log_callback* to a function with the following signature: + +function(query, err, res) +- query: the SQL query that lead to this result (note: if you pass a params array to the [query function](#query), multiple queries are fired to the database, hence the log callback will be called multiple times) +- err, res: Same as in the callback for the [query function](#query) + + # a name=q/aQ Integration Due to the huge popularity of the [Q module for NodeJS](https://www.npmjs.org/package/q), we decided to add native Q support, that wraps our API in a promise returning API that exists on top of the original API, so you can use both interchangeably. diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.2.2, + version: 0.2.3, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Extended nodejs client with an optional loggi...
Changeset: 276f3da3ae88 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=276f3da3ae88 Modified Files: clients/nodejs/Tests/nodetest.js clients/nodejs/monetdb/mapiclient.js Branch: default Log Message: Extended nodejs client with an optional logging callback diffs (65 lines): diff --git a/clients/nodejs/Tests/nodetest.js b/clients/nodejs/Tests/nodetest.js --- a/clients/nodejs/Tests/nodetest.js +++ b/clients/nodejs/Tests/nodetest.js @@ -87,8 +87,20 @@ conn.query('SELECT id from tables where ['connections', 0, false], function(err, res) { assert.equal(null, err); assert(res.rows 0); +}); -}); +/* Try the log callback functionality */ +var nr_log_callbacks = 0; +conn.log_callback = function(message, error, result) { + assert(message); // message must contain something + ++nr_log_callbacks; +} +conn.query('SELECT id FROM tables WHERE name=? AND type=? AND readonly=?', + ['connections', 0, false], function(err, res) { + assert.equal(null, err); + assert(nr_log_callbacks 0); + conn.log_callback = null; +}); /* some quoting fun, jesus */ conn.query(SELECT 'asdf','\', '\\\', '\', '\\'', function(err, res) { diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -8,7 +8,7 @@ function MonetDBConnection(options, conn this.read_leftover = 0; this.read_final = false; this.read_str = ''; - this.read_callback = undefined; + this.cur_op = undefined; // object { message (str), callback (fn)} this.conn_callback = conncallback; this.mapi_blocksize = 8192; this.do_close = false; @@ -223,11 +223,14 @@ function handle_message(message) { if (message.charAt(0) == '') { response = _parseresponse(message); } - - if (this.read_callback != undefined) { - this.read_callback(error, response); - this.read_callback = undefined; + if(typeof(this.log_callback) == function) { + this.log_callback(this.cur_op.message, error, response); } + if (this.cur_op.callback != undefined) { + this.cur_op.callback(error, response); + this.cur_op.callback = undefined; + } + this.cur_op.message = undefined; next_op.call(this); } @@ -243,7 +246,7 @@ function next_op() { var op = this.queryqueue.shift(); send_message.call(this, op.message); - this.read_callback = op.callback; + this.cur_op = op; } function cleanup() { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Added minimal MAL test that demonstrates wh...
Changeset: a589e25d2715 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a589e25d2715 Added Files: sql/backends/monet5/bam/Tests/seq_char.mal Branch: bamloader Log Message: Added minimal MAL test that demonstrates where coverage test case fails: misalignment in input BATs to manifold function diffs (33 lines): diff --git a/sql/backends/monet5/bam/Tests/seq_char.mal b/sql/backends/monet5/bam/Tests/seq_char.mal new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/seq_char.mal @@ -0,0 +1,28 @@ +seqs := bat.new(:oid, :str); +poss := bat.new(:oid, :int); +cgrs := bat.new(:oid, :str); +bat.append(seqs, ATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT); +bat.append(seqs, TATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT); +bat.append(seqs, GTACGTGGGAAAAAAGCTGGGCATCCAAGACATCACGTGTCAGTAGGTTCCGTGATGCCAAGCCGCCCAGCAAGTA); +bat.append(poss, 17922987); +bat.append(poss, 18225784); +bat.append(poss, 18226024); +bat.append(cgrs, 3=1X1=1X43=1X16=1X33=); +bat.append(cgrs, 5=1X31=1X62=); +bat.append(cgrs, 100=); +io.print(=== Input BATs:); +io.print(seqs); +io.print(poss); +io.print(cgrs); +i := 17922987; +res1:bat[:oid,:str] := mal.manifold(bam, seq_char, i, seqs, poss, cgrs); +io.print(=== Manifold result1:); +io.print(res1); +is := poss; +res2:bat[:oid,:str] := mal.manifold(bam, seq_char, is, seqs, poss, cgrs); +io.print(=== Manifold result2:); +io.print(res2); +is := bat.new(:oid, :int); +bat.append(is, 17922987); +res3:bat[:oid,:str] := mal.manifold(bam, seq_char, is, seqs, poss, cgrs); +io.print(res3); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Improve seq_length_bat by direct appending ...
Changeset: 15361647fecc for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=15361647fecc Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Improve seq_length_bat by direct appending instead of usign BUNappend and make code of reverse_seq_bat and reverse_qual_bat more generic diffs (296 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -283,15 +283,6 @@ seq_char(str * ret, int * ref_pos, str * output-T-nonil = output-T-nonil *cur_out != TPE##_nil; \ } -#define update_props_str() { \ - output-tsorted = output-tsorted \ - (c == 0 || strcmp(*cur_out, prev_out) = 0); \ - output-trevsorted = output-trevsorted \ - (c == 0 || strcmp(*cur_out, prev_out) = 0); \ - output-T-nil = output-T-nil || *cur_out == str_nil; \ - output-T-nonil = output-T-nonil *cur_out != str_nil; \ -} - #define finish_props() { \ BATsetcount(output, BATcount(input)); \ BATseqbase(output, input-hseqbase); \ @@ -344,221 +335,102 @@ bam_flag_bat(bat * ret, bat * bid, str * return MAL_SUCCEED; } -/* Commented out, since string heap addition needs to be added -str -reverse_seq_bat(bat * ret, bat * bid) -{ - BAT *input, *output; - str prev_out = 0; - str *cur_in; - str *cur_out; - BUN c; - str msg; - - assert(ret != NULL bid != NULL); - - if ((input = BATdescriptor(*bid)) == NULL) - throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING); -*/ - /* allocate result BAT */ -/* output = BATnew(TYPE_void, TYPE_str, BATcount(input), TRANSIENT); - if (output == NULL) { - throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); - } - - init_props(); - cur_in = (str *) Tloc(input, BUNfirst(input)); - cur_out = (str *) Tloc(output, BUNfirst(output)); - for(c = 0; c BATcount(input); ++c) { - if ((msg = reverse_seq(cur_out, cur_in)) != MAL_SUCCEED) { - BBPreleaseref(output-batCacheid); - return msg; - } - update_props_str(); - cur_in++; - prev_out = *cur_out++; - } - finish_props(); -*/ - /* release input BAT-descriptor */ -/* BBPreleaseref(input-batCacheid); - - BBPkeepref((*ret = output-batCacheid)); - - return MAL_SUCCEED; -}*/ +#define transform_strbat(transform_fn) { \ + BAT *input, *output; \ + BATiter li; \ + BUN p = 0, q = 0; \ + \ + assert(ret != NULL bid != NULL); \ + \ + if ((input = BATdescriptor(*bid)) == NULL) \ + throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING); \ + \ + /* allocate result BAT */ \ + output = BATnew(TYPE_void, TYPE_str, BATcount(input), TRANSIENT); \ + if (output == NULL) { \ + BBPreleaseref(input-batCacheid); \ + throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); \ + } \ + BATseqbase(output, input-hseqbase); \ + \ + li = bat_iterator(input); \ + \ + BATloop(input, p, q) { \ + str t = (str) BUNtail(li, p); \ + str r, msg; \ + \ + if ((msg = transform_fn(r, t)) != MAL_SUCCEED) { \ + BBPreleaseref(input-batCacheid); \ + BBPreleaseref(output-batCacheid); \ + return msg; \ + } \ + BUNappend(output, (ptr) r, FALSE); \ + GDKfree(r); \ + } \ + \ + /* release input BAT-descriptor */ \ + BBPreleaseref(input-batCacheid); \ + \ + BBPkeepref((*ret = output-batCacheid)); \ + \ + return MAL_SUCCEED; \ +} str reverse_seq_bat(bat * ret, bat * bid) { - BAT *seqs, *result; - BATiter li; - BUN p = 0, q = 0; - - assert(ret != NULL bid != NULL); - - if ((seqs = BATdescriptor(*bid)) == NULL) - throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING); - - /* allocate result BAT */ - result = BATnew(TYPE_void, TYPE_str, BATcount(seqs), TRANSIENT); - if (result == NULL) { - BBPreleaseref(seqs-batCacheid); - throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); - } - BATseqbase(result, seqs-hseqbase); - - li = bat_iterator(seqs); - - BATloop(seqs, p, q) { - str t = (str) BUNtail(li, p); - str r, msg; - - if ((msg = reverse_seq(r, t)) != MAL_SUCCEED) { - BBPreleaseref(seqs-batCacheid); - BBPreleaseref(result-batCacheid); - return msg; - } - BUNappend(result, (ptr) r, FALSE); - GDKfree(r); - } - - /* release input BAT-descriptor */ - BBPreleaseref(seqs-batCacheid); - -
MonetDB: bamloader - Added some MAL tests for testing the UDFs
Changeset: 1e897d944e34 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1e897d944e34 Added Files: sql/backends/monet5/bam/Tests/bam_lib_mal.mal sql/backends/monet5/bam/Tests/bam_lib_mal.stable.err sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out Removed Files: sql/backends/monet5/bam/Tests/seq_char.mal Modified Files: sql/backends/monet5/bam/Tests/All sql/backends/monet5/bam/Tests/coverage_usecase.sql Branch: bamloader Log Message: Added some MAL tests for testing the UDFs diffs (truncated from 360 to 300 lines): diff --git a/sql/backends/monet5/bam/Tests/All b/sql/backends/monet5/bam/Tests/All --- a/sql/backends/monet5/bam/Tests/All +++ b/sql/backends/monet5/bam/Tests/All @@ -25,5 +25,6 @@ HAVE_SAMTOOLS?query2.10 HAVE_SAMTOOLS?query2.11 HAVE_SAMTOOLS?query2.12 HAVE_SAMTOOLS?mergetable +HAVE_SAMTOOLS?bam_lib_mal HAVE_SAMTOOLS?bam_lib #HAVE_SAMTOOLS?coverage_usecase diff --git a/sql/backends/monet5/bam/Tests/bam_lib_mal.mal b/sql/backends/monet5/bam/Tests/bam_lib_mal.mal new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/bam_lib_mal.mal @@ -0,0 +1,72 @@ +sht1 := calc.sht(200); +sht2 := calc.sht(0); +sht3 := calc.sht(100); +f := bam.bam_flag(sht1, segm_unma); +io.print(f); +fs := bat.new(:oid, :sht); +bat.append(fs, sht1); +bat.append(fs, sht2); +bat.append(fs, sht3); +fsb := batbam.bam_flag(fs, segm_unma); +io.print(fsb); + +seq_rev := bam.reverse_seq(ACGTCGTA); +io.print(seq_rev); +seqs := bat.new(:oid, :str); +bat.append(seqs, seq_rev); +bat.append(seqs, TVGHCDMKNYSABWR); +bat.append(seqs, AAA); +io.print(seqs); +seqs_rev := batbam.reverse_seq(seqs); +io.print(seqs_rev); +seqs_revrev := batbam.reverse_seq(seqs_rev); +io.print(seqs_revrev); + +qual_rev := bam.reverse_qual(dJwn!@dJ!#jjjnsl); +io.print(qual_rev); +quals := bat.new(:oid, :str); +bat.append(quals, qual_rev); +bat.append(quals, dJwn!@dJ!#jjjnsl); +bat.append(quals, I_am_a_quality_string_without_a_purpose); +io.print(quals); +quals_rev := batbam.reverse_qual(quals); +io.print(quals_rev); +quals_revrev := batbam.reverse_qual(quals_rev); +io.print(quals_revrev); + +l := bam.seq_length(100M3I200D5M); +io.print(l); +cigars := bat.new(:oid, :str); +bat.append(cigars, 100M3I200D5M); +bat.append(cigars, *); +ls := batbam.seq_length(cigars); +io.print(ls); + + +seq_char := bam.seq_char(12, ACTAGAATGGCT, 5, 3M1I3M1D5M); +io.print(seq_char); +sc_seqs := bat.new(:oid, :str); +sc_poss := bat.new(:oid, :int); +sc_cgrs := bat.new(:oid, :str); +bat.append(sc_seqs, ATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT); +bat.append(sc_seqs, TATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT); +bat.append(sc_seqs, GTACGTGGGAAAAAAGCTGGGCATCCAAGACATCACGTGTCAGTAGGTTCCGTGATGCCAAGCCGCCCAGCAAGTA); +bat.append(sc_poss, 17922987); +bat.append(sc_poss, 18225784); +bat.append(sc_poss, 18226024); +bat.append(sc_cgrs, 3=1X1=1X43=1X16=1X33=); +bat.append(sc_cgrs, 5=1X31=1X62=); +bat.append(sc_cgrs, 100=); +io.print(sc_seqs); +io.print(sc_poss); +io.print(sc_cgrs); +i := 17922987; +res1:bat[:oid,:str] := mal.manifold(bam, seq_char, i, sc_seqs, sc_poss, sc_cgrs); +io.print(res1); +is := sc_poss; +res2:bat[:oid,:str] := mal.manifold(bam, seq_char, is, sc_seqs, sc_poss, sc_cgrs); +io.print(res2); +is := bat.new(:oid, :int); +bat.append(is, 17922987); +res3:bat[:oid,:str] := mal.manifold(bam, seq_char, is, sc_seqs, sc_poss, sc_cgrs); +io.print(res3); diff --git a/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.err b/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.err new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.err @@ -0,0 +1,29 @@ +stderr of test 'bam_lib_mal` in directory 'sql/backends/monet5/bam` itself: + + +# 16:51:57 +# 16:51:57 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=39916 --set mapi_usock=/var/tmp/mtest-16560/.s.monetdb.39916 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam bam_lib_mal.mal +# 16:51:57 + +# builtin opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 39916 +# cmdline opt mapi_usock =
MonetDB: bamloader - Merge with default
Changeset: 9654ce09f3a2 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9654ce09f3a2 Modified Files: sql/test/pg_regress/Tests/alter_table.stable.err sql/test/pg_regress/Tests/arrays.stable.err sql/test/pg_regress/Tests/boolean.stable.err sql/test/pg_regress/Tests/boolean.stable.out sql/test/pg_regress/Tests/case.stable.err sql/test/pg_regress/Tests/case.stable.out sql/test/pg_regress/Tests/char.stable.err sql/test/pg_regress/Tests/comments.stable.err sql/test/pg_regress/Tests/create_aggregate.stable.err sql/test/pg_regress/Tests/create_misc.stable.err sql/test/pg_regress/Tests/create_operator.stable.err sql/test/pg_regress/Tests/date.stable.err sql/test/pg_regress/Tests/date.stable.out sql/test/pg_regress/Tests/drop.stable.err sql/test/pg_regress/Tests/float4.stable.err sql/test/pg_regress/Tests/float8.stable.err sql/test/pg_regress/Tests/inet.stable.err sql/test/pg_regress/Tests/insert.stable.err sql/test/pg_regress/Tests/int2.stable.err sql/test/pg_regress/Tests/int4.stable.err sql/test/pg_regress/Tests/int8.stable.err sql/test/pg_regress/Tests/int8.stable.out sql/test/pg_regress/Tests/interval.stable.err sql/test/pg_regress/Tests/limit.stable.err sql/test/pg_regress/Tests/numeric.stable.err sql/test/pg_regress/Tests/numeric_big.stable.err sql/test/pg_regress/Tests/numerology.stable.err sql/test/pg_regress/Tests/oid.stable.err sql/test/pg_regress/Tests/select_into.stable.err sql/test/pg_regress/Tests/strings.stable.err sql/test/pg_regress/Tests/strings_cast.stable.err sql/test/pg_regress/Tests/strings_concat.stable.err sql/test/pg_regress/Tests/timestamp.stable.err sql/test/pg_regress/Tests/timestamptz.stable.err sql/test/pg_regress/Tests/timetz.stable.out sql/test/pg_regress/Tests/varchar.stable.err sql/test/pg_regress/Tests/without_oid.stable.err Branch: bamloader Log Message: Merge with default diffs (truncated from 6431 to 300 lines): diff --git a/sql/test/pg_regress/Tests/alter_table.stable.err b/sql/test/pg_regress/Tests/alter_table.stable.err --- a/sql/test/pg_regress/Tests/alter_table.stable.err +++ b/sql/test/pg_regress/Tests/alter_table.stable.err @@ -26,613 +26,613 @@ stderr of test 'alter_table` in director # cmdline opt gdk_debug = 536870922 -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = ALTER TABLE tmp RENAME TO tmp_new; ERROR = !syntax error, unexpected RENAME, expecting DROP or SET or ALTER or ADD in: alter table tmp rename -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = SELECT * FROM tmp_new; ERROR = !SELECT: no such table 'tmp_new' -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = ALTER TABLE tmp RENAME TO tmp_new2; ERROR = !syntax error, unexpected RENAME, expecting DROP or SET or ALTER or ADD in: alter table tmp rename -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = SELECT * FROM tmp_new; ERROR = !SELECT: no such table 'tmp_new' -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = SELECT * FROM tmp_new2; ERROR = !SELECT: no such table 'tmp_new2' -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = DROP TABLE tmp_new; ERROR = !DROP TABLE: no such table 'tmp_new' -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = DROP TABLE tmp_new2; ERROR = !DROP TABLE: no such table 'tmp_new2' -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = ALTER INDEX onek_unique1 RENAME TO tmp_onek_unique1; ERROR = !syntax error, unexpected INDEX, expecting USER or SEQUENCE or TABLE in: alter index -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = ALTER INDEX tmp_onek_unique1 RENAME TO onek_unique1; ERROR = !syntax error, unexpected INDEX, expecting USER or SEQUENCE or TABLE in: alter index -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = ALTER TABLE tmp_view RENAME TO tmp_view_new; ERROR = !syntax error, unexpected RENAME, expecting DROP or SET or ALTER or ADD in: alter table tmp_view rename -MAPI = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440 +MAPI = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812 QUERY = DROP VIEW tmp_view_new; ERROR = !DROP VIEW: unknown
MonetDB: bamloader - Merge with default
Changeset: 13b37e298c46 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=13b37e298c46 Modified Files: sql/backends/monet5/generator/generator.c Branch: bamloader Log Message: Merge with default diffs (89 lines): diff --git a/sql/backends/monet5/generator/generator.c b/sql/backends/monet5/generator/generator.c --- a/sql/backends/monet5/generator/generator.c +++ b/sql/backends/monet5/generator/generator.c @@ -30,6 +30,8 @@ #include math.h +#define IDENTITY(x)(x) + /* * The noop simply means that we keep the properties for the generator object. */ @@ -986,10 +988,10 @@ str VLTgenerator_join(Client cntxt, MalB } /* The operands of a join operation can either be defined on a generator */ -#define VLTrangejoin(TPE, ABS) \ +#define VLTrangejoin(TPE, ABS, FLOOR) \ { TPE f,f1,l,s; TPE *vlow,*vhgh; BUN w;\ - f = *getArgReference_bte(stk,p, 1);\ - l = *getArgReference_bte(stk,p, 2);\ + f = *getArgReference_##TPE(stk,p, 1);\ + l = *getArgReference_##TPE(stk,p, 2);\ if ( p-argc == 3) \ s = fl? (TPE) 1: (TPE)-1;\ else s = * getArgReference_##TPE(stk, p, 3); \ @@ -999,9 +1001,9 @@ str VLTgenerator_join(Client cntxt, MalB vlow = (TPE*) Tloc(blow,BUNfirst(blow));\ vhgh = (TPE*) Tloc(bhgh,BUNfirst(bhgh));\ for( ; cnt 0; cnt--, done++, o++,vlow++,vhgh++){\ - f1 = f + floor(ABS(*vlow-f)/ABS(s)) * s;\ + f1 = f + FLOOR(ABS(*vlow-f)/ABS(s)) * s;\ if ( f1 *vlow ) f1+= s;\ - w = (BUN) floor(ABS(f1-f)/ABS(s));\ + w = (BUN) FLOOR(ABS(f1-f)/ABS(s));\ for( ; (f1 *vlow || (li f1 == *vlow)) (f1 *vhgh || (ri f1 == *vhgh)); f1 += s, w++){\ if(c == limit)\ VLTrangeExpand();\ @@ -1058,43 +1060,15 @@ str VLTgenerator_rangejoin(Client cntxt, /* The actual join code for generators be injected here */ switch(tpe){ - case TYPE_bte: VLTrangejoin(bte,abs); break; - case TYPE_sht: VLTrangejoin(sht,abs); break; - case TYPE_int: VLTrangejoin(int,abs); break; - case TYPE_lng: //VLTrangejoin(lng,llabs); break; - { lng f,f1,l,s; lng *vlow,*vhgh; BUN w; - f = *getArgReference_lng(stk,p, 1); - l = *getArgReference_lng(stk,p, 2); - if ( p-argc == 3) - s = fl? (lng) 1: (lng)-1; - else s = * getArgReference_sht(stk, p, 3); - incr = s 0; - - if ( s == 0 || (f l s0) || (fl s 0)) - throw(MAL,generator.rangejoin,Illegal range); - - vlow = (lng*) Tloc(blow,BUNfirst(blow)); - vhgh = (lng*) Tloc(bhgh,BUNfirst(bhgh)); - for( ; cnt 0; done++,cnt--, o++,vlow++,vhgh++){ - f1 = f + floor(abs(*vlow-f)/abs(s)) * s; - if ( f1 *vlow ) f1+= s; - w = (BUN) floor(abs(f1-f)/abs(s)); - for( ; (f1 *vlow || (li f1 == *vlow)) (f1 *vhgh || (ri f1 == *vhgh)); f1 += s, w++){ - if(c == limit){ - limit += 0; - VLTrangeExpand(); - } - *ol++ = (oid) w; - *or++ = o; - c++; - } - } } - break; + case TYPE_bte: VLTrangejoin(bte,abs,IDENTITY); break; + case TYPE_sht: VLTrangejoin(sht,abs,IDENTITY); break; + case TYPE_int: VLTrangejoin(int,abs,IDENTITY); break; + case TYPE_lng: VLTrangejoin(lng,llabs,IDENTITY); break; #ifdef HAVE_HGE - case TYPE_hge: VLTrangejoin(hge,HGE_ABS); break; + case TYPE_hge: VLTrangejoin(hge,HGE_ABS,IDENTITY); break; #endif - case TYPE_flt: VLTrangejoin(flt,fabsf); break; - case TYPE_dbl: VLTrangejoin(dbl,fabs); break; + case TYPE_flt: VLTrangejoin(flt,fabsf,floorf); break; + case TYPE_dbl: VLTrangejoin(dbl,fabs,floor); break; default: if( tpe == TYPE_timestamp){ // it is easier to produce the timestamp series ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Hard code number of threads instead of lett...
Changeset: 9ad8b922c870 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9ad8b922c870 Modified Files: sql/backends/monet5/bam/85_bam.sql sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql sql/backends/monet5/bam/bam.mal sql/backends/monet5/bam/bam_loader.c Branch: bamloader Log Message: Hard code number of threads instead of letting the user decide this on SQL level diffs (81 lines): diff --git a/sql/backends/monet5/bam/85_bam.sql b/sql/backends/monet5/bam/85_bam.sql --- a/sql/backends/monet5/bam/85_bam.sql +++ b/sql/backends/monet5/bam/85_bam.sql @@ -1,9 +1,9 @@ CREATE SCHEMA bam; -CREATE PROCEDURE bam.bam_loader_repos(bam_repos STRING, dbschema SMALLINT, nr_threads SMALLINT) +CREATE PROCEDURE bam.bam_loader_repos(bam_repos STRING, dbschema SMALLINT) EXTERNAL NAME bam.bam_loader_repos; -CREATE PROCEDURE bam.bam_loader_files(bam_files STRING, dbschema SMALLINT, nr_threads SMALLINT) +CREATE PROCEDURE bam.bam_loader_files(bam_files STRING, dbschema SMALLINT) EXTERNAL NAME bam.bam_loader_files; CREATE PROCEDURE bam.bam_loader_file(bam_file STRING, dbschema SMALLINT) diff --git a/sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql b/sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql --- a/sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql +++ b/sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql @@ -1,3 +1,3 @@ -CALL bam.bam_loader_files('PWD/files.lst', 0, 4); -CALL bam.bam_loader_files('PWD/files_qname.lst', 1, 4); -CALL bam.bam_loader_files('PWD/files_many.lst', 0, 4); +CALL bam.bam_loader_files('PWD/files.lst', 0); +CALL bam.bam_loader_files('PWD/files_qname.lst', 1); +CALL bam.bam_loader_files('PWD/files_many.lst', 0); diff --git a/sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql b/sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql --- a/sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql +++ b/sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql @@ -1,3 +1,3 @@ # Load files -CALL bam.bam_loader_repos('PWD/files', 0, 4); -CALL bam.bam_loader_repos('PWD/files/queryname', 1, 4); \ No newline at end of file +CALL bam.bam_loader_repos('PWD/files', 0); +CALL bam.bam_loader_repos('PWD/files/queryname', 1); \ No newline at end of file diff --git a/sql/backends/monet5/bam/bam.mal b/sql/backends/monet5/bam/bam.mal --- a/sql/backends/monet5/bam/bam.mal +++ b/sql/backends/monet5/bam/bam.mal @@ -3,11 +3,11 @@ module bam; # Bam loader related signatures -pattern bam_loader_repos(bam_repos:str, dbschema:sht, nr_threads:sht):void +pattern bam_loader_repos(bam_repos:str, dbschema:sht):void address bam_loader_repos comment Read all bam files in the given bam_repos directory (non-recursive) and store them in the given dbschema; -pattern bam_loader_files(bam_files:str, dbschema:sht, nr_threads:sht):void +pattern bam_loader_files(bam_files:str, dbschema:sht):void address bam_loader_files comment Read all bam files in the file list stored in the file bam_files (separated by a newline) and store them in the given dbschema; diff --git a/sql/backends/monet5/bam/bam_loader.c b/sql/backends/monet5/bam/bam_loader.c --- a/sql/backends/monet5/bam/bam_loader.c +++ b/sql/backends/monet5/bam/bam_loader.c @@ -419,8 +419,10 @@ bam_loader_repos(Client cntxt, MalBlkPtr str bam_repos = *getArgReference_str(stk, pci, pci-retc); /* arg 2: dbschema to use */ sht dbschema = *getArgReference_sht(stk, pci, pci-retc + 1); - /* arg 3: max number of threads that will be used by bam_loader */ - sht nr_threads = *getArgReference_sht(stk, pci, pci-retc + 2); + /* For now, we hard code the number of threads, since the SQL + level should not bother with this */ + + sht nr_threads = 4; str *filenames = NULL; int nr_files = 0; @@ -539,8 +541,9 @@ bam_loader_files(Client cntxt, MalBlkPtr str bam_files = *getArgReference_str(stk, pci, pci-retc); /* arg 2: dbschema to use */ sht dbschema = *getArgReference_sht(stk, pci, pci-retc + 1); - /* arg 3: max number of threads that will be used by bam_loader */ - sht nr_threads = *getArgReference_sht(stk, pci, pci-retc + 2); + /* For now, we hard code the number of threads, since the SQL + level should not bother with this */ + sht nr_threads = 4; str *filenames = NULL; int nr_files = 0; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Replace sscanf with more efficient parsing ...
Changeset: c38437e1b729 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c38437e1b729 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Replace sscanf with more efficient parsing in seq_length diffs (52 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -183,31 +183,34 @@ reverse_qual(str * ret, str * qual) return MAL_SUCCEED; } +#define next_cigar_op(fn) { \ + str tmp; \ + cnt = strtol(s, tmp, 10); \ + if(cnt = 0 || s == tmp) { \ + *ret = -1; \ + throw(MAL, fn, Could not read CIGAR operation); \ + } \ + s = tmp; \ + op = *s++; \ +} + str seq_length(int * ret, str * cigar) { int result = 0; - str cigar_consumable = *cigar; + str s = *cigar; + long int cnt; + char op; - if (cigar_consumable[0] == '\0' || - (cigar_consumable[0] == '*' cigar_consumable[1] == '\0')) { + if (*s == '\0' || (*s == '*' *(s+1) == '\0')) { *ret = -1; return MAL_SUCCEED; } - while (cigar_consumable[0] != '\0') { - int cnt; - char op; - int nr_chars_read; - - if (sscanf - (cigar_consumable, %d%c%n, cnt, op, -nr_chars_read) != 2) - throw(MAL, seq_length, - Error parsing CIGAR string '%s'\n, *cigar); + while (*s != '\0') { + next_cigar_op(seq_length); if (op == 'M' || op == 'D' || op == 'N' || op == '=' || op == 'X') result += cnt; - cigar_consumable += nr_chars_read; } *ret = result; return MAL_SUCCEED; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Let seq_char also use the more efficient ve...
Changeset: c5fa74ef8100 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c5fa74ef8100 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Let seq_char also use the more efficient version of the CIGAR parser diffs (64 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -186,9 +186,8 @@ reverse_qual(str * ret, str * qual) #define next_cigar_op(fn) { \ str tmp; \ cnt = strtol(s, tmp, 10); \ - if(cnt = 0 || s == tmp) { \ - *ret = -1; \ - throw(MAL, fn, Could not read CIGAR operation); \ + if(cnt = 0 || s == tmp || *s == '\0') { \ + throw(MAL, fn, Could not parse CIGAR string); \ } \ s = tmp; \ op = *s++; \ @@ -219,30 +218,26 @@ seq_length(int * ret, str * cigar) str seq_char(str * ret, int * ref_pos, str * alg_seq, int * alg_pos, str * alg_cigar) { - str cigar_consumable = *alg_cigar; + str s = *alg_cigar; int seq_pos = -1; int cur_ref_pos = *alg_pos - 1; + + long int cnt; + char op; - if (cigar_consumable[0] == '\0' || - (cigar_consumable[0] == '*' cigar_consumable[1] == '\0')) { + if (*s == '\0' || (*s == '*' *(s+1) == '\0')) { *ret = GDKstrdup(str_nil); return MAL_SUCCEED; } while(TRUE) { - int cnt; - char op; - int nr_chars_read; bit advance_ref_pos; bit advance_seq_pos; - if (sscanf - (cigar_consumable, %d%c%n, cnt, op, -nr_chars_read) != 2) - throw(MAL, seq_char, - Error parsing CIGAR string '%s'\n, *alg_cigar); + next_cigar_op(seq_char); + advance_ref_pos = (op == 'M' || op == 'D' || op == 'N' || op == '=' || op == 'X'); - advance_seq_pos = (op == 'M' || op == 'I' || op == '='); // TODO: Find out which chars advance the seq pos + advance_seq_pos = (op == 'M' || op == 'I' || op == '='); // TODO: Find out which chars advance the seq pos exactly if(advance_seq_pos) { seq_pos += cnt; } @@ -257,8 +252,7 @@ seq_char(str * ret, int * ref_pos, str * break; } } - cigar_consumable += nr_chars_read; - if(cigar_consumable[0] == '\0') { + if(*s == '\0') { seq_pos = -1; break; } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Use GDKnr_threads within bam loader
Changeset: 72bd0543ee79 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=72bd0543ee79 Modified Files: sql/backends/monet5/bam/bam_loader.c Branch: bamloader Log Message: Use GDKnr_threads within bam loader diffs (21 lines): diff --git a/sql/backends/monet5/bam/bam_loader.c b/sql/backends/monet5/bam/bam_loader.c --- a/sql/backends/monet5/bam/bam_loader.c +++ b/sql/backends/monet5/bam/bam_loader.c @@ -422,7 +422,7 @@ bam_loader_repos(Client cntxt, MalBlkPtr /* For now, we hard code the number of threads, since the SQL level should not bother with this */ - sht nr_threads = 4; + sht nr_threads = GDKnr_threads; str *filenames = NULL; int nr_files = 0; @@ -543,7 +543,7 @@ bam_loader_files(Client cntxt, MalBlkPtr sht dbschema = *getArgReference_sht(stk, pci, pci-retc + 1); /* For now, we hard code the number of threads, since the SQL level should not bother with this */ - sht nr_threads = 4; + sht nr_threads = GDKnr_threads; str *filenames = NULL; int nr_files = 0; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Simplified and improved some tests, added c...
Changeset: 3f7305d007e1 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3f7305d007e1 Added Files: sql/backends/monet5/bam/Tests/bam_lib.reqtests sql/backends/monet5/bam/Tests/bam_lib.sql sql/backends/monet5/bam/Tests/coverage_usecase.reqtests sql/backends/monet5/bam/Tests/coverage_usecase.sql sql/backends/monet5/bam/Tests/mergetable.reqtests sql/backends/monet5/bam/Tests/mergetable.sql Removed Files: sql/backends/monet5/bam/Tests/bam_lib.SQL.py sql/backends/monet5/bam/Tests/mergetable.SQL.py sql/backends/monet5/bam/Tests/sql/bam_lib.sql sql/backends/monet5/bam/Tests/sql/mergetable.sql Modified Files: sql/backends/monet5/bam/Tests/All Branch: bamloader Log Message: Simplified and improved some tests, added coverage test based on BTW-paper usecase; however does not work yet diffs (truncated from 443 to 300 lines): diff --git a/sql/backends/monet5/bam/Tests/All b/sql/backends/monet5/bam/Tests/All --- a/sql/backends/monet5/bam/Tests/All +++ b/sql/backends/monet5/bam/Tests/All @@ -26,3 +26,4 @@ HAVE_SAMTOOLS?query2.11 HAVE_SAMTOOLS?query2.12 HAVE_SAMTOOLS?mergetable HAVE_SAMTOOLS?bam_lib +#HAVE_SAMTOOLS?coverage_usecase diff --git a/sql/backends/monet5/bam/Tests/bam_lib.SQL.py b/sql/backends/monet5/bam/Tests/bam_lib.SQL.py deleted file mode 100644 --- a/sql/backends/monet5/bam/Tests/bam_lib.SQL.py +++ /dev/null @@ -1,3 +0,0 @@ -import bam - -bam.exec_sql_file(bam_lib.sql, {'PWD': bam.SRCDIR}) \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/bam_lib.reqtests b/sql/backends/monet5/bam/Tests/bam_lib.reqtests new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/bam_lib.reqtests @@ -0,0 +1,1 @@ +bam_loader_file \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/bam_lib.sql b/sql/backends/monet5/bam/Tests/bam_lib.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/bam_lib.sql @@ -0,0 +1,133 @@ +SET SCHEMA bam; + +# BAM_FLAG + +# Should all give 'true' +SELECT bam_flag(1, 'mult_segm'); +SELECT bam_flag(2, 'prop_alig'); +SELECT bam_flag(4, 'segm_unma'); +SELECT bam_flag(8, 'next_unma'); +SELECT bam_flag(16, 'segm_reve'); +SELECT bam_flag(32, 'next_reve'); +SELECT bam_flag(64, 'firs_segm'); +SELECT bam_flag(128, 'last_segm'); +SELECT bam_flag(256, 'seco_alig'); +SELECT bam_flag(512, 'qual_cont'); +SELECT bam_flag(1024, 'opti_dupl'); +SELECT bam_flag(2048, 'supp_alig'); + +# Should all give 'false' +SELECT bam_flag(2, 'mult_segm'); +SELECT bam_flag(4, 'prop_alig'); +SELECT bam_flag(8, 'segm_unma'); +SELECT bam_flag(16, 'next_unma'); +SELECT bam_flag(4, 'segm_reve'); +SELECT bam_flag(2, 'next_reve'); +SELECT bam_flag(4, 'firs_segm'); +SELECT bam_flag(8, 'last_segm'); +SELECT bam_flag(16, 'seco_alig'); +SELECT bam_flag(32, 'qual_cont'); +SELECT bam_flag(64, 'opti_dupl'); +SELECT bam_flag(128, 'supp_alig'); + +# Should all give 'true' +SELECT bam_flag(2047, 'mult_segm'); +SELECT bam_flag(2047, 'prop_alig'); +SELECT bam_flag(2047, 'segm_unma'); +SELECT bam_flag(2047, 'next_unma'); +SELECT bam_flag(2047, 'segm_reve'); +SELECT bam_flag(2047, 'next_reve'); +SELECT bam_flag(2047, 'firs_segm'); +SELECT bam_flag(2047, 'last_segm'); +SELECT bam_flag(2047, 'seco_alig'); +SELECT bam_flag(2047, 'qual_cont'); +SELECT bam_flag(2047, 'opti_dupl'); +SELECT bam_flag(4095, 'supp_alig'); + +# Should fail +SELECT bam_flag(111, 'Fail-hard'); + +# Bulk +SELECT flag, + bam_flag(flag, 'mult_segm') AS mult_segm, + bam_flag(flag, 'prop_alig') AS prop_alig, + bam_flag(flag, 'segm_unma') AS segm_unma, + bam_flag(flag, 'next_unma') AS next_unma, + bam_flag(flag, 'segm_reve') AS segm_reve, + bam_flag(flag, 'next_reve') AS next_reve, + bam_flag(flag, 'firs_segm') AS firs_segm, + bam_flag(flag, 'last_segm') AS last_segm, + bam_flag(flag, 'seco_alig') AS seco_alig, + bam_flag(flag, 'qual_cont') AS qual_cont, + bam_flag(flag, 'opti_dupl') AS opti_dupl, + bam_flag(flag, 'supp_alig') AS supp_alig +FROM bam.alignments_1; + + + + +# REVERSE_SEQ +SELECT reverse_seq('A'); +SELECT reverse_seq('T'); +SELECT reverse_seq('C'); +SELECT reverse_seq('G'); +SELECT reverse_seq('R'); +SELECT reverse_seq('Y'); +SELECT reverse_seq('S'); +SELECT reverse_seq('W'); +SELECT reverse_seq('K'); +SELECT reverse_seq('M'); +SELECT reverse_seq('H'); +SELECT reverse_seq('D'); +SELECT reverse_seq('V'); +SELECT reverse_seq('B'); +SELECT reverse_seq('N'); +SELECT reverse_seq('ATCGRYSWKMHDVBN'); +SELECT reverse_seq('invalidchars'); + +# Bulk +SELECT seq, reverse_seq(seq) AS reverse_seq +FROM bam.alignments_1; + + +# REVERSE_QUAL +SELECT reverse_qual('Should be reversed'); + +# Bulk +SELECT qual, reverse_qual(qual) AS reverse_qual +FROM bam.alignments_1; + + + + +# SEQ_LENGTH +SELECT seq_length('18M'); +SELECT seq_length('3I5D6N'); +SELECT seq_length('3=1X1=1X43=1X16=1X33='); + +# Bulk +SELECT cigar, seq_length(cigar)
MonetDB: bamloader - Included bulk version of seq_char in tests
Changeset: 615ce04ed0c6 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=615ce04ed0c6 Modified Files: sql/backends/monet5/bam/Tests/bam_lib.stable.out sql/backends/monet5/bam/Tests/sql/bam_lib.sql Branch: bamloader Log Message: Included bulk version of seq_char in tests diffs (74 lines): diff --git a/sql/backends/monet5/bam/Tests/bam_lib.stable.out b/sql/backends/monet5/bam/Tests/bam_lib.stable.out --- a/sql/backends/monet5/bam/Tests/bam_lib.stable.out +++ b/sql/backends/monet5/bam/Tests/bam_lib.stable.out @@ -1,9 +1,9 @@ stdout of test 'bam_lib` in directory 'sql/backends/monet5/bam` itself: -# 10:07:17 -# 10:07:17 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=35421 --set mapi_usock=/var/tmp/mtest-25140/.s.monetdb.35421 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 --set embedded_r=yes -# 10:07:17 +# 08:43:53 +# 08:43:53 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=35352 --set mapi_usock=/var/tmp/mtest-9342/.s.monetdb.35352 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 --set embedded_r=yes +# 08:43:53 # MonetDB 5 server v11.20.0 # This is an unreleased version @@ -13,8 +13,8 @@ stdout of test 'bam_lib` in directory 's # Copyright (c) 1993-July 2008 CWI. # Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved # Visit http://www.monetdb.org/ for further information -# Listening for connection requests on mapi:monetdb://robin-xps13:35421/ -# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-25140/.s.monetdb.35421 +# Listening for connection requests on mapi:monetdb://robin-xps13:35352/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-9342/.s.monetdb.35352 # MonetDB/SQL module loaded # MonetDB/R module loaded @@ -51,9 +51,9 @@ Ready. # loading sql script: 90_generator.sql # loading sql script: 99_system.sql -# 10:07:18 -# 10:07:18 /usr/bin/python2 bam_lib.SQL.py bam_lib -# 10:07:18 +# 08:43:54 +# 08:43:54 /usr/bin/python2 bam_lib.SQL.py bam_lib +# 08:43:54 #SET SCHEMA bam; #SELECT bam_flag(1, 'mult_segm'); @@ -771,7 +771,15 @@ Ready. % char # type % 1 # length [ NULL ] +#SELECT 17922987 AS ref_pos, seq, pos, cigar, seq_char(17922987, seq, pos, cigar) AS seq_char +#FROM bam.alignments_1 +#WHERE seq_char(17922987, seq, pos, cigar) IS NOT NULL; +% .L, bam.alignments_1, bam.alignments_1, bam.alignments_1, .L # table_name +% ref_pos, seq,pos,cigar, seq_char # name +% int, clob, int,clob, char # type +% 8, 100,8, 21, 1 # length +[ 17922987, TATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT, 17922987, 3=1X1=1X43=1X16=1X33=,T ] -# 10:07:18 -# 10:07:18 Done. -# 10:07:18 +# 08:43:54 +# 08:43:54 Done. +# 08:43:54 diff --git a/sql/backends/monet5/bam/Tests/sql/bam_lib.sql b/sql/backends/monet5/bam/Tests/sql/bam_lib.sql --- a/sql/backends/monet5/bam/Tests/sql/bam_lib.sql +++ b/sql/backends/monet5/bam/Tests/sql/bam_lib.sql @@ -131,7 +131,7 @@ SELECT seq_char(11, 'ACTAGAATGGCT', 5, ' SELECT seq_char(16, 'ACTAGAATGGCT', 5, '3M1I3M1D5M'); SELECT seq_char(17, 'ACTAGAATGGCT', 5, '3M1I3M1D5M'); -# Bulk, does not work properly yet... -# SELECT 17922987 AS ref_pos, seq, pos, cigar, seq_char(17922987, seq, pos, cigar) AS seq_char -# FROM bam.alignments_1 -# WHERE seq_char(17922987, seq, pos, cigar) IS NOT NULL; +# Bulk +SELECT 17922987 AS ref_pos, seq, pos, cigar, seq_char(17922987, seq, pos, cigar) AS seq_char +FROM bam.alignments_1 +WHERE seq_char(17922987, seq, pos, cigar) IS NOT NULL; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Improved bam_flag bulk version by directly ...
Changeset: afdccf6751c0 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=afdccf6751c0 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Improved bam_flag bulk version by directly writing to BAT. Also attempted this for other functions, but there are still problems with those so the old versions are kept in tact. diffs (195 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -229,16 +229,44 @@ seq_char(str * ret, int * ref_pos, str * return MAL_SUCCEED; } +#define init_props() { \ + output-tsorted = TRUE; \ + output-trevsorted = TRUE; \ +} +#define update_props(TPE) { \ + output-tsorted = output-tsorted \ + (c == 0 || *cur_out = prev_out); \ + output-trevsorted = output-trevsorted \ + (c == 0 || *cur_out = prev_out); \ + output-T-nil = output-T-nil || *cur_out == TPE##_nil; \ + output-T-nonil = output-T-nonil *cur_out != TPE##_nil; \ +} +#define update_props_str() { \ + output-tsorted = output-tsorted \ + (c == 0 || strcmp(*cur_out, prev_out) = 0); \ + output-trevsorted = output-trevsorted \ + (c == 0 || strcmp(*cur_out, prev_out) = 0); \ + output-T-nil = output-T-nil || *cur_out == str_nil; \ + output-T-nonil = output-T-nonil *cur_out != str_nil; \ +} + +#define finish_props() { \ + BATsetcount(output, BATcount(input)); \ + BATseqbase(output, input-hseqbase); \ + output-tkey = FALSE; /* Tail values are not unique */ \ +} str bam_flag_bat(bat * ret, bat * bid, str * name) { - BAT *flags, *result; - BATiter li; - BUN p = 0, q = 0; + BAT *input, *output; + sht prev_out = 0; + sht *cur_in; + bit *cur_out; sht k; + BUN c; assert(ret != NULL bid != NULL name != NULL); @@ -247,33 +275,78 @@ bam_flag_bat(bat * ret, bat * bid, str * throw(MAL, bam_flag, Unknown flag name given: %s\n, *name); - if ((flags = BATdescriptor(*bid)) == NULL) + if ((input = BATdescriptor(*bid)) == NULL) throw(MAL, bam_flag_bat, RUNTIME_OBJECT_MISSING); /* allocate result BAT */ - result = BATnew(TYPE_void, TYPE_bit, BATcount(flags), TRANSIENT); - if (result == NULL) { + output = BATnew(TYPE_void, TYPE_bit, BATcount(input), TRANSIENT); + if (output == NULL) { throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL); } - BATseqbase(result, flags-hseqbase); - - li = bat_iterator(flags); - - BATloop(flags, p, q) { - sht t = *(sht *) BUNtail(li, p); - bit r = kth_bit(t, k); - - BUNappend(result, (ptr) r, FALSE); + + init_props(); + cur_in = (sht *) Tloc(input, BUNfirst(input)); + cur_out = (bit *) Tloc(output, BUNfirst(output)); + for(c = 0; c BATcount(input); ++c) { + *cur_out = kth_bit(*cur_in, k); + update_props(bit); + cur_in++; + prev_out = *cur_out++; } + finish_props(); /* release input BAT-descriptor */ - BBPreleaseref(flags-batCacheid); + BBPreleaseref(input-batCacheid); - BBPkeepref((*ret = result-batCacheid)); + BBPkeepref((*ret = output-batCacheid)); return MAL_SUCCEED; } +/* Commented out, since string heap addition needs to be added +str +reverse_seq_bat(bat * ret, bat * bid) +{ + BAT *input, *output; + str prev_out = 0; + str *cur_in; + str *cur_out; + BUN c; + str msg; + + assert(ret != NULL bid != NULL); + + if ((input = BATdescriptor(*bid)) == NULL) + throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING); +*/ + /* allocate result BAT */ +/* output = BATnew(TYPE_void, TYPE_str, BATcount(input), TRANSIENT); + if (output == NULL) { + throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); + } + + init_props(); + cur_in = (str *) Tloc(input, BUNfirst(input)); + cur_out = (str *) Tloc(output, BUNfirst(output)); + for(c = 0; c BATcount(input); ++c) { + if ((msg = reverse_seq(cur_out, cur_in)) != MAL_SUCCEED) { + BBPreleaseref(output-batCacheid); + return msg; + } + update_props_str(); + cur_in++; + prev_out = *cur_out++; + } + finish_props(); +*/ + /* release input BAT-descriptor */ +/* BBPreleaseref(input-batCacheid); + + BBPkeepref((*ret = output-batCacheid)); + + return MAL_SUCCEED; +}*/ + str reverse_seq_bat(bat * ret, bat * bid) { @@ -356,6 +429,50 @@ reverse_qual_bat(bat * ret, bat * bid) return MAL_SUCCEED; } +/*str Does not
MonetDB: bamloader - Replaced linear time case-statement by cons...
Changeset: 6a45c22ca31d for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6a45c22ca31d Modified Files: sql/backends/monet5/bam/Tests/bam_lib.stable.err sql/backends/monet5/bam/Tests/bam_lib.stable.out sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Replaced linear time case-statement by constant time dictionary lookup for reverse_seq udf, also improved its output diffs (296 lines): diff --git a/sql/backends/monet5/bam/Tests/bam_lib.stable.err b/sql/backends/monet5/bam/Tests/bam_lib.stable.err --- a/sql/backends/monet5/bam/Tests/bam_lib.stable.err +++ b/sql/backends/monet5/bam/Tests/bam_lib.stable.err @@ -1,9 +1,9 @@ stderr of test 'bam_lib` in directory 'sql/backends/monet5/bam` itself: -# 09:28:35 -# 09:28:35 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=33399 --set mapi_usock=/var/tmp/mtest-22883/.s.monetdb.33399 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 --set embedded_r=yes -# 09:28:35 +# 14:48:31 +# 14:48:31 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=35990 --set mapi_usock=/var/tmp/mtest-20228/.s.monetdb.35990 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 --set embedded_r=yes +# 14:48:31 # builtin opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo # builtin opt gdk_debug = 0 @@ -17,71 +17,23 @@ stderr of test 'bam_lib` in directory 's # builtin opt sql_debug = 0 # cmdline opt gdk_nr_threads = 0 # cmdline opt mapi_open = true -# cmdline opt mapi_port = 33399 -# cmdline opt mapi_usock = /var/tmp/mtest-22883/.s.monetdb.33399 +# cmdline opt mapi_port = 35990 +# cmdline opt mapi_usock = /var/tmp/mtest-20228/.s.monetdb.35990 # cmdline opt monet_prompt = # cmdline opt mal_listing = 2 # cmdline opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam # cmdline opt mal_listing = 0 # cmdline opt embedded_r = yes # cmdline opt gdk_debug = 536870922 -# bam_loader: Loader started for 1 BAM file... -# # bam_loader Retrieving next file id... -# # bam_loader Initializing BAM wrapper for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... -# # bam_loader Parsing header for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... -# # bam_loader Creating alignment tables for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... -# # CREATE TABLE bam.alignments_1 ( -# virtual_offset BIGINTNOT NULL, -# qnameSTRING NOT NULL, -# flag SMALLINT NOT NULL, -# rnameSTRING NOT NULL, -# posINT NOT NULL, -# mapq SMALLINT NOT NULL, -# cigarSTRING NOT NULL, -# rnextSTRING NOT NULL, -# pnextINT NOT NULL, -# tlen INT NOT NULL, -# seqSTRING NOT NULL, -# qual STRING NOT NULL, -# CONSTRAINT alignments_1_pkey_virtual_offset PRIMARY KEY (virtual_offset) -# ); -# -# CREATE TABLE bam.alignments_extra_1 ( -# tagCHAR(2) NOT NULL, -# virtual_offset BIGINTNOT NULL, -# type CHAR(1) NOT NULL, -# valueSTRING, -# CONSTRAINT alignments_extra_1_pkey_tag_virtual_offset PRIMARY KEY (tag, virtual_offset), -# CONSTRAINT alignments_extra_1_fkey_virtual_offset FOREIGN KEY (virtual_offset) -# REFERENCES bam.alignments_1 (virtual_offset) -# ); -# # bam_loader Creating reader threads... -# # bam_loader Waiting for reader threads to finish... -# # Thread 0 Starting on next file... -# # Thread 0 Processing alignments of file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam' (file id 1)... -# # Thread 0 All alignments in file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam' (file id 1)
MonetDB: bamloader - Always clean up on exceptions
Changeset: df1858231ff4 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=df1858231ff4 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Always clean up on exceptions diffs (127 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -264,6 +264,7 @@ bam_flag_bat(bat * ret, bat * bid, str * /* allocate result BAT */ output = BATnew(TYPE_void, TYPE_bit, BATcount(input), TRANSIENT); if (output == NULL) { + BBPreleaseref(input-batCacheid); throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL); } @@ -345,6 +346,7 @@ reverse_seq_bat(bat * ret, bat * bid) /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_str, BATcount(seqs), TRANSIENT); if (result == NULL) { + BBPreleaseref(seqs-batCacheid); throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); } BATseqbase(result, seqs-hseqbase); @@ -356,6 +358,7 @@ reverse_seq_bat(bat * ret, bat * bid) str r, msg; if ((msg = reverse_seq(r, t)) != MAL_SUCCEED) { + BBPreleaseref(seqs-batCacheid); BBPreleaseref(result-batCacheid); return msg; } @@ -386,6 +389,7 @@ reverse_qual_bat(bat * ret, bat * bid) /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_str, BATcount(quals), TRANSIENT); if (result == NULL) { + BBPreleaseref(quals-batCacheid); throw(MAL, reverse_qual_bat, MAL_MALLOC_FAIL); } BATseqbase(result, quals-hseqbase); @@ -397,6 +401,7 @@ reverse_qual_bat(bat * ret, bat * bid) str r, msg; if ((msg = reverse_qual(r, t)) != MAL_SUCCEED) { + BBPreleaseref(quals-batCacheid); BBPreleaseref(result-batCacheid); return msg; } @@ -471,6 +476,7 @@ seq_length_bat(bat * ret, bat * bid) /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_int, BATcount(cigars), TRANSIENT); if (result == NULL) { + BBPreleaseref(cigars-batCacheid); throw(MAL, seq_length_bat, MAL_MALLOC_FAIL); } BATseqbase(result, cigars-hseqbase); @@ -483,6 +489,7 @@ seq_length_bat(bat * ret, bat * bid) int r; if ((msg = seq_length(r, t)) != MAL_SUCCEED) { + BBPreleaseref(cigars-batCacheid); BBPreleaseref(result-batCacheid); return msg; } @@ -500,27 +507,33 @@ seq_length_bat(bat * ret, bat * bid) str seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * alg_cigar) { - BAT *seqs, *poss, *cigars, *result; + BAT *seqs = NULL, *poss = NULL, *cigars = NULL, *result = NULL; BUN seq = 0, pos = 0, cigar = 0, seq_end = 0; BATiter seq_it, pos_it, cigar_it; + str msg = MAL_SUCCEED; + assert(ret != NULL ref_pos != NULL alg_seq != NULL alg_pos != NULL alg_cigar != NULL); if ((seqs = BATdescriptor(*alg_seq)) == NULL || (poss = BATdescriptor(*alg_pos)) == NULL || - (cigars = BATdescriptor(*alg_cigar)) == NULL) - throw(MAL, seq_char_bat, RUNTIME_OBJECT_MISSING); + (cigars = BATdescriptor(*alg_cigar)) == NULL) { + msg = createException(MAL, seq_char_bat, RUNTIME_OBJECT_MISSING); + goto cleanup; + } if(BATcount(seqs) != BATcount(poss) || BATcount(seqs) != BATcount(cigars)) { - throw(MAL, seq_char_bat, + msg = createException(MAL, seq_char_bat, Misalignment in input BATs: BUNFMT/BUNFMT/BUNFMT, BATcount(poss), BATcount(seqs), BATcount(cigars)); + goto cleanup; } /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_str, BATcount(cigars), TRANSIENT); if (result == NULL) { - throw(MAL, seq_char_bat, MAL_MALLOC_FAIL); + msg = createException(MAL, seq_char_bat, MAL_MALLOC_FAIL); + goto cleanup; } BATseqbase(result, seqs-hseqbase); @@ -541,8 +554,7 @@ seq_char_bat(bat * ret, int * ref_pos, b str msg; if ((msg = seq_char(r, ref_pos, seq_val, pos_val, cigar_val)) != MAL_SUCCEED) { - BBPreleaseref(result-batCacheid); - return msg; + goto cleanup; } BUNappend(result, (ptr) r, FALSE); ++seq; @@ -550,12 +562,13 @@ seq_char_bat(bat * ret, int * ref_pos, b ++cigar; } +cleanup: /*
MonetDB: bamloader - Improved bam_flag udfs by using lookup tabl...
Changeset: 5b5cbb2c5217 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5b5cbb2c5217 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Improved bam_flag udfs by using lookup tables instead of serial string comparison diffs (96 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -26,26 +26,78 @@ #include monetdb_config.h #include bam_lib.h -#define flag_str2sht(flag_str) \ - (strcmp(flag_str, mult_segm) == 0 ? 0 : \ -(strcmp(flag_str, prop_alig) == 0 ? 1 : \ - (strcmp(flag_str, segm_unma) == 0 ? 2 : \ - (strcmp(flag_str, next_unma) == 0 ? 3 :\ - (strcmp(flag_str, segm_reve) == 0 ? 4 : \ -(strcmp(flag_str, next_reve) == 0 ? 5 : \ - (strcmp(flag_str, firs_segm) == 0 ? 6 : \ - (strcmp(flag_str, last_segm) == 0 ? 7 :\ - (strcmp(flag_str, seco_alig) == 0 ? 8 : \ -(strcmp(flag_str, qual_cont) == 0 ? 9 : \ - (strcmp(flag_str, opti_dupl) == 0 ? 10 :\ - (strcmp(flag_str, supp_alig) == 0 ? 11 : -1 +/* Map string to integer by adding all odd and subtracting all even numbers + * For the strings that we want to detect, this gives the following: + * mult_segm - 71 + * prop_alig - 101 + * segm_unma - 84 + * next_unma - 89 + * segm_reve - 73 + * next_reve - 78 + * firs_segm - 83 + * last_segm - 97 + * seco_alig - 106 + * qual_cont - 98 + * opti_dupl - 118 + * supp_alig - 102 + * + * These integers fit in an array of 48 positions + * Define two arrays: + * - One that maps int position to the bit position that we + * are looking for + * - One that maps bit position to string, so we can check if + * the input really was one of the valid strings (since many) + * strings will map to the same integer + */ + +sht intbit_map[] = { +0, -1, 4, -1, -1, -1, -1, 5, -1, -1, + -1, -1, 6, 2, -1, -1, -1, -1, 3, -1, + -1, -1, -1, -1, -1, -1, 7, 9, -1, -1, +1, 11, -1, -1, -1, 8, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 10 +}; + +str posstr_map[] = { + mult_segm, + prop_alig, + segm_unma, + next_unma, + segm_reve, + next_reve, + firs_segm, + last_segm, + seco_alig, + qual_cont, + opti_dupl, + supp_alig +}; + +static sht +flag_str2sht(str flag_str) { + int i = 0; + int mult = 1; + int strnum = -71; + char c; + sht k = -1; + while((c = flag_str[i++]) != '\0') { + strnum += mult * (int)c; + mult *= -1; + } + if(strnum 0 || strnum 47 || (k = intbit_map[strnum]) 0 || + strcmp(posstr_map[k], flag_str) != 0) { + k = -1; + } + return k; +} #define kth_bit(flag, k) ((flag (1 k)) == (1 k)) str bam_flag(bit * ret, sht * flag, str * name) { - sht k = flag_str2sht(*name); + sht k; + k = flag_str2sht(*name); if (k 0) throw(MAL, bam_flag, Unknown flag name given: %s\n, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Keep forward and backward pointers for reve...
Changeset: 86ac986f6f30 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=86ac986f6f30 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Keep forward and backward pointers for reverse_seq and reverse_qual to improve performance diffs (46 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -141,17 +141,21 @@ reverse_seq(str * ret, str * seq) unsigned int i; unsigned int len = strlen(*seq); sht map_index; + str forward = *seq; + str backward; result = GDKmalloc((len + 1) * sizeof(char)); if (result == NULL) throw(MAL, reverse_seq, MAL_MALLOC_FAIL); + backward = result[len-1]; for (i = 0; i len; ++i) { - map_index = (sht)((*seq)[i] - 'A'); + map_index = (sht)(*forward++ - 'A'); if(map_index 0 || map_index 24 || - (result[len - i - 1] = reverse_seq_map[map_index]) == 0) { - result[len - i - 1] = '?'; + (*backward = reverse_seq_map[map_index]) == 0) { + *backward = '?'; } + --backward; } result[len] = '\0'; *ret = result; @@ -164,12 +168,16 @@ reverse_qual(str * ret, str * qual) str result; unsigned int i; unsigned int len = strlen(*qual); + str forward = *qual; + str backward; result = GDKmalloc((len + 1) * sizeof(char)); if (result == NULL) throw(MAL, reverse_qual, MAL_MALLOC_FAIL); + + backward = result[len-1]; for (i = 0; i len; ++i) - result[len - i - 1] = (*qual)[i]; + *backward-- = *forward++; result[len] = '\0'; *ret = result; return MAL_SUCCEED; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Added thorough testing of bam_lib and added...
Changeset: 91d6d8630be1 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=91d6d8630be1 Added Files: sql/backends/monet5/bam/Tests/bam.pyc sql/backends/monet5/bam/Tests/bam_lib.SQL.py sql/backends/monet5/bam/Tests/bam_lib.stable.err sql/backends/monet5/bam/Tests/bam_lib.stable.out sql/backends/monet5/bam/Tests/sql/bam_lib.sql Removed Files: sql/backends/monet5/bam/Tests/sql/drop_last_files.sql Modified Files: sql/backends/monet5/bam/Tests/All sql/backends/monet5/bam/Tests/bam_loader_files.SQL.py sql/backends/monet5/bam/Tests/bam_loader_files.stable.err sql/backends/monet5/bam/Tests/bam_loader_files.stable.out sql/backends/monet5/bam/Tests/check_files.sql sql/backends/monet5/bam/Tests/check_files.stable.out sql/backends/monet5/bam/Tests/check_files_exact.sql sql/backends/monet5/bam/Tests/drop_last_files.SQL.py sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql sql/backends/monet5/bam/Tests/sql/sam_export.sql Branch: bamloader Log Message: Added thorough testing of bam_lib and added test for just removed bug that occurred when loading many files in one command diffs (truncated from 3716 to 300 lines): diff --git a/sql/backends/monet5/bam/Tests/All b/sql/backends/monet5/bam/Tests/All --- a/sql/backends/monet5/bam/Tests/All +++ b/sql/backends/monet5/bam/Tests/All @@ -25,3 +25,4 @@ HAVE_SAMTOOLS?query2.10 HAVE_SAMTOOLS?query2.11 HAVE_SAMTOOLS?query2.12 HAVE_SAMTOOLS?mergetable +HAVE_SAMTOOLS?bam_lib diff --git a/sql/backends/monet5/bam/Tests/bam.pyc b/sql/backends/monet5/bam/Tests/bam.pyc new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..920c4a9a9889e1cc40f6d04cb102a0f9b57a828a GIT binary patch literal 1282 zc$}?M2G~`5dPM7e%nA*^*}4)Pqm0yo4@(+E|FQtVc#B2bmuTWZIDwYRj9oKt$| zA$SNb+;|iofd_z@jZ?(ER(5t~clMk4W+vwEUiZhZpPrBUYtVW{tNcbHpr1etjBa- zkyzmMh2!E+B6WGNGyyjh++BmTX|sMKOdBqW0Ifslc%Wq9y_po1+$L!y==tjSiwN zBwe%-G%zYT0zjn4W+S)kf2()p)`7oDg_{GyFr}zAZqEfRT=`LZlkm-IvL~y7c zuf2h4F3jCMS24{)S*r4uIOFu0xXmb(hG2Z+#k#@C){0dtb76WmLoPw-(AESwNycv zQJO*mujiXqOjS|zV{Ik3*R%Ws!QYcQD|%_$SJjB%Jdskxr4FhN%|OQqV~alH3O zZhzoVJm+-mPh=XDPQrK3$W9#1inS;CR2LwXS0{^5D-3HexbS)F(CRK}e_zm^12f_* zpbSJNB4XR%LR8RPwuXz8+y_;`%Kr6T6@Km|3VS%o@1fMkVOzA!bcf4eo2zRg-s z^pIRG-e6b}bPG|NUFHJf;wgqhnvoC3eAD3^K9X|_vo?rAw4!}m+L!HBZMU^epch% zeU}TVcm@i)$AxBUy1H3Te6|*YC9jRbFD;K-718o(i6wLgw+moE2GH}t9qmDIKK zvn07pLw`)RZuzuy`L)aM?9MXWSr|(Vm5lQ|Oh4Kd@awkLmZ|H2bC?r8Ehh_dgqbh zo#*9lhk=VCucd!N-Y1;oeMa8U^UyK2MNc$oZHX;oU91WF9y8O0WEWk(LMlG3gDV0j zdW(VP`s^inR;WQykYsu?{SpgN0be-@6)!@6tB4!Mv28~eMX%YW$SNfunj@#v5 Uu9Mgnrr02sYetXc1EXX81w@VghX4Qo diff --git a/sql/backends/monet5/bam/Tests/bam_lib.SQL.py b/sql/backends/monet5/bam/Tests/bam_lib.SQL.py new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/bam_lib.SQL.py @@ -0,0 +1,3 @@ +import bam + +bam.exec_sql_file(bam_lib.sql, {'PWD': bam.SRCDIR}) \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/bam_lib.stable.err b/sql/backends/monet5/bam/Tests/bam_lib.stable.err new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/bam_lib.stable.err @@ -0,0 +1,87 @@ +stderr of test 'bam_lib` in directory 'sql/backends/monet5/bam` itself: + + +# 09:28:35 +# 09:28:35 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=33399 --set mapi_usock=/var/tmp/mtest-22883/.s.monetdb.33399 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 --set embedded_r=yes +# 09:28:35 + +# builtin opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 33399 +# cmdline opt mapi_usock = /var/tmp/mtest-22883/.s.monetdb.33399 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam +# cmdline opt mal_listing = 0 +# cmdline opt embedded_r = yes +# cmdline opt gdk_debug = 536870922 +# bam_loader: Loader started for 1 BAM file... +# # bam_loader Retrieving next file id... +# # bam_loader Initializing BAM wrapper for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... +# # bam_loader Parsing header for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... +# # bam_loader Creating alignment tables for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... +# # CREATE TABLE bam.alignments_1 ( +#
MonetDB: bamloader - Merge with default
Changeset: 85bbd04c51b0 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=85bbd04c51b0 Added Files: sql/test/BugTracker-2014/Tests/case-sqrt.Bug-3627.sql sql/test/BugTracker-2014/Tests/case-sqrt.Bug-3627.stable.err sql/test/BugTracker-2014/Tests/case-sqrt.Bug-3627.stable.out sql/test/BugTracker-2014/Tests/column-with-comma.Bug-3616.sql sql/test/BugTracker-2014/Tests/column-with-comma.Bug-3616.stable.err sql/test/BugTracker-2014/Tests/column-with-comma.Bug-3616.stable.out Modified Files: .hgtags MonetDB.spec NT/installer32/MonetDB-ODBC-Installer.vdproj NT/installer32/MonetDB5-Geom-Module.vdproj NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB-ODBC-Installer.vdproj NT/installer64/MonetDB5-Geom-Module.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj NT/monetdb_config.h.in NT/rules.msc clients/Tests/exports.stable.out clients/mapilib/mapi.rc clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc clients/python2/setup.py clients/python3/setup.py configure.ag debian/changelog gdk/gdk.h gdk/gdk_atoms.h gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_bbp.c gdk/gdk_private.h gdk/libbat.rc libversions monetdb5/modules/atoms/mtime.c monetdb5/modules/atoms/mtime.h monetdb5/modules/mal/sysmon.c monetdb5/tools/libmonetdb5.rc sql/backends/monet5/bam/bam_lib.c sql/backends/monet5/sql_result.c sql/jdbc/tests/Tests/Test_PSsqldata.stable.out sql/test/BugTracker-2014/Tests/All sql/test/pg_regress/Tests/alter_table.stable.err sql/test/pg_regress/Tests/arrays.stable.err sql/test/pg_regress/Tests/boolean.stable.err sql/test/pg_regress/Tests/case.stable.err sql/test/pg_regress/Tests/char.stable.err sql/test/pg_regress/Tests/comments.stable.err sql/test/pg_regress/Tests/create_aggregate.stable.err sql/test/pg_regress/Tests/create_misc.stable.err sql/test/pg_regress/Tests/create_operator.stable.err sql/test/pg_regress/Tests/date.stable.err sql/test/pg_regress/Tests/date.stable.out sql/test/pg_regress/Tests/drop.stable.err sql/test/pg_regress/Tests/float4.stable.err sql/test/pg_regress/Tests/float8.stable.err sql/test/pg_regress/Tests/inet.stable.err sql/test/pg_regress/Tests/inet.stable.out sql/test/pg_regress/Tests/insert.stable.err sql/test/pg_regress/Tests/int2.stable.err sql/test/pg_regress/Tests/int4.stable.err sql/test/pg_regress/Tests/int8.stable.err sql/test/pg_regress/Tests/interval.stable.err sql/test/pg_regress/Tests/limit.stable.err sql/test/pg_regress/Tests/numeric.stable.err sql/test/pg_regress/Tests/numeric_big.stable.err sql/test/pg_regress/Tests/numerology.stable.err sql/test/pg_regress/Tests/oid.stable.err sql/test/pg_regress/Tests/select_into.stable.err sql/test/pg_regress/Tests/strings.stable.err sql/test/pg_regress/Tests/strings_cast.stable.err sql/test/pg_regress/Tests/strings_concat.stable.err sql/test/pg_regress/Tests/timestamp.stable.err sql/test/pg_regress/Tests/timestamptz.stable.err sql/test/pg_regress/Tests/varchar.stable.err sql/test/pg_regress/Tests/without_oid.stable.err vertoo.data Branch: bamloader Log Message: Merge with default diffs (truncated from 7109 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -583,3 +583,6 @@ 4f894ebf1a93ae895ec23d54ce2112bf21a6427e 88a29a5a2bef5c6cef6b3bfa5ab7c4d3b855a2c8 Oct2014_release eb2967c408350131b63f5393704fd697c701fc49 Oct2014_5 2da8cbcb65ed56690bad29f089d996eac347b48d Oct2014_SP1_release +ee3d8e3af6aae6be6cfaa876a379a0751f164e97 Oct2014_7 +2da8cbcb65ed56690bad29f089d996eac347b48d Oct2014_SP1_release +6607367487d91eb02c5eb7dfb9d96606f8f7c6f0 Oct2014_SP1_release diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -960,6 +960,9 @@ mv $RPM_BUILD_ROOT%{_datadir}/doc/MonetD rm -fr $RPM_BUILD_ROOT %changelog +* Fri Nov 21 2014 Sjoerd Mullender sjo...@acm.org - 11.19.7-20141121 +- Rebuilt. + * Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 - Rebuilt. - BZ#3580: cosmetic change (append newline) diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -1658,7 +1658,7 @@ str MTIMEtime_add_msec_interval_wrap(day str MTIMEtime_sub_msec_interval_wrap(daytime *ret, const daytime *t, const lng *msec); str MTIMEtimestamp(timestamp *ret, const int *sec); str MTIMEtimestamp2timestamp(timestamp *ret, const timestamp *src); -str
MonetDB: bamloader - Revert bam_lib.c and bam.mal back to their ...
Changeset: 652329ec295a for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=652329ec295a Modified Files: sql/backends/monet5/bam/bam.mal sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Revert bam_lib.c and bam.mal back to their previous bulk operator for seq_char diffs (167 lines): diff --git a/sql/backends/monet5/bam/bam.mal b/sql/backends/monet5/bam/bam.mal --- a/sql/backends/monet5/bam/bam.mal +++ b/sql/backends/monet5/bam/bam.mal @@ -53,7 +53,9 @@ pattern bam_export(output_path:str):void address bam_exportf comment Export results in the bam.export table to a BAM file + # BAT signatures for bam_lib + module batbam; command bam_flag(flags:bat[:oid,:sht], name:str):bat[:oid,:bit] @@ -72,6 +74,6 @@ command seq_length(cigars:bat[:oid,:str] address seq_length_bat comment Calculate the real length of a bat of DNA sequences, given their CIGAR string. -command seq_char(ref_pos:bat[:oid,:int], alg_seq:bat[:oid,:str], alg_pos:bat[:oid,:int], alg_cigar:bat[:oid,:str]):bat[:oid,:str] +command seq_char(ref_pos:int, alg_seq:bat[:oid,:str], alg_pos:bat[:oid,:int], alg_cigar:bat[:oid,:str]):bat[:oid,:str] address seq_char_bat comment Calculate the character in the alignment string (alg_str) that is aligned to position 'ref_pos', conforming to the given cigar string (bat based version) diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -54,7 +54,6 @@ bam_flag(bit * ret, sht * flag, str * na return MAL_SUCCEED; } -// use a simple lookup table for these mappings str reverse_seq(str * ret, str * seq) { @@ -254,7 +253,6 @@ bam_flag_bat(bat * ret, bat * bid, str * /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_bit, BATcount(flags), TRANSIENT); if (result == NULL) { - BBPreleaseref(flags-batCacheid); throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL); } BATseqbase(result, flags-hseqbase); @@ -291,7 +289,6 @@ reverse_seq_bat(bat * ret, bat * bid) /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_str, BATcount(seqs), TRANSIENT); if (result == NULL) { - BBPreleaseref(seqs-batCacheid); throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); } BATseqbase(result, seqs-hseqbase); @@ -304,7 +301,6 @@ reverse_seq_bat(bat * ret, bat * bid) if ((msg = reverse_seq(r, t)) != MAL_SUCCEED) { BBPreleaseref(result-batCacheid); - BBPreleaseref(seqs-batCacheid); return msg; } BUNappend(result, (ptr) r, FALSE); @@ -334,7 +330,6 @@ reverse_qual_bat(bat * ret, bat * bid) /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_str, BATcount(quals), TRANSIENT); if (result == NULL) { - BBPreleaseref(quals-batCacheid); throw(MAL, reverse_qual_bat, MAL_MALLOC_FAIL); } BATseqbase(result, quals-hseqbase); @@ -346,7 +341,6 @@ reverse_qual_bat(bat * ret, bat * bid) str r, msg; if ((msg = reverse_qual(r, t)) != MAL_SUCCEED) { - BBPreleaseref(quals-batCacheid); BBPreleaseref(result-batCacheid); return msg; } @@ -377,7 +371,6 @@ seq_length_bat(bat * ret, bat * bid) /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_int, BATcount(cigars), TRANSIENT); if (result == NULL) { - BBPreleaseref(cigars-batCacheid); throw(MAL, seq_length_bat, MAL_MALLOC_FAIL); } BATseqbase(result, cigars-hseqbase); @@ -391,7 +384,6 @@ seq_length_bat(bat * ret, bat * bid) if ((msg = seq_length(r, t)) != MAL_SUCCEED) { BBPreleaseref(result-batCacheid); - BBPreleaseref(cigars-batCacheid); return msg; } BUNappend(result, (ptr) r, FALSE); @@ -409,26 +401,18 @@ seq_length_bat(bat * ret, bat * bid) str seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * alg_cigar) { - BAT *seqs = NULL, *poss = NULL, *refs = NULL, *cigars = NULL, *result = NULL; - BUN ref= 0, seq = 0, pos = 0, cigar = 0, seq_end = 0; - BATiter ref_it, seq_it, pos_it, cigar_it; + BAT *seqs, *poss, *cigars, *result; + BUN seq = 0, pos = 0, cigar = 0, seq_end = 0; + BATiter seq_it, pos_it, cigar_it; assert(ret != NULL ref_pos != NULL alg_seq != NULL alg_pos != NULL alg_cigar != NULL); if ((seqs = BATdescriptor(*alg_seq)) == NULL || (poss = BATdescriptor(*alg_pos)) == NULL || - (refs = BATdescriptor(*ref_pos)) == NULL || - (cigars = BATdescriptor(*alg_cigar)) == NULL) {
MonetDB: default - Add decent graphics of database schemas that ...
Changeset: 7c9cec2601fa for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7c9cec2601fa Added Files: sql/backends/monet5/bam/default_schema.dia sql/backends/monet5/bam/default_schema.png sql/backends/monet5/bam/metadata.dia sql/backends/monet5/bam/metadata.png sql/backends/monet5/bam/paired_schema.dia sql/backends/monet5/bam/paired_schema.png Branch: default Log Message: Add decent graphics of database schemas that we can use in the documentation diffs (truncated from 5842 to 300 lines): diff --git a/sql/backends/monet5/bam/default_schema.dia b/sql/backends/monet5/bam/default_schema.dia new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..682ce8e1628447e46f8f8dd2955adcac4bcf49ef GIT binary patch literal 1422 zc$@)_1#$WxiwFP!21MOUGZ=*O6{@!1KsI;HD1lXiWw%e}SJ71lyMr-`-~iL zh_8;F+9qwj+;2bgk{2K(0TSA#MrtKsn8zMJo*AZ7zI=TiV{HlN0eOZmSO4|Brf$3 z@h|m%?*IJQ)4yJweDM(YEPs3k#+rPB@bK(X9|w}+-f~NJsEh$fuPL5C@?r|{Rc1x zEh*H}uTC^=Re=Wt2(Kkqfe;K00s%Dw#_=f0QbRXG$5WHOC@qH#gu7NfG_paVRY!N zL`^HDXM^qu@Zo?#@KCJUlt;f`MfC)-tlIH}awJ8Y?2cx=dY%kx8Vu6e9lB2=w z9#V2)scYRp3qiyfFdvaTjgMq4PhHS+qR`KI{jYX?6$ks+$QDW2F=3ZX5m~aELn- z$^-*QRJ%?VhBi#Dpxk8T*XNwH1SsxgZp5`u0S{}5ol%LL-fi``as=lTqJnTJs zqCm?N^2Eq_K9IR$?Xv@EpCXP17_J*VB4TgqXZul~U7g5#FCVYThFDu+c_3#13Trz zgC8Id@+~w=%M@rDC1R~@R?o`qYMA^CRj%h7|b9`)ZZ+_w3aMz;C+K3N)Rct=NPP z{)MhcCi{m}JOCVOAGLcRAJyh|EEdcd7q2)8@)fP^y!jFlVm`U6870*!{D%(dl%u zR=F1B80p37+g0P!)*f?QZV!#qE;I)+^paSR2}xA5IVuBfIRVmNkO%1Ay9*`w7W zl*nP_EjMoeWQEcUpxiV2vR1N1_aGL5dedRW!K1bh}w=nMjuU!!yeX`0m_WWFB zh8QA@cYsKz*cE26g#o5vndUxLC6h3x(q);!_+$);;0{`wIF|vE`!G~hgjHMrJWLkg zY)Xwvu^9L=TlEhOBLf3u5fLT~$qaesJiysTuyndw25UgEw+h1#H7P=6vX7l1D$-! zrVYUu;hA*flu$kau8eA2%)-bhV_K}@o7{H@vg%ylj?Mw$3*g=Y`3J;PD3A+2-bi~@ z8$d#uvSY{(;r9Te$DsZ)Y2#o(aZRmc1_#w0GW$m809LwVOj-obzphzJumXCa?Y z@3$5=GCp*QmZ`(3OpQBdmG@P1I_^5keY!QR|{zx{Q;H@LiK-7Q#D!CLQJcdG}K zMW5oQL?@bD);JM9xEV`gV^Jw*1iBWNAAZ^pfOJWeSdfR-*7jdJjZg`_Tn^unwj zVFGNc6gk{(KLvTG9H2Y0Z?gR;)fcR!i-Taay@)Md*VQdJHCyD*I5hJ~CRn$OlYi zABxpSmQ%xLp;)b0eOx(pqpVixhobbsDJ?K0iqVSE$HQn2A64R^IDJ%{ehdIsq*kPE zgw*RhomPc-Wq+rR^2{d+cMgRZ}eS3x%gxm_W4=Q=)L7e-`Zw-b)CL#5RlVtE)PF z-v=xBbSZ=)$XgeO0@!YWG#`zOT96S2Wo=_C8)RFChAzLS5z;v(I|{=pX)s*zoh z{D?%s%?zY-+o-*sj`!*r5^Ynj{pu-yYCxjb^Va=wrZ(a0nSB?z(!aUALYZf5+h5 z=pMA5I!Ce0V_9m{oYyE=$Nmvo=TY*Igx=%o2{}~T%D~ZR(fufp*6)JNP=lwwW8k z@}9Nfo^l|AP|(%w0qxY8RBRyp2|1Be1B-Q-Qzfn}KFcSVH!(qTf0pBpqiho%k zoy-8PsMYx?V73`42i#ai8^5wvwPC0vCXZ*!wviZe+#+^Ss?59hKSht$S-@w#rQ zf@UXE=j-hTQX{vz?GQ+$*uY9c=`Nm{Ah=#xv@@~+k`OvfGVJmsq$~@f8e|su3 cq-@d?pGTjWx;jaHhwAFf3BbKDyvce082o?{{R30 diff --git a/sql/backends/monet5/bam/default_schema.png b/sql/backends/monet5/bam/default_schema.png new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f1aa49a5c4c81e2e3babd7aa127fe5df45701854 GIT binary patch literal 54296 zc$}oX1yq%7x9`0W1yni=|(|8N;;GZb3p3q`MoXB_#xu5RgU?kZz)3F!{$7Nj|I z`MmG9_jmTU_jkrP_wX662UgtIyypDR-+#^fHbhxb8tVq4Fm##B`YJLia?;QAP^`6 z7+2sYI}Oe%@E59sn5;Sm2FApk(k%S%bz2#22LuA=Bl16#kLF=62n01kR^pMmYtqJ) zo2Y`C1Pi}eEUP@;Qgdt7GqcOehCKat1=J~C#kX(p)dmGllpGlf77DKNcts4+) zhM_Gct7_KoTr)L8_5Czu#%;T9YRlWkQ{92c|U1cu;RnMSD!=5H5vrsh?b9^pF^|= z|JE(W)do@oB8vXDEHUDr;|C|-+4r$hliuE5hNpxG1a6ttM0v8Xm(D;YLhOO-g{^T zx@hGOrmdky1uIFw#{Ysyfqxg#tc=oCtHPm6a8lY79}{*w|QVOB6)?RSu5JnMY{V zUH|%ko1(20e;@hkBbc}T`Qh?%68~~AAKbDeIxMC59)lPG^5tz7#!kNx+NCwv z)2@qvTZn;V?HhkG-t6ox2hrQwa|je($Mpni|Yz2f7f8y(|_f{03xi)US0US6=e zc{5bI^rdXMAo8^t)A$PRa(pQ0@VN$s@RZGo3LF~`H@BwynDBclcR8-r`%72JMO)AJ z@$m5nrl%t+(6Pzyj5Q5X9cbxWJD^}PH(q_*oypGm0%nSY?S(ek1DX5Wc_t1c-1 zj`A1qF%{#!goTB1TMSoLR(hTvjGR-MDcBP8kqbLa3-QC@yyV`EX61shdhd}6|7 zYm(1#McLTc7!?)O#R#@?hPnJ3EbHNNDIn)KfXR?%~f*?Xdaz`BPF-T(+l;nSGOz z$d6;;;eG9rmeJ8Cl9H`mUFO@b$WVw7-@xtg$W4cyv-Zd3kFqE3U4tlT{8un@r3s zEKjTKmxA%=9334A2?;T=u=I3wBgpv%`uqDA+Lv{oQBhId2oS4fm%zx(%v6yP_B`_a zGr0P@vXa-uXFI$=$!lZ8EZn(b20{uWn3K6xG%yO%w`(YWzLCh_{3(k}J;drZcb zrBcaO9xfns6t?(z1UmqY)P|eIDgml%vM`5-)v(U^W)nLu6kKYr})?tUaebCLP zk{dtn77EndXkSbbOv{8B*bSpFcBXYiVdOL}X=WJ2Q?v`1!0N51URmXi9__zJbH zt*s?xbaZsfUI0bw#hHiURhner?ZnLm6nA?j(n=rB=XfOl)~g!or)In-G}-ZM_EX zRlF(tv~7ABGgMV{#B=mkA#IEvms=YGlNoaa#BBr2#=13tW8oN{42^50xRe zMP4sVP5J22qZX~%m6bcI^N4BxfB-y$)UvXPoZr)bFY9MlkBm=YExym2IKX(D+Hr zraybmd%QAO+k_T=FN?*ZrO{C({SqYHGGbR##W^`siN{R9@YEi;9JX^~-=ueUcI z2#1=9DLW%0BP*-z_fOA1Q!b0YJtxh*JUvG*w$qjG*xOuXubLNS@Y4OS60~X-o0B? z!{Fib8{=t9vvSqnG!MTz_r(GINQ6artUWrT0{1%?|KtZB}?x#;29l6}HU%!4` zSjfuf07b;FY`6XM9r{A1c5w%#`SpD`qF84#DCvj0%PiNS2nC7S;Na_`a#}b*4`J z_z{PDlLs}(zRy8c!{3FIreBu66r58MnE*ynIg#lhXM3xVeSJhxhN-*4CW2CS~b) zeJ%uxe=P*CaYqiqs7ilt*nw06SY-UOH95~!YQ*Fm^nE*Vp0ko{YVoJ2?GKv|LC zQVMwMQz^YBJJ((-(BpCsd3^Rad7!oT+h9QdG2_s;|^FSbPw3=H^O90zqcpB`?u zr0wkN1O^5|KERJ{W|U3JrCA?7ByUSxx9)b=i7t5zIO8#Zcvtd;$ENxtb@k!{KSd zk)@?2)K?G=mCnrsX`uy%c(x@Zf;I4N}x*Yex|QwO}OUJs4|6YXGJeBUX)g=4-** zl{{;Dot4E{YcgHy#%amGT|9pT3t;_!}Yw`-?;l-?+2dm9UU0k#0LhXvp2q0%8}i? zd2@b3Rz`;UaDQBCZ~wapfdCO9;SLL@@)gF#Re2?EBg2E-*3+}w4buDu-qlQVYf1r ztJZRN@gcsN+F*J4!;wv*X2^R^wlZxC6F==@c5D;j03p%Z-+vi%()GLsyYG_#5
MonetDB: bamloader - Merge with default
Changeset: 6f1a94ecdfcf for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6f1a94ecdfcf Added Files: sql/backends/monet5/bam/default_schema.dia sql/backends/monet5/bam/default_schema.png sql/backends/monet5/bam/metadata.dia sql/backends/monet5/bam/metadata.png sql/backends/monet5/bam/paired_schema.dia sql/backends/monet5/bam/paired_schema.png Modified Files: sql/server/rel_select.c Branch: bamloader Log Message: Merge with default diffs (truncated from 5854 to 300 lines): diff --git a/sql/backends/monet5/bam/default_schema.dia b/sql/backends/monet5/bam/default_schema.dia new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..682ce8e1628447e46f8f8dd2955adcac4bcf49ef GIT binary patch literal 1422 zc$@)_1#$WxiwFP!21MOUGZ=*O6{@!1KsI;HD1lXiWw%e}SJ71lyMr-`-~iL zh_8;F+9qwj+;2bgk{2K(0TSA#MrtKsn8zMJo*AZ7zI=TiV{HlN0eOZmSO4|Brf$3 z@h|m%?*IJQ)4yJweDM(YEPs3k#+rPB@bK(X9|w}+-f~NJsEh$fuPL5C@?r|{Rc1x zEh*H}uTC^=Re=Wt2(Kkqfe;K00s%Dw#_=f0QbRXG$5WHOC@qH#gu7NfG_paVRY!N zL`^HDXM^qu@Zo?#@KCJUlt;f`MfC)-tlIH}awJ8Y?2cx=dY%kx8Vu6e9lB2=w z9#V2)scYRp3qiyfFdvaTjgMq4PhHS+qR`KI{jYX?6$ks+$QDW2F=3ZX5m~aELn- z$^-*QRJ%?VhBi#Dpxk8T*XNwH1SsxgZp5`u0S{}5ol%LL-fi``as=lTqJnTJs zqCm?N^2Eq_K9IR$?Xv@EpCXP17_J*VB4TgqXZul~U7g5#FCVYThFDu+c_3#13Trz zgC8Id@+~w=%M@rDC1R~@R?o`qYMA^CRj%h7|b9`)ZZ+_w3aMz;C+K3N)Rct=NPP z{)MhcCi{m}JOCVOAGLcRAJyh|EEdcd7q2)8@)fP^y!jFlVm`U6870*!{D%(dl%u zR=F1B80p37+g0P!)*f?QZV!#qE;I)+^paSR2}xA5IVuBfIRVmNkO%1Ay9*`w7W zl*nP_EjMoeWQEcUpxiV2vR1N1_aGL5dedRW!K1bh}w=nMjuU!!yeX`0m_WWFB zh8QA@cYsKz*cE26g#o5vndUxLC6h3x(q);!_+$);;0{`wIF|vE`!G~hgjHMrJWLkg zY)Xwvu^9L=TlEhOBLf3u5fLT~$qaesJiysTuyndw25UgEw+h1#H7P=6vX7l1D$-! zrVYUu;hA*flu$kau8eA2%)-bhV_K}@o7{H@vg%ylj?Mw$3*g=Y`3J;PD3A+2-bi~@ z8$d#uvSY{(;r9Te$DsZ)Y2#o(aZRmc1_#w0GW$m809LwVOj-obzphzJumXCa?Y z@3$5=GCp*QmZ`(3OpQBdmG@P1I_^5keY!QR|{zx{Q;H@LiK-7Q#D!CLQJcdG}K zMW5oQL?@bD);JM9xEV`gV^Jw*1iBWNAAZ^pfOJWeSdfR-*7jdJjZg`_Tn^unwj zVFGNc6gk{(KLvTG9H2Y0Z?gR;)fcR!i-Taay@)Md*VQdJHCyD*I5hJ~CRn$OlYi zABxpSmQ%xLp;)b0eOx(pqpVixhobbsDJ?K0iqVSE$HQn2A64R^IDJ%{ehdIsq*kPE zgw*RhomPc-Wq+rR^2{d+cMgRZ}eS3x%gxm_W4=Q=)L7e-`Zw-b)CL#5RlVtE)PF z-v=xBbSZ=)$XgeO0@!YWG#`zOT96S2Wo=_C8)RFChAzLS5z;v(I|{=pX)s*zoh z{D?%s%?zY-+o-*sj`!*r5^Ynj{pu-yYCxjb^Va=wrZ(a0nSB?z(!aUALYZf5+h5 z=pMA5I!Ce0V_9m{oYyE=$Nmvo=TY*Igx=%o2{}~T%D~ZR(fufp*6)JNP=lwwW8k z@}9Nfo^l|AP|(%w0qxY8RBRyp2|1Be1B-Q-Qzfn}KFcSVH!(qTf0pBpqiho%k zoy-8PsMYx?V73`42i#ai8^5wvwPC0vCXZ*!wviZe+#+^Ss?59hKSht$S-@w#rQ zf@UXE=j-hTQX{vz?GQ+$*uY9c=`Nm{Ah=#xv@@~+k`OvfGVJmsq$~@f8e|su3 cq-@d?pGTjWx;jaHhwAFf3BbKDyvce082o?{{R30 diff --git a/sql/backends/monet5/bam/default_schema.png b/sql/backends/monet5/bam/default_schema.png new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f1aa49a5c4c81e2e3babd7aa127fe5df45701854 GIT binary patch literal 54296 zc$}oX1yq%7x9`0W1yni=|(|8N;;GZb3p3q`MoXB_#xu5RgU?kZz)3F!{$7Nj|I z`MmG9_jmTU_jkrP_wX662UgtIyypDR-+#^fHbhxb8tVq4Fm##B`YJLia?;QAP^`6 z7+2sYI}Oe%@E59sn5;Sm2FApk(k%S%bz2#22LuA=Bl16#kLF=62n01kR^pMmYtqJ) zo2Y`C1Pi}eEUP@;Qgdt7GqcOehCKat1=J~C#kX(p)dmGllpGlf77DKNcts4+) zhM_Gct7_KoTr)L8_5Czu#%;T9YRlWkQ{92c|U1cu;RnMSD!=5H5vrsh?b9^pF^|= z|JE(W)do@oB8vXDEHUDr;|C|-+4r$hliuE5hNpxG1a6ttM0v8Xm(D;YLhOO-g{^T zx@hGOrmdky1uIFw#{Ysyfqxg#tc=oCtHPm6a8lY79}{*w|QVOB6)?RSu5JnMY{V zUH|%ko1(20e;@hkBbc}T`Qh?%68~~AAKbDeIxMC59)lPG^5tz7#!kNx+NCwv z)2@qvTZn;V?HhkG-t6ox2hrQwa|je($Mpni|Yz2f7f8y(|_f{03xi)US0US6=e zc{5bI^rdXMAo8^t)A$PRa(pQ0@VN$s@RZGo3LF~`H@BwynDBclcR8-r`%72JMO)AJ z@$m5nrl%t+(6Pzyj5Q5X9cbxWJD^}PH(q_*oypGm0%nSY?S(ek1DX5Wc_t1c-1 zj`A1qF%{#!goTB1TMSoLR(hTvjGR-MDcBP8kqbLa3-QC@yyV`EX61shdhd}6|7 zYm(1#McLTc7!?)O#R#@?hPnJ3EbHNNDIn)KfXR?%~f*?Xdaz`BPF-T(+l;nSGOz z$d6;;;eG9rmeJ8Cl9H`mUFO@b$WVw7-@xtg$W4cyv-Zd3kFqE3U4tlT{8un@r3s zEKjTKmxA%=9334A2?;T=u=I3wBgpv%`uqDA+Lv{oQBhId2oS4fm%zx(%v6yP_B`_a zGr0P@vXa-uXFI$=$!lZ8EZn(b20{uWn3K6xG%yO%w`(YWzLCh_{3(k}J;drZcb zrBcaO9xfns6t?(z1UmqY)P|eIDgml%vM`5-)v(U^W)nLu6kKYr})?tUaebCLP zk{dtn77EndXkSbbOv{8B*bSpFcBXYiVdOL}X=WJ2Q?v`1!0N51URmXi9__zJbH zt*s?xbaZsfUI0bw#hHiURhner?ZnLm6nA?j(n=rB=XfOl)~g!or)In-G}-ZM_EX zRlF(tv~7ABGgMV{#B=mkA#IEvms=YGlNoaa#BBr2#=13tW8oN{42^50xRe zMP4sVP5J22qZX~%m6bcI^N4BxfB-y$)UvXPoZr)bFY9MlkBm=YExym2IKX(D+Hr zraybmd%QAO+k_T=FN?*ZrO{C({SqYHGGbR##W^`siN{R9@YEi;9JX^~-=ueUcI z2#1=9DLW%0BP*-z_fOA1Q!b0YJtxh*JUvG*w$qjG*xOuXubLNS@Y4OS60~X-o0B? z!{Fib8{=t9vvSqnG!MTz_r(GINQ6artUWrT0{1%?|KtZB}?x#;29l6}HU%!4` zSjfuf07b;FY`6XM9r{A1c5w%#`SpD`qF84#DCvj0%PiNS2nC7S;Na_`a#}b*4`J z_z{PDlLs}(zRy8c!{3FIreBu66r58MnE*ynIg#lhXM3xVeSJhxhN-*4CW2CS~b) zeJ%uxe=P*CaYqiqs7ilt*nw06SY-UOH95~!YQ*Fm^nE*Vp0ko{YVoJ2?GKv|LC zQVMwMQz^YBJJ((-(BpCsd3^Rad7!oT+h9QdG2_s;|^FSbPw3=H^O90zqcpB`?u zr0wkN1O^5|KERJ{W|U3JrCA?7ByUSxx9)b=i7t5zIO8#Zcvtd;$ENxtb@k!{KSd zk)@?2)K?G=mCnrsX`uy%c(x@Zf;I4N}x*Yex|QwO}OUJs4|6YXGJeBUX)g=4-** zl{{;Dot4E{YcgHy#%amGT|9pT3t;_!}Yw`-?;l-?+2dm9UU0k#0LhXvp2q0%8}i? zd2@b3Rz`;UaDQBCZ~wapfdCO9;SLL@@)gF#Re2?EBg2E-*3+}w4buDu-qlQVYf1r ztJZRN@gcsN+F*J4!;wv*X2^R^wlZxC6F==@c5D;j03p%Z-+vi%()GLsyYG_#5
MonetDB: bamloader - Treat = character in CIGAR string as the ...
Changeset: 95104df7abc8 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=95104df7abc8 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Treat = character in CIGAR string as the m character when calculating seq_char diffs (38 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -196,7 +196,7 @@ seq_char(str * ret, int * ref_pos, str * Error parsing CIGAR string '%s'\n, *alg_cigar); advance_ref_pos = (op == 'M' || op == 'D' || op == 'N' || op == '=' || op == 'X'); - advance_seq_pos = (op == 'M' || op == 'I'); // TODO: Find out which chars advance the seq pos + advance_seq_pos = (op == 'M' || op == 'I' || op == '='); // TODO: Find out which chars advance the seq pos if(advance_seq_pos) { seq_pos += cnt; } @@ -397,7 +397,6 @@ seq_length_bat(bat * ret, bat * bid) return MAL_SUCCEED; } - str seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * alg_cigar) { @@ -417,7 +416,7 @@ seq_char_bat(bat * ret, int * ref_pos, b Misalignment in input BATs: BUNFMT/BUNFMT/BUNFMT, BATcount(poss), BATcount(seqs), BATcount(cigars)); } - + /* allocate result BAT */ result = BATnew(TYPE_void, TYPE_str, BATcount(cigars), TRANSIENT); if (result == NULL) { @@ -450,7 +449,7 @@ seq_char_bat(bat * ret, int * ref_pos, b ++pos; ++cigar; } - + /* release input BAT-descriptors */ BBPreleaseref(seqs-batCacheid); BBPreleaseref(poss-batCacheid); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merge with default
Changeset: d08a2ca961f6 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d08a2ca961f6 Modified Files: .hgtags MonetDB.spec NT/installer32/MonetDB-ODBC-Installer.vdproj NT/installer32/MonetDB5-Geom-Module.vdproj NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB-ODBC-Installer.vdproj NT/installer64/MonetDB5-Geom-Module.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj NT/monetdb_config.h.in NT/rules.msc buildtools/ChangeLog-Archive buildtools/ChangeLog.Oct2014 clients/mapilib/mapi.rc clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc clients/python2/setup.py clients/python3/setup.py configure.ag debian/changelog gdk/ChangeLog-Archive gdk/ChangeLog.Oct2014 gdk/gdk_batop.c gdk/libbat.rc java/Makefile.ag java/build.properties java/pom.xml java/release.txt libversions monetdb5/ChangeLog-Archive monetdb5/ChangeLog.Oct2014 monetdb5/optimizer/opt_generator.c monetdb5/tools/libmonetdb5.rc sql/ChangeLog-Archive sql/ChangeLog.Oct2014 sql/backends/monet5/generator/generator.c vertoo.data Branch: bamloader Log Message: Merge with default diffs (truncated from 799 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -581,3 +581,5 @@ 4f894ebf1a93ae895ec23d54ce2112bf21a6427e e92e434d19b7308c2952d50978c2cfd78e1bdb80 Oct2014_3 4f894ebf1a93ae895ec23d54ce2112bf21a6427e Oct2014_release 88a29a5a2bef5c6cef6b3bfa5ab7c4d3b855a2c8 Oct2014_release +eb2967c408350131b63f5393704fd697c701fc49 Oct2014_5 +2da8cbcb65ed56690bad29f089d996eac347b48d Oct2014_SP1_release diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -91,7 +91,7 @@ Vendor: MonetDB BV i...@monetdb.org Group: Applications/Databases License: MPL - http://www.monetdb.org/Legal/MonetDBLicense URL: http://www.monetdb.org/ -Source: http://dev.monetdb.org/downloads/sources/Oct2014/%{name}-%{version}.tar.bz2 +Source: http://dev.monetdb.org/downloads/sources/Oct2014-SP1/%{name}-%{version}.tar.bz2 BuildRequires: bison BuildRequires: bzip2-devel @@ -960,6 +960,45 @@ mv $RPM_BUILD_ROOT%{_datadir}/doc/MonetD rm -fr $RPM_BUILD_ROOT %changelog +* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 +- Rebuilt. +- BZ#3580: cosmetic change (append newline) +- BZ#3609: Incorrect use of generate_series +- BZ#3611: quantile() and median() commands crash when used 2x on the + same variable on a null table +- BZ#3612: assertion failure when deleting rows from table to which a + FK constraint is defined +- BZ#3620: ORDER BY broken when using UNION ALL +- BZ#3621: Hexadecimal literal vs decimal literal + +* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 +- gdk: Implemented a change to the way in which string bats are appended. + We now try harder to limit the growth of the string heap. + +* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 +- monetdb5: Fixed adding of 0 intervals to dates. + +* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 +- sql: Fixed sys.queue() implementation to report on other queries being + executed. + +* Fri Nov 14 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 +- sql: A number of bugs were fixed in the code to upgrade a database from + previous releases. This version should fix the upgrade of a database + that had been upgraded to the Oct2014 release, but also properly + upgrade directly from Jan2014 and Feb2013 releases. + +* Fri Nov 7 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 +- buildtools: The libraries included in the Windows installers have been upgraded. + We now use libxml2-2.9.2, openssl-1.0.1j, pcre-8.36, and zlib-1.2.8. + +* Wed Nov 5 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 +- gdk: Fixed some problems with BATsample. It was possible for BATsample to + return a value that was just beyond the end of the sampled BAT. Also, + on some systems the range of the rand() function is rather limited + (0..32767) and trying to get a sample larger than this range would + result in an infinite loop. + * Tue Oct 28 2014 Sjoerd Mullender sjo...@acm.org - 11.19.3-20141028 - Rebuilt. diff --git a/buildtools/ChangeLog-Archive b/buildtools/ChangeLog-Archive --- a/buildtools/ChangeLog-Archive +++ b/buildtools/ChangeLog-Archive @@ -1,6 +1,10 @@ # DO NOT EDIT THIS FILE -- MAINTAINED AUTOMATICALLY # This file contains past ChangeLog entries +* Fri Nov 7 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120 +- The libraries included in the Windows installers have been upgraded. + We now use libxml2-2.9.2, openssl-1.0.1j, pcre-8.36, and zlib-1.2.8. + * Thu Apr 10 2014 Sjoerd Mullender sjo...@acm.org - 11.19.1-20141024 - Jacqueline, the
MonetDB: bamloader - Initialize BAT pointers to NULL, compiler d...
Changeset: ecbfa7f49a2e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ecbfa7f49a2e Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Initialize BAT pointers to NULL, compiler does not like it otherwise diffs (12 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -401,7 +401,7 @@ seq_length_bat(bat * ret, bat * bid) str seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * alg_cigar) { - BAT *seqs, *poss, *refs, *cigars, *result; + BAT *seqs = NULL, *poss = NULL, *refs = NULL, *cigars = NULL, *result = NULL; BUN ref= 0, seq = 0, pos = 0, cigar = 0, seq_end = 0; BATiter ref_it, seq_it, pos_it, cigar_it; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merge with default
Changeset: 5bb422b5d2e8 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5bb422b5d2e8 Added Files: sql/test/BugTracker-2014/Tests/typecast_error.Bug-3622.sql Modified Files: clients/mapiclient/tomograph.c gdk/ChangeLog.Oct2014 gdk/gdk_atoms.c gdk/gdk_batop.c gdk/gdk_private.h monetdb5/ChangeLog.Oct2014 monetdb5/mal/mal_interpreter.c monetdb5/modules/atoms/mtime.c sql/ChangeLog.Oct2014 sql/backends/monet5/generator/generator.c sql/backends/monet5/sql.c sql/server/rel_updates.c sql/test/BugTracker-2014/Tests/All sql/test/BugTracker-2014/Tests/queueError.Bug-3604.sql sql/test/BugTracker-2014/Tests/queueError.Bug-3604.stable.err sql/test/BugTracker-2014/Tests/queueError.Bug-3604.stable.out sql/test/testdb-upgrade-chain/Tests/dump.stable.out.Windows sql/test/testdb-upgrade/Tests/dump.stable.out.Windows Branch: bamloader Log Message: Merge with default diffs (truncated from 1012 to 300 lines): diff --git a/clients/mapiclient/tomograph.c b/clients/mapiclient/tomograph.c --- a/clients/mapiclient/tomograph.c +++ b/clients/mapiclient/tomograph.c @@ -2310,7 +2310,7 @@ main(int argc, char **argv) while (1) { int option_index = 0; - int c = getopt_long(argc, argv, d:u:p:h:?T:i:t:r:o:Db:B:A:s:m:a, + int c = getopt_long(argc, argv, d:u:p:P:h:?T:i:t:r:o:Db:B:A:s:m:a, long_options, option_index); if (c == -1) break; @@ -2339,7 +2339,7 @@ main(int argc, char **argv) case 'u': if (user) free(user); - user = optarg; + user = strdup(optarg); /* force password prompt */ if (password) free(password); @@ -2354,7 +2354,7 @@ main(int argc, char **argv) case 'P': if (password) free(password); - password = optarg; + password = strdup(optarg); break; case 'p': if (optarg) diff --git a/gdk/ChangeLog.Oct2014 b/gdk/ChangeLog.Oct2014 --- a/gdk/ChangeLog.Oct2014 +++ b/gdk/ChangeLog.Oct2014 @@ -1,6 +1,10 @@ # ChangeLog file for MonetDB # This file is updated with Maddlog +* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org +- Implemented a change to the way in which string bats are appended. + We now try harder to limit the growth of the string heap. + * Wed Nov 5 2014 Sjoerd Mullender sjo...@acm.org - Fixed some problems with BATsample. It was possible for BATsample to return a value that was just beyond the end of the sampled BAT. Also, diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c --- a/gdk/gdk_atoms.c +++ b/gdk/gdk_atoms.c @@ -1050,10 +1050,6 @@ strHash(const char *s) return res; } -/* if at least (2*SIZEOF_BUN), also store length (heaps are then - * incompatible) */ -#define EXTRALEN ((SIZEOF_BUN + GDK_VARALIGN - 1) ~(GDK_VARALIGN - 1)) - void strCleanHash(Heap *h, int rebuild) { diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -41,11 +41,12 @@ } \ } while (0) -/* - * BAT insert/delete/replace - * The content of a BAT can be appended to (removed from) another - * using BATins (BATdel). - */ +/* We try to be clever when appending one string bat to another. + * First of all, we try to actually share the string heap so that we + * don't need an extra copy, and if that can't be done, we see whether + * it makes sense to just quickly copy the whole string heap instead + * of inserting individual strings. See the comments in the code for + * more information. */ static BAT * insert_string_bat(BAT *b, BAT *n, int append, int force) { @@ -54,20 +55,17 @@ insert_string_bat(BAT *b, BAT *n, int ap size_t toff = ~(size_t) 0; /* tail offset */ BUN p, q; /* loop variables */ oid o = 0; /* in case we're appending */ - ptr hp, tp; /* head and tail value pointers */ + const void *hp, *tp;/* head and tail value pointers */ unsigned char tbv; /* tail value-as-bte */ - const unsigned char *tbp = NULL;/* tail value-as-bte */ unsigned short tsv; /* tail value-as-sht */ - const unsigned short *tsp = NULL; /* tail value-as-sht */ #if SIZEOF_VAR_T == 8 unsigned int tiv; /* tail value-as-int */ - const unsigned int *tip = NULL; /* tail value-as-int */ #endif var_t v;/* value */ - const var_t *tvp = NULL;/* value */ - int ntw, btw;
MonetDB: bamloader - Merge with default
Changeset: ec1329475b17 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec1329475b17 Added Files: sql/backends/monet5/generator/Tests/rangejoin.sql sql/backends/monet5/generator/Tests/rangejoin.stable.err sql/backends/monet5/generator/Tests/rangejoin.stable.out sql/test/BugTracker-2014/Tests/orderby_column_exp.Bug-3620.sql sql/test/BugTracker-2014/Tests/orderby_column_exp.Bug-3620.stable.err sql/test/BugTracker-2014/Tests/orderby_column_exp.Bug-3620.stable.out Modified Files: clients/Tests/MAL-signatures_all.stable.out clients/Tests/MAL-signatures_all.stable.out.int128 clients/Tests/MAL-signatures_fits_geom.stable.out clients/Tests/MAL-signatures_fits_geom.stable.out.int128 clients/Tests/MAL-signatures_geom.stable.out clients/Tests/MAL-signatures_geom.stable.out.int128 clients/Tests/MAL-signatures_none.stable.out clients/Tests/MAL-signatures_none.stable.out.int128 clients/Tests/exports.stable.out monetdb5/modules/kernel/algebra.c monetdb5/modules/kernel/algebra.h monetdb5/modules/kernel/algebra.mal sql/backends/monet5/generator/Tests/All sql/backends/monet5/generator/generator.c sql/backends/monet5/generator/generator.h sql/backends/monet5/generator/generator.mal sql/common/sql_types.c sql/common/sql_types.h sql/server/rel_psm.c sql/server/rel_select.c sql/server/sql_mvc.h sql/test/BugTracker-2014/Tests/All sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out.32bit Branch: bamloader Log Message: Merge with default diffs (truncated from 995 to 300 lines): diff --git a/clients/Tests/MAL-signatures_all.stable.out b/clients/Tests/MAL-signatures_all.stable.out --- a/clients/Tests/MAL-signatures_all.stable.out +++ b/clients/Tests/MAL-signatures_all.stable.out @@ -138,8 +138,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[ address AGGRavg12_dbl; comment Grouped tail average on bte -command aggr.avg(b:bat[:oid,:any_2]):dbl -address ALGavg; +pattern aggr.avg(b:bat[:oid,:any_2]):dbl +address CMDcalcavg; comment Gives the avg of all tail values function aggr.corr(e1:bat[:oid,:dbl],e2:bat[:oid,:dbl],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:dbl]; diff --git a/clients/Tests/MAL-signatures_all.stable.out.int128 b/clients/Tests/MAL-signatures_all.stable.out.int128 --- a/clients/Tests/MAL-signatures_all.stable.out.int128 +++ b/clients/Tests/MAL-signatures_all.stable.out.int128 @@ -149,8 +149,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[ address AGGRavg12_dbl; comment Grouped tail average on bte -command aggr.avg(b:bat[:oid,:any_2]):dbl -address ALGavg; +pattern aggr.avg(b:bat[:oid,:any_2]):dbl +address CMDcalcavg; comment Gives the avg of all tail values function aggr.corr(e1:bat[:oid,:hge],e2:bat[:oid,:hge],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:hge]; diff --git a/clients/Tests/MAL-signatures_fits_geom.stable.out b/clients/Tests/MAL-signatures_fits_geom.stable.out --- a/clients/Tests/MAL-signatures_fits_geom.stable.out +++ b/clients/Tests/MAL-signatures_fits_geom.stable.out @@ -138,8 +138,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[ address AGGRavg12_dbl; comment Grouped tail average on bte -command aggr.avg(b:bat[:oid,:any_2]):dbl -address ALGavg; +pattern aggr.avg(b:bat[:oid,:any_2]):dbl +address CMDcalcavg; comment Gives the avg of all tail values function aggr.corr(e1:bat[:oid,:dbl],e2:bat[:oid,:dbl],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:dbl]; diff --git a/clients/Tests/MAL-signatures_fits_geom.stable.out.int128 b/clients/Tests/MAL-signatures_fits_geom.stable.out.int128 --- a/clients/Tests/MAL-signatures_fits_geom.stable.out.int128 +++ b/clients/Tests/MAL-signatures_fits_geom.stable.out.int128 @@ -154,8 +154,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[ address AGGRavg12_dbl; comment Grouped tail average on bte -command aggr.avg(b:bat[:oid,:any_2]):dbl -address ALGavg; +pattern aggr.avg(b:bat[:oid,:any_2]):dbl +address CMDcalcavg; comment Gives the avg of all tail values function aggr.corr(e1:bat[:oid,:hge],e2:bat[:oid,:hge],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:hge]; diff --git a/clients/Tests/MAL-signatures_geom.stable.out b/clients/Tests/MAL-signatures_geom.stable.out --- a/clients/Tests/MAL-signatures_geom.stable.out +++ b/clients/Tests/MAL-signatures_geom.stable.out @@ -138,8 +138,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[ address AGGRavg12_dbl; comment Grouped tail average on bte -command aggr.avg(b:bat[:oid,:any_2]):dbl -address ALGavg; +pattern aggr.avg(b:bat[:oid,:any_2]):dbl +address CMDcalcavg; comment Gives the avg of all tail values function aggr.corr(e1:bat[:oid,:dbl],e2:bat[:oid,:dbl],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:dbl]; diff --git a/clients/Tests/MAL-signatures_geom.stable.out.int128
MonetDB: bamloader - Merge with default
Changeset: 04191ece031d for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=04191ece031d Modified Files: monetdb5/optimizer/opt_generator.c sql/backends/monet5/generator/generator.c sql/backends/monet5/generator/generator.mal Branch: bamloader Log Message: Merge with default diffs (79 lines): diff --git a/monetdb5/optimizer/opt_generator.c b/monetdb5/optimizer/opt_generator.c --- a/monetdb5/optimizer/opt_generator.c +++ b/monetdb5/optimizer/opt_generator.c @@ -63,7 +63,7 @@ int OPTgeneratorImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { InstrPtr p,q; - int i,j,k, actions=0, used= 0, cases, blocked; + int i,j,k, actions=0, used, cases, blocked; (void) cntxt; (void) stk; @@ -73,6 +73,7 @@ OPTgeneratorImplementation(Client cntxt, p = getInstrPtr(mb,i); if ( getModuleId(p) == generatorRef getFunctionId(p) == seriesRef){ /* found a target for propagation */ + used = 0; if ( assignedOnce(mb, getArg(p,0)) ){ cases = useCount(mb, getArg(p,0)); blocked = 0; @@ -116,8 +117,10 @@ OPTgeneratorImplementation(Client cntxt, else { // check for use without conversion for(k = q-retc; k q-argc; k++) - if( getArg(q,k) == getArg(p,0)) + if( getArg(q,k) == getArg(p,0)){ blocked++; + } + // materialize a copy and re-use where appropriate } } // fix the original, only when all use cases are replaced by the overloaded function @@ -125,8 +128,9 @@ OPTgeneratorImplementation(Client cntxt, setModuleId(p, generatorRef); setFunctionId(p, parametersRef); typeChecker(cntxt-fdout, cntxt-nspace, mb, p, TRUE); + } + if( used) actions++; - } else used = 0; #ifdef VLT_DEBUG mnstr_printf(cntxt-fdout,#generator target %d cases %d used %d error %d\n,getArg(p,0), cases, used, p-typechk); #endif diff --git a/sql/backends/monet5/generator/generator.c b/sql/backends/monet5/generator/generator.c --- a/sql/backends/monet5/generator/generator.c +++ b/sql/backends/monet5/generator/generator.c @@ -218,7 +218,7 @@ findGeneratorDefinition(MalBlkPtr mb, In for (i = 1; i mb-stop; i++) { q = getInstrPtr(mb, i); - if (q-argv[0] == target getModuleId(q) == generatorRef getFunctionId(q) == parametersRef) + if (q-argv[0] == target getModuleId(q) == generatorRef (getFunctionId(q) == parametersRef || getFunctionId(q) == seriesRef)) p = q; if (q == pci) return p; diff --git a/sql/backends/monet5/generator/generator.mal b/sql/backends/monet5/generator/generator.mal --- a/sql/backends/monet5/generator/generator.mal +++ b/sql/backends/monet5/generator/generator.mal @@ -223,15 +223,15 @@ comment Overloaded join operation; pattern join(gen:bat[:oid,:bte], low:bat[:oid,:bte], hgh:bat[:oid,:bte], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) address VLTgenerator_rangejoin; -pattern join(gen:bat[:oid,:sht], low:bat[:oid,:sht], hgh:bat[:oid,:bte], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) +pattern join(gen:bat[:oid,:sht], low:bat[:oid,:sht], hgh:bat[:oid,:sht], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) address VLTgenerator_rangejoin; -pattern join(gen:bat[:oid,:int], low:bat[:oid,:int], hgh:bat[:oid,:bte], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) +pattern join(gen:bat[:oid,:int], low:bat[:oid,:int], hgh:bat[:oid,:int], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) address VLTgenerator_rangejoin; -pattern join(gen:bat[:oid,:lng], low:bat[:oid,:lng], hgh:bat[:oid,:bte], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) +pattern join(gen:bat[:oid,:lng], low:bat[:oid,:lng], hgh:bat[:oid,:lng], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) address VLTgenerator_rangejoin; -pattern join(gen:bat[:oid,:flt], low:bat[:oid,:flt], hgh:bat[:oid,:bte], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) +pattern join(gen:bat[:oid,:flt], low:bat[:oid,:flt], hgh:bat[:oid,:flt], li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid]) address VLTgenerator_rangejoin;
MonetDB: bamloader - Merge with default
Changeset: 1901ba646704 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1901ba646704 Added Files: clients/mapilib/ChangeLog-Archive clients/nodejs/Tests/nodetest.js clients/nodejs/monetdb/README.md clients/nodejs/monetdb/mapiclient.js clients/nodejs/monetdb/package.json common/stream/ChangeLog-Archive debian/libmonetdb-client7.install debian/libmonetdb-stream6.install debian/libmonetdb11.install sql/backends/monet5/Tests/limithack.sql sql/backends/monet5/Tests/limithack.stable.err sql/backends/monet5/Tests/limithack.stable.out sql/backends/monet5/Tests/prepare.mal sql/backends/monet5/Tests/simpleSQL.mal sql/backends/monet5/Tests/simpleSQL.stable.err sql/backends/monet5/Tests/simpleSQL.stable.out sql/backends/monet5/generator/Tests/crash.Bug-3609.sql sql/backends/monet5/generator/Tests/crash.Bug-3609.stable.err sql/backends/monet5/generator/Tests/crash.Bug-3609.stable.out sql/backends/monet5/sql_assert.c sql/backends/monet5/sql_assert.h sql/backends/monet5/sql_execute.c sql/backends/monet5/sql_execute.h sql/backends/monet5/sql_upgrades.c sql/backends/monet5/sql_upgrades.h sql/test/BugTracker-2009/Tests/bit_and.SF-2850341.stable.err.int128 sql/test/BugTracker-2009/Tests/bit_and.SF-2850341.stable.out.int128 sql/test/BugTracker-2014/Tests/empty_intermediate_quantile.Bug-3611.sql sql/test/BugTracker-2014/Tests/empty_intermediate_quantile.Bug-3611.stable.err sql/test/BugTracker-2014/Tests/empty_intermediate_quantile.Bug-3611.stable.out sql/test/BugTracker-2014/Tests/fk-property-assert.Bug-3612.sql sql/test/BugTracker-2014/Tests/fk-property-assert.Bug-3612.stable.err sql/test/BugTracker-2014/Tests/fk-property-assert.Bug-3612.stable.out sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.sql sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.err sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.err.int128 sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.out sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.out.int128 sql/test/BugTracker-2014/Tests/nil_2dec_lng.Bug-3592.stable.out.single sql/test/BugTracker-2014/Tests/querylog.Bug-3607.sql sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.err sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.out sql/test/json/Tests/shop.sql sql/test/json/Tests/shop.stable.err sql/test/json/Tests/shop.stable.out sql/test/mergetables/Tests/doubletable.sql sql/test/mergetables/Tests/singlekeyconstraint.sql sql/test/pg_regress/Tests/loadwisconsin.sql sql/test/pg_regress/Tests/loadwisconsin.stable.err sql/test/pg_regress/Tests/loadwisconsin.stable.out Removed Files: buildtools/ChangeLog.Jan2014 clients/ChangeLog.Jan2014 clients/python2/ChangeLog.Jan2014 clients/python3/ChangeLog.Jan2014 debian/libmonetdb-client6.install debian/libmonetdb-stream5.install debian/libmonetdb10.install gdk/ChangeLog.Jan2014 geom/ChangeLog.Jan2014 java/ChangeLog.Jan2014 monetdb5/ChangeLog.Jan2014 sql/ChangeLog.Jan2014 testing/ChangeLog.Jan2014 tools/merovingian/ChangeLog.Jan2014 Modified Files: .hgtags MonetDB.spec NT/installer32/MonetDB-ODBC-Installer.vdproj NT/installer32/MonetDB5-Geom-Module.vdproj NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB-ODBC-Installer.vdproj NT/installer64/MonetDB5-Geom-Module.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj NT/monetdb_config.h.in NT/rules.msc buildtools/ChangeLog-Archive buildtools/ChangeLog.Oct2014 buildtools/doc/windowsbuild.rst clients/R/MonetDB.R/NAMESPACE clients/R/MonetDB.R/NEWS clients/R/MonetDB.R/R/dbi.R clients/R/MonetDB.R/R/dplyr.R clients/R/MonetDB.R/R/mapi.R clients/R/Tests/dplyr.R clients/R/Tests/install-dependencies.sh clients/R/Tests/survey.R clients/R/Tests/survey.stable.err clients/Tests/MAL-signatures_all.stable.err clients/Tests/MAL-signatures_all.stable.out clients/Tests/MAL-signatures_all.stable.out.int128 clients/Tests/MAL-signatures_fits_geom.stable.out clients/Tests/MAL-signatures_fits_geom.stable.out.int128 clients/Tests/MAL-signatures_geom.stable.out clients/Tests/MAL-signatures_geom.stable.out.int128 clients/Tests/MAL-signatures_none.stable.out clients/Tests/MAL-signatures_none.stable.out.int128 clients/Tests/SQL-dump_all.stable.err
MonetDB: bamloader - Delay opening of file handles to the point ...
Changeset: 2e254b0c071f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2e254b0c071f Modified Files: sql/backends/monet5/bam/bam_loader.c sql/backends/monet5/bam/bam_wrapper.c sql/backends/monet5/bam/bam_wrapper.h Branch: bamloader Log Message: Delay opening of file handles to the point where they are really needed, to prevent opening all file handles on beginning. On my machine this starts to fail if trying to import 23 files at once, since for every file tens of file handlers to binary files are opened. diffs (truncated from 378 to 300 lines): diff --git a/sql/backends/monet5/bam/bam_loader.c b/sql/backends/monet5/bam/bam_loader.c --- a/sql/backends/monet5/bam/bam_loader.c +++ b/sql/backends/monet5/bam/bam_loader.c @@ -209,10 +209,9 @@ bam_loader(Client cntxt, MalBlkPtr mb, s goto cleanup; } if (nr_threads = 0) { - msg = createException(MAL, bam_loader, - We can not get the work done with only %d threads, - nr_threads); - goto cleanup; + nr_threads = 1; + } else if(nr_threads 4) { + nr_threads = 4; } /* Get SQL context */ @@ -381,7 +380,6 @@ bam_loader(Client cntxt, MalBlkPtr mb, s TO_LOG(bam_loader Copying data into DB...\n); /* All threads finished succesfully, copy all data into DB */ for (i = 0; i nr_files; ++i) { - prepare_for_copy(bws + i); if ((msg = copy_into_db(cntxt, bws + i)) != MAL_SUCCEED) { goto cleanup; } diff --git a/sql/backends/monet5/bam/bam_wrapper.c b/sql/backends/monet5/bam/bam_wrapper.c --- a/sql/backends/monet5/bam/bam_wrapper.c +++ b/sql/backends/monet5/bam/bam_wrapper.c @@ -99,6 +99,44 @@ init_bam_wrapper(bam_wrapper * bw, filet file_location, flushdir, strerror(errno)); } + // Construct all the file paths to the binary files + for (i = 0; i 6; ++i) { + snprintf(bw-fp_files[i], BW_FP_BUF_SIZE, %s/files_%d, +flushdir, i); + } + for (i = 0; i 7; ++i) { + snprintf(bw-fp_sq[i], BW_FP_BUF_SIZE, %s/sq_%d, flushdir, +i); + } + for (i = 0; i 13; ++i) { + snprintf(bw-fp_rg[i], BW_FP_BUF_SIZE, %s/rg_%d, flushdir, +i); + } + for (i = 0; i 6; ++i) { + snprintf(bw-fp_pg[i], BW_FP_BUF_SIZE, %s/pg_%d, flushdir, +i); + } + for (i = 0; i 12; ++i) { + snprintf(bw-fp_alignments[i], BW_FP_BUF_SIZE, +%s/alignments_%d, flushdir, i); + } + for (i = 0; i 4; ++i) { + snprintf(bw-fp_alignments_extra[i], BW_FP_BUF_SIZE, +%s/alignments_extra_%d, flushdir, i); + } + if (dbschema == 1) { + for (i = 0; i 23; ++i) { + snprintf(bw-fp_alignments_paired_primary[i], +BW_FP_BUF_SIZE, +%s/alignments_paired_primary_%d, flushdir, +i); + snprintf(bw-fp_alignments_paired_secondary[i], +BW_FP_BUF_SIZE, +%s/alignments_paired_secondary_%d, +flushdir, i); + } + } + if (type == BAM) { /* Open BAM file and read its header */ if ((bw-bam.input = bam_open(file_location, r)) == NULL) { @@ -184,105 +222,102 @@ init_bam_wrapper(bam_wrapper * bw, filet bw-cnt_alignments_paired_secondary = 0; bw-cnt_alignments_total = 0; + + return MAL_SUCCEED; +} + + +static str +open_header_write_streams(bam_wrapper * bw) +{ + int i; + for (i = 0; i 6; ++i) { - snprintf(bw-fp_files[i], BW_FP_BUF_SIZE, %s/files_%d, -flushdir, i); if ((bw-files[i] = bsopen(bw-fp_files[i])) == NULL) { throw(MAL, init_bam_wrapper, ERR_INIT_BAM_WRAPPER Binary file '%s' could not be opened, - file_location, bw-fp_files[i]); + bw-file_location, bw-fp_files[i]); } } for (i = 0; i 7; ++i) { - snprintf(bw-fp_sq[i], BW_FP_BUF_SIZE, %s/sq_%d, flushdir, -i); if ((bw-sq[i] = bsopen(bw-fp_sq[i])) == NULL) { throw(MAL, init_bam_wrapper, ERR_INIT_BAM_WRAPPER Binary file '%s' could not be opened, - file_location, bw-fp_sq[i]); +
MonetDB: default - Handle null in param array correctly
Changeset: 7fbf1caaeabe for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7fbf1caaeabe Modified Files: clients/nodejs/monetdb/mapiclient.js Branch: default Log Message: Handle null in param array correctly diffs (13 lines): diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -123,6 +123,9 @@ MonetDBConnection.prototype.prepare = fu if (!error) { var execfun = function(bindparams, ecallback) { var quoted = bindparams.map(function(param) { + if(param === null) { + return NULL; + } var type = typeof param; switch(type) { case 'boolean': ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - NodeJS connector: update package version
Changeset: 9cc5f7d3867c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9cc5f7d3867c Modified Files: clients/nodejs/monetdb/package.json Branch: default Log Message: NodeJS connector: update package version diffs (11 lines): diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.2.1, + version: 0.2.2, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Cleaned up argument handling
Changeset: 5facdb3d0476 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5facdb3d0476 Modified Files: clients/nodejs/monetdb/mapiclient.js clients/nodejs/monetdb/package.json Branch: default Log Message: Cleaned up argument handling diffs (27 lines): diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -80,11 +80,7 @@ MonetDBConnection.prototype.query = func return; } - if (Array.isArray(arguments[1])) { - params = arguments[1]; - callback = arguments[2]; - raw = arguments[3]; - + if (params.length 0) { this.prepare(message, function(err, resp) { if (err) { if (callback != undefined) { diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.1.8, + version: 0.2.0, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - NodeJS connector: Fixed bug that prevented th...
Changeset: 93da657eae35 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=93da657eae35 Modified Files: clients/nodejs/monetdb/mapiclient.js clients/nodejs/monetdb/package.json Branch: default Log Message: NodeJS connector: Fixed bug that prevented the callback from being called when undefined was passed as second parameter to query diffs (31 lines): diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -59,7 +59,6 @@ MonetDBConnection.prototype.query = func var params = []; var callback = undefined; var raw = false; - for (argi in arguments) { if (typeof arguments[argi] == 'function') { callback = arguments[argi]; @@ -103,8 +102,6 @@ MonetDBConnection.prototype.query = func }); } else { - callback = arguments[1]; - raw = arguments[2]; if (!raw) { message = 's' + message + ';'; } diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.2.0, + version: 0.2.1, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Version 0.1.8
Changeset: 24f875d842a7 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=24f875d842a7 Modified Files: clients/nodejs/Tests/nodetest.js clients/nodejs/monetdb/package.json Branch: default Log Message: Version 0.1.8 diffs (21 lines): diff --git a/clients/nodejs/Tests/nodetest.js b/clients/nodejs/Tests/nodetest.js --- a/clients/nodejs/Tests/nodetest.js +++ b/clients/nodejs/Tests/nodetest.js @@ -109,6 +109,3 @@ conn.prepare('SELECT id from tables wher }); conn.close(); - - - diff --git a/clients/nodejs/monetdb/package.json b/clients/nodejs/monetdb/package.json --- a/clients/nodejs/monetdb/package.json +++ b/clients/nodejs/monetdb/package.json @@ -1,6 +1,6 @@ { name: monetdb, - version: 0.1.7, + version: 0.1.8, description: Connect MonetDB and node.js, main: mapiclient.js, author: Hannes Mühleisen han...@cwi.nl, ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merge with default
Changeset: 13957da3e185 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=13957da3e185 Added Files: sql/test/mergetables/Tests/emptymergequery.stable.err sql/test/mergetables/Tests/emptymergequery.stable.out sql/test/mergetables/Tests/mergeloadidx.sql sql/test/mergetables/Tests/mergeloadidx.stable.err sql/test/mergetables/Tests/mergeloadidx.stable.out Modified Files: sql/backends/monet5/bam/Tests/mergetable.stable.err sql/backends/monet5/bam/Tests/mergetable.stable.out sql/backends/monet5/bam/Tests/sql/mergetable.sql sql/server/rel_schema.c sql/server/rel_select.c sql/storage/store.c sql/test/mergetables/Tests/All Branch: bamloader Log Message: Merge with default diffs (truncated from 361 to 300 lines): diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.err b/sql/backends/monet5/bam/Tests/mergetable.stable.err --- a/sql/backends/monet5/bam/Tests/mergetable.stable.err +++ b/sql/backends/monet5/bam/Tests/mergetable.stable.err @@ -118,21 +118,13 @@ stderr of test 'mergetable` in directory # 08:29:32 /usr/bin/python2 mergetable.SQL.py mergetable # 08:29:32 -MAPI = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321 +MAPI = (monetdb) /var/tmp/mtest-17563/.s.monetdb.34280 QUERY = SELECT COUNT(*) FROM alignments; -ERROR = !Unable to query empty Merge or Replica tables -MAPI = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321 -QUERY = ALTER TABLE alignments ADD TABLE alignments_1; -ERROR = !ALTER MERGE TABLE: to be added table index doesn't match MERGE TABLE definition -MAPI = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321 -QUERY = SELECT COUNT(*) FROM alignments; -ERROR = !Unable to query empty Merge or Replica tables -MAPI = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321 -QUERY = ALTER TABLE alignments ADD TABLE alignments_2; -ERROR = !ALTER MERGE TABLE: to be added table index doesn't match MERGE TABLE definition -MAPI = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321 -QUERY = SELECT COUNT(*) FROM alignments; -ERROR = !Unable to query empty Merge or Replica tables +ERROR = !MERGE or REPLICA TABLE should have at least one table associated + +# 18:59:22 +# 18:59:22 Done. +# 18:59:22 # 08:29:33 # 08:29:33 Done. diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.out b/sql/backends/monet5/bam/Tests/mergetable.stable.out --- a/sql/backends/monet5/bam/Tests/mergetable.stable.out +++ b/sql/backends/monet5/bam/Tests/mergetable.stable.out @@ -68,7 +68,25 @@ Ready. # pnext INT NOT NULL, # tlen INT NOT NULL, # seqSTRINGNOT NULL, -# qual STRINGNOT NULL +# qual STRINGNOT NULL, +#ALTER TABLE alignments ADD TABLE alignments_1; +#SELECT COUNT(*) FROM alignments; +% bam.L1 # table_name +% L1 # name +% wrd # type +% 2 # length +[ 71 ] +#ALTER TABLE alignments ADD TABLE alignments_2; +#SELECT COUNT(*) FROM alignments; +% .L1 # table_name +% L1 # name +% wrd # type +% 3 # length +[ 142 ] + +# 18:59:22 +# 18:59:22 Done. +# 18:59:22 # 08:29:33 # 08:29:33 Done. diff --git a/sql/backends/monet5/bam/Tests/sql/mergetable.sql b/sql/backends/monet5/bam/Tests/sql/mergetable.sql --- a/sql/backends/monet5/bam/Tests/sql/mergetable.sql +++ b/sql/backends/monet5/bam/Tests/sql/mergetable.sql @@ -16,7 +16,8 @@ CREATE MERGE TABLE alignments ( pnext INT NOT NULL, tlen INT NOT NULL, seqSTRINGNOT NULL, - qual STRINGNOT NULL + qual STRINGNOT NULL, + CONSTRAINT alignments_pkey_virtual_offset PRIMARY KEY (virtual_offset) ); SELECT COUNT(*) FROM alignments; diff --git a/sql/server/rel_schema.c b/sql/server/rel_schema.c --- a/sql/server/rel_schema.c +++ b/sql/server/rel_schema.c @@ -1161,7 +1161,7 @@ rel_alter_table(mvc *sql, dlist *qname, /* check tables */ if (nnt) { - if (rel_check_tables(sql, nt, nnt) 0) + if (rel_check_tables(sql, t, nnt) 0) return NULL; cs_add(nt-tables, nnt, TR_NEW); } diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c --- a/sql/server/rel_select.c +++ b/sql/server/rel_select.c @@ -1756,7 +1756,7 @@ table_ref(mvc *sql, sql_rel *rel, symbol return rel; } if ((isMergeTable(t) || isReplicaTable(t)) list_empty(t-tables.set)) - return sql_error(sql, 02, Unable to query empty Merge or Replica tables); + return sql_error(sql, 02, MERGE or REPLICA TABLE should have at least one table associated); return rel_basetable(sql, t,
MonetDB: bamloader - Merge with default
Changeset: cd686acf3134 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cd686acf3134 Removed Files: clients/R/MonetDB.R/src/Makevars.win clients/R/MonetDB.R/src/mapi.c Modified Files: clients/R/MonetDB.R/NEWS clients/R/MonetDB.R/R/dbi.R clients/R/MonetDB.R/R/mapi.R clients/R/Tests/dbi.R clients/R/Tests/dbi.stable.out clients/R/Tests/dplyr.R clients/R/Tests/survey.R clients/Tests/MAL-signatures_all.stable.out clients/Tests/MAL-signatures_fits_geom.stable.out clients/Tests/MAL-signatures_geom.stable.out clients/Tests/MAL-signatures_none.stable.out clients/Tests/exports.stable.out gdk/gdk_firstn.c monetdb5/modules/kernel/bat5.mal monetdb5/modules/mal/Tests/inspect05.stable.out monetdb5/modules/mal/Tests/pqueue.stable.out monetdb5/modules/mal/Tests/pqueue2.stable.out monetdb5/modules/mal/mdb.c monetdb5/modules/mal/mdb.h monetdb5/optimizer/opt_mergetable.c monetdb5/tests/gdkTests/Tests/All monetdb5/tests/gdkTests/Tests/firstn.stable.out sql/backends/monet5/bam/bam_db_interface.h sql/backends/monet5/sql_result.c sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_scenario.h sql/backends/monet5/sql_statistics.c sql/backends/monet5/vaults/fits.c sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out sql/test/testdb-upgrade/Tests/upgrade.stable.out Branch: bamloader Log Message: Merge with default diffs (truncated from 1909 to 300 lines): diff --git a/clients/R/MonetDB.R/NEWS b/clients/R/MonetDB.R/NEWS --- a/clients/R/MonetDB.R/NEWS +++ b/clients/R/MonetDB.R/NEWS @@ -1,3 +1,7 @@ +0.9.6 +- Fixed non-ASCII character handling (thanks, Roman!) +- Fully removed C-based socket code + 0.9.5 - Removed package date (Thanks, Dimitar) - Added sys. schema name to internal queries, so SET SCHEMA xx would not break things (Thanks again, Dimitar) diff --git a/clients/R/MonetDB.R/R/dbi.R b/clients/R/MonetDB.R/R/dbi.R --- a/clients/R/MonetDB.R/R/dbi.R +++ b/clients/R/MonetDB.R/R/dbi.R @@ -1,12 +1,6 @@ +# C library that contains our MAPI string splitting state machine C_LIBRARY - MonetDB.R -.onLoad - function(lib, pkg) { - if (getOption(monetdb.clib, FALSE)) { -library.dynam( C_LIBRARY, pkg, lib ) -.Call(mapiInit, PACKAGE=C_LIBRARY) - } -} - # Make S4 aware of S3 classes setOldClass(c(sockconn, connection, monetdb_mapi_conn)) @@ -234,6 +228,8 @@ setMethod(dbSendQuery, signature(conn= } conn@connenv$exception - list() env - NULL +# Auto-convert? +# statement - enc2utf8(statement) if (getOption(monetdb.debug.query, F)) message(QQ: ', statement, ') resp - .mapiParseResponse(.mapiRequest(conn, paste0(s, statement, ;), async=async)) @@ -539,8 +535,10 @@ setMethod(dbFetch, signature(res=Mone col - ct[[j]] if (col == .CT_NUM) df[[j]] - as.numeric(parts[[j]]) -if (col == .CT_CHRR) +if (col == .CT_CHRR) { df[[j]] - parts[[j]] + Encoding(df[[j]]) - UTF-8 +} if (col == .CT_BOOL) df[[j]] - parts[[j]]==true if (col == .CT_CHR) { diff --git a/clients/R/MonetDB.R/R/mapi.R b/clients/R/MonetDB.R/R/mapi.R --- a/clients/R/MonetDB.R/R/mapi.R +++ b/clients/R/MonetDB.R/R/mapi.R @@ -86,20 +86,11 @@ REPLY_SIZE- 100 # Apparently, -1 me .mapiConnect - function(host, port, timeout) { - if (getOption(monetdb.clib, FALSE)) { -return(.Call(mapiConnect, host, port, timeout, PACKAGE=C_LIBRARY)) - } else { -return(socketConnection(host = host, port = port, blocking = TRUE, open=r+b, timeout = timeout)) - } + socketConnection(host = host, port = port, blocking = TRUE, open=r+b, timeout = timeout) } .mapiDisconnect - function(socket) { - if (getOption(monetdb.clib, FALSE)) { -.Call(mapiDisconnect, socket, PACKAGE=C_LIBRARY) - } else { - # close, don't care about the errors... - tryCatch(close(socket), error=function(e){}, warning=function(w){}) - } + tryCatch(close(socket), error=function(e){}, warning=function(w){}) } .mapiCleanup - function(conObj) { @@ -110,69 +101,48 @@ REPLY_SIZE- 100 # Apparently, -1 me } .mapiRead - function(con) { - if (getOption(monetdb.clib, FALSE)) { -if (!identical(class(con)[[1]], externalptr)) - stop(I can only be called with a MonetDB connection object as parameter.) -respstr - .Call(mapiRead, con, PACKAGE=C_LIBRARY) -if (getOption(monetdb.debug.mapi, F)) { - dstr - respstr - if (nchar(dstr) 300) { -dstr - paste0(substring(dstr, 1, 200), ..., substring(dstr, nchar(dstr)-100, nchar(dstr))) - } - message(RX: ', dstr, ') + if (!identical(class(con)[[1]], sockconn)) +stop(I can only be called with a MonetDB connection object as parameter.) + resp - list()
MonetDB: default - Let close accept a callback that gets called ...
Changeset: e1d4ab77171c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1d4ab77171c Modified Files: clients/nodejs/monetdb/mapiclient.js Branch: default Log Message: Let close accept a callback that gets called when socket.destroy is called, and create Q based disconnect and close function diffs (38 lines): diff --git a/clients/nodejs/monetdb/mapiclient.js b/clients/nodejs/monetdb/mapiclient.js --- a/clients/nodejs/monetdb/mapiclient.js +++ b/clients/nodejs/monetdb/mapiclient.js @@ -12,6 +12,7 @@ function MonetDBConnection(options, conn this.conn_callback = conncallback; this.mapi_blocksize = 8192; this.do_close = false; + this.close_callback = null; this.alldone = false; this.queryqueue = []; @@ -163,7 +164,8 @@ MonetDBConnection.prototype.prepare = fu } MonetDBConnection.prototype.disconnect = -MonetDBConnection.prototype.close = function() { +MonetDBConnection.prototype.close = function(callback) { + this.close_callback = callback; this.do_close = true; if (this.queryqueue.length 1) { next_op.call(this); @@ -238,6 +240,7 @@ function next_op() { this.alldone = true; if (this.do_close) { this.socket.destroy(); + this.close_callback this.close_callback(null); } return; } @@ -483,7 +486,7 @@ function __get_connect_args(options) { } /* Q integration, robin.cij...@monetdbsolutions.com */ -['request', 'query', 'prepare'].forEach(function(funToQ) { +['request', 'query', 'prepare', 'close', 'disconnect'].forEach(function(funToQ) { var funQ = funToQ + 'Q'; MonetDBConnection.prototype[funQ] = function() { if (this.options.q == undefined) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merge with default
Changeset: 7588bd33711f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7588bd33711f Added Files: sql/backends/monet5/bam/Tests/mergetable.SQL.py sql/backends/monet5/bam/Tests/sql/mergetable.sql Modified Files: sql/backends/monet5/bam/Tests/All sql/backends/monet5/sql.c sql/server/rel_schema.c sql/storage/sql_storage.h sql/storage/store.c sql/storage/store_dependency.c sql/test/mergetables/Tests/All Branch: bamloader Log Message: Merge with default diffs (196 lines): diff --git a/sql/backends/monet5/bam/Tests/All b/sql/backends/monet5/bam/Tests/All --- a/sql/backends/monet5/bam/Tests/All +++ b/sql/backends/monet5/bam/Tests/All @@ -24,3 +24,4 @@ HAVE_SAMTOOLS?query2.9 HAVE_SAMTOOLS?query2.10 HAVE_SAMTOOLS?query2.11 HAVE_SAMTOOLS?query2.12 +HAVE_SAMTOOLS?mergetable \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/mergetable.SQL.py b/sql/backends/monet5/bam/Tests/mergetable.SQL.py new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/mergetable.SQL.py @@ -0,0 +1,3 @@ +import bam + +bam.exec_sql_file(mergetable.sql, {'PWD': bam.SRCDIR}) \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/sql/mergetable.sql b/sql/backends/monet5/bam/Tests/sql/mergetable.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/sql/mergetable.sql @@ -0,0 +1,30 @@ +SET SCHEMA bam; + +# Load files +CALL bam_loader_repos('PWD/files', 0, 4); + +# Add a merge table over these files +CREATE MERGE TABLE alignments ( + virtual_offset BIGINTNOT NULL, + qname STRINGNOT NULL, + flag SMALLINT NOT NULL, + rname STRINGNOT NULL, + posINT NOT NULL, + mapq SMALLINT NOT NULL, + cigar STRINGNOT NULL, + rnext STRINGNOT NULL, + pnext INT NOT NULL, + tlen INT NOT NULL, + seqSTRINGNOT NULL, + qual STRINGNOT NULL +); + +SELECT COUNT(*) FROM alignments; + +ALTER TABLE alignments ADD TABLE alignments_1; + +SELECT COUNT(*) FROM alignments; + +ALTER TABLE alignments ADD TABLE alignments_2; + +SELECT COUNT(*) FROM alignments; diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -564,7 +564,7 @@ alter_table(mvc *sql, char *sname, sql_t sql_column *c = n-data; sql_column *nc = mvc_bind_column(sql, nt, c-base.name); - if (c-null != nc-null) { + if (c-null != nc-null isTable(nt)) { mvc_null(sql, nc, c-null); /* for non empty check for nulls */ if (c-null == 0) { diff --git a/sql/server/rel_schema.c b/sql/server/rel_schema.c --- a/sql/server/rel_schema.c +++ b/sql/server/rel_schema.c @@ -604,7 +604,7 @@ table_element(mvc *sql, symbol *s, sql_s { int res = SQL_OK; - if (alter (isView(t) || ((isMergeTable(t) || isReplicaTable(t)) s-token != SQL_TABLE s-token != SQL_DROP_TABLE) || (isTable(t) (s-token == SQL_TABLE || s-token == SQL_DROP_TABLE)) )){ + if (alter (isView(t) || ((isMergeTable(t) || isReplicaTable(t)) (s-token != SQL_TABLE s-token != SQL_DROP_TABLE cs_size(t-tables)0)) || (isTable(t) (s-token == SQL_TABLE || s-token == SQL_DROP_TABLE)) )){ char *msg = ; switch (s-token) { @@ -1072,6 +1072,21 @@ rel_check_tables(mvc *sql, sql_table *nt return -2; } } + if (cs_size(nt-idxs) != cs_size(nnt-idxs)) { + (void) sql_error(sql, 02, 3F000!ALTER MERGE TABLE: to be added table index doesn't match MERGE TABLE definition); + return -1; + } + if (cs_size(nt-idxs)) + for (n = nt-idxs.set-h, m = nnt-idxs.set-h; n m; n = n-next, m = m-next) { + sql_idx *ni = n-data; + sql_idx *mi = m-data; + + /* todo check def */ + if (ni-type != mi-type) { + (void) sql_error(sql, 02, 3F000!ALTER MERGE TABLE: to be added table index type doesn't match MERGE TABLE definition); + return -2; + } + } return 0; } @@ -1120,6 +1135,9 @@ rel_alter_table(mvc *sql, dlist *qname, if (t-s !nt-s) nt-s = t-s; + if (nt-type == tt_merge_table) + return sql_error(sql, 02, 42S02!ALTER TABLE: read only MERGE TABLES are not supported); + if (state == tr_readonly) { nt = mvc_readonly(sql, nt, 1); } else { @@ -1162,6 +1180,9 @@
MonetDB: default - Newline at end of files
Changeset: 37c1b2ce60e9 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=37c1b2ce60e9 Modified Files: sql/backends/monet5/bam/Tests/mergetable.stable.err sql/backends/monet5/bam/Tests/mergetable.stable.out Branch: default Log Message: Newline at end of files diffs (20 lines): diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.err b/sql/backends/monet5/bam/Tests/mergetable.stable.err --- a/sql/backends/monet5/bam/Tests/mergetable.stable.err +++ b/sql/backends/monet5/bam/Tests/mergetable.stable.err @@ -136,4 +136,4 @@ ERROR = !Unable to query empty Merge or # 08:29:33 # 08:29:33 Done. -# 08:29:33 \ No newline at end of file +# 08:29:33 diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.out b/sql/backends/monet5/bam/Tests/mergetable.stable.out --- a/sql/backends/monet5/bam/Tests/mergetable.stable.out +++ b/sql/backends/monet5/bam/Tests/mergetable.stable.out @@ -72,4 +72,4 @@ Ready. # 08:29:33 # 08:29:33 Done. -# 08:29:33 \ No newline at end of file +# 08:29:33 ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Added key constraint on mergetable definition...
Changeset: bf3858286beb for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bf3858286beb Modified Files: sql/backends/monet5/bam/Tests/sql/mergetable.sql Branch: default Log Message: Added key constraint on mergetable definition in mergetable test diffs (13 lines): diff --git a/sql/backends/monet5/bam/Tests/sql/mergetable.sql b/sql/backends/monet5/bam/Tests/sql/mergetable.sql --- a/sql/backends/monet5/bam/Tests/sql/mergetable.sql +++ b/sql/backends/monet5/bam/Tests/sql/mergetable.sql @@ -16,7 +16,8 @@ CREATE MERGE TABLE alignments ( pnext INT NOT NULL, tlen INT NOT NULL, seqSTRINGNOT NULL, - qual STRINGNOT NULL + qual STRINGNOT NULL, + CONSTRAINT alignments_pkey_virtual_offset PRIMARY KEY (virtual_offset) ); SELECT COUNT(*) FROM alignments; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merge with default
Changeset: a42705485166 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a42705485166 Added Files: sql/backends/monet5/bam/Tests/mergetable.stable.err sql/backends/monet5/bam/Tests/mergetable.stable.out Branch: bamloader Log Message: Merge with default diffs (224 lines): diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.err b/sql/backends/monet5/bam/Tests/mergetable.stable.err new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/mergetable.stable.err @@ -0,0 +1,139 @@ +stderr of test 'mergetable` in directory 'sql/backends/monet5/bam` itself: + + +# 08:29:32 +# 08:29:32 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=37321 --set mapi_usock=/var/tmp/mtest-10986/.s.monetdb.37321 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 --set embedded_r=yes +# 08:29:32 + +# builtin opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 37321 +# cmdline opt mapi_usock = /var/tmp/mtest-10986/.s.monetdb.37321 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam +# cmdline opt mal_listing = 0 +# cmdline opt embedded_r = yes +# cmdline opt gdk_debug = 536870922 +# bam_loader: Loader started for 2 BAM files... +# # bam_loader Retrieving next file id... +# # bam_loader Initializing BAM wrapper for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file2.sam'... +# # bam_loader Initializing BAM wrapper for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... +# # bam_loader Parsing header for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file2.sam'... +# # bam_loader Parsing header for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... +# # bam_loader Creating alignment tables for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file2.sam'... +# # CREATE TABLE bam.alignments_1 ( +# virtual_offset BIGINTNOT NULL, +# qnameSTRING NOT NULL, +# flag SMALLINT NOT NULL, +# rnameSTRING NOT NULL, +# posINT NOT NULL, +# mapq SMALLINT NOT NULL, +# cigarSTRING NOT NULL, +# rnextSTRING NOT NULL, +# pnextINT NOT NULL, +# tlen INT NOT NULL, +# seqSTRING NOT NULL, +# qual STRING NOT NULL, +# CONSTRAINT alignments_1_pkey_virtual_offset PRIMARY KEY (virtual_offset) +# ); +# +# CREATE TABLE bam.alignments_extra_1 ( +# tagCHAR(2) NOT NULL, +# virtual_offset BIGINTNOT NULL, +# type CHAR(1) NOT NULL, +# valueSTRING, +# CONSTRAINT alignments_extra_1_pkey_tag_virtual_offset PRIMARY KEY (tag, virtual_offset), +# CONSTRAINT alignments_extra_1_fkey_virtual_offset FOREIGN KEY (virtual_offset) +# REFERENCES bam.alignments_1 (virtual_offset) +# ); +# # bam_loader Creating alignment tables for file '/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'... +# # CREATE TABLE bam.alignments_2 ( +# virtual_offset BIGINTNOT NULL, +# qnameSTRING NOT NULL, +# flag SMALLINT NOT NULL, +# rnameSTRING NOT NULL, +# pos
MonetDB: bamloader - Added UDF for calculating the character fro...
Changeset: 84ebb8ecd572 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=84ebb8ecd572 Modified Files: sql/backends/monet5/bam/85_bam.sql sql/backends/monet5/bam/bam.mal sql/backends/monet5/bam/bam_lib.c sql/backends/monet5/bam/bam_lib.h Branch: bamloader Log Message: Added UDF for calculating the character from an alignment that maps to some position in the reference string, given this position and the alignment pos, cigar and seq fields diffs (182 lines): diff --git a/sql/backends/monet5/bam/85_bam.sql b/sql/backends/monet5/bam/85_bam.sql --- a/sql/backends/monet5/bam/85_bam.sql +++ b/sql/backends/monet5/bam/85_bam.sql @@ -25,6 +25,9 @@ RETURNS STRING EXTERNAL NAME bam.reverse CREATE FUNCTION bam.seq_length(cigar STRING) RETURNS INT EXTERNAL NAME bam.seq_length; +CREATE FUNCTION bam.seq_char(ref_pos INT, alg_seq STRING, alg_pos INT, alg_cigar STRING) +RETURNS CHAR(1) EXTERNAL NAME bam.seq_char; + CREATE PROCEDURE bam.sam_export(output_path STRING) EXTERNAL NAME bam.sam_export; diff --git a/sql/backends/monet5/bam/bam.mal b/sql/backends/monet5/bam/bam.mal --- a/sql/backends/monet5/bam/bam.mal +++ b/sql/backends/monet5/bam/bam.mal @@ -38,6 +38,10 @@ command seq_length(cigar:str):int address seq_length comment Calculate the real length of a DNA sequence, given its CIGAR string. +command seq_char(ref_pos:int, alg_seq:str, alg_pos:int, alg_cigar:str):str +address seq_char +comment Calculate the character in the alignment string (alg_str) that is aligned to position 'ref_pos', conforming to the given cigar string + # Export signatures @@ -69,3 +73,7 @@ comment Reverse a bat of DNA Quality st command seq_length(cigars:bat[:oid,:str]):bat[:oid,:int] address seq_length_bat comment Calculate the real length of a bat of DNA sequences, given their CIGAR string. + +command seq_char(ref_pos:int, alg_seq:bat[:oid,:str], alg_pos:bat[:oid,:int], alg_cigar:bat[:oid,:str]):bat[:oid,:str] +address seq_char +comment Calculate the character in the alignment string (alg_str) that is aligned to position 'ref_pos', conforming to the given cigar string (bat based version) diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -141,7 +141,7 @@ reverse_qual(str * ret, str * qual) } str -seq_length(int *ret, str * cigar) +seq_length(int * ret, str * cigar) { int result = 0; str cigar_consumable = *cigar; @@ -169,6 +169,52 @@ seq_length(int *ret, str * cigar) return MAL_SUCCEED; } +str +seq_char(str * ret, int * ref_pos, str * alg_seq, int * alg_pos, str * alg_cigar) +{ + str cigar_consumable = *alg_cigar; + int seq_pos = 0; + bit at_character = false; + int iterate_until = *ref_pos - *alg_pos; + str result; + + if((result = GDKmalloc(2 * sizeof(char))) == NULL) { + throw(MAL, seq_char, MAL_MALLOC_FAIL); + } + result[1] = '\0'; + + if (cigar_consumable[0] == '*' cigar_consumable[1] == '\0') { + result[0] = '\0'; + *ret = result; + return MAL_SUCCEED; + } + while(cigar_consumable[0] != '\0' seq_pos iterate_until) { + int cnt; + char op; + int nr_chars_read; + + if (sscanf + (cigar_consumable, %d%c%n, cnt, op, +nr_chars_read) != 2) + throw(MAL, seq_char, + Error parsing CIGAR string '%s'\n, *alg_cigar); + if (op == 'M' || op == 'D' || op == 'N' || op == '=' + || op == 'X') { + seq_pos += cnt; + if(seq_pos iterate_until) + seq_pos = iterate_until; + at_character = true; + } else { + at_character = false; + } + cigar_consumable += nr_chars_read; + } + result[0] = at_character ? (*alg_seq)[seq_pos] : '\0'; + *ret = result; + return MAL_SUCCEED; +} + + str @@ -335,3 +381,65 @@ seq_length_bat(bat * ret, bat * bid) return MAL_SUCCEED; } + + +str +seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * alg_cigar) +{ + BAT *seqs, *poss, *cigars, *result; + BUN seq = 0, pos = 0, cigar = 0, seq_end = 0; + BATiter seq_it, pos_it, cigar_it; + + assert(ret != NULL ref_pos != NULL alg_seq != NULL alg_pos != NULL alg_cigar != NULL); + + if ((seqs = BATdescriptor(*alg_seq)) == NULL || + (poss = BATdescriptor(*alg_pos)) == NULL || + (cigars = BATdescriptor(*alg_cigar)) == NULL) + throw(MAL, seq_char_bat, RUNTIME_OBJECT_MISSING); + + if(BATcount(seqs) != BATcount(poss) || BATcount(seqs) != BATcount(cigars)) {
MonetDB: bamloader - Fixed erroneous function reference
Changeset: cd4153c81e49 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cd4153c81e49 Modified Files: sql/backends/monet5/bam/bam.mal Branch: bamloader Log Message: Fixed erroneous function reference diffs (10 lines): diff --git a/sql/backends/monet5/bam/bam.mal b/sql/backends/monet5/bam/bam.mal --- a/sql/backends/monet5/bam/bam.mal +++ b/sql/backends/monet5/bam/bam.mal @@ -75,5 +75,5 @@ address seq_length_bat comment Calculate the real length of a bat of DNA sequences, given their CIGAR string. command seq_char(ref_pos:int, alg_seq:bat[:oid,:str], alg_pos:bat[:oid,:int], alg_cigar:bat[:oid,:str]):bat[:oid,:str] -address seq_char +address seq_char_bat comment Calculate the character in the alignment string (alg_str) that is aligned to position 'ref_pos', conforming to the given cigar string (bat based version) ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Make seq_char also work properly on inserti...
Changeset: 3ed973da6129 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3ed973da6129 Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Make seq_char also work properly on insertions/deletions that occur in the CIGAR string. Furthermore, add some extra error checks diffs (114 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -146,7 +146,8 @@ seq_length(int * ret, str * cigar) int result = 0; str cigar_consumable = *cigar; - if (cigar_consumable[0] == '*' cigar_consumable[1] == '\0') { + if (cigar_consumable[0] == '\0' || + (cigar_consumable[0] == '*' cigar_consumable[1] == '\0')) { *ret = -1; return MAL_SUCCEED; } @@ -173,44 +174,58 @@ str seq_char(str * ret, int * ref_pos, str * alg_seq, int * alg_pos, str * alg_cigar) { str cigar_consumable = *alg_cigar; - int seq_pos = 0; - bit at_character = false; - int iterate_until = *ref_pos - *alg_pos; - str result; - - if((result = GDKmalloc(2 * sizeof(char))) == NULL) { - throw(MAL, seq_char, MAL_MALLOC_FAIL); - } - result[1] = '\0'; + int seq_pos = -1; + int cur_ref_pos = *alg_pos - 1; - if (cigar_consumable[0] == '*' cigar_consumable[1] == '\0') { - result[0] = '\0'; - *ret = result; + if (cigar_consumable[0] == '\0' || + (cigar_consumable[0] == '*' cigar_consumable[1] == '\0')) { + *ret = GDKstrdup(str_nil); return MAL_SUCCEED; } - while(cigar_consumable[0] != '\0' seq_pos iterate_until) { + while(TRUE) { int cnt; char op; int nr_chars_read; + bit advance_ref_pos; + bit advance_seq_pos; if (sscanf (cigar_consumable, %d%c%n, cnt, op, nr_chars_read) != 2) throw(MAL, seq_char, Error parsing CIGAR string '%s'\n, *alg_cigar); - if (op == 'M' || op == 'D' || op == 'N' || op == '=' - || op == 'X') { + advance_ref_pos = (op == 'M' || op == 'D' || + op == 'N' || op == '=' || op == 'X'); + advance_seq_pos = (op == 'M' || op == 'I'); // TODO: Find out which chars advance the seq pos + if(advance_seq_pos) { seq_pos += cnt; - if(seq_pos iterate_until) - seq_pos = iterate_until; - at_character = true; - } else { - at_character = false; + } + if (advance_ref_pos) { + cur_ref_pos += cnt; + if(cur_ref_pos = *ref_pos) { + if(!advance_seq_pos) { + seq_pos = -1; + } else { + seq_pos -= (cur_ref_pos - *ref_pos); + } + break; + } } cigar_consumable += nr_chars_read; + if(cigar_consumable[0] == '\0') { + seq_pos = -1; + break; + } } - result[0] = at_character ? (*alg_seq)[seq_pos] : '\0'; - *ret = result; + if(seq_pos 0 || seq_pos = (int)strlen(*alg_seq)) { + *ret = GDKstrdup(str_nil); + return MAL_SUCCEED; + } + if(((*ret) = GDKmalloc(2 * sizeof(char))) == NULL) { + throw(MAL, seq_char, MAL_MALLOC_FAIL); + } + (*ret)[0] = (*alg_seq)[seq_pos]; + (*ret)[1] = '\0'; return MAL_SUCCEED; } @@ -398,7 +413,9 @@ seq_char_bat(bat * ret, int * ref_pos, b throw(MAL, seq_char_bat, RUNTIME_OBJECT_MISSING); if(BATcount(seqs) != BATcount(poss) || BATcount(seqs) != BATcount(cigars)) { - throw(MAL, seq_char_bat, Misalignment in input BATs); + throw(MAL, seq_char_bat, + Misalignment in input BATs: BUNFMT/BUNFMT/BUNFMT, + BATcount(poss), BATcount(seqs), BATcount(cigars)); } /* allocate result BAT */ @@ -428,7 +445,7 @@ seq_char_bat(bat * ret, int * ref_pos, b BBPreleaseref(result-batCacheid); return msg; } - BUNappend(result, (ptr) r, FALSE); + BUNappend(result, (ptr) r, FALSE); ++seq; ++pos; ++cigar;
MonetDB: bamloader - Merge with default
Changeset: 0eef8c674c73 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eef8c674c73 Added Files: sql/test/BugTracker-2014/Tests/queueError.Bug-3604.sql sql/test/BugTracker-2014/Tests/queueError.Bug-3604.stable.err sql/test/BugTracker-2014/Tests/queueError.Bug-3604.stable.out sql/test/mergetables/Tests/alter.sql sql/test/mergetables/Tests/emptymergequery.sql sql/test/mergetables/Tests/mergekey.sql sql/test/mergetables/Tests/types.sql Modified Files: clients/R/Tests/install-dependencies.sh clients/Tests/MAL-signatures_all.stable.out clients/Tests/MAL-signatures_all.stable.out.int128 clients/Tests/MAL-signatures_fits_geom.stable.out clients/Tests/MAL-signatures_fits_geom.stable.out.int128 clients/Tests/MAL-signatures_geom.stable.out clients/Tests/MAL-signatures_geom.stable.out.int128 clients/Tests/MAL-signatures_none.stable.out clients/Tests/MAL-signatures_none.stable.out.int128 clients/Tests/exports.stable.out gdk/gdk.h gdk/gdk_align.c gdk/gdk_bat.c gdk/gdk_bbp.c gdk/gdk_cross.c gdk/gdk_group.c gdk/gdk_heap.c gdk/gdk_imprints.c gdk/gdk_join.c gdk/gdk_private.h gdk/gdk_search.c gdk/gdk_select.c gdk/gdk_storage.c gdk/gdk_unique.c java/src/nl/cwi/monetdb/jdbc/MonetDatabaseMetaData.java monetdb5/extras/rdf/rdf.h monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_linker.c monetdb5/mal/mal_linker.h monetdb5/mal/mal_private.h monetdb5/modules/atoms/blob.c monetdb5/modules/atoms/mtime.c monetdb5/modules/atoms/mtime.h monetdb5/modules/atoms/str.c monetdb5/modules/atoms/str.h monetdb5/modules/kernel/aggr.c monetdb5/modules/kernel/algebra.c monetdb5/modules/kernel/algebra.h monetdb5/modules/kernel/bat5.c monetdb5/modules/kernel/bat5.h monetdb5/modules/kernel/batcolor.mal monetdb5/modules/kernel/batstr.c monetdb5/modules/kernel/counters.c monetdb5/modules/kernel/counters.mal monetdb5/modules/mal/batExtensions.c monetdb5/modules/mal/batExtensions.h monetdb5/modules/mal/batcalc.c monetdb5/modules/mal/bbp.h monetdb5/modules/mal/factories.c monetdb5/modules/mal/factories.h monetdb5/modules/mal/inspect.h monetdb5/modules/mal/language.c monetdb5/modules/mal/language.h monetdb5/modules/mal/mal_mapi.c monetdb5/modules/mal/mal_mapi.h monetdb5/modules/mal/mat.c monetdb5/modules/mal/mat.h monetdb5/modules/mal/mdb.c monetdb5/modules/mal/mdb.h monetdb5/modules/mal/pcre.c monetdb5/modules/mal/pcre.mal monetdb5/modules/mal/profiler.c monetdb5/modules/mal/profiler.h monetdb5/modules/mal/transaction.c sql/backends/monet5/Tests/All sql/backends/monet5/datacell/basket.h sql/backends/monet5/datacell/petrinet.c sql/backends/monet5/generator/generator.c sql/backends/monet5/rest/rest_jsonstore_handle_get.h sql/backends/monet5/sql.c sql/backends/monet5/sql.h sql/backends/monet5/sql.mal sql/backends/monet5/sql_cast.c sql/backends/monet5/sql_cast.h sql/backends/monet5/sql_cast_impl_down_from_flt.h sql/backends/monet5/sql_cast_impl_down_from_int.h sql/backends/monet5/sql_cast_impl_up_to_flt.h sql/backends/monet5/sql_cast_impl_up_to_int.h sql/backends/monet5/sql_gencode.h sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_scenario.h sql/benchmarks/ssbm/Tests/01-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/02-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/03-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/04-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/05-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/06-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/07-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/08-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/09-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/10-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/11-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/12-explain.stable.out.int128 sql/benchmarks/ssbm/Tests/13-explain.stable.out.int128 sql/server/rel_planner.c sql/server/rel_schema.c sql/test/BugTracker-2014/Tests/All sql/test/leaks/Tests/check0_nogeom.stable.out sql/test/leaks/Tests/check3_nogeom.stable.out sql/test/leaks/Tests/check4_nogeom.stable.out sql/test/leaks/Tests/check5_nogeom.stable.out sql/test/leaks/Tests/select1_nogeom.stable.out
MonetDB: default - Add test case for creating a merge table over...
Changeset: 6cb7f375f65d for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6cb7f375f65d Added Files: sql/backends/monet5/bam/Tests/mergetable.SQL.py sql/backends/monet5/bam/Tests/sql/mergetable.sql Modified Files: sql/backends/monet5/bam/Tests/All Branch: default Log Message: Add test case for creating a merge table over two BAM files, which causes a server crash diffs (53 lines): diff --git a/sql/backends/monet5/bam/Tests/All b/sql/backends/monet5/bam/Tests/All --- a/sql/backends/monet5/bam/Tests/All +++ b/sql/backends/monet5/bam/Tests/All @@ -24,3 +24,4 @@ HAVE_SAMTOOLS?query2.9 HAVE_SAMTOOLS?query2.10 HAVE_SAMTOOLS?query2.11 HAVE_SAMTOOLS?query2.12 +HAVE_SAMTOOLS?mergetable \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/mergetable.SQL.py b/sql/backends/monet5/bam/Tests/mergetable.SQL.py new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/mergetable.SQL.py @@ -0,0 +1,3 @@ +import bam + +bam.exec_sql_file(mergetable.sql, {'PWD': bam.SRCDIR}) \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/sql/mergetable.sql b/sql/backends/monet5/bam/Tests/sql/mergetable.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/bam/Tests/sql/mergetable.sql @@ -0,0 +1,30 @@ +SET SCHEMA bam; + +# Load files +CALL bam_loader_repos('PWD/files', 0, 4); + +# Add a merge table over these files +CREATE MERGE TABLE alignments ( + virtual_offset BIGINTNOT NULL, + qname STRINGNOT NULL, + flag SMALLINT NOT NULL, + rname STRINGNOT NULL, + posINT NOT NULL, + mapq SMALLINT NOT NULL, + cigar STRINGNOT NULL, + rnext STRINGNOT NULL, + pnext INT NOT NULL, + tlen INT NOT NULL, + seqSTRINGNOT NULL, + qual STRINGNOT NULL +); + +SELECT COUNT(*) FROM alignments; + +ALTER TABLE alignments ADD TABLE alignments_1; + +SELECT COUNT(*) FROM alignments; + +ALTER TABLE alignments ADD TABLE alignments_2; + +SELECT COUNT(*) FROM alignments; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Fixed test query 2.4
Changeset: 8228693a6130 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8228693a6130 Modified Files: sql/backends/monet5/bam/Tests/query2.4.sql sql/backends/monet5/bam/Tests/query2.4.stable.out Branch: default Log Message: Fixed test query 2.4 diffs (200 lines): diff --git a/sql/backends/monet5/bam/Tests/query2.4.sql b/sql/backends/monet5/bam/Tests/query2.4.sql --- a/sql/backends/monet5/bam/Tests/query2.4.sql +++ b/sql/backends/monet5/bam/Tests/query2.4.sql @@ -3,29 +3,36 @@ FROM bam.alignments_1 WHERE qname IN ( SELECT qname FROM ( -SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, -bam.bam_flag(flag, 'firs_segm') AS firs_segm -FROM bam.alignments_1 -WHERE bam.bam_flag(flag, 'firs_segm') bam.bam_flag(flag, 'last_segm') +SELECT qname, SUM(segm_unma), SUM(seco_alig), COUNT(*) +FROM ( +SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, +bam.bam_flag(flag, 'firs_segm') AS firs_segm +FROM bam.alignments_1 +WHERE bam.bam_flag(flag, 'firs_segm') bam.bam_flag(flag, 'last_segm') +) AS sub +GROUP BY qname, firs_segm +HAVING SUM(segm_unma) COUNT(*) + AND (COUNT(*) - SUM(seco_alig)) 1 ) AS qnames -GROUP BY qname, firs_segm -HAVING SUM(segm_unma) COUNT(*) - AND (COUNT(*) - SUM(seco_alig)) 1 -) -ORDER BY qname; +); + SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual FROM bam.unpaired_all_alignments_3 WHERE qname IN ( SELECT qname FROM ( -SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, -bam.bam_flag(flag, 'firs_segm') AS firs_segm -FROM bam.unpaired_all_alignments_3 -WHERE bam.bam_flag(flag, 'firs_segm') bam.bam_flag(flag, 'last_segm') +SELECT qname, SUM(segm_unma), SUM(seco_alig), COUNT(*) +FROM ( +SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, +bam.bam_flag(flag, 'firs_segm') AS firs_segm +FROM bam.unpaired_all_alignments_3 +WHERE bam.bam_flag(flag, 'firs_segm') bam.bam_flag(flag, 'last_segm') +) AS sub +GROUP BY qname, firs_segm +HAVING SUM(segm_unma) COUNT(*) + AND (COUNT(*) - SUM(seco_alig)) 1 ) AS qnames -GROUP BY qname, firs_segm -HAVING SUM(segm_unma) COUNT(*) - AND (COUNT(*) - SUM(seco_alig)) 1 -) -ORDER BY qname; +); + + diff --git a/sql/backends/monet5/bam/Tests/query2.4.stable.out b/sql/backends/monet5/bam/Tests/query2.4.stable.out --- a/sql/backends/monet5/bam/Tests/query2.4.stable.out +++ b/sql/backends/monet5/bam/Tests/query2.4.stable.out @@ -1,9 +1,9 @@ stdout of test 'query2.4` in directory 'sql/backends/monet5/bam` itself: -# 10:22:42 -# 10:22:42 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=32986 --set mapi_usock=/var/tmp/mtest-16074/.s.monetdb.32986 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 -# 10:22:42 +# 17:36:06 +# 17:36:06 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=33676 --set mapi_usock=/var/tmp/mtest-11149/.s.monetdb.33676 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 +# 17:36:06 # MonetDB 5 server v11.18.0 # This is an unreleased version @@ -13,29 +13,27 @@ stdout of test 'query2.4` in directory ' # Copyright (c) 1993-July 2008 CWI. # Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved # Visit http://www.monetdb.org/ for further information -# Listening for connection requests on mapi:monetdb://robin-xps13:32986/ -# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-16074/.s.monetdb.32986 +# Listening for connection requests on mapi:monetdb://robin-xps13:33676/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-11149/.s.monetdb.33676 # MonetDB/SQL module loaded Ready. -# 10:22:42 -# 10:22:42 mclient -lsql -ftest -Eutf-8 -i -e --host=/var/tmp/mtest-16074 --port=32986 -# 10:22:42 +# 17:36:06 +# 17:36:06 mclient -lsql -ftest -Eutf-8 -i -e --host=/var/tmp/mtest-11149 --port=33676 +# 17:36:06 #SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual #FROM bam.alignments_1 #WHERE qname IN ( #SELECT qname #FROM ( -#SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, -#bam.bam_flag(flag, 'firs_segm') AS firs_segm -#FROM
MonetDB: Oct2014 - Fixed test query 2.4
Changeset: bcc095a70a73 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bcc095a70a73 Modified Files: sql/backends/monet5/bam/Tests/query2.4.sql sql/backends/monet5/bam/Tests/query2.4.stable.out Branch: Oct2014 Log Message: Fixed test query 2.4 (grafted from 8228693a61305d8bdf68082a366ab5c7cd2ada10) diffs (200 lines): diff --git a/sql/backends/monet5/bam/Tests/query2.4.sql b/sql/backends/monet5/bam/Tests/query2.4.sql --- a/sql/backends/monet5/bam/Tests/query2.4.sql +++ b/sql/backends/monet5/bam/Tests/query2.4.sql @@ -3,29 +3,36 @@ FROM bam.alignments_1 WHERE qname IN ( SELECT qname FROM ( -SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, -bam.bam_flag(flag, 'firs_segm') AS firs_segm -FROM bam.alignments_1 -WHERE bam.bam_flag(flag, 'firs_segm') bam.bam_flag(flag, 'last_segm') +SELECT qname, SUM(segm_unma), SUM(seco_alig), COUNT(*) +FROM ( +SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, +bam.bam_flag(flag, 'firs_segm') AS firs_segm +FROM bam.alignments_1 +WHERE bam.bam_flag(flag, 'firs_segm') bam.bam_flag(flag, 'last_segm') +) AS sub +GROUP BY qname, firs_segm +HAVING SUM(segm_unma) COUNT(*) + AND (COUNT(*) - SUM(seco_alig)) 1 ) AS qnames -GROUP BY qname, firs_segm -HAVING SUM(segm_unma) COUNT(*) - AND (COUNT(*) - SUM(seco_alig)) 1 -) -ORDER BY qname; +); + SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual FROM bam.unpaired_all_alignments_3 WHERE qname IN ( SELECT qname FROM ( -SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, -bam.bam_flag(flag, 'firs_segm') AS firs_segm -FROM bam.unpaired_all_alignments_3 -WHERE bam.bam_flag(flag, 'firs_segm') bam.bam_flag(flag, 'last_segm') +SELECT qname, SUM(segm_unma), SUM(seco_alig), COUNT(*) +FROM ( +SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, +bam.bam_flag(flag, 'firs_segm') AS firs_segm +FROM bam.unpaired_all_alignments_3 +WHERE bam.bam_flag(flag, 'firs_segm') bam.bam_flag(flag, 'last_segm') +) AS sub +GROUP BY qname, firs_segm +HAVING SUM(segm_unma) COUNT(*) + AND (COUNT(*) - SUM(seco_alig)) 1 ) AS qnames -GROUP BY qname, firs_segm -HAVING SUM(segm_unma) COUNT(*) - AND (COUNT(*) - SUM(seco_alig)) 1 -) -ORDER BY qname; +); + + diff --git a/sql/backends/monet5/bam/Tests/query2.4.stable.out b/sql/backends/monet5/bam/Tests/query2.4.stable.out --- a/sql/backends/monet5/bam/Tests/query2.4.stable.out +++ b/sql/backends/monet5/bam/Tests/query2.4.stable.out @@ -1,9 +1,9 @@ stdout of test 'query2.4` in directory 'sql/backends/monet5/bam` itself: -# 10:22:42 -# 10:22:42 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=32986 --set mapi_usock=/var/tmp/mtest-16074/.s.monetdb.32986 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 -# 10:22:42 +# 17:36:06 +# 17:36:06 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=33676 --set mapi_usock=/var/tmp/mtest-11149/.s.monetdb.33676 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 +# 17:36:06 # MonetDB 5 server v11.18.0 # This is an unreleased version @@ -13,29 +13,27 @@ stdout of test 'query2.4` in directory ' # Copyright (c) 1993-July 2008 CWI. # Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved # Visit http://www.monetdb.org/ for further information -# Listening for connection requests on mapi:monetdb://robin-xps13:32986/ -# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-16074/.s.monetdb.32986 +# Listening for connection requests on mapi:monetdb://robin-xps13:33676/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-11149/.s.monetdb.33676 # MonetDB/SQL module loaded Ready. -# 10:22:42 -# 10:22:42 mclient -lsql -ftest -Eutf-8 -i -e --host=/var/tmp/mtest-16074 --port=32986 -# 10:22:42 +# 17:36:06 +# 17:36:06 mclient -lsql -ftest -Eutf-8 -i -e --host=/var/tmp/mtest-11149 --port=33676 +# 17:36:06 #SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual #FROM bam.alignments_1 #WHERE qname IN ( #SELECT qname #FROM ( -#SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, bam.bam_flag(flag, 'segm_unma') AS segm_unma, -#
MonetDB: bamloader - Fixed most tests in sql/test, the only ones...
Changeset: b95d9e5134d0 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b95d9e5134d0 Added Files: sql/test/BugConstraints/Tests/add_constraint_1643522_bam.sql.src sql/test/BugConstraints/Tests/add_constraint_1643522_bam.stable.err sql/test/BugConstraints/Tests/add_constraint_1643522_bam.stable.out sql/test/BugDay_2005-10-06_2.8/Tests/MapiClient-dump.SF-905851_bam.SQL.py.src sql/test/BugDay_2005-10-06_2.8/Tests/MapiClient-dump.SF-905851_bam.stable.err sql/test/BugDay_2005-10-06_2.8/Tests/MapiClient-dump.SF-905851_bam.stable.out sql/test/BugDay_2005-11-09_2.8/Tests/ORDER_BY_evaluation_error.SF-1023658_bam.sql.src sql/test/BugDay_2005-11-09_2.8/Tests/ORDER_BY_evaluation_error.SF-1023658_bam.stable.err sql/test/BugDay_2005-11-09_2.8/Tests/ORDER_BY_evaluation_error.SF-1023658_bam.stable.out sql/test/BugTracker-2008/Tests/sql_command_kills_db.SF-2233677_bam.sql.src sql/test/BugTracker-2008/Tests/sql_command_kills_db.SF-2233677_bam.stable.err sql/test/BugTracker-2008/Tests/sql_command_kills_db.SF-2233677_bam.stable.out sql/test/BugTracker-2009/Tests/mclient-lsql-D_bam.SQL.sh.src sql/test/BugTracker-2009/Tests/mclient-lsql-D_bam.stable.err sql/test/BugTracker-2009/Tests/mclient-lsql-D_bam.stable.out sql/test/BugTracker-2009/Tests/name_clash_with_dump.SF-2780395_bam.sql.src sql/test/BugTracker-2009/Tests/name_clash_with_dump.SF-2780395_bam.stable.err sql/test/BugTracker-2009/Tests/name_clash_with_dump.SF-2780395_bam.stable.out sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552_bam.sql.src sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552_bam.stable.err sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552_bam.stable.out sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552_bam.stable.out.32bit sql/test/BugTracker-2011/Tests/count-count-distinct.Bug-2808_bam.sql.src sql/test/BugTracker-2011/Tests/count-count-distinct.Bug-2808_bam.stable.err sql/test/BugTracker-2011/Tests/count-count-distinct.Bug-2808_bam.stable.out sql/test/BugTracker-2011/Tests/exists-select.Bug-2933_bam.sql.src sql/test/BugTracker-2011/Tests/exists-select.Bug-2933_bam.stable.err sql/test/BugTracker-2011/Tests/exists-select.Bug-2933_bam.stable.out sql/test/BugTracker-2011/Tests/interrupted-initialization.Bug-2875_bam.py.src sql/test/BugTracker-2011/Tests/interrupted-initialization.Bug-2875_bam.stable.err sql/test/BugTracker-2011/Tests/interrupted-initialization.Bug-2875_bam.stable.out sql/test/BugTracker-2012/Tests/aggregate_vs_positional_column_crash.Bug-3085_bam.sql.src sql/test/BugTracker-2012/Tests/aggregate_vs_positional_column_crash.Bug-3085_bam.stable.err sql/test/BugTracker-2012/Tests/aggregate_vs_positional_column_crash.Bug-3085_bam.stable.out sql/test/BugTracker-2013/Tests/qualified_aggrname.Bug-3332_bam.sql.src sql/test/BugTracker-2013/Tests/qualified_aggrname.Bug-3332_bam.stable.err sql/test/BugTracker-2013/Tests/qualified_aggrname.Bug-3332_bam.stable.out sql/test/BugTracker-2014/Tests/select-having.Bug-3458_bam.sql.src sql/test/BugTracker-2014/Tests/select-having.Bug-3458_bam.stable.err sql/test/BugTracker-2014/Tests/select-having.Bug-3458_bam.stable.out sql/test/BugTracker/Tests/multi-column-constraint.SF-1964587_bam.sql.src sql/test/BugTracker/Tests/multi-column-constraint.SF-1964587_bam.stable.err sql/test/BugTracker/Tests/multi-column-constraint.SF-1964587_bam.stable.out sql/test/BugTracker/Tests/rank_over.SF-1691098_bam.sql.src sql/test/BugTracker/Tests/rank_over.SF-1691098_bam.stable.err sql/test/BugTracker/Tests/rank_over.SF-1691098_bam.stable.out sql/test/BugTracker/Tests/with_only_once.SF-1720293_bam.sql.src sql/test/BugTracker/Tests/with_only_once.SF-1720293_bam.stable.err sql/test/BugTracker/Tests/with_only_once.SF-1720293_bam.stable.out sql/test/DropCascade/Tests/drop_columns_cascade_bam.sql.src sql/test/DropCascade/Tests/drop_columns_cascade_bam.stable.err sql/test/DropCascade/Tests/drop_columns_cascade_bam.stable.out sql/test/DropCascade/Tests/drop_pkey_cascade_bam.sql.src sql/test/DropCascade/Tests/drop_pkey_cascade_bam.stable.err sql/test/DropCascade/Tests/drop_pkey_cascade_bam.stable.out sql/test/Dump/Tests/dump-empty_bam.SQL.py.src sql/test/Dump/Tests/dump-empty_bam.stable.err sql/test/Dump/Tests/dump-empty_bam.stable.out sql/test/Dump/Tests/dump_bam.SQL.py.src sql/test/Dump/Tests/dump_bam.stable.err sql/test/Dump/Tests/dump_bam.stable.out sql/test/UserDump/Tests/create_bam.SQL.py.src sql/test/UserDump/Tests/create_bam.stable.err
MonetDB: bamloader - Merged local repos with branch
Changeset: 4194703b0d57 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4194703b0d57 Added Files: sql/test/BugTracker-2014/Tests/oid-table-assert.Bug-3514.sql sql/test/BugTracker-2014/Tests/oid-table-assert.Bug-3514.stable.err sql/test/BugTracker-2014/Tests/oid-table-assert.Bug-3514.stable.out sql/test/BugTracker-2014/Tests/round-properties.Bug-3515.sql sql/test/BugTracker-2014/Tests/round-properties.Bug-3515.stable.err sql/test/BugTracker-2014/Tests/round-properties.Bug-3515.stable.out Modified Files: .hgtags MonetDB.spec NT/installer32/MonetDB-ODBC-Installer.vdproj NT/installer32/MonetDB5-Geom-Module.vdproj NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB-ODBC-Installer.vdproj NT/installer64/MonetDB5-Geom-Module.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj NT/monetdb_config.h.in NT/rules.msc buildtools/ChangeLog-Archive buildtools/ChangeLog.Jan2014 clients/R/MonetDB.R/DESCRIPTION clients/Tests/exports.stable.out clients/mapilib/mapi.rc clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc clients/python2/setup.py clients/python3/setup.py configure.ag debian/changelog gdk/gdk.h gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_imprints.c gdk/gdk_join.c gdk/gdk_join_legacy.c gdk/gdk_logger.c gdk/gdk_sample.c gdk/gdk_select.c gdk/gdk_unique.c gdk/gdk_utils.c gdk/libbat.rc libversions monetdb5/extras/sphinx/sphinx.c monetdb5/mal/mal_authorize.c monetdb5/mal/mal_builder.c monetdb5/mal/mal_function.c monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_profiler.c monetdb5/modules/atoms/json.c monetdb5/modules/atoms/mtime.c monetdb5/modules/kernel/algebra.c monetdb5/modules/kernel/array.c monetdb5/modules/kernel/bat5.c monetdb5/modules/kernel/batmmath.c monetdb5/modules/kernel/batstr.c monetdb5/modules/kernel/microbenchmark.c monetdb5/modules/mal/clients.c monetdb5/modules/mal/cluster.c monetdb5/modules/mal/mal_mapi.c monetdb5/modules/mal/mat.c monetdb5/modules/mal/mdb.c monetdb5/modules/mal/pcre.c monetdb5/modules/mal/remote.c monetdb5/modules/mal/sample.c monetdb5/modules/mal/sample.h monetdb5/modules/mal/zorder.c monetdb5/optimizer/opt_wrapper.c monetdb5/tools/libmonetdb5.rc sql/backends/monet5/datacell/basket.c sql/backends/monet5/sql.c sql/backends/monet5/sql_round_impl.h sql/benchmarks/ssbm/Tests/01-13.reqtests sql/benchmarks/ssbm/Tests/01-explain.reqtests sql/benchmarks/ssbm/Tests/01-plan.reqtests sql/benchmarks/ssbm/Tests/01.reqtests sql/benchmarks/ssbm/Tests/02-explain.reqtests sql/benchmarks/ssbm/Tests/02-plan.reqtests sql/benchmarks/ssbm/Tests/02.reqtests sql/benchmarks/ssbm/Tests/03-explain.reqtests sql/benchmarks/ssbm/Tests/03-plan.reqtests sql/benchmarks/ssbm/Tests/03.reqtests sql/benchmarks/ssbm/Tests/04-explain.reqtests sql/benchmarks/ssbm/Tests/04-plan.reqtests sql/benchmarks/ssbm/Tests/04.reqtests sql/benchmarks/ssbm/Tests/05-explain.reqtests sql/benchmarks/ssbm/Tests/05-plan.reqtests sql/benchmarks/ssbm/Tests/05.reqtests sql/benchmarks/ssbm/Tests/06-explain.reqtests sql/benchmarks/ssbm/Tests/06-plan.reqtests sql/benchmarks/ssbm/Tests/06.reqtests sql/benchmarks/ssbm/Tests/07-explain.reqtests sql/benchmarks/ssbm/Tests/07-plan.reqtests sql/benchmarks/ssbm/Tests/07.reqtests sql/benchmarks/ssbm/Tests/08-explain.reqtests sql/benchmarks/ssbm/Tests/08-plan.reqtests sql/benchmarks/ssbm/Tests/08.reqtests sql/benchmarks/ssbm/Tests/09-explain.reqtests sql/benchmarks/ssbm/Tests/09-plan.reqtests sql/benchmarks/ssbm/Tests/09.reqtests sql/benchmarks/ssbm/Tests/10-explain.reqtests sql/benchmarks/ssbm/Tests/10-plan.reqtests sql/benchmarks/ssbm/Tests/10.reqtests sql/benchmarks/ssbm/Tests/11-explain.reqtests sql/benchmarks/ssbm/Tests/11-plan.reqtests sql/benchmarks/ssbm/Tests/11.reqtests sql/benchmarks/ssbm/Tests/12-explain.reqtests sql/benchmarks/ssbm/Tests/12-plan.reqtests sql/benchmarks/ssbm/Tests/12.reqtests sql/benchmarks/ssbm/Tests/13-explain.reqtests sql/benchmarks/ssbm/Tests/13-plan.reqtests sql/benchmarks/ssbm/Tests/13.reqtests sql/benchmarks/ssbm/Tests/check1.reqtests sql/benchmarks/ssbm/Tests/load.reqtests sql/storage/bat/bat_logger.c sql/storage/bat/bat_utils.c
MonetDB: bamloader - Merge with default
Changeset: bc4ce7f2abb9 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bc4ce7f2abb9 Modified Files: sql/test/BugTracker/Tests/rank_over.SF-1691098.sql sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out testing/Mtest.py.in Branch: bamloader Log Message: Merge with default diffs (167 lines): diff --git a/sql/test/BugTracker/Tests/rank_over.SF-1691098.sql b/sql/test/BugTracker/Tests/rank_over.SF-1691098.sql --- a/sql/test/BugTracker/Tests/rank_over.SF-1691098.sql +++ b/sql/test/BugTracker/Tests/rank_over.SF-1691098.sql @@ -1,1 +1,9 @@ -select RANK () OVER () as foo from tables; +create table rankbug (id int); +insert into rankbug values (42); +insert into rankbug select * from rankbug; +insert into rankbug select * from rankbug; +insert into rankbug select * from rankbug; +insert into rankbug select * from rankbug; +insert into rankbug select * from rankbug; +select RANK () OVER () as foo from rankbug; +drop table rankbug; diff --git a/sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out b/sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out --- a/sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out +++ b/sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out @@ -24,8 +24,21 @@ Ready. # 16:03:46 Mtimeout -timeout 60 MapiClient -lsql -umonetdb -Pmonetdb --host=localhost --port=38533 # 16:03:46 -#select RANK () OVER () as foo from tables; -% .L # table_name +#create table rankbug (id int); +#insert into rankbug values (42); +[ 1] +#insert into rankbug select * from rankbug; +[ 1] +#insert into rankbug select * from rankbug; +[ 2] +#insert into rankbug select * from rankbug; +[ 4] +#insert into rankbug select * from rankbug; +[ 8] +#insert into rankbug select * from rankbug; +[ 16 ] +#select RANK () OVER () as foo from rankbug; +% sys.L # table_name % foo # name % int # type % 2 # length @@ -61,14 +74,7 @@ Ready. [ 30 ] [ 31 ] [ 32 ] -[ 33 ] -[ 34 ] -[ 35 ] -[ 36 ] -[ 37 ] -[ 38 ] -[ 39 ] -[ 40 ] +#drop table rankbug; # 17:29:50 # 17:29:50 Done. diff --git a/sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql b/sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql --- a/sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql +++ b/sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql @@ -1,2 +1,10 @@ -WITH t AS ( SELECT count(*) FROM tables ) SELECT * FROM t; -WITH t AS ( SELECT count(*) FROM tables ) SELECT * FROM t; +create table withonlybug (id int); +insert into withonlybug values (42); +insert into withonlybug select * from withonlybug; +insert into withonlybug select * from withonlybug; +insert into withonlybug select * from withonlybug; +insert into withonlybug select * from withonlybug; +insert into withonlybug select * from withonlybug; +WITH t AS ( SELECT count(*) FROM withonlybug ) SELECT * FROM t; +WITH t AS ( SELECT count(*) FROM withonlybug ) SELECT * FROM t; +drop table withonlybug; diff --git a/sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out b/sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out --- a/sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out +++ b/sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out @@ -14,27 +14,63 @@ stdout of test 'with_only_once.SF-172029 # MonetDB/SQL module v2.25.0 loaded Ready. -#function user.main():void; -#clients.quit(); -#end main; +# SQL catalog created, loading sql scripts once +# loading sql script: 09_like.sql +# loading sql script: 10_math.sql +# loading sql script: 11_times.sql +# loading sql script: 12_url.sql +# loading sql script: 13_date.sql +# loading sql script: 14_inet.sql +# loading sql script: 15_querylog.sql +# loading sql script: 16_tracelog.sql +# loading sql script: 17_compress.sql +# loading sql script: 18_dictionary.sql +# loading sql script: 19_cluster.sql +# loading sql script: 20_vacuum.sql +# loading sql script: 21_dependency_functions.sql +# loading sql script: 22_clients.sql +# loading sql script: 23_skyserver.sql +# loading sql script: 24_zorder.sql +# loading sql script: 25_debug.sql +# loading sql script: 26_sysmon.sql +# loading sql script: 39_analytics.sql +# loading sql script: 40_geom.sql +# loading sql script: 46_gsl.sql +# loading sql script: 75_storagemodel.sql +# loading sql script: 80_statistics.sql +# loading sql script: 80_udf.sql +# loading sql script: 99_system.sql +# 17:55:39 +# 17:55:39 mclient -lsql -ftest -Eutf-8 -i -e --host=/var/tmp/mtest-30978 --port=32166 +# 17:55:39 -# 22:41:45 -# 22:41:45 Mtimeout -timeout 60 MapiClient -lsql -umonetdb -Pmonetdb --host=localhost --port=32869 -# 22:41:45 - -#WITH t AS ( SELECT count(*) FROM tables ) SELECT * FROM t; -% .t # table_name +#create table withonlybug (id int); +#insert into withonlybug values (42);
MonetDB: bamloader - Adjusted jdbc tests to comply with bamloade...
Changeset: f4280bc65958 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f4280bc65958 Added Files: sql/jdbc/tests/Tests/Test_Dobjects_bam.SQL.sh sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.err sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.out sql/jdbc/tests/Tests/Test_JdbcClient_bam.SQL.sh sql/jdbc/tests/Tests/Test_JdbcClient_bam.sh sql/jdbc/tests/Tests/Test_JdbcClient_bam.sh.src sql/jdbc/tests/Tests/Test_JdbcClient_bam.stable.err sql/jdbc/tests/Tests/Test_JdbcClient_bam.stable.out Modified Files: sql/jdbc/tests/Tests/All Branch: bamloader Log Message: Adjusted jdbc tests to comply with bamloader branch diffs (truncated from 590 to 300 lines): diff --git a/sql/jdbc/tests/Tests/All b/sql/jdbc/tests/Tests/All --- a/sql/jdbc/tests/Tests/All +++ b/sql/jdbc/tests/Tests/All @@ -5,7 +5,8 @@ HAVE_JDBCTESTS?Test_Creplysize HAVE_JDBCTESTS?Test_Csavepoints HAVE_JDBCTESTS?Test_Ctransaction #HAVE_JDBCTESTS?Test_Csendthread # unfortunately has runtime-dependant output -HAVE_JDBCTESTS?Test_Dobjects +HAVE_JDBCTESTS!HAVE_SAMTOOLS?Test_Dobjects +HAVE_JDBCTESTSHAVE_SAMTOOLS?Test_Dobjects_bam HAVE_JDBCTESTS?Test_PSgeneratedkeys HAVE_JDBCTESTS?Test_PSsomeamount HAVE_JDBCTESTS?Test_PSlargebatchval @@ -21,7 +22,8 @@ HAVE_JDBCTESTS?Test_Rpositioning HAVE_JDBCTESTS?Test_Rtimedate HAVE_JDBCTESTS?Test_Rsqldata HAVE_JDBCTESTS?Test_Sbatching -HAVE_JDBCCLIENT_JAR?Test_JdbcClient +HAVE_JDBCCLIENT_JAR!HAVE_SAMTOOLS?Test_JdbcClient +HAVE_JDBCCLIENT_JARHAVE_SAMTOOLS?Test_JdbcClient_bam HAVE_JDBCTESTS?BugConcurrent_clients_SF_1504657 HAVE_JDBCTESTS?BugConcurrent_sequences HAVE_JDBCTESTS?BugDatabaseMetaData_Bug_3356 diff --git a/sql/jdbc/tests/Tests/Test_Dobjects_bam.SQL.sh b/sql/jdbc/tests/Tests/Test_Dobjects_bam.SQL.sh new file mode 100755 --- /dev/null +++ b/sql/jdbc/tests/Tests/Test_Dobjects_bam.SQL.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +TST=${TST/_bam} +$TSTSRCDIR/Test.SQL.sh ${*/_bam} diff --git a/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.err b/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.err new file mode 100644 --- /dev/null +++ b/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.err @@ -0,0 +1,40 @@ +stderr of test 'Test_Dobjects_bam` in directory 'sql/jdbc/tests` itself: + + +# 15:28:55 +# 15:28:55 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=37247 --set mapi_usock=/var/tmp/mtest-21432/.s.monetdb.37247 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_jdbc_tests --set mal_listing=0 +# 15:28:55 + +# builtin opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 37247 +# cmdline opt mapi_usock = /var/tmp/mtest-21432/.s.monetdb.37247 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_jdbc_tests +# cmdline opt mal_listing = 0 +# cmdline opt gdk_debug = 536870922 + +# 15:28:56 +# 15:28:56 ./Test_Dobjects_bam.SQL.sh Test_Dobjects_bam +# 15:28:56 + + +# 15:28:56 +# 15:28:56 java Test_Dobjects jdbc:monetdb://robin-xps13:37247/mTests_sql_jdbc_tests?user=monetdbpassword=monetdb +# 15:28:56 + + +# 15:28:56 +# 15:28:56 Done. +# 15:28:56 diff --git a/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.out b/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.out new file mode 100644 --- /dev/null +++ b/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.out @@ -0,0 +1,133 @@ +stdout of test 'Test_Dobjects_bam` in directory 'sql/jdbc/tests` itself: + + +# 15:28:55 +# 15:28:55 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=37247 --set mapi_usock=/var/tmp/mtest-21432/.s.monetdb.37247 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_jdbc_tests --set mal_listing=0 +# 15:28:55 + +# MonetDB 5 server v11.18.0 +# This is an unreleased version +# Serving database 'mTests_sql_jdbc_tests', using 4 threads +# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically linked +# Found 7.333 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://robin-xps13:37247/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-21432/.s.monetdb.37247 +# MonetDB/SQL module loaded + +Ready. +# SQL
MonetDB: bamloader - Extended some more tests
Changeset: 5d30a783ecdd for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5d30a783ecdd Added Files: sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.sql.src sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.err sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.out sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-1.Bug-3063_bam.sql.src sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-1.Bug-3063_bam.stable.err sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-1.Bug-3063_bam.stable.out sql/test/Dependencies/Tests/Dependencies_bam.SQL.py.src sql/test/Dependencies/Tests/Dependencies_bam.stable.err sql/test/Dependencies/Tests/Dependencies_bam.stable.out sql/test/Tests/keys_bam.sql.src sql/test/Tests/keys_bam.stable.err sql/test/Tests/keys_bam.stable.out Modified Files: sql/test/BugTracker-2012/Tests/All sql/test/Dependencies/Tests/All sql/test/Tests/All sql/test/bugs/Tests/All Branch: bamloader Log Message: Extended some more tests diffs (truncated from 1327 to 300 lines): diff --git a/sql/test/BugTracker-2012/Tests/All b/sql/test/BugTracker-2012/Tests/All --- a/sql/test/BugTracker-2012/Tests/All +++ b/sql/test/BugTracker-2012/Tests/All @@ -29,8 +29,10 @@ name_resolution_in_proc.Bug-3074 segfault_incorrect_head.Bug-3028 cast_varchar_to_double.Bug-3071 table_returning_func_returns_too_many_columns.Bug-3077 -table_functions_fail_after_restart-0.Bug-3063 -table_functions_fail_after_restart-1.Bug-3063 +!HAVE_SAMTOOLS?table_functions_fail_after_restart-0.Bug-3063 +HAVE_SAMTOOLS?table_functions_fail_after_restart-0.Bug-3063_bam +!HAVE_SAMTOOLS?table_functions_fail_after_restart-1.Bug-3063 +HAVE_SAMTOOLS?table_functions_fail_after_restart-1.Bug-3063_bam user_defined_decimal_function.Bug-2992 non_keyword_no.Bug-3079 round-type-inference.Bug-2987 diff --git a/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.sql.src b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.sql.src new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.sql.src @@ -0,0 +1,1 @@ +$RELSRCDIR/table_functions_fail_after_restart-0.Bug-3063.sql diff --git a/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.err b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.err new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.err @@ -0,0 +1,35 @@ +stderr of test 'table_functions_fail_after_restart-0.Bug-3063_bam` in directory 'sql/test/BugTracker-2012` itself: + + +# 09:28:52 +# 09:28:52 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=36187 --set mapi_usock=/var/tmp/mtest-1669/.s.monetdb.36187 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_test_BugTracker-2012 --set mal_listing=0 +# 09:28:52 + +# builtin opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 36187 +# cmdline opt mapi_usock = /var/tmp/mtest-1669/.s.monetdb.36187 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_test_BugTracker-2012 +# cmdline opt mal_listing = 0 +# cmdline opt gdk_debug = 536870922 + +# 09:28:52 +# 09:28:52 mclient -lsql -ftest -Eutf-8 -i -e --host=/var/tmp/mtest-1669 --port=36187 +# 09:28:52 + + +# 09:28:53 +# 09:28:53 Done. +# 09:28:53 diff --git a/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.out b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.out new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.out @@ -0,0 +1,43 @@ +stdout of test 'table_functions_fail_after_restart-0.Bug-3063_bam` in directory 'sql/test/BugTracker-2012` itself: + + +# 09:28:52 +# 09:28:52 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=36187 --set mapi_usock=/var/tmp/mtest-1669/.s.monetdb.36187 --set monet_prompt= --forcemito --set mal_listing=2
MonetDB: bamloader - Moved creation of static SQL tables from c ...
Changeset: 99e071e2ea6d for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=99e071e2ea6d Modified Files: sql/backends/monet5/bam/85_bam.sql sql/backends/monet5/bam/bam_db_interface.c sql/backends/monet5/bam/bam_db_interface.h sql/backends/monet5/bam/bam_loader.c Branch: bamloader Log Message: Moved creation of static SQL tables from c code to 85_bam.sql diffs (truncated from 304 to 300 lines): diff --git a/sql/backends/monet5/bam/85_bam.sql b/sql/backends/monet5/bam/85_bam.sql --- a/sql/backends/monet5/bam/85_bam.sql +++ b/sql/backends/monet5/bam/85_bam.sql @@ -31,3 +31,70 @@ EXTERNAL NAME bam.sam_export; CREATE PROCEDURE bam.bam_export(output_path STRING) EXTERNAL NAME bam.bam_export; + + + +CREATE TABLE bam.files ( + file_id BIGINT NOT NULL, + file_location STRING NOT NULL, + dbschemaSMALLINT NOT NULL, + format_version VARCHAR(7), + sorting_order VARCHAR(10), + commentsSTRING, + CONSTRAINT files_pkey_file_id PRIMARY KEY (file_id) +); + +CREATE TABLE bam.sq ( + sn STRING NOT NULL, + file_id BIGINT NOT NULL, + ln INT, + asINT, + m5 STRING, + sp STRING, + ur STRING, + CONSTRAINT sq_pkey_sn_file_id PRIMARY KEY (sn, file_id), + CONSTRAINT sq_fkey_file_id FOREIGN KEY (file_id) REFERENCES bam.files (file_id) +); + +CREATE TABLE bam.rg ( + id STRING NOT NULL, + file_id BIGINT NOT NULL, + cn STRING, + ds STRING, + dt TIMESTAMP, + fo STRING, + ks STRING, + lb STRING, + pg STRING, + pi INT, + pl STRING, + pu STRING, + sm STRING, + CONSTRAINT rg_pkey_id_file_id PRIMARY KEY (id, file_id), + CONSTRAINT rg_fkey_file_id FOREIGN KEY (file_id) REFERENCES bam.files (file_id) +); + +CREATE TABLE bam.pg ( + id STRING NOT NULL, + file_id BIGINT NOT NULL, + pn STRING, + cl STRING, + pp STRING, + vn STRING, + CONSTRAINT pg_pkey_id_file_id PRIMARY KEY (id, file_id), + CONSTRAINT pg_fkey_file_id FOREIGN KEY (file_id) REFERENCES bam.files (file_id) +); + +CREATE TABLE bam.export ( + qname STRING NOT NULL, + flagSMALLINT NOT NULL, + rname STRING NOT NULL, + pos INT NOT NULL, + mapqSMALLINT NOT NULL, + cigar STRING NOT NULL, + rnext STRING NOT NULL, + pnext INT NOT NULL, + tlenINT NOT NULL, + seq STRING NOT NULL, + qualSTRING NOT NULL +); diff --git
MonetDB: bamloader - Fix system absolute paths in test outputs
Changeset: 9cb7c7c1cf07 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9cb7c7c1cf07 Modified Files: sql/backends/monet5/bam/Tests/bam_loader_file.stable.err sql/backends/monet5/bam/Tests/bam_loader_file.stable.out sql/backends/monet5/bam/Tests/bam_loader_files.stable.err sql/backends/monet5/bam/Tests/bam_loader_files.stable.out sql/backends/monet5/bam/Tests/check_files.sql sql/backends/monet5/bam/Tests/check_files.stable.err sql/backends/monet5/bam/Tests/check_files.stable.out sql/backends/monet5/bam/Tests/check_files_exact.sql sql/backends/monet5/bam/Tests/check_files_exact.stable.err sql/backends/monet5/bam/Tests/check_files_exact.stable.out sql/backends/monet5/bam/Tests/drop_last_files.stable.err sql/backends/monet5/bam/Tests/drop_last_files.stable.out sql/backends/monet5/bam/Tests/query1.1.stable.err sql/backends/monet5/bam/Tests/query1.1.stable.out sql/backends/monet5/bam/Tests/query1.2.stable.err sql/backends/monet5/bam/Tests/query1.2.stable.out sql/backends/monet5/bam/Tests/query1.3.stable.err sql/backends/monet5/bam/Tests/query1.3.stable.out sql/backends/monet5/bam/Tests/query1.4.stable.err sql/backends/monet5/bam/Tests/query1.4.stable.out sql/backends/monet5/bam/Tests/query1.5.stable.err sql/backends/monet5/bam/Tests/query1.5.stable.out sql/backends/monet5/bam/Tests/query2.1.stable.err sql/backends/monet5/bam/Tests/query2.1.stable.out sql/backends/monet5/bam/Tests/query2.10.stable.err sql/backends/monet5/bam/Tests/query2.10.stable.out sql/backends/monet5/bam/Tests/query2.11.stable.err sql/backends/monet5/bam/Tests/query2.11.stable.out sql/backends/monet5/bam/Tests/query2.12.stable.err sql/backends/monet5/bam/Tests/query2.12.stable.out sql/backends/monet5/bam/Tests/query2.2.stable.err sql/backends/monet5/bam/Tests/query2.2.stable.out sql/backends/monet5/bam/Tests/query2.3.stable.err sql/backends/monet5/bam/Tests/query2.3.stable.out sql/backends/monet5/bam/Tests/query2.4.stable.err sql/backends/monet5/bam/Tests/query2.4.stable.out sql/backends/monet5/bam/Tests/query2.5.stable.err sql/backends/monet5/bam/Tests/query2.5.stable.out sql/backends/monet5/bam/Tests/query2.6.stable.err sql/backends/monet5/bam/Tests/query2.6.stable.out sql/backends/monet5/bam/Tests/query2.7.stable.err sql/backends/monet5/bam/Tests/query2.7.stable.out sql/backends/monet5/bam/Tests/query2.8.stable.err sql/backends/monet5/bam/Tests/query2.8.stable.out sql/backends/monet5/bam/Tests/query2.9.stable.err sql/backends/monet5/bam/Tests/query2.9.stable.out sql/backends/monet5/bam/Tests/sam_export.stable.err sql/backends/monet5/bam/Tests/sam_export.stable.out sql/backends/monet5/bam/Tests/verify_load.stable.err sql/backends/monet5/bam/Tests/verify_load.stable.out Branch: bamloader Log Message: Fix system absolute paths in test outputs diffs (truncated from 2583 to 300 lines): diff --git a/sql/backends/monet5/bam/Tests/bam_loader_file.stable.err b/sql/backends/monet5/bam/Tests/bam_loader_file.stable.err --- a/sql/backends/monet5/bam/Tests/bam_loader_file.stable.err +++ b/sql/backends/monet5/bam/Tests/bam_loader_file.stable.err @@ -1,9 +1,9 @@ stderr of test 'bam_loader_file` in directory 'sql/backends/monet5/bam` itself: -# 13:29:48 -# 13:29:48 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=37021 --set mapi_usock=/var/tmp/mtest-21062/.s.monetdb.37021 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 -# 13:29:48 +# 10:22:37 +# 10:22:37 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=32986 --set mapi_usock=/var/tmp/mtest-16074/.s.monetdb.32986 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 +# 10:22:37 # builtin opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo # builtin opt gdk_debug = 0 @@ -17,16 +17,14 @@ stderr of test 'bam_loader_file` in dire # builtin opt sql_debug = 0 # cmdline opt gdk_nr_threads = 0 # cmdline opt mapi_open = true -# cmdline opt mapi_port = 37021 -# cmdline opt mapi_usock = /var/tmp/mtest-21062/.s.monetdb.37021 +# cmdline opt mapi_port = 32986 +# cmdline opt mapi_usock = /var/tmp/mtest-16074/.s.monetdb.32986 # cmdline opt monet_prompt = # cmdline opt mal_listing = 2 # cmdline opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam # cmdline opt mal_listing = 0 # cmdline opt gdk_debug = 536870922 # bam_loader: Loader
MonetDB: bamloader - Fix file permissions
Changeset: 5b99954e0ebb for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5b99954e0ebb Modified Files: sql/backends/monet5/bam/Tests/All sql/backends/monet5/bam/Tests/query1.1.sql sql/backends/monet5/bam/Tests/query1.2.sql sql/backends/monet5/bam/Tests/query1.3.sql sql/backends/monet5/bam/Tests/query1.4.sql sql/backends/monet5/bam/Tests/query1.5.sql sql/backends/monet5/bam/Tests/query2.1.sql sql/backends/monet5/bam/Tests/query2.10.sql sql/backends/monet5/bam/Tests/query2.11.sql sql/backends/monet5/bam/Tests/query2.12.sql sql/backends/monet5/bam/Tests/query2.2.sql sql/backends/monet5/bam/Tests/query2.3.sql sql/backends/monet5/bam/Tests/query2.4.sql sql/backends/monet5/bam/Tests/query2.5.sql sql/backends/monet5/bam/Tests/query2.6.sql sql/backends/monet5/bam/Tests/query2.7.sql sql/backends/monet5/bam/Tests/query2.8.sql sql/backends/monet5/bam/Tests/query2.9.sql sql/backends/monet5/bam/Tests/test.modules Branch: bamloader Log Message: Fix file permissions diffs (57 lines): diff --git a/sql/backends/monet5/bam/Tests/All b/sql/backends/monet5/bam/Tests/All old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query1.1.sql b/sql/backends/monet5/bam/Tests/query1.1.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query1.2.sql b/sql/backends/monet5/bam/Tests/query1.2.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query1.3.sql b/sql/backends/monet5/bam/Tests/query1.3.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query1.4.sql b/sql/backends/monet5/bam/Tests/query1.4.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query1.5.sql b/sql/backends/monet5/bam/Tests/query1.5.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.1.sql b/sql/backends/monet5/bam/Tests/query2.1.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.10.sql b/sql/backends/monet5/bam/Tests/query2.10.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.11.sql b/sql/backends/monet5/bam/Tests/query2.11.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.12.sql b/sql/backends/monet5/bam/Tests/query2.12.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.2.sql b/sql/backends/monet5/bam/Tests/query2.2.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.3.sql b/sql/backends/monet5/bam/Tests/query2.3.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.4.sql b/sql/backends/monet5/bam/Tests/query2.4.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.5.sql b/sql/backends/monet5/bam/Tests/query2.5.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.6.sql b/sql/backends/monet5/bam/Tests/query2.6.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.7.sql b/sql/backends/monet5/bam/Tests/query2.7.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.8.sql b/sql/backends/monet5/bam/Tests/query2.8.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/query2.9.sql b/sql/backends/monet5/bam/Tests/query2.9.sql old mode 100755 new mode 100644 diff --git a/sql/backends/monet5/bam/Tests/test.modules b/sql/backends/monet5/bam/Tests/test.modules old mode 100755 new mode 100644 ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Added required role parameter to calls to B...
Changeset: c8c0548c31cc for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c8c0548c31cc Modified Files: sql/backends/monet5/bam/bam_lib.c Branch: bamloader Log Message: Added required role parameter to calls to BATnew diffs (39 lines): diff --git a/sql/backends/monet5/bam/bam_lib.c b/sql/backends/monet5/bam/bam_lib.c --- a/sql/backends/monet5/bam/bam_lib.c +++ b/sql/backends/monet5/bam/bam_lib.c @@ -190,7 +190,7 @@ bam_flag_bat(bat * ret, bat * bid, str * throw(MAL, bam_flag_bat, RUNTIME_OBJECT_MISSING); /* allocate result BAT */ - result = BATnew(TYPE_void, TYPE_bit, BATcount(flags)); + result = BATnew(TYPE_void, TYPE_bit, BATcount(flags), TRANSIENT); if (result == NULL) { throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL); } @@ -226,7 +226,7 @@ reverse_seq_bat(bat * ret, bat * bid) throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING); /* allocate result BAT */ - result = BATnew(TYPE_void, TYPE_str, BATcount(seqs)); + result = BATnew(TYPE_void, TYPE_str, BATcount(seqs), TRANSIENT); if (result == NULL) { throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); } @@ -267,7 +267,7 @@ reverse_qual_bat(bat * ret, bat * bid) throw(MAL, reverse_qual_bat, RUNTIME_OBJECT_MISSING); /* allocate result BAT */ - result = BATnew(TYPE_void, TYPE_str, BATcount(quals)); + result = BATnew(TYPE_void, TYPE_str, BATcount(quals), TRANSIENT); if (result == NULL) { throw(MAL, reverse_qual_bat, MAL_MALLOC_FAIL); } @@ -308,7 +308,7 @@ seq_length_bat(bat * ret, bat * bid) throw(MAL, seq_length_bat, RUNTIME_OBJECT_MISSING); /* allocate result BAT */ - result = BATnew(TYPE_void, TYPE_int, BATcount(cigars)); + result = BATnew(TYPE_void, TYPE_int, BATcount(cigars), TRANSIENT); if (result == NULL) { throw(MAL, seq_length_bat, MAL_MALLOC_FAIL); } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Fix stack smashing error by allocating a bi...
Changeset: 6c764974f9f0 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6c764974f9f0 Modified Files: sql/backends/monet5/bam/bam_loader.c Branch: bamloader Log Message: Fix stack smashing error by allocating a big enough buffer diffs (30 lines): diff --git a/sql/backends/monet5/bam/bam_loader.c b/sql/backends/monet5/bam/bam_loader.c --- a/sql/backends/monet5/bam/bam_loader.c +++ b/sql/backends/monet5/bam/bam_loader.c @@ -187,7 +187,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s sql_schema *s = NULL; sql_table *files_table = NULL; lng cur_file_id; - char buf_threads_msg[] = There were reader threads that contained errors:\n; + char buf_threads_msg[4096] = There were reader threads that contained errors:\n; int threads_msg_len = strlen(buf_threads_msg); int i, errnr; str msg = MAL_SUCCEED; @@ -355,7 +355,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s TO_LOG(bam_loader Waiting for reader threads to finish...\n); /* Wait until all threads finish and collect their * messages. Though it is not very likely, it could be the -* case that more than 1 thread generate an error message (not +* case that more than 1 thread generates an error message (not * likely because threads exit once they notice that another * thread has failed). Therefore, we collect all error * messages in one big error string @@ -382,7 +382,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s * use MAX to make sure we don't add a * negative amount to threads_msg_len */ step = snprintf(msg + threads_msg_len, - 2048 - threads_msg_len, * %s\n, + 4096 - threads_msg_len, * %s\n, r_thread_data[i].msg); threads_msg_len += MAX(0, step); GDKfree(r_thread_data[i].msg); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - test.modules is useless after changes in All
Changeset: 0b8e5a569aa3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0b8e5a569aa3 Removed Files: sql/backends/monet5/bam/Tests/test.modules Branch: bamloader Log Message: test.modules is useless after changes in All diffs (6 lines): diff --git a/sql/backends/monet5/bam/Tests/test.modules b/sql/backends/monet5/bam/Tests/test.modules deleted file mode 100644 --- a/sql/backends/monet5/bam/Tests/test.modules +++ /dev/null @@ -1,1 +0,0 @@ -bam ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Splitted test into many small tests
Changeset: b3b973d8c3da for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b3b973d8c3da Added Files: sql/backends/monet5/bam/Tests/bam.py sql/backends/monet5/bam/Tests/bam_export.SQL.py sql/backends/monet5/bam/Tests/bam_loader_file.SQL.py sql/backends/monet5/bam/Tests/bam_loader_file.stable.err sql/backends/monet5/bam/Tests/bam_loader_file.stable.out sql/backends/monet5/bam/Tests/bam_loader_files.SQL.py sql/backends/monet5/bam/Tests/bam_loader_files.stable.err sql/backends/monet5/bam/Tests/bam_loader_files.stable.out sql/backends/monet5/bam/Tests/bam_loader_repos.SQL.py sql/backends/monet5/bam/Tests/bam_loader_repos.stable.err sql/backends/monet5/bam/Tests/bam_loader_repos.stable.out sql/backends/monet5/bam/Tests/check_files.sql sql/backends/monet5/bam/Tests/check_files.stable.err sql/backends/monet5/bam/Tests/check_files.stable.out sql/backends/monet5/bam/Tests/check_files_exact.sql sql/backends/monet5/bam/Tests/check_files_exact.stable.err sql/backends/monet5/bam/Tests/check_files_exact.stable.out sql/backends/monet5/bam/Tests/drop_last_files.SQL.py sql/backends/monet5/bam/Tests/drop_last_files.stable.err sql/backends/monet5/bam/Tests/drop_last_files.stable.out sql/backends/monet5/bam/Tests/query1.1.sql sql/backends/monet5/bam/Tests/query1.1.stable.err sql/backends/monet5/bam/Tests/query1.1.stable.out sql/backends/monet5/bam/Tests/query1.2.sql sql/backends/monet5/bam/Tests/query1.2.stable.err sql/backends/monet5/bam/Tests/query1.2.stable.out sql/backends/monet5/bam/Tests/query1.3.sql sql/backends/monet5/bam/Tests/query1.3.stable.err sql/backends/monet5/bam/Tests/query1.3.stable.out sql/backends/monet5/bam/Tests/query1.4.sql sql/backends/monet5/bam/Tests/query1.4.stable.err sql/backends/monet5/bam/Tests/query1.4.stable.out sql/backends/monet5/bam/Tests/query1.5.sql sql/backends/monet5/bam/Tests/query1.5.stable.err sql/backends/monet5/bam/Tests/query1.5.stable.out sql/backends/monet5/bam/Tests/query1.6.sql sql/backends/monet5/bam/Tests/query2.1.sql sql/backends/monet5/bam/Tests/query2.1.stable.err sql/backends/monet5/bam/Tests/query2.1.stable.out sql/backends/monet5/bam/Tests/query2.10.sql sql/backends/monet5/bam/Tests/query2.10.stable.err sql/backends/monet5/bam/Tests/query2.10.stable.out sql/backends/monet5/bam/Tests/query2.11.sql sql/backends/monet5/bam/Tests/query2.11.stable.err sql/backends/monet5/bam/Tests/query2.11.stable.out sql/backends/monet5/bam/Tests/query2.12.sql sql/backends/monet5/bam/Tests/query2.12.stable.err sql/backends/monet5/bam/Tests/query2.12.stable.out sql/backends/monet5/bam/Tests/query2.2.sql sql/backends/monet5/bam/Tests/query2.2.stable.err sql/backends/monet5/bam/Tests/query2.2.stable.out sql/backends/monet5/bam/Tests/query2.3.sql sql/backends/monet5/bam/Tests/query2.3.stable.err sql/backends/monet5/bam/Tests/query2.3.stable.out sql/backends/monet5/bam/Tests/query2.4.sql sql/backends/monet5/bam/Tests/query2.4.stable.err sql/backends/monet5/bam/Tests/query2.4.stable.out sql/backends/monet5/bam/Tests/query2.5.sql sql/backends/monet5/bam/Tests/query2.5.stable.err sql/backends/monet5/bam/Tests/query2.5.stable.out sql/backends/monet5/bam/Tests/query2.6.sql sql/backends/monet5/bam/Tests/query2.6.stable.err sql/backends/monet5/bam/Tests/query2.6.stable.out sql/backends/monet5/bam/Tests/query2.7.sql sql/backends/monet5/bam/Tests/query2.7.stable.err sql/backends/monet5/bam/Tests/query2.7.stable.out sql/backends/monet5/bam/Tests/query2.8.sql sql/backends/monet5/bam/Tests/query2.8.stable.err sql/backends/monet5/bam/Tests/query2.8.stable.out sql/backends/monet5/bam/Tests/query2.9.sql sql/backends/monet5/bam/Tests/query2.9.stable.err sql/backends/monet5/bam/Tests/query2.9.stable.out sql/backends/monet5/bam/Tests/sam_export.SQL.py sql/backends/monet5/bam/Tests/sam_export.stable.err sql/backends/monet5/bam/Tests/sam_export.stable.out sql/backends/monet5/bam/Tests/sql/bam_export.sql sql/backends/monet5/bam/Tests/sql/bam_loader_file.sql sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql sql/backends/monet5/bam/Tests/sql/drop_last_files.sql sql/backends/monet5/bam/Tests/sql/sam_export.sql sql/backends/monet5/bam/Tests/verify_load.sql sql/backends/monet5/bam/Tests/verify_load.stable.err sql/backends/monet5/bam/Tests/verify_load.stable.out Removed Files:
MonetDB: bamloader - Attach all BAM related SQL functions and pr...
Changeset: c11e529234ec for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c11e529234ec Modified Files: sql/backends/monet5/bam/85_bam.sql sql/backends/monet5/bam/Tests/query1.1.sql sql/backends/monet5/bam/Tests/query1.2.sql sql/backends/monet5/bam/Tests/query2.1.sql sql/backends/monet5/bam/Tests/query2.10.sql sql/backends/monet5/bam/Tests/query2.11.sql sql/backends/monet5/bam/Tests/query2.12.sql sql/backends/monet5/bam/Tests/query2.2.sql sql/backends/monet5/bam/Tests/query2.3.sql sql/backends/monet5/bam/Tests/query2.4.sql sql/backends/monet5/bam/Tests/query2.8.sql sql/backends/monet5/bam/Tests/query2.9.sql sql/backends/monet5/bam/Tests/sql/bam_export.sql sql/backends/monet5/bam/Tests/sql/bam_loader_file.sql sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql sql/backends/monet5/bam/Tests/sql/drop_last_files.sql sql/backends/monet5/bam/Tests/sql/sam_export.sql Branch: bamloader Log Message: Attach all BAM related SQL functions and procedures to the bam schema instead of the sys schema diffs (truncated from 573 to 300 lines): diff --git a/sql/backends/monet5/bam/85_bam.sql b/sql/backends/monet5/bam/85_bam.sql --- a/sql/backends/monet5/bam/85_bam.sql +++ b/sql/backends/monet5/bam/85_bam.sql @@ -1,31 +1,33 @@ -CREATE PROCEDURE bam_loader_repos(bam_repos STRING, dbschema SMALLINT, nr_threads SMALLINT) +CREATE SCHEMA bam; + +CREATE PROCEDURE bam.bam_loader_repos(bam_repos STRING, dbschema SMALLINT, nr_threads SMALLINT) EXTERNAL NAME bam.bam_loader_repos; -CREATE PROCEDURE bam_loader_files(bam_files STRING, dbschema SMALLINT, nr_threads SMALLINT) +CREATE PROCEDURE bam.bam_loader_files(bam_files STRING, dbschema SMALLINT, nr_threads SMALLINT) EXTERNAL NAME bam.bam_loader_files; -CREATE PROCEDURE bam_loader_file(bam_file STRING, dbschema SMALLINT) +CREATE PROCEDURE bam.bam_loader_file(bam_file STRING, dbschema SMALLINT) EXTERNAL NAME bam.bam_loader_file; -CREATE PROCEDURE bam_drop_file(file_id BIGINT, dbschema SMALLINT) +CREATE PROCEDURE bam.bam_drop_file(file_id BIGINT, dbschema SMALLINT) EXTERNAL NAME bam.bam_drop_file; -CREATE FUNCTION bam_flag(flag SMALLINT, name STRING) +CREATE FUNCTION bam.bam_flag(flag SMALLINT, name STRING) RETURNS BOOLEAN EXTERNAL NAME bam.bam_flag; -CREATE FUNCTION reverse_seq(seq STRING) +CREATE FUNCTION bam.reverse_seq(seq STRING) RETURNS STRING EXTERNAL NAME bam.reverse_seq; -CREATE FUNCTION reverse_qual(qual STRING) +CREATE FUNCTION bam.reverse_qual(qual STRING) RETURNS STRING EXTERNAL NAME bam.reverse_qual; -CREATE FUNCTION seq_length(cigar STRING) +CREATE FUNCTION bam.seq_length(cigar STRING) RETURNS INT EXTERNAL NAME bam.seq_length; -CREATE PROCEDURE sam_export(output_path STRING) +CREATE PROCEDURE bam.sam_export(output_path STRING) EXTERNAL NAME bam.sam_export; -CREATE PROCEDURE bam_export(output_path STRING) +CREATE PROCEDURE bam.bam_export(output_path STRING) EXTERNAL NAME bam.bam_export; diff --git a/sql/backends/monet5/bam/Tests/query1.1.sql b/sql/backends/monet5/bam/Tests/query1.1.sql --- a/sql/backends/monet5/bam/Tests/query1.1.sql +++ b/sql/backends/monet5/bam/Tests/query1.1.sql @@ -1,6 +1,6 @@ SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual FROM bam.alignments_1 -WHERE bam_flag(flag, 'seco_alig') = False +WHERE bam.bam_flag(flag, 'seco_alig') = False ORDER BY qname; SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual @@ -8,5 +8,5 @@ FROM bam.unpaired_primary_alignments_3 UNION SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual FROM bam.unpaired_alignments_3 -WHERE bam_flag(flag, 'seco_alig') = False +WHERE bam.bam_flag(flag, 'seco_alig') = False ORDER BY qname; diff --git a/sql/backends/monet5/bam/Tests/query1.2.sql b/sql/backends/monet5/bam/Tests/query1.2.sql --- a/sql/backends/monet5/bam/Tests/query1.2.sql +++ b/sql/backends/monet5/bam/Tests/query1.2.sql @@ -1,6 +1,6 @@ SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual FROM bam.alignments_1 -WHERE bam_flag(flag, 'seco_alig') = False +WHERE bam.bam_flag(flag, 'seco_alig') = False ORDER BY rname, pos; SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual @@ -8,5 +8,5 @@ FROM bam.unpaired_primary_alignments_3 UNION SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual FROM bam.unpaired_alignments_3 -WHERE bam_flag(flag, 'seco_alig') = False +WHERE bam.bam_flag(flag, 'seco_alig') = False ORDER BY rname, pos; diff --git a/sql/backends/monet5/bam/Tests/query2.1.sql b/sql/backends/monet5/bam/Tests/query2.1.sql --- a/sql/backends/monet5/bam/Tests/query2.1.sql +++ b/sql/backends/monet5/bam/Tests/query2.1.sql @@ -1,41 +1,41 @@ WITH alig AS ( SELECT qname, flag, seq, qual FROM bam.alignments_1 -
MonetDB: bamloader - Merge with default branch
Changeset: 1ce852d1a673 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1ce852d1a673 Added Files: gdk/gdk_firstn.c monetdb5/tests/gdkTests/Tests/firstn.mal monetdb5/tests/gdkTests/Tests/firstn.stable.err monetdb5/tests/gdkTests/Tests/firstn.stable.out Removed Files: monetdb5/modules/mal/pqueue.c monetdb5/modules/mal/pqueue.h monetdb5/modules/mal/pqueue.mal Modified Files: NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures_gsl.stable.out clients/Tests/MAL-signatures_nocfitsio.stable.out clients/Tests/MAL-signatures_sphinxclient.stable.out clients/Tests/exports.stable.out gdk/Makefile.ag gdk/gdk.h java/src/nl/cwi/monetdb/jdbc/MonetDatabaseMetaData.java monetdb5/modules/kernel/algebra.c monetdb5/modules/kernel/algebra.h monetdb5/modules/kernel/algebra.mal monetdb5/modules/mal/Makefile.ag monetdb5/modules/mal/Tests/pqueue.mal monetdb5/modules/mal/Tests/pqueue.stable.out monetdb5/modules/mal/Tests/pqueue2.mal monetdb5/modules/mal/Tests/pqueue2.stable.out monetdb5/modules/mal/Tests/pqueue3.mal monetdb5/modules/mal/Tests/pqueue3.stable.out monetdb5/modules/mal/mal_init.mal monetdb5/optimizer/opt_centipede.c monetdb5/optimizer/opt_prelude.c monetdb5/optimizer/opt_prelude.h monetdb5/optimizer/opt_support.c monetdb5/tests/gdkTests/Tests/All sql/backends/monet5/rel_bin.c sql/backends/monet5/sql.c sql/backends/monet5/sql_gencode.c sql/benchmarks/tpch/Tests/02-explain.stable.out sql/benchmarks/tpch/Tests/03-explain.stable.out sql/benchmarks/tpch/Tests/10-explain.stable.out sql/benchmarks/tpch/Tests/18-explain.stable.out sql/benchmarks/tpch/Tests/21-explain.stable.out sql/benchmarks/tpch/Tests/21-explain.stable.out.32bit sql/server/rel_select.c sql/server/sql_parser.y sql/test/BugTracker-2010/Tests/LIMIT_OFFSET_big-endian.Bug-2622.stable.out sql/test/pg_regress/Tests/numeric.sql sql/test/pg_regress/Tests/numeric.stable.err sql/test/pg_regress/Tests/numeric.stable.out sql/test/pg_regress/Tests/oid.sql sql/test/pg_regress/Tests/oid.stable.err sql/test/pg_regress/Tests/oid.stable.out sql/test/pg_regress/postgresql2sql99.sh Branch: bamloader Log Message: Merge with default branch diffs (truncated from 11401 to 300 lines): diff --git a/NT/installer32/MonetDB5-SQL-Installer.vdproj b/NT/installer32/MonetDB5-SQL-Installer.vdproj --- a/NT/installer32/MonetDB5-SQL-Installer.vdproj +++ b/NT/installer32/MonetDB5-SQL-Installer.vdproj @@ -927,12 +927,6 @@ } Entry { -MsmKey = 8:_F4DDA57B13344198B902859413BF2B9D -OwnerKey = 8:_UNDEFINED -MsmSig = 8:_UNDEFINED -} -Entry -{ MsmKey = 8:_F671CE9CE9A64006AFFFAFBCBDA8EDCE OwnerKey = 8:_UNDEFINED MsmSig = 8:_UNDEFINED @@ -4052,26 +4046,6 @@ IsDependency = 11:FALSE IsolateTo = 8: } - {1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_F4DDA57B13344198B902859413BF2B9D -{ -SourcePath = 8:lib\\monetdb5\\pqueue.mal -TargetName = 8:pqueue.mal -Tag = 8: -Folder = 8:_B593DAA0787744D6A44D17884E0C3951 -Condition = 8: -Transitive = 11:FALSE -Vital = 11:TRUE -ReadOnly = 11:FALSE -Hidden = 11:FALSE -System = 11:FALSE -Permanent = 11:FALSE -SharedLegacy = 11:FALSE -PackageAs = 3:1 -Register = 3:1 -Exclude = 11:FALSE -IsDependency = 11:FALSE -IsolateTo = 8: -} {1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_F671CE9CE9A64006AFFFAFBCBDA8EDCE { SourcePath = 8:lib\\monetdb5\\createdb\\23_skyserver.sql diff --git a/NT/installer64/MonetDB5-SQL-Installer.vdproj b/NT/installer64/MonetDB5-SQL-Installer.vdproj --- a/NT/installer64/MonetDB5-SQL-Installer.vdproj +++ b/NT/installer64/MonetDB5-SQL-Installer.vdproj @@ -927,12 +927,6 @@ } Entry { -MsmKey = 8:_F4DDA57B13344198B902859413BF2B9D -OwnerKey = 8:_UNDEFINED -MsmSig = 8:_UNDEFINED -} -Entry -{ MsmKey = 8:_F671CE9CE9A64006AFFFAFBCBDA8EDCE OwnerKey = 8:_UNDEFINED MsmSig = 8:_UNDEFINED @@ -4052,26 +4046,6 @@ IsDependency = 11:FALSE IsolateTo = 8: } - {1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_F4DDA57B13344198B902859413BF2B9D -{ -SourcePath =
MonetDB: bamloader - Fixed some memory issues
Changeset: f123faa305d8 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f123faa305d8 Modified Files: sql/backends/monet5/bam/bam_loader.c sql/backends/monet5/bam/bam_wrapper.h Branch: bamloader Log Message: Fixed some memory issues diffs (100 lines): diff --git a/sql/backends/monet5/bam/bam_loader.c b/sql/backends/monet5/bam/bam_loader.c --- a/sql/backends/monet5/bam/bam_loader.c +++ b/sql/backends/monet5/bam/bam_loader.c @@ -571,7 +571,8 @@ bam_loader_files(Client cntxt, MalBlkPtr char cur; int line_size; - size_t line_buf_size = 128; + str line = NULL; + size_t line_buf_size = 0; int nr_lines = 1; str msg = MAL_SUCCEED; @@ -607,39 +608,35 @@ bam_loader_files(Client cntxt, MalBlkPtr goto cleanup; } /* Enables cleanup to check individual files */ - memset(filenames, 0, sizeof(nr_lines * sizeof(str))); - - /* malloc 128 chars for every file; this is safe since call to -* getline automatically reallocs when buffer is too small */ - for (i = 0; i nr_lines; ++i) { - if ((filenames[i] = -GDKmalloc(line_buf_size * sizeof(char))) == NULL) { - msg = createException(MAL, bam_loader_files, - MAL_MALLOC_FAIL); - goto cleanup; - } - } + memset(filenames, 0, nr_lines * sizeof(str)); /* Reset file pointer to beginning of file */ rewind(f); /* Read BAM filenames into files array */ while ((line_size = - getline(filenames[nr_files], line_buf_size, f)) != -1) { - str l = filenames[nr_files]; + getline(line, line_buf_size, f)) = 0) { - if (l[0] == '\n' || l[0] == '#') { + if (line_size == 0 || line[0] == '\n' || line[0] == '\r' || line[0] == '#') { /* Skip empty lines or lines that start with a hash */ + free(line); + line = NULL; + /* We need to reset buf size to 0, since getline will have updated it */ + line_buf_size = 0; continue; } - if (l[line_size - 1] == '\n') { + if (line[line_size - 1] == '\n') { /* Newline character is not part of the filepath */ - l[line_size - 1] = '\0'; + line[line_size - 1] = '\0'; } - ++nr_files; - /* We need to reset it to 128, since getline might -* have changed its value */ - line_buf_size = 128; + if(line_size 1 line[line_size - 2] == '\r') { + /* Return character is also not a part of the filepath */ + line[line_size - 2] = '\0'; + } + filenames[nr_files++] = line; + line = NULL; + /* We need to reset it to 0, since getline will have updated this val */ + line_buf_size = 0; } if (ferror(f)) { @@ -661,10 +658,16 @@ bam_loader_files(Client cntxt, MalBlkPtr nr_threads); cleanup: + if(line) { + //if line still contains anything, this is something that was put there + //by getline on the last, failed attempt to read a line + free(line); + } + if (filenames) { for (i = 0; i nr_files; ++i) { if (filenames[i]) { - GDKfree(filenames[i]); + free(filenames[i]); } } GDKfree(filenames); diff --git a/sql/backends/monet5/bam/bam_wrapper.h b/sql/backends/monet5/bam/bam_wrapper.h --- a/sql/backends/monet5/bam/bam_wrapper.h +++ b/sql/backends/monet5/bam/bam_wrapper.h @@ -31,7 +31,7 @@ #include samtools/sam.h -#define BW_FP_BUF_SIZE 128 +#define BW_FP_BUF_SIZE 1024 typedef sht ordering; typedef enum {SAM, BAM} filetype; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Merge with default branch
Changeset: c0cf13acf65a for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c0cf13acf65a Modified Files: MonetDB.spec clients/Tests/SQL-dump.stable.out.32bit configure.ag gdk/gdk_logger.c gdk/gdk_posix.c gdk/gdk_private.h gdk/gdk_storage.c sql/server/sql_atom.c sql/server/sql_parser.y sql/test/pg_regress/Tests/char.sql sql/test/pg_regress/Tests/char.stable.err sql/test/pg_regress/Tests/char.stable.out sql/test/pg_regress/Tests/float4.sql sql/test/pg_regress/Tests/float4.stable.err sql/test/pg_regress/Tests/float4.stable.out sql/test/pg_regress/Tests/float8.sql sql/test/pg_regress/Tests/float8.stable.err sql/test/pg_regress/Tests/float8.stable.out sql/test/pg_regress/Tests/int2.sql sql/test/pg_regress/Tests/int2.stable.err sql/test/pg_regress/Tests/int2.stable.out sql/test/pg_regress/Tests/int4.sql sql/test/pg_regress/Tests/int4.stable.err sql/test/pg_regress/Tests/int4.stable.out sql/test/pg_regress/Tests/int8.sql sql/test/pg_regress/Tests/int8.stable.err sql/test/pg_regress/Tests/int8.stable.out sql/test/pg_regress/Tests/numerology.sql sql/test/pg_regress/Tests/polygon.sql sql/test/pg_regress/postgresql2sql99.sh testing/Mtest.py.in Branch: bamloader Log Message: Merge with default branch diffs (truncated from 3786 to 300 lines): diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -597,9 +597,9 @@ used from the MAL level. Summary: MonetDB5 SQL server modules Group: Applications/Databases Requires: MonetDB5-server = %{version}-%{release} -%if %{?rhel:0}%{!?rhel:1} -# for systemd-tmpfiles -Requires: systemd-units +%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} = 7 +# RHEL = 7, and all current Fedora +Requires: %{_bindir}/systemd-tmpfiles %endif Obsoletes: MonetDB-SQL-devel Obsoletes: %{name}-SQL @@ -613,7 +613,7 @@ accelerators. It also has an SQL fronte This package contains the SQL frontend for MonetDB. If you want to use SQL with MonetDB, you will need to install this package. -%if %{?rhel:0}%{!?rhel:1} +%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} = 7 %post SQL-server5 systemd-tmpfiles --create %{_sysconfdir}/tmpfiles.d/monetdbd.conf %endif @@ -623,11 +623,11 @@ systemd-tmpfiles --create %{_sysconfdir} %{_bindir}/monetdb %{_bindir}/monetdbd %dir %attr(775,monetdb,monetdb) %{_localstatedir}/log/monetdb -%if %{?rhel:0}%{!?rhel:1} -# Fedora 15 and newer +%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} = 7 +# RHEL = 7, and all current Fedora %{_sysconfdir}/tmpfiles.d/monetdbd.conf %else -# RedHat Enterprise Linux +# RedHat Enterprise Linux 7 %dir %attr(775,monetdb,monetdb) %{_localstatedir}/run/monetdb %exclude %{_sysconfdir}/tmpfiles.d/monetdbd.conf %endif diff --git a/clients/Tests/SQL-dump.stable.out.32bit b/clients/Tests/SQL-dump.stable.out.32bit --- a/clients/Tests/SQL-dump.stable.out.32bit +++ b/clients/Tests/SQL-dump.stable.out.32bit @@ -6998,8 +6998,8 @@ 6302 text create function json.text(j 6306 text create function json.text(js string)\nreturns string external name json.text; json true1 false false false 6230 6310 text create function json.text(js int)\nreturns string external name json.text;json true1 false false false 6230 6314 output-- The remainder awaits the implementation \n\ncreate aggregate json.output(js json)\nreturns string external name json.output; json true3 false false false 6230 -6318 tojsonarray -- create function json.object(*) returns json external name json.objectrender;\n\n-- create function json.array(*) returns json external name json.arrayrender;\n\n-- unnesting the JSON structure\n\n-- create function json.unnest(js json)\n-- returns table( id integer, k string, v string) external name json.unnest;\n\n-- create function json.unnest(js json)\n-- returns table( k string, v string) external name json.unnest;\n\n-- create function json.unnest(js json)\n-- returns table( v string) external name json.unnest;\n\n-- create function json.nest table( id integer, k string, v string)\n-- returns json external name json.nest;\n\ncreate aggregate json.tojsonarray( x string ) returns string external name json.aggr; json true3 false false false 6230 -6322 tojsonarray create aggregate json.tojsonarray( x double ) returns string external name json.aggr; json true3 false false false 6230 +6318 tojsonarray -- create function json.object(*) returns json external name json.objectrender;\n\n-- create function json.array(*) returns json external name json.arrayrender;\n\n-- unnesting the JSON structure\n\n-- create function json.unnest(js json)\n-- returns table( id integer, k string, v string) external name json.unnest;\n\n-- create function
MonetDB: bamloader - Merge with changes Sjoerd
Changeset: 350c99a6a6f8 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=350c99a6a6f8 Modified Files: configure.ag Branch: bamloader Log Message: Merge with changes Sjoerd ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Fixed test
Changeset: e43b1e7b7612 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e43b1e7b7612 Modified Files: sql/backends/monet5/bam/Tests/test.SQL.py sql/backends/monet5/bam/Tests/test.stable.err sql/backends/monet5/bam/Tests/test.stable.out Branch: bamloader Log Message: Fixed test diffs (truncated from 2855 to 300 lines): diff --git a/sql/backends/monet5/bam/Tests/test.SQL.py b/sql/backends/monet5/bam/Tests/test.SQL.py --- a/sql/backends/monet5/bam/Tests/test.SQL.py +++ b/sql/backends/monet5/bam/Tests/test.SQL.py @@ -177,7 +177,7 @@ for (file_id, dbschema) in to_query: # Insert both the SAM and BAM files into the database. Note that we always insert into # dbschema 0, since the outputted files aren't necessarily ordered by qname. c.stdin.write(CALL bam_loader_file('%s', 0);\n% tmp_output(file_id)) -c.stdin.write(CALL bam_loader_file('%s', 0);\n% tmp_output(file_id, sam=False)) +#c.stdin.write(CALL bam_loader_file('%s', 0);\n% tmp_output(file_id, sam=False)) # We now have the alignment data for this file in three different alignment tables. The ultimate # export/import test is now to see if the data in them is exactly the same (except for virtual_offset) @@ -206,4 +206,4 @@ sys.stderr.write(err) # All that is left to do is delete the temporary SAM/BAM files for (file_id, dbschema) in to_query: os.remove(tmp_output(file_id)) -os.remove(tmp_output(file_id, sam=False)) \ No newline at end of file +#os.remove(tmp_output(file_id, sam=False)) \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/test.stable.err b/sql/backends/monet5/bam/Tests/test.stable.err --- a/sql/backends/monet5/bam/Tests/test.stable.err +++ b/sql/backends/monet5/bam/Tests/test.stable.err @@ -1,1596 +1,18 @@ stderr of test 'test` in directory 'sql/backends/monet5/bam` itself: -# 16:03:13 -# 16:03:13 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=31808 --set mapi_usock=/var/tmp/mtest-26892/.s.monetdb.31808 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 -# 16:03:13 -# builtin opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo -# builtin opt gdk_debug = 0 -# builtin opt gdk_vmtrim = no -# builtin opt monet_prompt = -# builtin opt monet_daemon = no -# builtin opt mapi_port = 5 -# builtin opt mapi_open = false -# builtin opt mapi_autosense = false -# builtin opt sql_optimizer = default_pipe -# builtin opt sql_debug = 0 -# cmdline opt gdk_nr_threads = 0 -# cmdline opt mapi_open = true -# cmdline opt mapi_port = 31808 -# cmdline opt mapi_usock = /var/tmp/mtest-26892/.s.monetdb.31808 -# cmdline opt monet_prompt = -# cmdline opt mal_listing = 2 -# cmdline opt gdk_dbpath = /home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam -# cmdline opt mal_listing = 0 -# cmdline opt gdk_debug = 536870922 -# bam_loader: Loader started for 2 BAM files... -# # bam_loader Creating schema 'bam'... -# # CREATE SCHEMA bam; -# # bam_loader Creating table 'files'... -# # CREATE TABLE bam.files ( -# file_id BIGINTNOT NULL, -# file_location STRING NOT NULL, -# dbschema SMALLINT NOT NULL, -# format_version VARCHAR(7), -# sorting_order VARCHAR(10), -# comments STRING, -# CONSTRAINT files_pkey_file_id PRIMARY KEY (file_id) -# ); -# # bam_loader Creating table 'sq'... -# # CREATE TABLE bam.sq ( -# sn STRING NOT NULL, -# file_id BIGINTNOT NULL, -# ln INT, -# as INT, -# m5 STRING, -# sp STRING, -# ur STRING, -# CONSTRAINT sq_pkey_sn_file_id PRIMARY KEY (sn, file_id), -# CONSTRAINT sq_fkey_file_id FOREIGN KEY (file_id) REFERENCES bam.files (file_id) -# ); -# # bam_loader Creating table 'rg'... -# # CREATE TABLE bam.rg ( -# id STRING NOT NULL, -# file_id BIGINTNOT NULL, -# cn STRING, -# ds
MonetDB: bamloader - Improved test greatly
Changeset: e1dd2e16216e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1dd2e16216e Removed Files: sql/backends/monet5/bam/Tests/load.sql Modified Files: sql/backends/monet5/bam/Tests/test.SQL.py sql/backends/monet5/bam/Tests/test.stable.err sql/backends/monet5/bam/Tests/test.stable.out Branch: bamloader Log Message: Improved test greatly diffs (truncated from 4768 to 300 lines): diff --git a/sql/backends/monet5/bam/Tests/load.sql b/sql/backends/monet5/bam/Tests/load.sql deleted file mode 100644 --- a/sql/backends/monet5/bam/Tests/load.sql +++ /dev/null @@ -1,2 +0,0 @@ -CALL bam_loader_repos('PWD/files', 0, 32); -CALL bam_loader_repos('PWD/files/queryname', 1, 32); diff --git a/sql/backends/monet5/bam/Tests/test.SQL.py b/sql/backends/monet5/bam/Tests/test.SQL.py --- a/sql/backends/monet5/bam/Tests/test.SQL.py +++ b/sql/backends/monet5/bam/Tests/test.SQL.py @@ -13,38 +13,119 @@ def new_client(): def set_var(c, name, value, numeric=True): if numeric: -c.stdin.write('DECLARE %s integer;'% name) -c.stdin.write('SET %s=%s;'%(name, value)) +c.stdin.write('DECLARE %s integer;\n'% name) +c.stdin.write('SET %s=%s;\n'%(name, value)) else: -c.stdin.write('DECLARE %s varchar(32);'% name) -c.stdin.write('SET %s=\'%s\';'% (name, value)) +c.stdin.write('DECLARE %s varchar(32);\n'% name) +c.stdin.write('SET %s=\'%s\';\n'% (name, value)) -# start with loading bam files into the database + + + + +# start with loading bam files into the database with all loader functions we have built +# we assume that all bam or sam files are stored into the files directory. +# these files will all be loaded into the database with all loading methods that we have. +# files in the directory files/queryname are assumed to be ordered by queryname and are +# therefore also loaded into the pairwise storage schema + c = new_client() -for ln in open(os.path.join(SRCDIR,load.sql)): -c.stdin.write(ln.replace('PWD', SRCDIR)) + +# get files +files_path = os.path.join(SRCDIR, files) +files_qname_path = os.path.join(files_path, queryname) +files = [ f for f in os.listdir(files_path) if os.path.isfile(os.path.join(files_path, f)) ] +files_qname = [ f for f in os.listdir(files_qname_path) if os.path.isfile(os.path.join(files_qname_path, f)) ] + +# use bam_loader_repos +c.stdin.write(CALL bam_loader_repos('%s', 0, 8);\n% files_path) +c.stdin.write(CALL bam_loader_repos('%s', 1, 8);\n% files_qname_path) + +# use bam_loader_files by first writing temporary files and then invoking these loaders +f_tmp_path = os.path.join(SRCDIR, tmp_list.txt) +f_tmp_qname_path = os.path.join(SRCDIR, tmp_list_qname.txt) +f_tmp = open(f_tmp_path, w) +f_tmp_qname = open(f_tmp_qname_path, w) + +for file in files: +f_tmp.write(%s\n% os.path.join(files_path, file)) + +for file in files_qname: +f_tmp_qname.write(%s\n% os.path.join(files_qname_path, file)) + +f_tmp.close() +f_tmp_qname.close() + +c.stdin.write(CALL bam_loader_files('%s', 0, 8);\n% f_tmp_path) +c.stdin.write(CALL bam_loader_files('%s', 1, 8);\n% f_tmp_qname_path) + +# use bam_loader_file on all separate files +for file in files: +c.stdin.write(CALL bam_loader_file('%s', 0);\n% os.path.join(files_path, file)) + +for file in files_qname: +c.stdin.write(CALL bam_loader_file('%s', 1);\n% os.path.join(files_qname_path, file)) out, err = c.communicate() sys.stdout.write(out) sys.stderr.write(err) +# clean up the temporary files +os.remove(f_tmp_path) +os.remove(f_tmp_qname_path) -# now retrieve the file ids that have been inserted -# (should be 1,2,..,nr_files, but still we extract it from the -# db to be flexible to logical changes in the bam loader as -# much as possible) + +# extract the files that have been inserted into the database +# gives us a list of (file_id, dbschema) tuples c = new_client() -c.stdin.write(SELECT file_id, dbschema FROM bam.files;) +c.stdin.write(SELECT file_id, dbschema FROM bam.files;\n) out, err = c.communicate() -p = re.compile('^\s*\[\s*(\d)\s*,\s*(\d)\s*\]\s*$', re.MULTILINE) -files_to_test = [] +sys.stdout.write(out) +sys.stderr.write(err) + +p = re.compile('^\s*\[\s*(\d+)\s*,\s*(\d)\s*\]\s*$', re.MULTILINE) # Parses raw DB output +inserted = [] for match in p.finditer(out): -files_to_test.append((int(match.group(1)), int(match.group(2 +inserted.append((int(match.group(1)), int(match.group(2 -# Now we will execute all benchmark queries on all BAM files in the bam.files table -# and output all data contained in the aux table. -# Furthermore, we transfer every file to the export table, use sam_export to write -# the contents to a SAM file and then print the raw contents of this SAM file + + +# now we will test the drop functionality by removing all duplicate bam/sam files from the database +# we know that only the first len(files) + len(files_qname) files are unique, so we remove +#
MonetDB: bamloader - finished bam_export. For this, switched who...
Changeset: d149e885d77d for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d149e885d77d Modified Files: sql/backends/monet5/bam/Makefile.ag sql/backends/monet5/bam/bam_export.c sql/backends/monet5/bam/bam_wrapper.c sql/backends/monet5/bam/bam_wrapper.h Branch: bamloader Log Message: finished bam_export. For this, switched whole bam library code from libbam to libhts since it provides better functionality for our purpose. diffs (truncated from 612 to 300 lines): diff --git a/sql/backends/monet5/bam/Makefile.ag b/sql/backends/monet5/bam/Makefile.ag --- a/sql/backends/monet5/bam/Makefile.ag +++ b/sql/backends/monet5/bam/Makefile.ag @@ -30,8 +30,7 @@ INCLUDES = .. \ ../../../../common/stream \ ../../../../gdk \ ../../../../tools/merovingian \ - ../../../../tools/merovingian/daemon \ - $(SAMTOOLS_CFLAGS) + ../../../../tools/merovingian/daemon lib__bam = { MODULE @@ -44,7 +43,7 @@ lib__bam = { bam_export.c bam_export.h LIBS = ../../../../monetdb5/tools/libmonetdb5 \ ../../../../gdk/libbat \ - $(SAMTOOLS_LIBS) + -lhts } headers_mal = { diff --git a/sql/backends/monet5/bam/bam_export.c b/sql/backends/monet5/bam/bam_export.c --- a/sql/backends/monet5/bam/bam_export.c +++ b/sql/backends/monet5/bam/bam_export.c @@ -27,10 +27,11 @@ #include monetdb_config.h -#include samtools/bam.h +#include htslib/sam.h +#include htslib/kstring.h #include bam_globals.h -#include bam_db_interface.h +//#include bam_db_interface.h #include bam_export.h @@ -43,189 +44,6 @@ typedef struct bam_field { } bam_field; -/** - * Copied directly from bam.h/bam_import.c for use by fill_bam_alig - * Can not change the calls to realloc to GDKrealloc, since - * bam_destroy1 does not use GDKfree... - */ -#ifndef kroundup32 -/*! @function - @abstract Round an integer to the next closest power-2 integer. - @param x integer to be rounded (in place) - @discussion x will be modified. - */ -#define kroundup32(x) (--(x), (x)|=(x)1, (x)|=(x)2, (x)|=(x)4, (x)|=(x)8, (x)|=(x)16, ++(x)) -#endif -static inline uint8_t * -alloc_data(bam1_t *b, int size) -{ - if (b-m_data size) { - b-m_data = size; - kroundup32(b-m_data); - b-data = (uint8_t*)realloc(b-data, b-m_data); - } - return b-data; -} - -/* Error macro */ -#define FILL_BAM_ALIG_ERR Error processing alignment '%d': - -/** - * I had to write this function, which contains much low level - * function calls to the Samtools library, since the sam_read1 function - * contained in bam.h/bam_import.c only works on files. This function is - * an adjusted variant of the sam_read1 function that gets its fields - * directly instead of reads it from a tamFile. - * I wrote this function since I did not feel like first writing everything - * to a SAM file and then applying sam_read1 since that is just a waste of I/O. - * Note that I did do a write to/read from SAM for the BAM header, but headers are - * insignificant compared to alignment data. - * - * I also added realloc checking, since this was not done in the Samtools code - * and i properly throw a MAL_MALLOC_FAIL whenever applicable. - */ -static str -fill_bam_alig(str qname, sht flag, str rname, int pos, - sht mapq, str cigar, str rnext, int pnext, - int tlen, str seq, str qual, - bam_header_t *header, bam1_t *b, int alignment_nr) -{ - int doff = 0; - bam1_core_t *c = b-core; - - /* Note: in sam_read1, str-s is often used. This string is not NULL terminated! */ - { // qname - c-l_qname = strlen(qname) + 1; - if(alloc_data(b, doff + c-l_qname) == NULL) { - throw(MAL, fill_bam_alig, - FILL_BAM_ALIG_ERR MAL_MALLOC_FAIL, alignment_nr); - } - memcpy(b-data + doff, qname, c-l_qname); - doff += c-l_qname; - } - { // flag - c-flag = flag; - } - { // rname, pos, mapq - c-tid = bam_get_tid(header, rname); - if (c-tid 0 strcmp(rname, *) != 0) { - /* Should not happen, since we built the header structure ourselves */ - throw(MAL, fill_bam_alig, - FILL_BAM_ALIG_ERR SQ entry '%s' not found in header table, - alignment_nr, rname); - } - c-pos = pos - 1; - c-qual = mapq; - } - { // cigar - str s, t; - int i, op; - long x; - c-n_cigar = 0; - if (cigar[0] != '*') { - uint32_t *cigar_enc; - for (s = cigar; *s != '\0'; ++s) { - if ((isalpha(*s)) || (*s=='=')) { -
MonetDB: bamloader - Merge with default branch
Changeset: 805d4f32d3b6 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=805d4f32d3b6 Added Files: geom/BugTracker/Tests/copy_into_mbr.3492.modules geom/BugTracker/Tests/copy_into_mbr.3492.sql geom/BugTracker/Tests/copy_into_mbr.3492.stable.err geom/BugTracker/Tests/copy_into_mbr.3492.stable.out Modified Files: MonetDB.spec clients/R/MonetDB.R/DESCRIPTION clients/R/MonetDB.R/NEWS clients/R/MonetDB.R/R/control.R clients/R/MonetDB.R/R/monetdb.R clients/R/MonetDB.R/src/mapi.c clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures_gsl.stable.out clients/Tests/MAL-signatures_nocfitsio.stable.out clients/Tests/MAL-signatures_sphinxclient.stable.out clients/Tests/SQL-dump.stable.out clients/Tests/SQL-dump.stable.out.64bit.oid32 clients/Tests/SQL-dump_gsl.stable.out clients/Tests/SQL-dump_gsl.stable.out.oid32 clients/Tests/SQL-dump_nogeom.stable.out clients/Tests/exports.stable.out clients/mapiclient/dump.c clients/mapiclient/mclient.c clients/odbc/driver/ODBCConvert.c clients/odbc/driver/ODBCUtil.h clients/python2/monetdb/sql/monetize.py clients/python2/test/test_monetize.py clients/python3/monetdb/sql/monetize.py clients/python3/test/test_monetize.py gdk/gdk_logger.c gdk/gdk_logger.h gdk/gdk_sample.c gdk/gdk_unique.c geom/BugTracker/Tests/All geom/monetdb5/geom.c monetdb5/mal/mal_import.c monetdb5/modules/atoms/batxml.c monetdb5/modules/atoms/json.c monetdb5/modules/atoms/json.mal monetdb5/optimizer/opt_centipede.c monetdb5/optimizer/opt_pushselect.c sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_statistics.c sql/benchmarks/tpch/Tests/17-plan.stable.out sql/benchmarks/tpch/Tests/22-plan.stable.out sql/common/sql_types.c sql/scripts/40_json.sql sql/storage/bat/bat_logger.c sql/storage/store.c sql/test/BugTracker/Tests/between_with_column.SF-1959410.stable.err sql/test/Tests/systemfunctions.stable.out sql/test/Tests/systemfunctions_gsl.stable.out sql/test/Tests/systemfunctions_nogeom.stable.out sql/test/centipede/Tests/olap.stable.out sql/test/json/Tests/aggregate01.sql sql/test/json/Tests/aggregate01.stable.out sql/test/pg_regress/Tests/All sql/test/pg_regress/Tests/aggregates.stable.err sql/test/pg_regress/Tests/aggregates.stable.out sql/test/pg_regress/Tests/alter_table.sql sql/test/pg_regress/Tests/alter_table.stable.err sql/test/pg_regress/Tests/alter_table.stable.out sql/test/pg_regress/Tests/arrays.sql sql/test/pg_regress/Tests/arrays.stable.err sql/test/pg_regress/Tests/arrays.stable.out sql/test/pg_regress/Tests/bit.stable.err sql/test/pg_regress/Tests/bit.stable.out sql/test/pg_regress/Tests/boolean.sql sql/test/pg_regress/Tests/boolean.stable.err sql/test/pg_regress/Tests/boolean.stable.out sql/test/pg_regress/Tests/box.stable.err sql/test/pg_regress/Tests/box.stable.out sql/test/pg_regress/Tests/btree_index.stable.err sql/test/pg_regress/Tests/btree_index.stable.out sql/test/pg_regress/Tests/case.stable.err sql/test/pg_regress/Tests/case.stable.out sql/test/pg_regress/Tests/char.stable.err sql/test/pg_regress/Tests/char.stable.out sql/test/pg_regress/Tests/circle.stable.err sql/test/pg_regress/Tests/circle.stable.out sql/test/pg_regress/Tests/cluster.sql sql/test/pg_regress/Tests/cluster.stable.err sql/test/pg_regress/Tests/cluster.stable.out sql/test/pg_regress/Tests/comments.stable.err sql/test/pg_regress/Tests/comments.stable.out sql/test/pg_regress/Tests/conversion.stable.err sql/test/pg_regress/Tests/conversion.stable.out sql/test/pg_regress/Tests/copy2.stable.err sql/test/pg_regress/Tests/copy2.stable.out sql/test/pg_regress/Tests/create_aggregate.stable.err sql/test/pg_regress/Tests/create_aggregate.stable.out sql/test/pg_regress/Tests/create_index.stable.err sql/test/pg_regress/Tests/create_index.stable.out sql/test/pg_regress/Tests/create_misc.stable.err sql/test/pg_regress/Tests/create_misc.stable.out sql/test/pg_regress/Tests/create_operator.stable.err sql/test/pg_regress/Tests/create_operator.stable.out sql/test/pg_regress/Tests/create_table.sql sql/test/pg_regress/Tests/create_table.stable.err sql/test/pg_regress/Tests/create_table.stable.out sql/test/pg_regress/Tests/create_type.stable.err sql/test/pg_regress/Tests/create_type.stable.out
MonetDB: bamloader - Adapted .ag files, MonetDB.spec and debian/...
Changeset: 43f84e370374 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=43f84e370374 Modified Files: MonetDB.spec configure.ag debian/rules sql/backends/monet5/Makefile.ag sql/backends/monet5/bam/Makefile.ag Branch: bamloader Log Message: Adapted .ag files, MonetDB.spec and debian/rules to look for htslib instead of samtools diffs (202 lines): diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -36,8 +36,8 @@ %define with_geos 1 %endif -%if %{?_with_samtools:1}%{!?_with_samtools:0} -%define with_samtools 1 +%if %{?_with_hts:1}%{!?_with_hts:0} +%define with_hts 1 %endif Name: %{name} @@ -82,8 +82,8 @@ BuildRequires: rubygems-devel %endif BuildRequires: unixODBC-devel BuildRequires: zlib-devel -%if %{?with_samtools:1}%{!?with_samtools:0} -BuildRequires: samtools-devel +%if %{?with_hts:1}%{!?with_hts:0} +BuildRequires: hts-devel %endif # need to define python_sitelib on RHEL 5 and older @@ -456,7 +456,7 @@ numerical analysis (gsl). %{_libdir}/monetdb5/gsl.mal %{_libdir}/monetdb5/lib_gsl.so -%if %{?_with_samtools:1}%{!?_with_samtools:0} +%if %{?_with_hts:1}%{!?_with_hts:0} %package bam-MonetDB5 Summary: MonetDB5 SQL interface to the bam library Group: Applications/Databases @@ -542,7 +542,7 @@ fi %exclude %{_libdir}/monetdb5/lib_geom.so %endif %exclude %{_libdir}/monetdb5/lib_gsl.so -%if %{?_with_samtools:1}%{!?_with_samtools:0} +%if %{?_with_hts:1}%{!?_with_hts:0} %exclude %{_libdir}/monetdb5/bam.mal %exclude %{_libdir}/monetdb5/autoload/*_bam.mal %exclude %{_libdir}/monetdb5/lib_bam.so @@ -640,7 +640,7 @@ systemd-tmpfiles --create %{_sysconfdir} %exclude %{_libdir}/monetdb5/createdb/*_geom.sql %endif %exclude %{_libdir}/monetdb5/createdb/*_gsl.sql -%if %{?_with_samtools:1}%{!?_with_samtools:0} +%if %{?_with_hts:1}%{!?_with_hts:0} %exclude %{_libdir}/monetdb5/createdb/*_bam.sql %endif # %exclude %{_libdir}/monetdb5/createdb/*_rdf.sql @@ -800,7 +800,7 @@ developer, but if you do want to test, t --with-readline=yes \ --with-rubygem=%{?rhel:no}%{!?rhel:yes} \ --with-rubygem-dir=%{?rhel:no}%{!?rhel:%{gem_dir}} \ - --with-samtools=%{?with_samtools:yes}%{!?with_samtools:no} \ + --with-hts=%{?with_hts:yes}%{!?with_hts:no} \ --with-sphinxclient=no \ --with-unixodbc=yes \ --with-valgrind=no \ diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -2362,49 +2362,51 @@ AC_SUBST(MSEED_CFLAGS, $MSEED_CFLAGS) AC_SUBST(MSEED_LIBS, $MSEED_LIBS) AM_CONDITIONAL(HAVE_MSEED, test x$have_mseed != xno) -# samtools, only used by sql -org_have_samtools=auto -have_samtools=$org_have_samtools -AC_ARG_WITH(samtools, - AS_HELP_STRING([--with-samtools=DIR], - [samtools (providing bam library) are installed in DIR]), - [have_samtools=$withval], [have_samtools=auto]) -if test x$have_samtools != xno; then - case $have_samtools in +# hts (required by BAM library), only used by sql +org_have_bam=auto +have_bam=$org_have_bam +AC_ARG_WITH(hts, + AS_HELP_STRING([--with-hts=DIR], + [hts (providing bam library) are installed in DIR]), + [have_bam=$withval], [have_bam=auto]) +if test x$have_bam != xno; then + case $have_bam in auto|yes) ;; *) - SAMTOOLS_CFLAGS=-I$have_samtools/include - SAMTOOLS_LIBS=-L$have_samtools/lib -lbam + BAM_CFLAGS=-I$have_bam/include + BAM_LIBS=-L$have_bam/lib -lhts ;; esac save_CPPFLAGS=$CPPFLAGS save_LDFLAGS=$LDFLAGS - CPPFLAGS=$CPPFLAGS $SAMTOOLS_CFLAGS - LDFLAGS=$LDFLAGS $SAMTOOLS_LIBS - AC_CHECK_HEADER(samtools/bam.h, - AC_CHECK_HEADER(samtools/bgzf.h, - AC_CHECK_LIB(bam, bam_header_read, - AC_DEFINE(HAVE_SAMTOOLS, 1, [Define if you have the samtools (providing the bam library)]), - [ if test x$have_samtools != xauto; then AC_MSG_ERROR([-lbam library not found]); fi - have_samtools=no; why_have_samtools=(bam library not found) ], - [-lm -lpthread -lz]), - [ if test x$have_samtools != xauto; then AC_MSG_ERROR([samtools/bgzf.h header not found]); fi - have_samtools=no; why_have_samtools=(samtools/bgzf.h header not found) ]), - [ if test x$have_samtools != xauto; then AC_MSG_ERROR([samtools/bam.h header not found]); fi - have_samtools=no; why_have_samtools=(samtools/bam.h header not found) ]) + CPPFLAGS=$CPPFLAGS $BAM_CFLAGS + LDFLAGS=$LDFLAGS $BAM_LIBS + AC_CHECK_HEADER(htslib/sam.h, + AC_CHECK_HEADER(htslib/bgzf.h, + AC_CHECK_HEADER(htslib/kstring.h, + AC_CHECK_LIB(hts,
MonetDB: bamloader - Fixed indentation (spaces to tabs) and remo...
Changeset: aeed76d1d22e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=aeed76d1d22e Modified Files: sql/backends/monet5/bam/Makefile.ag sql/backends/monet5/bam/bam_db_interface.c sql/backends/monet5/bam/bam_db_interface.h sql/backends/monet5/bam/bam_export.c sql/backends/monet5/bam/bam_export.h sql/backends/monet5/bam/bam_globals.h sql/backends/monet5/bam/bam_lib.c sql/backends/monet5/bam/bam_loader.c sql/backends/monet5/bam/bam_loader.h sql/backends/monet5/bam/bam_wrapper.c sql/backends/monet5/bam/bam_wrapper.h Branch: bamloader Log Message: Fixed indentation (spaces to tabs) and removed whitespace at the end of lines diffs (truncated from 4809 to 300 lines): diff --git a/sql/backends/monet5/bam/Makefile.ag b/sql/backends/monet5/bam/Makefile.ag --- a/sql/backends/monet5/bam/Makefile.ag +++ b/sql/backends/monet5/bam/Makefile.ag @@ -37,14 +37,14 @@ lib__bam = { MODULE DIR = libdir/monetdb5 SOURCES = bam_loader.c bam_loader.h \ - bam_wrapper.c bam_wrapper.h \ - bam_db_interface.c bam_db_interface.h \ - bam_globals.c bam_globals.h \ - bam_lib.c bam_lib.h \ - bam_export.c bam_export.h + bam_wrapper.c bam_wrapper.h \ + bam_db_interface.c bam_db_interface.h \ + bam_globals.c bam_globals.h \ + bam_lib.c bam_lib.h \ + bam_export.c bam_export.h LIBS = ../../../../monetdb5/tools/libmonetdb5 \ - ../../../../gdk/libbat \ - $(SAMTOOLS_LIBS) + ../../../../gdk/libbat \ + $(SAMTOOLS_LIBS) } headers_mal = { diff --git a/sql/backends/monet5/bam/bam_db_interface.c b/sql/backends/monet5/bam/bam_db_interface.c --- a/sql/backends/monet5/bam/bam_db_interface.c +++ b/sql/backends/monet5/bam/bam_db_interface.c @@ -28,172 +28,172 @@ #include bam_db_interface.h #define SQL_CREATE_STORAGE_0 \ -CREATE TABLE bam.alignments_LLFMT ( \n\ -virtual_offsetBIGINT NOT NULL, \n\ -qname STRING NOT NULL, \n\ -flag SMALLINTNOT NULL, \n\ -rname STRING NOT NULL, \n\ -pos INT NOT NULL, \n\ -mapq SMALLINTNOT NULL, \n\ -cigar STRING NOT NULL, \n\ -rnext STRING NOT NULL, \n\ -pnext INT NOT NULL, \n\ -tlen INT NOT NULL, \n\ -seq STRING NOT NULL, \n\ -qual STRING NOT NULL, \n\ -CONSTRAINT alignments_LLFMT_pkey_virtual_offset PRIMARY KEY (virtual_offset) \n\ -); \n\ -\n\ -CREATE TABLE bam.alignments_extra_LLFMT ( \n\ -tag CHAR(2) NOT NULL, \n\ -virtual_offsetBIGINT NOT NULL, \n\ -type CHAR(1) NOT NULL, \n\ -value STRING, \n\ -CONSTRAINT alignments_extra_LLFMT_pkey_tag_virtual_offset PRIMARY KEY (tag, virtual_offset), \n\ -CONSTRAINT alignments_extra_LLFMT_fkey_virtual_offset FOREIGN KEY (virtual_offset) \n\ -REFERENCES bam.alignments_LLFMT (virtual_offset) \n\ -); + CREATE TABLE bam.alignments_LLFMT ( \n\ + virtual_offset BIGINTNOT NULL, \n\ + qnameSTRING NOT NULL, \n\ + flag SMALLINT NOT NULL, \n\ + rnameSTRING NOT NULL, \n\ + posINT NOT NULL, \n\ + mapq SMALLINT NOT NULL, \n\ + cigarSTRING NOT NULL, \n\ + rnextSTRING NOT NULL, \n\ + pnextINT NOT NULL, \n\ + tlen INT NOT NULL, \n\ + seqSTRING NOT NULL, \n\ + qual STRING NOT NULL, \n\ + CONSTRAINT alignments_LLFMT_pkey_virtual_offset PRIMARY KEY (virtual_offset) \n\ + ); \n\ + \n\ + CREATE TABLE bam.alignments_extra_LLFMT ( \n\ + tag
MonetDB: bamloader - Finished SAM loader functionality. Turned o...
Changeset: 19aee3b937e8 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=19aee3b937e8 Modified Files: sql/backends/monet5/bam/bam_loader.c sql/backends/monet5/bam/bam_wrapper.c sql/backends/monet5/bam/bam_wrapper.h Branch: bamloader Log Message: Finished SAM loader functionality. Turned out to be a lot of work, since Samtools wouldn't cooperate. Therefore, a SAM file is now entirely parsed by this BAM library, without any help of Samtools. Test update will follow soon. diffs (truncated from 1146 to 300 lines): diff --git a/sql/backends/monet5/bam/bam_loader.c b/sql/backends/monet5/bam/bam_loader.c --- a/sql/backends/monet5/bam/bam_loader.c +++ b/sql/backends/monet5/bam/bam_loader.c @@ -283,7 +283,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s /* Parse all headers */ for (i = 0; i nr_files; ++i) { - TO_LOG(bam_loader Parsing BAM header for file '%s'...\n, + TO_LOG(bam_loader Parsing header for file '%s'...\n, filenames[i]); if ((msg = process_header(bws + i)) != MAL_SUCCEED) { goto cleanup; @@ -310,7 +310,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s /* Create alignment storage */ for (i = 0; i nr_files; ++i) { - TO_LOG(bam_loader Creating alignment tables for BAM file '%s'...\n, filenames[i]); + TO_LOG(bam_loader Creating alignment tables for file '%s'...\n, filenames[i]); if ((dbschema == 0 create_alignment_storage_0(cntxt, bam.create_storage_0, diff --git a/sql/backends/monet5/bam/bam_wrapper.c b/sql/backends/monet5/bam/bam_wrapper.c --- a/sql/backends/monet5/bam/bam_wrapper.c +++ b/sql/backends/monet5/bam/bam_wrapper.c @@ -106,22 +106,63 @@ init_bam_wrapper(bam_wrapper * bw, filet ERR_INIT_BAM_WRAPPER BAM file could not be opened, file_location); } - if ((bw-header = bam_header_read(bw-bam.input)) == NULL) { + if ((bw-bam.header = bam_header_read(bw-bam.input)) == NULL) { throw(MAL, init_bam_wrapper, ERR_INIT_BAM_WRAPPER Unable to read header from file, file_location); } } else { /* Open SAM file and read its header */ -if ((bw-sam.input = samopen(file_location, r, NULL)) == NULL) { +int bufsize = 4096; +lng header_len = 0; +if ((bw-sam.input = open_rastream(file_location)) == NULL) { throw(MAL, init_bam_wrapper, ERR_INIT_BAM_WRAPPER SAM file could not be opened, file_location); } - if ((bw-header = bw-sam.input-header) == NULL) { - throw(MAL, init_bam_wrapper, - ERR_INIT_BAM_WRAPPER Unable to read header from file, - file_location); +if ((bw-sam.header = (str)GDKmalloc(bufsize * sizeof(char))) == NULL) { +throw(MAL, init_bam_wrapper, + ERR_INIT_BAM_WRAPPER MAL_MALLOC_FAIL, file_location); +} +while (TRUE) { +int read = mnstr_readline(bw-sam.input, bw-sam.header + header_len, bufsize - header_len); + +if (read = 0) { +throw(MAL, init_bam_wrapper, + ERR_INIT_BAM_WRAPPER Could not read line of SAM header, + file_location); +} + +if (bw-sam.header[header_len] != '@') { +/* This is not a header line, we assume that the header is finished. + * Rewind stream to start of line and stop reading */ +if (mnstr_fsetpos(bw-sam.input, header_len) 0) { +throw(MAL, init_bam_wrapper, + ERR_INIT_BAM_WRAPPER Could not read last line of SAM header, + file_location); +} +bw-sam.header[header_len] = '\0'; /* Truncate alignment data */ +break; +} + +if (bw-sam.header[header_len+read-1] != '\n') { +/* This line was not completed. Increase buffer size, rewind stream + * and try again */ +bufsize *= 2; +if ((bw-sam.header = (str)GDKrealloc(bw-sam.header, bufsize * sizeof(char))) == NULL) { +throw(MAL, init_bam_wrapper, + ERR_INIT_BAM_WRAPPER MAL_MALLOC_FAIL, file_location); +} +if (mnstr_fsetpos(bw-sam.input, header_len) 0) { +throw(MAL, init_bam_wrapper, + ERR_INIT_BAM_WRAPPER Could not read last line of SAM header, + file_location); +} +continue; +} + +
MonetDB: bamloader - Merged
Changeset: 793acf412199 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=793acf412199 Added Files: clients/mapilib/ChangeLog common/stream/ChangeLog geom/sql/Tests/T1.modules geom/sql/Tests/T10.modules geom/sql/Tests/T11.modules geom/sql/Tests/T12.modules geom/sql/Tests/T13.modules geom/sql/Tests/T14.modules geom/sql/Tests/T15.modules geom/sql/Tests/T16.modules geom/sql/Tests/T17.modules geom/sql/Tests/T18.modules geom/sql/Tests/T19.modules geom/sql/Tests/T2.modules geom/sql/Tests/T20.modules geom/sql/Tests/T21.modules geom/sql/Tests/T22.modules geom/sql/Tests/T23.modules geom/sql/Tests/T24.modules geom/sql/Tests/T25.modules geom/sql/Tests/T26.modules geom/sql/Tests/T27.modules geom/sql/Tests/T28.modules geom/sql/Tests/T29.modules geom/sql/Tests/T3.modules geom/sql/Tests/T30.modules geom/sql/Tests/T31.modules geom/sql/Tests/T32.modules geom/sql/Tests/T33.modules geom/sql/Tests/T34.modules geom/sql/Tests/T35.modules geom/sql/Tests/T36.modules geom/sql/Tests/T37.modules geom/sql/Tests/T38.modules geom/sql/Tests/T39.modules geom/sql/Tests/T4.modules geom/sql/Tests/T40.modules geom/sql/Tests/T41.modules geom/sql/Tests/T42.modules geom/sql/Tests/T43.modules geom/sql/Tests/T44.modules geom/sql/Tests/T45.modules geom/sql/Tests/T46.modules geom/sql/Tests/T47.modules geom/sql/Tests/T48.modules geom/sql/Tests/T49.modules geom/sql/Tests/T5.modules geom/sql/Tests/T50.modules geom/sql/Tests/T51.modules geom/sql/Tests/T52.modules geom/sql/Tests/T6.modules geom/sql/Tests/T7.modules geom/sql/Tests/T8.modules geom/sql/Tests/T9.modules geom/sql/Tests/conformance_drop.modules geom/sql/Tests/conformance_load.modules monetdb5/modules/mal/Tests/pqueue2.mal monetdb5/modules/mal/Tests/pqueueElim.mal monetdb5/modules/mal/Tests/pqueueElim.stable.err monetdb5/modules/mal/Tests/pqueueElim.stable.out sql/backends/monet5/bam/Tests/test.modules sql/backends/monet5/datacell/Tests/datacell99.sql sql/backends/monet5/datacell/Tests/datacell99.stable.err sql/backends/monet5/datacell/Tests/datacell99.stable.out sql/backends/monet5/datacell/Tests/emili.sql sql/backends/monet5/datacell/Tests/emili.txt sql/test/BugTracker-2014/Tests/crash_on_groupby_distinct_serial.Bug-3463.sql sql/test/BugTracker-2014/Tests/crash_on_groupby_distinct_serial.Bug-3463.stable.err sql/test/BugTracker-2014/Tests/crash_on_groupby_distinct_serial.Bug-3463.stable.out sql/test/BugTracker-2014/Tests/crash_on_or_with_in.Bug-3461.sql sql/test/BugTracker-2014/Tests/crash_on_or_with_in.Bug-3461.stable.err sql/test/BugTracker-2014/Tests/crash_on_or_with_in.Bug-3461.stable.out sql/test/BugTracker-2014/Tests/in_incorrect_multi.Bug-3462.sql sql/test/BugTracker-2014/Tests/in_incorrect_multi.Bug-3462.stable.err sql/test/BugTracker-2014/Tests/in_incorrect_multi.Bug-3462.stable.out sql/test/BugTracker-2014/Tests/select-having.Bug-3458.sql sql/test/BugTracker-2014/Tests/select-having.Bug-3458.stable.err sql/test/BugTracker-2014/Tests/select-having.Bug-3458.stable.out sql/test/BugTracker-2014/Tests/setreadonly_forgets_changes.Bug-3446.sql sql/test/BugTracker-2014/Tests/setreadonly_forgets_changes.Bug-3446.stable.err sql/test/BugTracker-2014/Tests/setreadonly_forgets_changes.Bug-3446.stable.out Removed Files: sql/backends/monet5/datacell/Tests/emili/README sql/backends/monet5/datacell/Tests/emili/emili.sql sql/backends/monet5/datacell/Tests/emili/sensor.log.gz sql/backends/monet5/datacell/Tests/pair00.mal sql/backends/monet5/datacell/Tests/pair00.stable.err sql/backends/monet5/datacell/Tests/pair00.stable.out sql/backends/monet5/datacell/Tests/petrinet00.mal sql/backends/monet5/datacell/Tests/petrinet00.stable.err sql/backends/monet5/datacell/Tests/petrinet00.stable.out Modified Files: NT/installer32/MonetDB5-Geom-Module.vdproj NT/installer64/MonetDB5-Geom-Module.vdproj buildtools/ChangeLog.Jan2014 buildtools/doc/windowsbuild.rst clients/ChangeLog.Jan2014 clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures_gsl.stable.out clients/Tests/MAL-signatures_nocfitsio.stable.out clients/Tests/MAL-signatures_sphinxclient.stable.out clients/Tests/SQL-dump.stable.out.32bit clients/Tests/SQL-dump.stable.out.64bit.oid32
MonetDB: bamloader - Small bug fix
Changeset: fb17f0c487cc for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fb17f0c487cc Modified Files: sql/backends/monet5/bam/bam_wrapper.c sql/backends/monet5/bam/bam_wrapper.h Branch: bamloader Log Message: Small bug fix diffs (40 lines): diff --git a/sql/backends/monet5/bam/bam_wrapper.c b/sql/backends/monet5/bam/bam_wrapper.c --- a/sql/backends/monet5/bam/bam_wrapper.c +++ b/sql/backends/monet5/bam/bam_wrapper.c @@ -182,6 +182,7 @@ init_bam_wrapper(bam_wrapper * bw, filet bw-cnt_alignments_extra = 0; bw-cnt_alignments_paired_primary = 0; bw-cnt_alignments_paired_secondary = 0; +bw-cnt_alignments_total = 0; for (i = 0; i 6; ++i) { snprintf(bw-fp_files[i], BW_FP_BUF_SIZE, %s/files_%d, @@ -1968,7 +1969,7 @@ complete_qname_group(alignment ** alignm return MAL_SUCCEED; } -#define BAMSAM_TELL(bw) (bw-type == BAM ? bam_tell(bw-bam.input) : (bw-cnt_alignments + 1)) +#define BAMSAM_TELL(bw) (bw-type == BAM ? bam_tell(bw-bam.input) : (bw-cnt_alignments_total + 1)) str process_alignments(bam_wrapper * bw, bit * some_thread_failed) @@ -2048,6 +2049,8 @@ process_alignments(bam_wrapper * bw, bit break; } } + +++bw-cnt_alignments_total; if (bw-dbschema == 1 alig_index 0 strcmp((bw-type == BAM ? bam1_qname(alig) : a-qname), diff --git a/sql/backends/monet5/bam/bam_wrapper.h b/sql/backends/monet5/bam/bam_wrapper.h --- a/sql/backends/monet5/bam/bam_wrapper.h +++ b/sql/backends/monet5/bam/bam_wrapper.h @@ -69,6 +69,7 @@ typedef struct bam_wrapper { lng cnt_alignments_extra; lng cnt_alignments_paired_primary; lng cnt_alignments_paired_secondary; +lng cnt_alignments_total; /* File paths to binary files */ char fp_files[6][BW_FP_BUF_SIZE]; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: bamloader - Included tests for sam_export function
Changeset: 0a3325139576 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0a3325139576 Modified Files: sql/backends/monet5/bam/Tests/test.SQL.py sql/backends/monet5/bam/Tests/test.stable.out Branch: bamloader Log Message: Included tests for sam_export function diffs (truncated from 508 to 300 lines): diff --git a/sql/backends/monet5/bam/Tests/test.SQL.py b/sql/backends/monet5/bam/Tests/test.SQL.py --- a/sql/backends/monet5/bam/Tests/test.SQL.py +++ b/sql/backends/monet5/bam/Tests/test.SQL.py @@ -43,6 +43,8 @@ for match in p.finditer(out): # Now we will execute all benchmark queries on all BAM files in the bam.files table # and output all data contained in the aux table. +# Furthermore, we transfer every file to the export table, use sam_export to write +# the contents to a SAM file and then print the raw contents of this SAM file c = new_client() set_var(c, 'rname_1_3', 'chr22', numeric=False) set_var(c, 'pos_1_3_1', 100) @@ -56,6 +58,7 @@ set_var(c, 'rname_2_10', 'chr22', numeri set_var(c, 'pos_2_10', 8000) set_var(c, 'distance_2_12', 1000) +output_files = [] for f in files_to_test: # benchmark 1 for uc in range(1, 6): @@ -71,6 +74,15 @@ for f in files_to_test: #write all aux data c.stdin.write(SELECT * FROM bam.alignments_extra_%d;% f[0]); +#load into export table +c.stdin.write(INSERT INTO bam.export (SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual FROM bam.%salignments_%d);\ +% (('unpaired_all_' if f[1] == 1 else ''), f[0])) +output_files.append(os.path.join(SRCDIR, (output_%d.sam% f[0]))) +c.stdin.write(CALL sam_export('%s');% output_files[-1]) + + + + out, err = c.communicate() # The output will contain explicit file_ids in the table names, making the table names @@ -83,3 +95,11 @@ out = (re.subn(p, replace, out))[0] sys.stdout.write(out) sys.stderr.write(err) + + +# All that is left to do is write the contents of the exported SAM files and delete them +for f in output_files: +sys.stdout.write(\n\nContents of exported SAM file '%s':\n% f) +for ln in open(f): +sys.stdout.write(ln) +os.remove(f) \ No newline at end of file diff --git a/sql/backends/monet5/bam/Tests/test.stable.out b/sql/backends/monet5/bam/Tests/test.stable.out --- a/sql/backends/monet5/bam/Tests/test.stable.out +++ b/sql/backends/monet5/bam/Tests/test.stable.out @@ -1,9 +1,9 @@ stdout of test 'test` in directory 'sql/backends/monet5/bam` itself: -# 21:48:24 -# 21:48:24 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=30475 --set mapi_usock=/var/tmp/mtest-32164/.s.monetdb.30475 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/export/scratch2/robin/Monet/PREFIX/DFT/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 -# 21:48:24 +# 23:17:55 +# 23:17:55 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=35794 --set mapi_usock=/var/tmp/mtest-20301/.s.monetdb.35794 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/export/scratch2/robin/Monet/PREFIX/DFT/var/MonetDB/mTests_sql_backends_monet5_bam --set mal_listing=0 +# 23:17:55 # MonetDB 5 server v11.18.0 # This is an unreleased version @@ -13,8 +13,8 @@ stdout of test 'test` in directory 'sql/ # Copyright (c) 1993-July 2008 CWI. # Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved # Visit http://www.monetdb.org/ for further information -# Listening for connection requests on mapi:monetdb://essen.ins.cwi.nl:30475/ -# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-32164/.s.monetdb.30475 +# Listening for connection requests on mapi:monetdb://essen.ins.cwi.nl:35794/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-20301/.s.monetdb.35794 # MonetDB/GIS module loaded # MonetDB/JAQL module loaded # MonetDB/SQL module loaded @@ -61,9 +61,9 @@ Ready. #WARNING To speedup calc.min a bulk operator implementation is needed #Interpreting lng as wrd. -# 21:48:24 -# 21:48:24 /usr/bin/python2 test.SQL.py test -# 21:48:24 +# 23:17:55 +# 23:17:55 /usr/bin/python2 test.SQL.py test +# 23:17:55 #DECLARE rname_1_3 varchar(32);SET rname_1_3='chr22';DECLARE pos_1_3_1 integer;SET pos_1_3_1=100;DECLARE pos_1_3_2 integer;SET pos_1_3_2=2000;DECLARE qname_1_4 varchar(32);SET qname_1_4='sim_22_1_1';DECLARE mapq_1_5 integer;SET mapq_1_5=200;DECLARE mapq_2_1 integer;SET mapq_2_1=100;DECLARE rname_2_9 varchar(32);SET rname_2_9='chr22';DECLARE pos_2_9 integer;SET pos_2_9=39996433;DECLARE rname_2_10 varchar(32);SET rname_2_10='chr22';DECLARE pos_2_10 integer;SET pos_2_10=8000;DECLARE distance_2_12 integer;SET distance_2_12=1000;SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual #DECLARE rname_1_3 varchar(32);SET rname_1_3='chr22';DECLARE pos_1_3_1 integer;SET
MonetDB: bamloader - Merged
Changeset: ba874198a299 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ba874198a299 Added Files: monetdb5/modules/atoms/Tests/strpad.mal monetdb5/modules/atoms/Tests/strpad.stable.err monetdb5/modules/atoms/Tests/strpad.stable.out monetdb5/modules/atoms/Tests/strtrim.mal monetdb5/modules/atoms/Tests/strtrim.stable.err monetdb5/modules/atoms/Tests/strtrim.stable.out sql/backends/monet5/datacell/Tests/emilifinish.stable.err sql/backends/monet5/datacell/Tests/emilifinish.stable.out sql/backends/monet5/datacell/Tests/emilistart.sql sql/backends/monet5/datacell/Tests/emilistart.stable.err sql/backends/monet5/datacell/Tests/emilistart.stable.out sql/test/Tests/str-pad.sql sql/test/Tests/str-pad.stable.err sql/test/Tests/str-pad.stable.out sql/test/Tests/str-trim.sql sql/test/Tests/str-trim.stable.err sql/test/Tests/str-trim.stable.out Removed Files: monetdb5/modules/mal/Tests/pqueueElim.mal monetdb5/modules/mal/Tests/pqueueElim.stable.err monetdb5/modules/mal/Tests/pqueueElim.stable.out sql/backends/monet5/datacell/Tests/emili.sql Modified Files: clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures_gsl.stable.out clients/Tests/MAL-signatures_nocfitsio.stable.out clients/Tests/MAL-signatures_sphinxclient.stable.out clients/Tests/SQL-dump.stable.out clients/Tests/SQL-dump.stable.out.64bit.oid32 clients/Tests/SQL-dump_gsl.stable.out clients/Tests/SQL-dump_gsl.stable.out.oid32 clients/Tests/SQL-dump_nogeom.stable.out clients/Tests/exports.stable.out clients/mapiclient/dump.c clients/mapiclient/mclient.c clients/mapiclient/tomograph.c clients/odbc/driver/SQLColAttribute.c clients/odbc/driver/SQLColumnPrivileges.c clients/odbc/driver/SQLColumns.c clients/odbc/driver/SQLForeignKeys.c clients/odbc/driver/SQLPrimaryKeys.c clients/odbc/driver/SQLProcedures.c clients/odbc/driver/SQLSpecialColumns.c clients/odbc/driver/SQLStatistics.c clients/odbc/driver/SQLTablePrivileges.c clients/odbc/driver/SQLTables.c common/stream/stream.c gdk/gdk.h gdk/gdk_imprints.c gdk/gdk_storage.c gdk/gdk_system.c gdk/gdk_utils.c monetdb5/extras/sphinx/sphinx.c monetdb5/mal/mal_builder.c monetdb5/mal/mal_debugger.c monetdb5/mal/mal_function.c monetdb5/mal/mal_profiler.c monetdb5/mal/mal_profiler.h monetdb5/mal/mal_readline.c monetdb5/mal/mal_recycle.c monetdb5/mal/mal_resolve.c monetdb5/modules/atoms/inet.c monetdb5/modules/atoms/str.c monetdb5/modules/atoms/str.h monetdb5/modules/atoms/str.mal monetdb5/modules/atoms/url.c monetdb5/modules/kernel/aggr.c monetdb5/modules/kernel/batstr.c monetdb5/modules/kernel/batstr.mal monetdb5/modules/mal/Tests/All monetdb5/modules/mal/Tests/pqueue.mal monetdb5/modules/mal/Tests/pqueue.stable.out monetdb5/modules/mal/Tests/pqueue2.mal monetdb5/modules/mal/bbp.c monetdb5/modules/mal/mal_io.c monetdb5/modules/mal/pqueue.c monetdb5/modules/mal/pqueue.h monetdb5/modules/mal/pqueue.mal monetdb5/modules/mal/profiler.c monetdb5/modules/mal/remote.c monetdb5/modules/mal/sysmon.c monetdb5/optimizer/opt_cluster.c monetdb5/optimizer/opt_dataflow.c monetdb5/optimizer/opt_macro.c monetdb5/optimizer/opt_mergetable.c monetdb5/optimizer/opt_multiplex.c monetdb5/optimizer/opt_pushselect.c monetdb5/optimizer/opt_remap.c monetdb5/optimizer/opt_remoteQueries.c sql/ChangeLog sql/backends/monet5/datacell/Tests/All sql/backends/monet5/datacell/Tests/dcload.sql sql/backends/monet5/datacell/Tests/dcload.stable.err sql/backends/monet5/datacell/opt_datacell.c sql/backends/monet5/datacell/opt_datacell.h sql/backends/monet5/datacell/petrinet.c sql/backends/monet5/sql.c sql/backends/monet5/sql.h sql/backends/monet5/sql.mal sql/backends/monet5/sql_cast.c sql/backends/monet5/sql_optimizer.c sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_statistics.c sql/backends/monet5/vaults/fits.c sql/common/sql_types.c sql/server/rel_exp.c sql/server/rel_exp.h sql/server/rel_optimizer.c sql/server/rel_planner.c sql/server/rel_select.c sql/server/sql_privileges.c sql/test/Tests/50ways.stable.err sql/test/Tests/coalesce.stable.err sql/test/Tests/create_in_schema.stable.err sql/test/Tests/decimal.stable.err
MonetDB: bamloader - Finished sam_export and properly catch the ...
Changeset: 19686dfee569 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=19686dfee569 Modified Files: sql/backends/monet5/bam/bam_export.c Branch: bamloader Log Message: Finished sam_export and properly catch the not yet working bam_export diffs (75 lines): diff --git a/sql/backends/monet5/bam/bam_export.c b/sql/backends/monet5/bam/bam_export.c --- a/sql/backends/monet5/bam/bam_export.c +++ b/sql/backends/monet5/bam/bam_export.c @@ -30,6 +30,7 @@ #include samtools/bam.h #include bam_globals.h +#include bam_db_interface.h #include bam_export.h @@ -44,7 +45,7 @@ typedef struct bam_field { /** * Copied directly from bam.h/bam_import.c for use by fill_bam_alig - * Can not change the call to realloc to GDKrealloc, since + * Can not change the calls to realloc to GDKrealloc, since * bam_destroy1 does not use GDKfree... */ #ifndef kroundup32 @@ -393,6 +394,7 @@ cleanup_fields(bam_field fields[11]) { } } + #define CUR_STR(field, i) ((str) BUNtail(field.iter, (field.cur+i))) #define CUR_SHT(field, i) (*(sht *) BUNtail(field.iter, (field.cur+i))) #define CUR_INT(field, i) (*(int *) BUNtail(field.iter, (field.cur+i))) @@ -409,6 +411,7 @@ sam_export(Client cntxt, MalBlkPtr mb, M int tuple_count = 0; int i; +str sql; str msg = MAL_SUCCEED; memset(fields, 0, 11 * sizeof(bam_field)); @@ -433,6 +436,13 @@ sam_export(Client cntxt, MalBlkPtr mb, M CUR_STR(fields[9], i), CUR_STR(fields[10], i)); } +/* If we got here, we succesfully exported the result. Drop all data in export table */ +sql = DELETE FROM bam.export;; +RUN_SQL(cntxt, sql, bam.drop_export, msg); +if (msg != MAL_SUCCEED) { +REUSE_EXCEPTION(msg, MAL, sam_export, Could not clear the export table after exporting: %s, msg); +} + (void)stk; (void)pci; @@ -465,7 +475,10 @@ bam_export(Client cntxt, MalBlkPtr mb, M bam1_t *alig = NULL; int i; +str sql; str msg = MAL_SUCCEED; + +throw(MAL, bam_export, Exporting to BAM files is not implemented yet. This is our first priority for the next release of the BAM library.); if ((output = bam_open(output_path, wb)) == NULL) { msg = createException(MAL, bam_export, Could not open output file '%s' for writing, output_path); @@ -518,6 +531,13 @@ bam_export(Client cntxt, MalBlkPtr mb, M bam_write1(output, alig); } +/* If we got here, we succesfully exported the result. Drop all data in export table */ +sql = DELETE FROM bam.export;; +RUN_SQL(cntxt, sql, bam.drop_export, msg); +if (msg != MAL_SUCCEED) { +REUSE_EXCEPTION(msg, MAL, sam_export, Could not clear the export table after exporting: %s, msg); +} + (void)stk; (void)pci; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list