MonetDB: default - Fix compiler "variable may be used uninitiali...

2016-04-26 Thread Robin Cijvat
Changeset: 3d370112d4b8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3d370112d4b8
Modified Files:
monetdb5/optimizer/opt_reorder.c
Branch: default
Log Message:

Fix compiler "variable may be used uninitialized" error


diffs (12 lines):

diff --git a/monetdb5/optimizer/opt_reorder.c b/monetdb5/optimizer/opt_reorder.c
--- a/monetdb5/optimizer/opt_reorder.c
+++ b/monetdb5/optimizer/opt_reorder.c
@@ -206,7 +206,7 @@ OPTbreadthfirst(Client cntxt, MalBlkPtr 
 static int
 OPTpostponeAppends(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
 {
-   int i,j,k=0, actions =0, last;
+   int i,j,k=0, actions =0, last=-1;
InstrPtr *old, *appends;
int limit;
(void) cntxt;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Fix preprocessor compilation issue

2015-11-22 Thread Robin Cijvat
Changeset: 37403127e328 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=37403127e328
Modified Files:
sql/backends/monet5/vaults/bam/bam_wrapper.c
Branch: default
Log Message:

Fix preprocessor compilation issue


diffs (12 lines):

diff --git a/sql/backends/monet5/vaults/bam/bam_wrapper.c 
b/sql/backends/monet5/vaults/bam/bam_wrapper.c
--- a/sql/backends/monet5/vaults/bam/bam_wrapper.c
+++ b/sql/backends/monet5/vaults/bam/bam_wrapper.c
@@ -23,7 +23,7 @@
 
 #ifdef HAVE_SAMTOOLS_KSTRING_H
 #include 
-#elif HAVE_BAM_KSTRING_H
+#elif defined (HAVE_BAM_KSTRING_H)
 #include 
 #else
 /* Ubuntu doesn't distribute samtools/kstring.h, so we need our own
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: profiler - Flush stdout after every JSON object

2015-11-10 Thread Robin Cijvat
Changeset: 7f1d8af451e8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7f1d8af451e8
Modified Files:
clients/mapiclient/stethoscope.c
Branch: profiler
Log Message:

Flush stdout after every JSON object


diffs (18 lines):

diff --git a/clients/mapiclient/stethoscope.c b/clients/mapiclient/stethoscope.c
--- a/clients/mapiclient/stethoscope.c
+++ b/clients/mapiclient/stethoscope.c
@@ -342,10 +342,12 @@ main(int argc, char **argv)
buffer[len + n] = 0;
response = buffer;
if(json) {
-   if(trace != NULL)
+   if(trace != NULL) {
fprintf(trace, "%s", response);
-   else
+   } else {
printf("%s", response);
+   fflush(stdout);
+   }
}
while ((e = strchr(response, '\n')) != NULL) {
*e = 0;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: profiler - Changes in stethoscope:

2015-11-05 Thread Robin Cijvat
Changeset: 997c7811bf34 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=997c7811bf34
Modified Files:
clients/mapiclient/stethoscope.c
Branch: profiler
Log Message:

Changes in stethoscope:
- Always either write to stdout or to specified file
- Add json flag that, when set, makes stethoscope output json
- Fix help messages


diffs (255 lines):

diff --git a/clients/mapiclient/stethoscope.c b/clients/mapiclient/stethoscope.c
--- a/clients/mapiclient/stethoscope.c
+++ b/clients/mapiclient/stethoscope.c
@@ -69,10 +69,12 @@
 
 static stream *conn = NULL;
 static char hostname[128];
-static char *basefilename = "stethoscope";
+static char *filename = NULL;
 static int beat = 50;
+static int json = 0;
 static Mapi dbh;
 static MapiHdl hdl = NULL;
+static FILE *trace = NULL;
 
 /*
  * Tuple level reformatting
@@ -80,51 +82,56 @@ static MapiHdl hdl = NULL;
 
 static void
 renderEvent(EventRecord *ev){
+   FILE *s;
+   if(trace != NULL) 
+   s = trace;
+   else
+   s = stdout;
if( ev->eventnr ==0 && ev->version){
-   printf("[ ");
-   printf("0,  ");
-   printf("0,  ");
-   printf("\"\",   " );
-   printf("0,  ");
-   printf("\"system\", "); 
-   printf("0,  ");
-   printf("0,  ");
-   printf("0,  ");
-   printf("0,  ");
-   printf("0,  ");
-   printf("0,  ");
-   printf("\"");
-   printf("version:%s, release:%s, threads:%s, memory:%s, host:%s, 
oid:%d, package:%s ", 
+   fprintf(s, "[ ");
+   fprintf(s, "0,  ");
+   fprintf(s, "0,  ");
+   fprintf(s, "\"\",   " );
+   fprintf(s, "0,  ");
+   fprintf(s, "\"system\", "); 
+   fprintf(s, "0,  ");
+   fprintf(s, "0,  ");
+   fprintf(s, "0,  ");
+   fprintf(s, "0,  ");
+   fprintf(s, "0,  ");
+   fprintf(s, "0,  ");
+   fprintf(s, "\"");
+   fprintf(s, "version:%s, release:%s, threads:%s, memory:%s, 
host:%s, oid:%d, package:%s ", 
ev->version, ev->release, ev->threads, ev->memory, 
ev->host, ev->oid, ev->package);
-   printf("\"  ]\n");
+   fprintf(s, "\"  ]\n");
return ;
}
if( ev->eventnr < 0)
return;
-   printf("[ ");
-   printf(LLFMT",  ", ev->eventnr);
-   printf("\"%s\", ", ev->clk);
+   fprintf(s, "[ ");
+   fprintf(s, LLFMT",  ", ev->eventnr);
+   fprintf(s, "\"%s\", ", ev->clk);
if( ev->function && *ev->function)
-   printf("\"%s[%d]%d\",   ", ev->function, ev->pc, ev->tag);
+   fprintf(s, "\"%s[%d]%d\",   ", ev->function, ev->pc, 
ev->tag);
else
-   printf("\"\",   ");
-   printf("%d, ", ev->thread);
+   fprintf(s, "\"\",   ");
+   fprintf(s, "%d, ", ev->thread);
switch(ev->state){
-   case MDB_START: printf("\"start\",  "); break;
-   case MDB_DONE: printf("\"done \",   "); break;
-   case MDB_WAIT: printf("\"wait \",   "); break;
-   case MDB_PING: printf("\"ping \",   "); break;
-   case MDB_SYSTEM: printf("\"system\","); 
+   case MDB_START: fprintf(s, "\"start\",  "); break;
+   case MDB_DONE: fprintf(s, "\"done \",   "); break;
+   case MDB_WAIT: fprintf(s, "\"wait \",   "); break;
+   case MDB_PING: fprintf(s, "\"ping \",   "); break;
+   case MDB_SYSTEM: fprintf(s, "\"system\","); 
}
-   printf(LLFMT",  ", ev->ticks);
-   printf(LLFMT",  ", ev->rss);
-   printf(LLFMT",  ", ev->size);
-   printf(LLFMT",  ", ev->inblock);
-   printf(LLFMT",  ", ev->oublock);
-   printf(LLFMT",  ", ev->majflt);
-   printf(LLFMT",  ", ev->swaps);
-   printf(LLFMT",  ", ev->csw);
-   printf("\"%s\"  ]\n", ev->stmt);
+   fprintf(s, LLFMT",  ", ev->ticks);
+   fprintf(s, LLFMT",  ", ev->rss);
+   fprintf(s, LLFMT",  ", ev->size);
+   fprintf(s, LLFMT",  ", ev->inblock);
+   fprintf(s, LLFMT",  ", ev->oublock);
+   fprintf(s, LLFMT",  ", ev->majflt);
+   fprintf(s, LLFMT",  ", ev->swaps);
+   fprintf(s, LLFMT",  ", ev->csw);
+   fprintf(s, "\"%s\"  ]\n", ev->stmt);
 }
 
 static void
@@ -133,10 +140,13 @@ usageStethoscope(void)
 fprintf(stderr, "stethoscope [options] \n");
 fprintf(stderr, "  -d | --dbname=\n");
 fprintf(stderr, "  -u | --user=\n");
+fprintf(stderr, "  -P | --password=\n");
 fprintf(stderr, "  -p | --port=\n");
 fprintf(stderr, "  -h | --host=\n");
+fprintf(stderr, "  -j | --json\n");
 fprintf(stderr, "  -o | --output=\n");
fprintf(stderr, "  -b | --beat= in milliseconds 

MonetDB: profiler - Enable beautified statements

2015-11-05 Thread Robin Cijvat
Changeset: 795a98661f1b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=795a98661f1b
Modified Files:
monetdb5/mal/mal_profiler.c
Branch: profiler
Log Message:

Enable beautified statements


diffs (21 lines):

diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c
--- a/monetdb5/mal/mal_profiler.c
+++ b/monetdb5/mal/mal_profiler.c
@@ -212,7 +212,7 @@ renderProfilerEvent(MalBlkPtr mb, MalStk
//GDKfree(stmt);
 
// ship the beautified version as well
-/*
+
stmt = shortStmtRendering(mb, stk, pci);
stmtq = mal_quote(stmt, strlen(stmt));
if (stmtq != NULL) {
@@ -220,7 +220,7 @@ renderProfilerEvent(MalBlkPtr mb, MalStk
GDKfree(stmtq);
} 
GDKfree(stmt);
-*/
+
 
// collect the prerequisite pre-requisite statements
prereq[0]='[';
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: profiler - Change help message in stable.err

2015-11-05 Thread Robin Cijvat
Changeset: 03dbc027b047 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=03dbc027b047
Modified Files:
clients/mapiclient/Tests/stethoscope--help.stable.err
Branch: profiler
Log Message:

Change help message in stable.err


diffs (41 lines):

diff --git a/clients/mapiclient/Tests/stethoscope--help.stable.err 
b/clients/mapiclient/Tests/stethoscope--help.stable.err
--- a/clients/mapiclient/Tests/stethoscope--help.stable.err
+++ b/clients/mapiclient/Tests/stethoscope--help.stable.err
@@ -1,25 +1,26 @@
 stderr of test 'stethoscope--help` in directory 'clients/mapiclient` itself:
 
 
-# 11:43:47 >  
-# 11:43:47 >  Mtimeout -timeout 120 ./stethoscope--help stethoscope--help 
-# 11:43:47 >  
+# 17:08:24 >  
+# 17:08:24 >  "./stethoscope--help" "stethoscope--help"
+# 17:08:24 >  
 
 
-# 10:36:09 >  
-# 10:36:09 >  Mtimeout -timeout 60 stethoscope --help
-# 10:36:09 >  
+# 17:08:24 >  
+# 17:08:24 >  Mtimeout -timeout 60 stethoscope --help
+# 17:08:24 >  
 
 stethoscope [options] 
   -d | --dbname=
   -u | --user=
+  -P | --password=
   -p | --port=
   -h | --host=
+  -j | --json
   -o | --output=
   -b | --beat= in milliseconds (default 50)
-  -? | --help
+  -D | --debug  -? | --help
 
-
-# 19:24:05 >  
-# 19:24:05 >  "Done."
-
+# 17:08:24 >  
+# 17:08:24 >  "Done."
+# 17:08:24 >  
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - NodeJS client source code is now hosted on gi...

2015-11-02 Thread Robin Cijvat
Changeset: 938ce957c0eb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=938ce957c0eb
Removed Files:
clients/nodejs/Tests/nodetest.js
clients/nodejs/monetdb/README.md
clients/nodejs/monetdb/mapiclient.js
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

NodeJS client source code is now hosted on github 
(https://github.com/MonetDB/monetdb-nodejs) and can be found on npm as well 
(https://www.npmjs.com/package/monetdb)


diffs (truncated from 843 to 300 lines):

diff --git a/clients/nodejs/Tests/nodetest.js b/clients/nodejs/Tests/nodetest.js
deleted file mode 100644
--- a/clients/nodejs/Tests/nodetest.js
+++ /dev/null
@@ -1,147 +0,0 @@
-var monetdb = require('../monetdb');
-var assert = require('assert');
-
-var dbport = parseInt(process.argv[2]);
-var dbname = process.argv[3];
-
-/* lets first check some failing connection attempts */
-monetdb.connect({host:'veryinvalidhostnamethathopefullyresolvesnowhere'}, 
function(err) {
-   assert(err);
-});
-
-monetdb.connect({dbname:'nonexist', port:dbport}, function(err) {
-   assert(err);
-});
-
-var failedconn = monetdb.connect({dbname:dbname, user:'nonexist', 
port:dbport}, function(err) {
-   assert(err);
-});
-/* try to query on a failed connection, we need this callback */
-failedconn.query('SELECT 1',function(err,res) {
-   assert(err);
-});
-
-/* now actually connect */
-var conn = monetdb.connect({dbname:dbname, port:dbport, debug: false}, 
function(err) {
-   assert.equal(null, err);
-   assert.equal(conn.env.gdk_dbname, dbname);
-});
-
-
-/* some querying, call chaining */
-conn.query('start transaction').
-   query('create table foo(a int, b float, c clob)').
-   query("insert into foo values 
(42,4.2,'42'),(43,4.3,'43'),(44,4.4,'44'),(45,4.5,'45')");
-
-conn.query('select * from foo', function(err, res) {
-   assert.equal(null, err);
-
-   assert.equal('table', res.type);
-   assert.equal(4, res.rows);
-   assert.equal(3, res.cols);
-
-   assert.equal(3, res.structure.length);
-
-   assert.equal('a', res.structure[0].column);
-   assert.equal('int', res.structure[0].type);
-
-   assert.equal(4, res.data.length);
-
-   assert.equal(42, res.data[0][res.structure[0].index]);
-   assert.equal(4.3, res.data[1][res.col['b']]);
-   assert.equal('44', res.data[2][2]);
-});
-
-/* we can also just put the queries in one .query call */
-conn.query('delete from foo; drop table foo; rollback');
-
-/* query that will stress multi-block operations */
-function rep(str,n) {
-   ret = '';
-   for (var i = 0; i< n; i++) {
-   ret += str;
-   }
-   return ret;
-}
-
-var longstr = rep('ABCDEFGHIJKLMNOP', 10);
-conn.query("SELECT '" + longstr + "'", function(err, res) {
-   assert.equal(null, err);
-   assert.equal(longstr,res.data[0][0]);
-});
-
-/* failing query */
-conn.query('MEHR BIER', function(err, res) {
-   assert(err);
-});
-
-/* prepared statements */
-conn.query('MEHR BIER',
-   ['connections', 0, false], function(err, res) {
-   assert(err);
-}); 
-
-
-/* fire-and-forget query with parameters */
-conn.query('SELECT id from tables where name=? and type=? and temporary=?',
-   ['connections', 0, 0], function(err, res) {
-   assert.equal(null, err);
-   assert(res.rows > 0);
-}); 
-
-/* Try the log callback functionality */
-var nr_log_callbacks = 0;
-conn.log_callback = function(message, error, result) {
-   assert(message); // message must contain something
-   ++nr_log_callbacks;
-}
-conn.query('SELECT id FROM tables WHERE name=? AND type=? AND temporary=?',
-   ['connections', 0, 0], function(err, res) {
-   assert.equal(null, err);
-   assert(nr_log_callbacks > 0);
-   conn.log_callback = null;   
-});
-
-/* some quoting fun, jesus */
-conn.query("SELECT 'asdf','\"', '\\\"', '\"', '\\''", function(err, 
res) {
-   assert.equal(null, err);
-   assert.equal('\\asdf', res.data[0][0]);
-   assert.equal('"', res.data[0][1]);
-   assert.equal('\"', res.data[0][2]);
-   assert.equal('\\"', res.data[0][3]);
-   assert.equal("'", res.data[0][4]);
-});
-
-/* Retrieving escaped characters from the database */
-conn.query('start transaction').
-   query('create table foo(a string)').
-   query("insert into foo values ('\t\n\r\n\tlalala\t\n\r')");
-
-conn.query("select * from foo", function(err, res) {
-   assert.equal(null, err);
-   assert.equal(1, res.rows);
-   assert.equal('\t\n\r\n\tlalala\t\n\r', res.data[0][0]);
-});
-
-conn.query('delete from foo; drop table foo; rollback');
-
-
-/* prepared statements can also be re-used  */
-conn.prepare('SELECT id from tables where name=? and type=? and temporary=?', 
function(err, res){
-   assert.equal(null, err);
-
-   /* parameters can also be given as array */
-   

MonetDB: default - Update npm version

2015-07-01 Thread Robin Cijvat
Changeset: 6b31db1e14d5 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6b31db1e14d5
Modified Files:
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

Update npm version


diffs (11 lines):

diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.2.7,
+  version: 0.2.8,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Also support dates in prepared statement exec...

2015-07-01 Thread Robin Cijvat
Changeset: 41c633c2aefe for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=41c633c2aefe
Modified Files:
clients/nodejs/monetdb/mapiclient.js
Branch: default
Log Message:

Also support dates in prepared statement execution


diffs (18 lines):

diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -147,10 +147,10 @@ MonetDBConnection.prototype.prepare = fu
}
var colData = 
resp.data[resp.rows-bindparams.length+paramIndex];
if(colData) {
-   if(colData[0] == timestamp) {
-   s = timestamp +s;
-   } else if(colData[0] == 
timestamptz) {
-   s = timestamptz +s
+   var toCheck = ['timestamp', 
'timestamptz', 'date'];
+   var i = 
toCheck.indexOf(colData[0]);
+   if(i = 0) {
+   s = toCheck[i] + ' ' + 
s;
}
}
return s;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Extend the prepared statement handling to sup...

2015-06-30 Thread Robin Cijvat
Changeset: 28883be78b39 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=28883be78b39
Modified Files:
clients/nodejs/monetdb/mapiclient.js
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

Extend the prepared statement handling to support json and timestamptz


diffs (38 lines):

diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -137,13 +137,21 @@ MonetDBConnection.prototype.prepare = fu
/* escape single quotes except 
if they are already escaped */
s = ' + 
param.replace(/([^\\])'/g,$1\\') + ';
break
+   case 'object':
+   case 'array':
+   s = json ' + 
JSON.stringify(param).replace(/([^\\])'/g,$1\\') + ';
+   break;
default:
s = param;
break;
}
var colData = 
resp.data[resp.rows-bindparams.length+paramIndex];
-   if(colData  colData[0] == 
timestamp) {
-   s = timestamp +s;
+   if(colData) {
+   if(colData[0] == timestamp) {
+   s = timestamp +s;
+   } else if(colData[0] == 
timestamptz) {
+   s = timestamptz +s
+   }
}
return s;
}).join(', ');
diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.2.6,
+  version: 0.2.7,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Properly reconstruct escaped characters \t, \...

2015-06-03 Thread Robin Cijvat
Changeset: 8df3a20c7058 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8df3a20c7058
Modified Files:
clients/nodejs/Tests/nodetest.js
clients/nodejs/monetdb/mapiclient.js
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

Properly reconstruct escaped characters \t, \n, and \r, when retrieved from the 
database


diffs (52 lines):

diff --git a/clients/nodejs/Tests/nodetest.js b/clients/nodejs/Tests/nodetest.js
--- a/clients/nodejs/Tests/nodetest.js
+++ b/clients/nodejs/Tests/nodetest.js
@@ -112,6 +112,20 @@ conn.query(SELECT 'asdf','\', '\\\
assert.equal(', res.data[0][4]);
 });
 
+/* Retrieving escaped characters from the database */
+conn.query('start transaction').
+   query('create table foo(a string)').
+   query(insert into foo values ('\t\n\r\n\tlalala\t\n\r'));
+
+conn.query(select * from foo, function(err, res) {
+   assert.equal(null, err);
+   assert.equal(1, res.rows);
+   assert.equal('\t\n\r\n\tlalala\t\n\r', res.data[0][0]);
+});
+
+conn.query('delete from foo; drop table foo; rollback');
+
+
 /* prepared statements can also be re-used  */
 conn.prepare('SELECT id from tables where name=? and type=? and temporary=?', 
function(err, res){
assert.equal(null, err);
diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -386,7 +386,12 @@ exports.parsetuples =  _parsetuples = fu
break;
case 'ESCAPED':
state = 'INQUOTES';
-   curtok += chr;
+switch(chr) {
+case 't': curtok += '\t'; break;
+case 'n': curtok += '\n'; break;
+case 'r': curtok += '\r'; break;
+default: curtok += chr;
+}
break;
case 'INQUOTES':
if (chr == '') {
diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.2.5,
+  version: 0.2.6,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merge with default

2015-04-29 Thread Robin Cijvat
Changeset: de97b2973b94 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=de97b2973b94
Added Files:
clients/mapiclient/Tests/tachograph--help
clients/mapiclient/Tests/tachograph--help.bat
clients/mapiclient/Tests/tachograph--help.stable.err
clients/mapiclient/Tests/tachograph--help.stable.out

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows.single
sql/backends/monet5/vaults/74_netcdf.mal
sql/backends/monet5/vaults/74_netcdf.sql
sql/backends/monet5/vaults/netcdf.c
sql/backends/monet5/vaults/netcdf.mal
sql/backends/monet5/vaults/netcdf_vault.h
sql/test/mergetables/Tests/doubletable.stable.err
sql/test/mergetables/Tests/doubletable.stable.out
sql/test/mergetables/Tests/forex1.stable.err
sql/test/mergetables/Tests/forex1.stable.out
sql/test/mergetables/Tests/mergedropcrash.sql
sql/test/mergetables/Tests/mergedropcrash.stable.err
sql/test/mergetables/Tests/mergedropcrash.stable.out
sql/test/remote/Tests/ssbm.stable.out.int128
Removed Files:
clients/Tests/SQL-dump.stable.out.oid32
clients/Tests/SQL-dump.stable.out.oid32.int128
Modified Files:
.hgtags
MonetDB.spec
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
buildtools/ChangeLog-Archive
buildtools/ChangeLog.Oct2014
buildtools/autogen/autogen/am.py
clients/Tests/All
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
clients/Tests/exports.stable.out
clients/mapiclient/Tests/All
clients/mapiclient/Tests/tomograph--help.stable.err
clients/mapiclient/eventparser.c
clients/mapiclient/tachograph.c
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
clients/python2/setup.py
clients/python3/setup.py
configure.ag
debian/changelog
gdk/ChangeLog-Archive
gdk/ChangeLog.Oct2014
gdk/gdk_aggr.c
gdk/gdk_align.c
gdk/gdk_atomic.h
gdk/gdk_calc.c
gdk/gdk_imprints.c
gdk/gdk_join.c
gdk/gdk_logger.c
gdk/gdk_private.h
gdk/gdk_select.c
gdk/gdk_system.c
gdk/gdk_system.h
gdk/gdk_utils.c
gdk/libbat.rc
geom/monetdb5/geom.mal
java/Makefile.ag
java/build.properties
java/pom.xml
java/release.txt
libversions
monetdb5/mal/Tests/performanceTests/performanceLog
monetdb5/mal/Tests/performanceTests/tst901a.mal
monetdb5/mal/Tests/performanceTests/tst901b.mal
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_parser.c
monetdb5/modules/atoms/color.mal
monetdb5/modules/atoms/inet.mal
monetdb5/modules/atoms/json.mal
monetdb5/modules/atoms/mtime.mal
monetdb5/modules/atoms/url.mal
monetdb5/modules/atoms/uuid.mal
monetdb5/modules/atoms/xml.mal
monetdb5/tools/libmonetdb5.rc
sql/backends/monet5/sql.c
sql/backends/monet5/sql.mal
sql/backends/monet5/sql_upgrades.c
sql/backends/monet5/vaults/Makefile.ag
sql/benchmarks/tpch/Tests/01-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/04-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/05-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/16-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/20-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/21-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/22-explain.stable.out.32bit
sql/jdbc/tests/Tests/All
sql/jdbc/tests/Tests/Test_Dobjects.stable.out
sql/scripts/75_storagemodel.sql
sql/server/rel_optimizer.c
sql/server/rel_schema.c
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_table.c
sql/storage/store.c
sql/test/BugTracker-2013/Tests/between.Bug-3259.stable.out.int128.single
sql/test/BugTracker-2013/Tests/between.Bug-3259.stable.out.single
sql/test/BugTracker-2014/Tests/nil_2dec_lng.Bug-3592.stable.out.single
sql/test/BugTracker-2014/Tests/typecast_error.Bug-3622.sql
sql/test/BugTracker-2014/Tests/typecast_error.Bug-3622.stable.out
sql/test/BugTracker-2015/Tests/readonly.Bug-3709.stable.err
sql/test/Tests/All
sql/test/Tests/systemfunctions.stable.out
sql/test/Tests/systemfunctions.stable.out.int128
  

MonetDB: bamloader - Merge with default

2015-04-22 Thread Robin Cijvat
Changeset: c3b0b8e5956b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c3b0b8e5956b
Added Files:
clients/R/build-for-cran.sh
clients/Tests/MAL-signatures.malC
clients/Tests/MAL-signatures.stable.err
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/SQL-dump.sql
clients/Tests/SQL-dump.stable.err
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
clients/Tests/SQL-dump.stable.out.oid32
clients/Tests/SQL-dump.stable.out.oid32.int128
debian/monetdb5-server-hugeint.install
debian/monetdb5-sql-hugeint.install
monetdb5/mal/Tests/tst062.stable.err
monetdb5/mal/Tests/tst062.stable.out
monetdb5/modules/mal/Tests/remote07.malC
monetdb5/optimizer/Tests/joinchain.mal
monetdb5/optimizer/Tests/joinchain.stable.err
monetdb5/optimizer/Tests/joinchain.stable.out
sql/benchmarks/ATIS/Tests/select_group.stable.out.int128
sql/benchmarks/tpch/15a.sql
sql/benchmarks/tpch/LOCKED/Tests/17.stable.out.int128
sql/benchmarks/tpch/Tests/05-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/17.stable.out.int128
sql/benchmarks/tpch/Tests/22.stable.out.int128
sql/benchmarks/tpch/c.sql-primary
sql/benchmarks/tpch/c.sql-primary-foreign
sql/scripts/41_md5sum.sql

sql/test/BugTracker-2009/Tests/case_evaluates_all_branches.SF-2893484.stable.out.int128

sql/test/BugTracker-2010/Tests/crash-with-huge-query.Bug-2647.stable.out.int128

sql/test/BugTracker-2011/Tests/subquery_in_from_clause.Bug-2812.stable.out.int128

sql/test/BugTracker-2012/Tests/aggregate_incorrect_precision_scale.Bug-3182.stable.out.int128
sql/test/BugTracker-2012/Tests/currenttime.Bug-2781.SQL.py
sql/test/BugTracker-2015/Tests/alias.Bug-3704.sql
sql/test/BugTracker-2015/Tests/alias.Bug-3704.stable.err
sql/test/BugTracker-2015/Tests/alias.Bug-3704.stable.out
sql/test/BugTracker-2015/Tests/assertHead.Bug-3706.sql
sql/test/BugTracker-2015/Tests/assertHead.Bug-3706.stable.err
sql/test/BugTracker-2015/Tests/assertHead.Bug-3706.stable.out
sql/test/BugTracker-2015/Tests/case_aggr.Bug-3705.sql
sql/test/BugTracker-2015/Tests/case_aggr.Bug-3705.stable.err
sql/test/BugTracker-2015/Tests/case_aggr.Bug-3705.stable.out
sql/test/BugTracker-2015/Tests/find_fk.Bug-3690.sql
sql/test/BugTracker-2015/Tests/find_fk.Bug-3690.stable.err
sql/test/BugTracker-2015/Tests/find_fk.Bug-3690.stable.out
sql/test/BugTracker-2015/Tests/float-space-parse.Bug-3691.sql
sql/test/BugTracker-2015/Tests/float-space-parse.Bug-3691.stable.err
sql/test/BugTracker-2015/Tests/float-space-parse.Bug-3691.stable.out
sql/test/BugTracker-2015/Tests/ifthenelse.Bug-3699.sql
sql/test/BugTracker-2015/Tests/ifthenelse.Bug-3699.stable.err
sql/test/BugTracker-2015/Tests/ifthenelse.Bug-3699.stable.out
sql/test/BugTracker-2015/Tests/incorrect_result_type.Bug-3687.sql
sql/test/BugTracker-2015/Tests/incorrect_result_type.Bug-3687.stable.err
sql/test/BugTracker-2015/Tests/incorrect_result_type.Bug-3687.stable.out

sql/test/BugTracker-2015/Tests/incorrect_result_type.Bug-3687.stable.out.int128
sql/test/BugTracker-2015/Tests/inf-nan-handling.Bug-3696.sql
sql/test/BugTracker-2015/Tests/inf-nan-handling.Bug-3696.stable.err
sql/test/BugTracker-2015/Tests/inf-nan-handling.Bug-3696.stable.out

sql/test/BugTracker-2015/Tests/insert_into_merge_table_crash.Bug-3703.sql

sql/test/BugTracker-2015/Tests/insert_into_merge_table_crash.Bug-3703.stable.err

sql/test/BugTracker-2015/Tests/insert_into_merge_table_crash.Bug-3703.stable.out
sql/test/BugTracker-2015/Tests/is_null_limit.Bug-3684.sql
sql/test/BugTracker-2015/Tests/is_null_limit.Bug-3684.stable.err
sql/test/BugTracker-2015/Tests/is_null_limit.Bug-3684.stable.out
sql/test/BugTracker-2015/Tests/operator-priority.Bug-3686.sql
sql/test/BugTracker-2015/Tests/operator-priority.Bug-3686.stable.err
sql/test/BugTracker-2015/Tests/operator-priority.Bug-3686.stable.out
sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693.sql
sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693.stable.err
sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693.stable.out
sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693_2.sql
sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693_2.stable.err
sql/test/BugTracker-2015/Tests/project_rewrite.Bug-3693_2.stable.out
sql/test/BugTracker-2015/Tests/readonly.Bug-3709.sql
sql/test/BugTracker-2015/Tests/readonly.Bug-3709.stable.err

MonetDB: bamloader - Merge with default

2015-03-12 Thread Robin Cijvat
Changeset: 04e4a6a7dac6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=04e4a6a7dac6
Modified Files:
clients/nodejs/monetdb/mapiclient.js
clients/nodejs/monetdb/package.json
clients/odbc/driver/ODBCStmt.h
clients/odbc/driver/SQLExtendedFetch.c
clients/odbc/driver/SQLFetch.c
clients/odbc/driver/SQLFetchScroll.c
configure.ag
gdk/gdk_atomic.h
gdk/gdk_system.c
gdk/gdk_system.h
gdk/gdk_utils.c
monetdb5/mal/mal_instruction.c
monetdb5/optimizer/Tests/manifold2.stable.out.single
sql/backends/monet5/UDF/Tests/udf-fuse.stable.out
sql/backends/monet5/UDF/Tests/udf-reverse.stable.out
sql/backends/monet5/rest/Tests/jsonstore00.stable.out
sql/backends/monet5/sql_scenario.c

sql/test/BugTracker-2010/Tests/LIMIT_OFFSET_big-endian.Bug-2622.stable.out

sql/test/BugTracker-2010/Tests/group-by_ordered_column.Bug-2564.stable.out
sql/test/BugTracker/Tests/insert_not_exists.SF-1380287.stable.out
sql/test/Skyserver/Tests/Skyserver.stable.out
sql/test/Tests/decimal2.stable.out
sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_MapiClient.stable.out
testing/Mtest.py.in
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 838 to 300 lines):

diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -122,26 +122,31 @@ MonetDBConnection.prototype.prepare = fu
thizz.query(query, function(error, resp) {
if (!error) {
var execfun = function(bindparams, ecallback) {
-   var quoted = bindparams.map(function(param) {
+   var quoted = bindparams.map(function(param, 
paramIndex) {
if(param === null) {
return NULL;
}
var type = typeof param;
+   var s;
switch(type) {
case 'boolean':
case 'number':
-   return '' + param;
+   s = '' + param;
break
case 'string':
/* escape single quotes except 
if they are already escaped */
-   return ' + 
param.replace(/([^\\])'/g,$1\\') + ';
+   s = ' + 
param.replace(/([^\\])'/g,$1\\') + ';
break
default:
-   return param;
+   s = param;
break;
}
+   var colData = 
resp.data[resp.rows-bindparams.length+paramIndex];
+   if(colData  colData[0] == 
timestamp) {
+   s = timestamp +s;
+   }
+   return s;
}).join(', ');
-
var execquery = 'EXEC ' + resp.queryid + '(' + 
quoted + ')';
thizz.query(execquery, ecallback);
}
diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.2.4,
+  version: 0.2.5,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
diff --git a/clients/odbc/driver/ODBCStmt.h b/clients/odbc/driver/ODBCStmt.h
--- a/clients/odbc/driver/ODBCStmt.h
+++ b/clients/odbc/driver/ODBCStmt.h
@@ -199,9 +199,9 @@ SQLRETURN MNDBColAttribute(ODBCStmt *stm
 SQLRETURN MNDBExecDirect(ODBCStmt *stmt, SQLCHAR *szSqlStr,
 SQLINTEGER nSqlStr);
 SQLRETURN MNDBExecute(ODBCStmt *stmt);
-SQLRETURN MNDBFetch(ODBCStmt *stmt);
+SQLRETURN MNDBFetch(ODBCStmt *stmt, SQLUSMALLINT *RowStatusArray);
 SQLRETURN MNDBFetchScroll(ODBCStmt *stmt, SQLSMALLINT nOrientation,
- SQLLEN nOffset);
+ SQLLEN nOffset, SQLUSMALLINT *RowStatusArray);
 SQLRETURN MNDBFreeStmt(ODBCStmt *stmt, SQLUSMALLINT option);
 

MonetDB: default - MonetDB/NodeJS: Update package version

2015-03-11 Thread Robin Cijvat
Changeset: ec75240aad7f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec75240aad7f
Modified Files:
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

MonetDB/NodeJS: Update package version


diffs (11 lines):

diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.2.4,
+  version: 0.2.5,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merge with default

2015-03-10 Thread Robin Cijvat
Changeset: c66c91916d41 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c66c91916d41
Added Files:
clients/R/Makefile.ag
clients/R/MonetDB.R/Makefile.ag
clients/R/MonetDB.R/src/Makefile.ag
clients/R/MonetDB.R/src/mapisplit-r.c
clients/R/MonetDB.R/src/mapisplit.h
clients/R/MonetDB.R/src/profiler-r.c
clients/R/MonetDB.R/src/profiler.h
clients/mapiclient/mapisplit-dummy.c
clients/mapiclient/profiler-dummy.c

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.single
monetdb5/optimizer/Tests/manifold2.stable.out.single
sql/backends/monet5/Tests/rapi12.sql
sql/backends/monet5/Tests/rapi12.stable.err
sql/backends/monet5/Tests/rapi12.stable.out
sql/backends/monet5/generator/90_generator_hge.mal
sql/backends/monet5/generator/90_generator_hge.sql
sql/test/bugs/Tests/apply_with_union_assertion.sql
sql/test/bugs/Tests/apply_with_union_assertion.stable.err
sql/test/bugs/Tests/apply_with_union_assertion.stable.out
sql/test/copy/Tests/format_date.sql
sql/test/copy/Tests/format_date.stable.err
sql/test/copy/Tests/format_date.stable.out
sql/test/mergetables/Tests/crash0.stable.err
sql/test/mergetables/Tests/crash0.stable.out
sql/test/mergetables/Tests/mergedrop.stable.err
sql/test/mergetables/Tests/mergedrop.stable.out
sql/test/pg_regress/Tests/int4.stable.out.int128
sql/test/pg_regress/Tests/int8.stable.err.int128
sql/test/pg_regress/Tests/int8.stable.out.int128
sql/test/pg_regress/Tests/numeric_big.stable.err.int128
sql/test/pg_regress/Tests/strings.stable.err.int128
Removed Files:
monetdb5/mal/Tests/LICENSE
monetdb5/modules/atoms/Tests/LICENSE
monetdb5/modules/mal/Tests/LICENSE
monetdb5/scheduler/Tests/LICENSE
monetdb5/tests/BugsViaSourgeforce/Tests/All
monetdb5/tests/gdkTests/Tests/casts.stable.out.Windows
monetdb5/tools/Tests/LICENSE
sql/backends/monet5/generator/89_generator_hge.mal
sql/backends/monet5/generator/89_generator_hge.sql
sql/backends/monet5/vaults/Tests/All
sql/test/BugTracker-2009/Tests/overflow.SF-2853458.stable.err.Windows
sql/test/Dump/Tests/dump.stable.out.Windows
Modified Files:
COPYING
HowToStart.rst
Makefile.ag
MonetDB.spec
NT/Makefile
NT/installer32/cv64-32.sh
NT/installer64/cv32-64.sh
NT/installer64/fixmsi.py
NT/monetdb_config.h.in
NT/rules.msc
NT/updatecodes.py
NT/wincompile.py
NT/winconfig.py
README
bootstrap
buildtools/ChangeLog
buildtools/Makefile.ag
buildtools/autogen/autogen.py
buildtools/autogen/autogen/am.py
buildtools/autogen/autogen/codegen.py
buildtools/autogen/autogen/msc.py
buildtools/autogen/autogen/var.py
buildtools/autogen/setup.py
buildtools/conf/Maddlog
buildtools/conf/Makefile.ag
buildtools/conf/fixlicense.py
buildtools/conf/license.rtf
buildtools/conf/rules.mk
buildtools/conf/website.html
buildtools/doc/HowToRelease.rst
buildtools/doc/windowsbuild.rst
clients/Makefile.ag
clients/NT/Makefile.ag
clients/NT/mclient.bat
clients/NT/msqldump.bat
clients/NT/stethoscope.bat
clients/R/MonetDB.R/DESCRIPTION
clients/R/MonetDB.R/NAMESPACE
clients/R/MonetDB.R/R/control.R
clients/R/MonetDB.R/R/dbi.R
clients/R/MonetDB.R/R/dplyr.R
clients/R/MonetDB.R/man/control.Rd
clients/R/MonetDB.R/man/src_monetdb.Rd
clients/R/MonetDB.R/src/Makevars.win
clients/R/MonetDB.R/src/mapisplit.c
clients/R/MonetDB.R/src/profiler.c
clients/R/Tests/dbi.R
clients/R/Tests/dplyr.R
clients/R/Tests/dplyr.stable.out
clients/R/Tests/survey.R
clients/Tests/MAL-signatures_all.stable.out
clients/Tests/MAL-signatures_all.stable.out.int128
clients/Tests/MAL-signatures_fits_geom.stable.out
clients/Tests/MAL-signatures_fits_geom.stable.out.int128
clients/Tests/MAL-signatures_geom.stable.out
clients/Tests/MAL-signatures_geom.stable.out.int128
clients/Tests/MAL-signatures_none.stable.out
clients/Tests/MAL-signatures_none.stable.out.int128
clients/Tests/SQL-dump_all.stable.out
clients/Tests/SQL-dump_all.stable.out.int128
clients/Tests/SQL-dump_all.stable.out.oid32
clients/Tests/SQL-dump_all.stable.out.oid32.int128
clients/Tests/SQL-dump_geom.stable.out
clients/Tests/SQL-dump_geom.stable.out.32bit
clients/Tests/SQL-dump_geom.stable.out.64bit.oid32
clients/Tests/SQL-dump_geom.stable.out.int128

MonetDB: bamloader - Do not track pyc file

2015-03-10 Thread Robin Cijvat
Changeset: f6f83b8f23f3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f6f83b8f23f3
Removed Files:
sql/backends/monet5/bam/Tests/bam.pyc
Branch: bamloader
Log Message:

Do not track pyc file


diffs (7 lines):

diff --git a/sql/backends/monet5/bam/Tests/bam.pyc 
b/sql/backends/monet5/bam/Tests/bam.pyc
deleted file mode 100644
index 
920c4a9a9889e1cc40f6d04cb102a0f9b57a828a..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
literal 0
Hc$@O1
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Add timestamp prefix to timestamp arguments i...

2015-03-10 Thread Robin Cijvat
Changeset: f84672e52d8f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f84672e52d8f
Modified Files:
clients/nodejs/monetdb/mapiclient.js
Branch: default
Log Message:

Add timestamp prefix to timestamp arguments in the execution of prepared 
statements


diffs (40 lines):

diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -122,26 +122,31 @@ MonetDBConnection.prototype.prepare = fu
thizz.query(query, function(error, resp) {
if (!error) {
var execfun = function(bindparams, ecallback) {
-   var quoted = bindparams.map(function(param) {
+   var quoted = bindparams.map(function(param, 
paramIndex) {
if(param === null) {
return NULL;
}
var type = typeof param;
+   var s;
switch(type) {
case 'boolean':
case 'number':
-   return '' + param;
+   s = '' + param;
break
case 'string':
/* escape single quotes except 
if they are already escaped */
-   return ' + 
param.replace(/([^\\])'/g,$1\\') + ';
+   s = ' + 
param.replace(/([^\\])'/g,$1\\') + ';
break
default:
-   return param;
+   s = param;
break;
}
+   var colData = 
resp.data[resp.rows-bindparams.length+paramIndex];
+   if(colData  colData[0] == 
timestamp) {
+   s = timestamp +s;
+   }
+   return s;
}).join(', ');
-
var execquery = 'EXEC ' + resp.queryid + '(' + 
quoted + ')';
thizz.query(execquery, ecallback);
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merged with default

2015-02-20 Thread Robin Cijvat
Changeset: b847a55bc9f2 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b847a55bc9f2
Added Files:
clients/R/MonetDB.R/src/Makevars.win
clients/R/MonetDB.R/src/profiler.c
clients/mapiclient/eventparser.c
clients/mapiclient/eventparser.h
clients/mapiclient/tachograph.c
monetdb5/mal/Tests/tst880.stable.err
monetdb5/mal/Tests/tst880.stable.out
sql/test/BugTracker-2015/Tests/inet-funcs.Bug-3645.sql
sql/test/BugTracker-2015/Tests/inet-funcs.Bug-3645.stable.err
sql/test/BugTracker-2015/Tests/inet-funcs.Bug-3645.stable.out
sql/test/BugTracker-2015/Tests/reserved_keywords.Bug-3613.sql
sql/test/BugTracker-2015/Tests/reserved_keywords.Bug-3613.stable.err
sql/test/BugTracker-2015/Tests/reserved_keywords.Bug-3613.stable.out
Removed Files:
monetdb5/extras/rdf/30_rdf.mal
monetdb5/extras/rdf/Makefile.ag
monetdb5/extras/rdf/rdf.h
monetdb5/extras/rdf/rdf_shredder.c
monetdb5/extras/rdf/rdfalgebra.c
monetdb5/extras/rdf/rdfalgebra.mal
monetdb5/mal/Tests/saveload.mal
monetdb5/mal/Tests/saveload.stable.err
monetdb5/mal/Tests/saveload.stable.out
monetdb5/mal/Tests/tst090.mal
monetdb5/mal/Tests/tst260.mal
monetdb5/mal/Tests/tst260.stable.err
monetdb5/mal/Tests/tst260.stable.out
monetdb5/mal/Tests/tst612.mal
monetdb5/mal/Tests/tst612.stable.err
monetdb5/mal/Tests/tst612.stable.out
monetdb5/mal/Tests/tst805.mal
monetdb5/mal/Tests/tst805.stable.err
monetdb5/mal/Tests/tst805.stable.out
monetdb5/mal/Tests/tst867.mal
monetdb5/mal/Tests/tst867.stable.err
monetdb5/mal/Tests/tst867.stable.out
monetdb5/mal/Tests/tst950.mal
monetdb5/modules/kernel/Tests/vacuum.mal
monetdb5/modules/kernel/Tests/vacuum.stable.err
monetdb5/modules/kernel/Tests/vacuum.stable.out
monetdb5/modules/kernel/counters.c
monetdb5/modules/kernel/counters.h
monetdb5/modules/kernel/counters.mal
monetdb5/modules/mal/Tests/cluster00.mal
monetdb5/modules/mal/Tests/cluster00.stable.err
monetdb5/modules/mal/Tests/cluster00.stable.out
monetdb5/modules/mal/trader.c
monetdb5/modules/mal/trader.h
monetdb5/modules/mal/trader.mal
monetdb5/optimizer/Tests/ESexample.mal
monetdb5/optimizer/Tests/ESexample.stable.err
monetdb5/optimizer/Tests/ESexample.stable.out
monetdb5/optimizer/Tests/SRexample.mal
monetdb5/optimizer/Tests/SRexample.stable.err
monetdb5/optimizer/Tests/SRexample.stable.out
monetdb5/optimizer/Tests/emptyset00.mal
monetdb5/optimizer/Tests/emptyset00.stable.err
monetdb5/optimizer/Tests/emptyset00.stable.out
monetdb5/optimizer/Tests/emptyset01.mal
monetdb5/optimizer/Tests/emptyset01.stable.err
monetdb5/optimizer/Tests/emptyset01.stable.out
monetdb5/optimizer/Tests/groups00.mal
monetdb5/optimizer/Tests/groups00.stable.err
monetdb5/optimizer/Tests/groups00.stable.out
monetdb5/optimizer/Tests/octopus00.mal
monetdb5/optimizer/Tests/octopus01.mal
monetdb5/optimizer/Tests/rangePush00.mal
monetdb5/optimizer/Tests/rangePush00.stable.err
monetdb5/optimizer/Tests/rangePush00.stable.out
monetdb5/optimizer/Tests/rangePush01.mal
monetdb5/optimizer/Tests/rangePush01.stable.err
monetdb5/optimizer/Tests/rangePush01.stable.out
monetdb5/optimizer/Tests/tst4820.mal
monetdb5/optimizer/Tests/tst4820.stable.err
monetdb5/optimizer/Tests/tst4820.stable.out
monetdb5/optimizer/opt_centipede.c
monetdb5/optimizer/opt_centipede.h
monetdb5/optimizer/opt_cluster.c
monetdb5/optimizer/opt_cluster.h
monetdb5/optimizer/opt_emptySet.c
monetdb5/optimizer/opt_emptySet.h
monetdb5/optimizer/opt_groups.c
monetdb5/optimizer/opt_groups.h
monetdb5/optimizer/opt_mapreduce.c
monetdb5/optimizer/opt_mapreduce.h
monetdb5/optimizer/opt_octopus.c
monetdb5/optimizer/opt_octopus.h
monetdb5/optimizer/opt_pushranges.c
monetdb5/optimizer/opt_pushranges.h
monetdb5/optimizer/opt_strengthReduction.c
monetdb5/optimizer/opt_strengthReduction.h
monetdb5/scheduler/run_octopus.c
monetdb5/scheduler/run_octopus.h
monetdb5/scheduler/run_octopus.mal

monetdb5/tests/BugDay_2005-10-06_4.9.3/Tests/return_types_of_head_and_tail.SF-701641.stable.out.int128
monetdb5/tests/BugReports/Tests/no.050.mal
monetdb5/tests/BugReports/Tests/no.050.stable.err
monetdb5/tests/BugReports/Tests/no.050.stable.out
monetdb5/tests/BugReports/Tests/no.051.mal
monetdb5/tests/BugReports/Tests/no.051.stable.err
monetdb5/tests/BugReports/Tests/no.051.stable.out
  

MonetDB: bamloader - Fixed test

2015-02-12 Thread Robin Cijvat
Changeset: d96643f0932f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d96643f0932f
Modified Files:
sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out
Branch: bamloader
Log Message:

Fixed test


diffs (12 lines):

diff --git a/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out 
b/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out
--- a/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out
+++ b/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out
@@ -84,7 +84,7 @@ function user.main():void;
 bat.append(is,17922987);
 res3:bat[:oid,:str]  := 
mal.manifold(bam,seq_char,is,sc_seqs,sc_poss,sc_cgrs);
 io.print(res3);
-end main;
+end user.main;
 [ false ]
 #--#
 # ht  # name
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Replace BBPreleaseref with BBPunfix

2015-02-12 Thread Robin Cijvat
Changeset: a81783156d70 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a81783156d70
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Replace BBPreleaseref with BBPunfix


diffs (81 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -312,7 +312,7 @@ bam_flag_bat(bat * ret, bat * bid, str *
/* allocate result BAT */
output = BATnew(TYPE_void, TYPE_bit, BATcount(input), TRANSIENT);
if (output == NULL) {
-   BBPreleaseref(input-batCacheid);
+   BBPunfix(input-batCacheid);
throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL);
}

@@ -328,7 +328,7 @@ bam_flag_bat(bat * ret, bat * bid, str *
finish_props();
 
/* release input BAT-descriptor */
-   BBPreleaseref(input-batCacheid);
+   BBPunfix(input-batCacheid);
 
BBPkeepref((*ret = output-batCacheid));
 
@@ -348,7 +348,7 @@ bam_flag_bat(bat * ret, bat * bid, str *
/* allocate result BAT */ \
output = BATnew(TYPE_void, TYPE_str, BATcount(input), TRANSIENT); \
if (output == NULL) { \
-   BBPreleaseref(input-batCacheid); \
+   BBPunfix(input-batCacheid); \
throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); \
} \
BATseqbase(output, input-hseqbase); \
@@ -360,8 +360,8 @@ bam_flag_bat(bat * ret, bat * bid, str *
str r, msg; \
  \
if ((msg = transform_fn(r, t)) != MAL_SUCCEED) { \
-   BBPreleaseref(input-batCacheid); \
-   BBPreleaseref(output-batCacheid); \
+   BBPunfix(input-batCacheid); \
+   BBPunfix(output-batCacheid); \
return msg; \
} \
BUNappend(output, (ptr) r, FALSE); \
@@ -369,7 +369,7 @@ bam_flag_bat(bat * ret, bat * bid, str *
} \
  \
/* release input BAT-descriptor */ \
-   BBPreleaseref(input-batCacheid); \
+   BBPunfix(input-batCacheid); \
  \
BBPkeepref((*ret = output-batCacheid)); \
  \
@@ -416,7 +416,7 @@ seq_length_bat(bat * ret, bat * bid)
BATloop(input, p, q) {
cur_in = (str) BUNtail(li, p);
if ((msg = seq_length(cur_out, cur_in)) != MAL_SUCCEED) {
-   BBPreleaseref(output-batCacheid);
+   BBPunfix(output-batCacheid);
return msg;
}
update_props(int);
@@ -426,7 +426,7 @@ seq_length_bat(bat * ret, bat * bid)
finish_props();
 
/* release input BAT-descriptor */
-   BBPreleaseref(input-batCacheid);
+   BBPunfix(input-batCacheid);
 
BBPkeepref((*ret = output-batCacheid));
 
@@ -493,9 +493,9 @@ seq_char_bat(bat * ret, int * ref_pos, b

 cleanup:
/* release input BAT-descriptors */
-   if(seqs) BBPreleaseref(seqs-batCacheid);
-   if(poss) BBPreleaseref(poss-batCacheid);
-   if(cigars) BBPreleaseref(cigars-batCacheid);
+   if(seqs) BBPunfix(seqs-batCacheid);
+   if(poss) BBPunfix(poss-batCacheid);
+   if(cigars) BBPunfix(cigars-batCacheid);
 
if(result) BBPkeepref((*ret = result-batCacheid));
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Updated readme and version number

2015-02-12 Thread Robin Cijvat
Changeset: 57cf0e1e0c2e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=57cf0e1e0c2e
Modified Files:
clients/nodejs/monetdb/README.md
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

Updated readme and version number


diffs (29 lines):

diff --git a/clients/nodejs/monetdb/README.md b/clients/nodejs/monetdb/README.md
--- a/clients/nodejs/monetdb/README.md
+++ b/clients/nodejs/monetdb/README.md
@@ -93,6 +93,14 @@ Empty the query queue and then close the
 
 
 
+# a name=log/aQuery logging
+Every time a query result is returned from the database, the callback function 
*conn.log_callback*, which defaults to null, will be executed right before the 
query callback function is executed (conn refers to a connection object 
returned by the [connect function](#connect)). If you have some general 
behavior that you want to be executed every time a query finishes, you can set 
*conn.log_callback* to a function with the following signature:
+
+function(query, err, res)
+- query: the SQL query that lead to this result (note: if you pass a params 
array to the [query function](#query), multiple queries are fired to the 
database, hence the log callback will be called multiple times)
+- err, res: Same as in the callback for the [query function](#query)
+
+
 # a name=q/aQ Integration
 Due to the huge popularity of the [Q module for 
NodeJS](https://www.npmjs.org/package/q), we decided to add native Q support, 
that wraps our API in a promise returning API that exists on top of the 
original API, so you can use both interchangeably. 
 
diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.2.2,
+  version: 0.2.3,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Extended nodejs client with an optional loggi...

2015-02-12 Thread Robin Cijvat
Changeset: 276f3da3ae88 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=276f3da3ae88
Modified Files:
clients/nodejs/Tests/nodetest.js
clients/nodejs/monetdb/mapiclient.js
Branch: default
Log Message:

Extended nodejs client with an optional logging callback


diffs (65 lines):

diff --git a/clients/nodejs/Tests/nodetest.js b/clients/nodejs/Tests/nodetest.js
--- a/clients/nodejs/Tests/nodetest.js
+++ b/clients/nodejs/Tests/nodetest.js
@@ -87,8 +87,20 @@ conn.query('SELECT id from tables where 
['connections', 0, false], function(err, res) {
assert.equal(null, err);
assert(res.rows  0);
+}); 
 
-}); 
+/* Try the log callback functionality */
+var nr_log_callbacks = 0;
+conn.log_callback = function(message, error, result) {
+   assert(message); // message must contain something
+   ++nr_log_callbacks;
+}
+conn.query('SELECT id FROM tables WHERE name=? AND type=? AND readonly=?',
+   ['connections', 0, false], function(err, res) {
+   assert.equal(null, err);
+   assert(nr_log_callbacks  0);
+   conn.log_callback = null;   
+});
 
 /* some quoting fun, jesus */
 conn.query(SELECT 'asdf','\', '\\\', '\', '\\'', function(err, 
res) {
diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -8,7 +8,7 @@ function MonetDBConnection(options, conn
this.read_leftover = 0;
this.read_final = false;
this.read_str = '';
-   this.read_callback = undefined;
+   this.cur_op = undefined; // object { message (str), callback (fn)}
this.conn_callback = conncallback;
this.mapi_blocksize = 8192;
this.do_close = false;
@@ -223,11 +223,14 @@ function handle_message(message) {
if (message.charAt(0) == '') {
response = _parseresponse(message);
}
-
-   if (this.read_callback != undefined) {
-   this.read_callback(error, response);
-   this.read_callback = undefined;
+   if(typeof(this.log_callback) == function) {
+   this.log_callback(this.cur_op.message, error, response);
}
+   if (this.cur_op.callback != undefined) {
+   this.cur_op.callback(error, response);
+   this.cur_op.callback = undefined;
+   }
+   this.cur_op.message = undefined;
next_op.call(this);
 }
 
@@ -243,7 +246,7 @@ function next_op() {
 
var op = this.queryqueue.shift();
send_message.call(this, op.message);
-   this.read_callback = op.callback;
+   this.cur_op = op;
 }
 
 function cleanup() {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Added minimal MAL test that demonstrates wh...

2014-11-28 Thread Robin Cijvat
Changeset: a589e25d2715 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a589e25d2715
Added Files:
sql/backends/monet5/bam/Tests/seq_char.mal
Branch: bamloader
Log Message:

Added minimal MAL test that demonstrates where coverage test case fails: 
misalignment in input BATs to manifold function


diffs (33 lines):

diff --git a/sql/backends/monet5/bam/Tests/seq_char.mal 
b/sql/backends/monet5/bam/Tests/seq_char.mal
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/seq_char.mal
@@ -0,0 +1,28 @@
+seqs := bat.new(:oid, :str);
+poss := bat.new(:oid, :int);
+cgrs := bat.new(:oid, :str);
+bat.append(seqs, 
ATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT);
+bat.append(seqs, 
TATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT);
+bat.append(seqs, 
GTACGTGGGAAAAAAGCTGGGCATCCAAGACATCACGTGTCAGTAGGTTCCGTGATGCCAAGCCGCCCAGCAAGTA);
+bat.append(poss, 17922987);
+bat.append(poss, 18225784);
+bat.append(poss, 18226024);
+bat.append(cgrs, 3=1X1=1X43=1X16=1X33=);
+bat.append(cgrs, 5=1X31=1X62=);
+bat.append(cgrs, 100=);
+io.print(===
 Input BATs:);
+io.print(seqs);
+io.print(poss);
+io.print(cgrs);
+i := 17922987;
+res1:bat[:oid,:str] := mal.manifold(bam, seq_char, i, seqs, poss, cgrs);
+io.print(===
 Manifold result1:);
+io.print(res1);
+is := poss;
+res2:bat[:oid,:str] := mal.manifold(bam, seq_char, is, seqs, poss, cgrs);
+io.print(===
 Manifold result2:);
+io.print(res2);
+is := bat.new(:oid, :int);
+bat.append(is, 17922987);
+res3:bat[:oid,:str] := mal.manifold(bam, seq_char, is, seqs, poss, cgrs);
+io.print(res3);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Improve seq_length_bat by direct appending ...

2014-11-28 Thread Robin Cijvat
Changeset: 15361647fecc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=15361647fecc
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Improve seq_length_bat by direct appending instead of usign BUNappend and make 
code of reverse_seq_bat and reverse_qual_bat more generic


diffs (296 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -283,15 +283,6 @@ seq_char(str * ret, int * ref_pos, str *
output-T-nonil = output-T-nonil  *cur_out != TPE##_nil; \
 }
 
-#define update_props_str() { \
-   output-tsorted = output-tsorted  \
-   (c == 0 || strcmp(*cur_out, prev_out) = 0); \
-   output-trevsorted = output-trevsorted  \
-   (c == 0 || strcmp(*cur_out, prev_out) = 0); \
-   output-T-nil = output-T-nil || *cur_out == str_nil; \
-   output-T-nonil = output-T-nonil  *cur_out != str_nil; \
-}
-
 #define finish_props() { \
BATsetcount(output, BATcount(input)); \
BATseqbase(output, input-hseqbase); \
@@ -344,221 +335,102 @@ bam_flag_bat(bat * ret, bat * bid, str *
return MAL_SUCCEED;
 }
 
-/* Commented out, since string heap addition needs to be added
-str
-reverse_seq_bat(bat * ret, bat * bid)
-{
-   BAT *input, *output;
-   str prev_out = 0;
-   str *cur_in;
-   str *cur_out;
-   BUN c;
-   str msg;
-
-   assert(ret != NULL  bid != NULL);
-
-   if ((input = BATdescriptor(*bid)) == NULL)
-   throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING);
-*/
-   /* allocate result BAT */
-/* output = BATnew(TYPE_void, TYPE_str, BATcount(input), TRANSIENT);
-   if (output == NULL) {
-   throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL);
-   }
-
-   init_props();
-   cur_in = (str *) Tloc(input, BUNfirst(input));
-   cur_out = (str *) Tloc(output, BUNfirst(output));
-   for(c = 0; c  BATcount(input); ++c) {
-   if ((msg = reverse_seq(cur_out, cur_in)) != MAL_SUCCEED) {
-   BBPreleaseref(output-batCacheid);
-   return msg;
-   }
-   update_props_str();
-   cur_in++;
-   prev_out = *cur_out++;
-   }
-   finish_props();
-*/
-   /* release input BAT-descriptor */
-/* BBPreleaseref(input-batCacheid);
-
-   BBPkeepref((*ret = output-batCacheid));
-
-   return MAL_SUCCEED;
-}*/
+#define transform_strbat(transform_fn) { \
+   BAT *input, *output; \
+   BATiter li; \
+   BUN p = 0, q = 0; \
+ \
+   assert(ret != NULL  bid != NULL); \
+ \
+   if ((input = BATdescriptor(*bid)) == NULL) \
+   throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING); \
+ \
+   /* allocate result BAT */ \
+   output = BATnew(TYPE_void, TYPE_str, BATcount(input), TRANSIENT); \
+   if (output == NULL) { \
+   BBPreleaseref(input-batCacheid); \
+   throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL); \
+   } \
+   BATseqbase(output, input-hseqbase); \
+ \
+   li = bat_iterator(input); \
+ \
+   BATloop(input, p, q) { \
+   str t = (str) BUNtail(li, p); \
+   str r, msg; \
+ \
+   if ((msg = transform_fn(r, t)) != MAL_SUCCEED) { \
+   BBPreleaseref(input-batCacheid); \
+   BBPreleaseref(output-batCacheid); \
+   return msg; \
+   } \
+   BUNappend(output, (ptr) r, FALSE); \
+   GDKfree(r); \
+   } \
+ \
+   /* release input BAT-descriptor */ \
+   BBPreleaseref(input-batCacheid); \
+ \
+   BBPkeepref((*ret = output-batCacheid)); \
+ \
+   return MAL_SUCCEED; \
+}
 
 str
 reverse_seq_bat(bat * ret, bat * bid)
 {
-   BAT *seqs, *result;
-   BATiter li;
-   BUN p = 0, q = 0;
-
-   assert(ret != NULL  bid != NULL);
-
-   if ((seqs = BATdescriptor(*bid)) == NULL)
-   throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING);
-
-   /* allocate result BAT */
-   result = BATnew(TYPE_void, TYPE_str, BATcount(seqs), TRANSIENT);
-   if (result == NULL) {
-   BBPreleaseref(seqs-batCacheid);
-   throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL);
-   }
-   BATseqbase(result, seqs-hseqbase);
-
-   li = bat_iterator(seqs);
-
-   BATloop(seqs, p, q) {
-   str t = (str) BUNtail(li, p);
-   str r, msg;
-
-   if ((msg = reverse_seq(r, t)) != MAL_SUCCEED) {
-   BBPreleaseref(seqs-batCacheid);
-   BBPreleaseref(result-batCacheid);
-   return msg;
-   }
-   BUNappend(result, (ptr) r, FALSE);
-   GDKfree(r);
-   }
-
-   /* release input BAT-descriptor */
-   BBPreleaseref(seqs-batCacheid);
-
-  

MonetDB: bamloader - Added some MAL tests for testing the UDFs

2014-11-28 Thread Robin Cijvat
Changeset: 1e897d944e34 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1e897d944e34
Added Files:
sql/backends/monet5/bam/Tests/bam_lib_mal.mal
sql/backends/monet5/bam/Tests/bam_lib_mal.stable.err
sql/backends/monet5/bam/Tests/bam_lib_mal.stable.out
Removed Files:
sql/backends/monet5/bam/Tests/seq_char.mal
Modified Files:
sql/backends/monet5/bam/Tests/All
sql/backends/monet5/bam/Tests/coverage_usecase.sql
Branch: bamloader
Log Message:

Added some MAL tests for testing the UDFs


diffs (truncated from 360 to 300 lines):

diff --git a/sql/backends/monet5/bam/Tests/All 
b/sql/backends/monet5/bam/Tests/All
--- a/sql/backends/monet5/bam/Tests/All
+++ b/sql/backends/monet5/bam/Tests/All
@@ -25,5 +25,6 @@ HAVE_SAMTOOLS?query2.10
 HAVE_SAMTOOLS?query2.11
 HAVE_SAMTOOLS?query2.12
 HAVE_SAMTOOLS?mergetable
+HAVE_SAMTOOLS?bam_lib_mal
 HAVE_SAMTOOLS?bam_lib
 #HAVE_SAMTOOLS?coverage_usecase
diff --git a/sql/backends/monet5/bam/Tests/bam_lib_mal.mal 
b/sql/backends/monet5/bam/Tests/bam_lib_mal.mal
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/bam_lib_mal.mal
@@ -0,0 +1,72 @@
+sht1 := calc.sht(200);
+sht2 := calc.sht(0);
+sht3 := calc.sht(100);
+f := bam.bam_flag(sht1, segm_unma);
+io.print(f);
+fs := bat.new(:oid, :sht);
+bat.append(fs, sht1);
+bat.append(fs, sht2);
+bat.append(fs, sht3);
+fsb := batbam.bam_flag(fs, segm_unma);
+io.print(fsb);
+
+seq_rev := bam.reverse_seq(ACGTCGTA);
+io.print(seq_rev);
+seqs := bat.new(:oid, :str);
+bat.append(seqs, seq_rev);
+bat.append(seqs, TVGHCDMKNYSABWR);
+bat.append(seqs, 
AAA);
+io.print(seqs);
+seqs_rev := batbam.reverse_seq(seqs);
+io.print(seqs_rev);
+seqs_revrev := batbam.reverse_seq(seqs_rev);
+io.print(seqs_revrev);
+
+qual_rev := bam.reverse_qual(dJwn!@dJ!#jjjnsl);
+io.print(qual_rev);
+quals := bat.new(:oid, :str);
+bat.append(quals, qual_rev);
+bat.append(quals, dJwn!@dJ!#jjjnsl);
+bat.append(quals, I_am_a_quality_string_without_a_purpose);
+io.print(quals);
+quals_rev := batbam.reverse_qual(quals);
+io.print(quals_rev);
+quals_revrev := batbam.reverse_qual(quals_rev);
+io.print(quals_revrev);
+
+l := bam.seq_length(100M3I200D5M);
+io.print(l);
+cigars := bat.new(:oid, :str);
+bat.append(cigars, 100M3I200D5M);
+bat.append(cigars, *);
+ls := batbam.seq_length(cigars);
+io.print(ls);
+
+
+seq_char := bam.seq_char(12, ACTAGAATGGCT, 5, 3M1I3M1D5M);
+io.print(seq_char);
+sc_seqs := bat.new(:oid, :str);
+sc_poss := bat.new(:oid, :int);
+sc_cgrs := bat.new(:oid, :str);
+bat.append(sc_seqs, 
ATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT);
+bat.append(sc_seqs, 
TATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT);
+bat.append(sc_seqs, 
GTACGTGGGAAAAAAGCTGGGCATCCAAGACATCACGTGTCAGTAGGTTCCGTGATGCCAAGCCGCCCAGCAAGTA);
+bat.append(sc_poss, 17922987);
+bat.append(sc_poss, 18225784);
+bat.append(sc_poss, 18226024);
+bat.append(sc_cgrs, 3=1X1=1X43=1X16=1X33=);
+bat.append(sc_cgrs, 5=1X31=1X62=);
+bat.append(sc_cgrs, 100=);
+io.print(sc_seqs);
+io.print(sc_poss);
+io.print(sc_cgrs);
+i := 17922987;
+res1:bat[:oid,:str] := mal.manifold(bam, seq_char, i, sc_seqs, sc_poss, 
sc_cgrs);
+io.print(res1);
+is := sc_poss;
+res2:bat[:oid,:str] := mal.manifold(bam, seq_char, is, sc_seqs, sc_poss, 
sc_cgrs);
+io.print(res2);
+is := bat.new(:oid, :int);
+bat.append(is, 17922987);
+res3:bat[:oid,:str] := mal.manifold(bam, seq_char, is, sc_seqs, sc_poss, 
sc_cgrs);
+io.print(res3);
diff --git a/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.err 
b/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/bam_lib_mal.stable.err
@@ -0,0 +1,29 @@
+stderr of test 'bam_lib_mal` in directory 'sql/backends/monet5/bam` itself:
+
+
+# 16:51:57   
+# 16:51:57   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=39916 --set 
mapi_usock=/var/tmp/mtest-16560/.s.monetdb.39916 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 bam_lib_mal.mal
+# 16:51:57   
+
+# builtin opt  gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = 
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 39916
+# cmdline opt  mapi_usock = 

MonetDB: bamloader - Merge with default

2014-11-28 Thread Robin Cijvat
Changeset: 9654ce09f3a2 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9654ce09f3a2
Modified Files:
sql/test/pg_regress/Tests/alter_table.stable.err
sql/test/pg_regress/Tests/arrays.stable.err
sql/test/pg_regress/Tests/boolean.stable.err
sql/test/pg_regress/Tests/boolean.stable.out
sql/test/pg_regress/Tests/case.stable.err
sql/test/pg_regress/Tests/case.stable.out
sql/test/pg_regress/Tests/char.stable.err
sql/test/pg_regress/Tests/comments.stable.err
sql/test/pg_regress/Tests/create_aggregate.stable.err
sql/test/pg_regress/Tests/create_misc.stable.err
sql/test/pg_regress/Tests/create_operator.stable.err
sql/test/pg_regress/Tests/date.stable.err
sql/test/pg_regress/Tests/date.stable.out
sql/test/pg_regress/Tests/drop.stable.err
sql/test/pg_regress/Tests/float4.stable.err
sql/test/pg_regress/Tests/float8.stable.err
sql/test/pg_regress/Tests/inet.stable.err
sql/test/pg_regress/Tests/insert.stable.err
sql/test/pg_regress/Tests/int2.stable.err
sql/test/pg_regress/Tests/int4.stable.err
sql/test/pg_regress/Tests/int8.stable.err
sql/test/pg_regress/Tests/int8.stable.out
sql/test/pg_regress/Tests/interval.stable.err
sql/test/pg_regress/Tests/limit.stable.err
sql/test/pg_regress/Tests/numeric.stable.err
sql/test/pg_regress/Tests/numeric_big.stable.err
sql/test/pg_regress/Tests/numerology.stable.err
sql/test/pg_regress/Tests/oid.stable.err
sql/test/pg_regress/Tests/select_into.stable.err
sql/test/pg_regress/Tests/strings.stable.err
sql/test/pg_regress/Tests/strings_cast.stable.err
sql/test/pg_regress/Tests/strings_concat.stable.err
sql/test/pg_regress/Tests/timestamp.stable.err
sql/test/pg_regress/Tests/timestamptz.stable.err
sql/test/pg_regress/Tests/timetz.stable.out
sql/test/pg_regress/Tests/varchar.stable.err
sql/test/pg_regress/Tests/without_oid.stable.err
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 6431 to 300 lines):

diff --git a/sql/test/pg_regress/Tests/alter_table.stable.err 
b/sql/test/pg_regress/Tests/alter_table.stable.err
--- a/sql/test/pg_regress/Tests/alter_table.stable.err
+++ b/sql/test/pg_regress/Tests/alter_table.stable.err
@@ -26,613 +26,613 @@ stderr of test 'alter_table` in director
 # cmdline opt  gdk_debug = 536870922
 
 
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = ALTER TABLE tmp RENAME TO tmp_new;
 ERROR = !syntax error, unexpected RENAME, expecting DROP or SET or ALTER or 
ADD in: alter table tmp rename
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = SELECT * FROM tmp_new;
 ERROR = !SELECT: no such table 'tmp_new'
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = ALTER TABLE tmp RENAME TO tmp_new2;
 ERROR = !syntax error, unexpected RENAME, expecting DROP or SET or ALTER or 
ADD in: alter table tmp rename
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = SELECT * FROM tmp_new;
 ERROR = !SELECT: no such table 'tmp_new'
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = SELECT * FROM tmp_new2;
 ERROR = !SELECT: no such table 'tmp_new2'
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = DROP TABLE tmp_new;
 ERROR = !DROP TABLE: no such table 'tmp_new'
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = DROP TABLE tmp_new2;
 ERROR = !DROP TABLE: no such table 'tmp_new2'
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = ALTER INDEX onek_unique1 RENAME TO tmp_onek_unique1;
 ERROR = !syntax error, unexpected INDEX, expecting USER or SEQUENCE or TABLE 
in: alter index
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = ALTER INDEX tmp_onek_unique1 RENAME TO onek_unique1;
 ERROR = !syntax error, unexpected INDEX, expecting USER or SEQUENCE or TABLE 
in: alter index
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = ALTER TABLE tmp_view RENAME TO tmp_view_new;
 ERROR = !syntax error, unexpected RENAME, expecting DROP or SET or ALTER or 
ADD in: alter table tmp_view rename
-MAPI  = (monetdb) /var/tmp/mtest-5635/.s.monetdb.38440
+MAPI  = (monetdb) /var/tmp/mtest-32157/.s.monetdb.35812
 QUERY = DROP VIEW tmp_view_new;
 ERROR = !DROP VIEW: unknown 

MonetDB: bamloader - Merge with default

2014-11-27 Thread Robin Cijvat
Changeset: 13b37e298c46 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=13b37e298c46
Modified Files:
sql/backends/monet5/generator/generator.c
Branch: bamloader
Log Message:

Merge with default


diffs (89 lines):

diff --git a/sql/backends/monet5/generator/generator.c 
b/sql/backends/monet5/generator/generator.c
--- a/sql/backends/monet5/generator/generator.c
+++ b/sql/backends/monet5/generator/generator.c
@@ -30,6 +30,8 @@
 #include math.h
 
 
+#define IDENTITY(x)(x)
+
 /*
  * The noop simply means that we keep the properties for the generator object.
  */
@@ -986,10 +988,10 @@ str VLTgenerator_join(Client cntxt, MalB
 }
 
 /* The operands of a join operation can either be defined on a generator */
-#define VLTrangejoin(TPE, ABS) \
+#define VLTrangejoin(TPE, ABS, FLOOR) \
 { TPE f,f1,l,s; TPE *vlow,*vhgh; BUN w;\
-   f = *getArgReference_bte(stk,p, 1);\
-   l = *getArgReference_bte(stk,p, 2);\
+   f = *getArgReference_##TPE(stk,p, 1);\
+   l = *getArgReference_##TPE(stk,p, 2);\
if ( p-argc == 3) \
s = fl? (TPE) 1: (TPE)-1;\
else s = * getArgReference_##TPE(stk, p, 3); \
@@ -999,9 +1001,9 @@ str VLTgenerator_join(Client cntxt, MalB
vlow = (TPE*) Tloc(blow,BUNfirst(blow));\
vhgh = (TPE*) Tloc(bhgh,BUNfirst(bhgh));\
for( ; cnt 0; cnt--, done++, o++,vlow++,vhgh++){\
-   f1 = f + floor(ABS(*vlow-f)/ABS(s)) * s;\
+   f1 = f + FLOOR(ABS(*vlow-f)/ABS(s)) * s;\
if ( f1  *vlow ) f1+= s;\
-   w = (BUN) floor(ABS(f1-f)/ABS(s));\
+   w = (BUN) FLOOR(ABS(f1-f)/ABS(s));\
for( ; (f1  *vlow || (li  f1 == *vlow))  (f1  *vhgh || 
(ri  f1 == *vhgh)); f1 += s, w++){\
if(c == limit)\
VLTrangeExpand();\
@@ -1058,43 +1060,15 @@ str VLTgenerator_rangejoin(Client cntxt,
 
/* The actual join code for generators be injected here */
switch(tpe){
-   case TYPE_bte: VLTrangejoin(bte,abs); break; 
-   case TYPE_sht: VLTrangejoin(sht,abs); break;
-   case TYPE_int: VLTrangejoin(int,abs); break;
-   case TYPE_lng: //VLTrangejoin(lng,llabs); break;
-   { lng f,f1,l,s; lng *vlow,*vhgh; BUN w;
-   f = *getArgReference_lng(stk,p, 1);
-   l = *getArgReference_lng(stk,p, 2);
-   if ( p-argc == 3)
-   s = fl? (lng) 1: (lng)-1;
-   else s = * getArgReference_sht(stk, p, 3); 
-   incr = s  0;
-
-   if ( s == 0 || (f l  s0) || (fl  s  0))
-   throw(MAL,generator.rangejoin,Illegal range);
-
-   vlow = (lng*) Tloc(blow,BUNfirst(blow));
-   vhgh = (lng*) Tloc(bhgh,BUNfirst(bhgh));
-   for( ; cnt 0; done++,cnt--, o++,vlow++,vhgh++){
-   f1 = f + floor(abs(*vlow-f)/abs(s)) * s;
-   if ( f1  *vlow ) f1+= s;
-   w = (BUN) floor(abs(f1-f)/abs(s));
-   for( ; (f1  *vlow || (li  f1 == *vlow))  (f1  *vhgh || 
(ri  f1 == *vhgh)); f1 += s, w++){
-   if(c == limit){
-   limit += 0;
-   VLTrangeExpand();
-   }
-   *ol++ = (oid) w;
-   *or++ = o;
-   c++;
-   }
-   } }
-   break;
+   case TYPE_bte: VLTrangejoin(bte,abs,IDENTITY); break;
+   case TYPE_sht: VLTrangejoin(sht,abs,IDENTITY); break;
+   case TYPE_int: VLTrangejoin(int,abs,IDENTITY); break;
+   case TYPE_lng: VLTrangejoin(lng,llabs,IDENTITY); break;
 #ifdef HAVE_HGE
-   case TYPE_hge: VLTrangejoin(hge,HGE_ABS); break;
+   case TYPE_hge: VLTrangejoin(hge,HGE_ABS,IDENTITY); break;
 #endif
-   case TYPE_flt: VLTrangejoin(flt,fabsf); break;
-   case TYPE_dbl: VLTrangejoin(dbl,fabs); break;
+   case TYPE_flt: VLTrangejoin(flt,fabsf,floorf); break;
+   case TYPE_dbl: VLTrangejoin(dbl,fabs,floor); break;
default:
if( tpe == TYPE_timestamp){ 
// it is easier to produce the timestamp series
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Hard code number of threads instead of lett...

2014-11-27 Thread Robin Cijvat
Changeset: 9ad8b922c870 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9ad8b922c870
Modified Files:
sql/backends/monet5/bam/85_bam.sql
sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql
sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql
sql/backends/monet5/bam/bam.mal
sql/backends/monet5/bam/bam_loader.c
Branch: bamloader
Log Message:

Hard code number of threads instead of letting the user decide this on SQL level


diffs (81 lines):

diff --git a/sql/backends/monet5/bam/85_bam.sql 
b/sql/backends/monet5/bam/85_bam.sql
--- a/sql/backends/monet5/bam/85_bam.sql
+++ b/sql/backends/monet5/bam/85_bam.sql
@@ -1,9 +1,9 @@
 CREATE SCHEMA bam;
 
-CREATE PROCEDURE bam.bam_loader_repos(bam_repos STRING, dbschema SMALLINT, 
nr_threads SMALLINT)
+CREATE PROCEDURE bam.bam_loader_repos(bam_repos STRING, dbschema SMALLINT)
 EXTERNAL NAME bam.bam_loader_repos;
 
-CREATE PROCEDURE bam.bam_loader_files(bam_files STRING, dbschema SMALLINT, 
nr_threads SMALLINT)
+CREATE PROCEDURE bam.bam_loader_files(bam_files STRING, dbschema SMALLINT)
 EXTERNAL NAME bam.bam_loader_files;
 
 CREATE PROCEDURE bam.bam_loader_file(bam_file STRING, dbschema SMALLINT)
diff --git a/sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql 
b/sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql
--- a/sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql
+++ b/sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql
@@ -1,3 +1,3 @@
-CALL bam.bam_loader_files('PWD/files.lst', 0, 4);
-CALL bam.bam_loader_files('PWD/files_qname.lst', 1, 4);
-CALL bam.bam_loader_files('PWD/files_many.lst', 0, 4);
+CALL bam.bam_loader_files('PWD/files.lst', 0);
+CALL bam.bam_loader_files('PWD/files_qname.lst', 1);
+CALL bam.bam_loader_files('PWD/files_many.lst', 0);
diff --git a/sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql 
b/sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql
--- a/sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql
+++ b/sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql
@@ -1,3 +1,3 @@
 # Load files
-CALL bam.bam_loader_repos('PWD/files', 0, 4);
-CALL bam.bam_loader_repos('PWD/files/queryname', 1, 4);
\ No newline at end of file
+CALL bam.bam_loader_repos('PWD/files', 0);
+CALL bam.bam_loader_repos('PWD/files/queryname', 1);
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/bam.mal b/sql/backends/monet5/bam/bam.mal
--- a/sql/backends/monet5/bam/bam.mal
+++ b/sql/backends/monet5/bam/bam.mal
@@ -3,11 +3,11 @@ module bam;
 
 # Bam loader related signatures
 
-pattern bam_loader_repos(bam_repos:str, dbschema:sht, nr_threads:sht):void
+pattern bam_loader_repos(bam_repos:str, dbschema:sht):void
 address bam_loader_repos
 comment Read all bam files in the given bam_repos directory (non-recursive) 
and store them in the given dbschema;
 
-pattern bam_loader_files(bam_files:str, dbschema:sht, nr_threads:sht):void
+pattern bam_loader_files(bam_files:str, dbschema:sht):void
 address bam_loader_files
 comment Read all bam files in the file list stored in the file bam_files 
(separated by a newline) and store them in the given dbschema;
 
diff --git a/sql/backends/monet5/bam/bam_loader.c 
b/sql/backends/monet5/bam/bam_loader.c
--- a/sql/backends/monet5/bam/bam_loader.c
+++ b/sql/backends/monet5/bam/bam_loader.c
@@ -419,8 +419,10 @@ bam_loader_repos(Client cntxt, MalBlkPtr
str bam_repos = *getArgReference_str(stk, pci, pci-retc);
/* arg 2: dbschema to use */
sht dbschema = *getArgReference_sht(stk, pci, pci-retc + 1);
-   /* arg 3: max number of threads that will be used by bam_loader */
-   sht nr_threads = *getArgReference_sht(stk, pci, pci-retc + 2);
+   /* For now, we hard code the number of threads, since the SQL
+  level should not bother with this */
+
+   sht nr_threads = 4;
 
str *filenames = NULL;
int nr_files = 0;
@@ -539,8 +541,9 @@ bam_loader_files(Client cntxt, MalBlkPtr
str bam_files = *getArgReference_str(stk, pci, pci-retc);
/* arg 2: dbschema to use */
sht dbschema = *getArgReference_sht(stk, pci, pci-retc + 1);
-   /* arg 3: max number of threads that will be used by bam_loader */
-   sht nr_threads = *getArgReference_sht(stk, pci, pci-retc + 2);
+   /* For now, we hard code the number of threads, since the SQL
+  level should not bother with this */
+   sht nr_threads = 4;
 
str *filenames = NULL;
int nr_files = 0;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Replace sscanf with more efficient parsing ...

2014-11-27 Thread Robin Cijvat
Changeset: c38437e1b729 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c38437e1b729
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Replace sscanf with more efficient parsing in seq_length


diffs (52 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -183,31 +183,34 @@ reverse_qual(str * ret, str * qual)
return MAL_SUCCEED;
 }
 
+#define next_cigar_op(fn) { \
+   str tmp; \
+   cnt = strtol(s, tmp, 10); \
+   if(cnt = 0 || s == tmp) { \
+   *ret = -1; \
+   throw(MAL, fn, Could not read CIGAR operation); \
+   } \
+   s = tmp; \
+   op = *s++; \
+}
+
 str
 seq_length(int * ret, str * cigar)
 {
int result = 0;
-   str cigar_consumable = *cigar;
+   str s = *cigar;
+   long int cnt;
+   char op;
 
-   if (cigar_consumable[0] == '\0' || 
-   (cigar_consumable[0] == '*'  cigar_consumable[1] == 
'\0')) {
+   if (*s == '\0' || (*s == '*'  *(s+1) == '\0')) {
*ret = -1;
return MAL_SUCCEED;
}
-   while (cigar_consumable[0] != '\0') {
-   int cnt;
-   char op;
-   int nr_chars_read;
-
-   if (sscanf
-   (cigar_consumable, %d%c%n, cnt, op,
-nr_chars_read) != 2)
-   throw(MAL, seq_length,
- Error parsing CIGAR string '%s'\n, *cigar);
+   while (*s != '\0') {
+   next_cigar_op(seq_length);
if (op == 'M' || op == 'D' || op == 'N' || op == '='
|| op == 'X')
result += cnt;
-   cigar_consumable += nr_chars_read;
}
*ret = result;
return MAL_SUCCEED;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Let seq_char also use the more efficient ve...

2014-11-27 Thread Robin Cijvat
Changeset: c5fa74ef8100 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c5fa74ef8100
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Let seq_char also use the more efficient version of the CIGAR parser


diffs (64 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -186,9 +186,8 @@ reverse_qual(str * ret, str * qual)
 #define next_cigar_op(fn) { \
str tmp; \
cnt = strtol(s, tmp, 10); \
-   if(cnt = 0 || s == tmp) { \
-   *ret = -1; \
-   throw(MAL, fn, Could not read CIGAR operation); \
+   if(cnt = 0 || s == tmp || *s == '\0') { \
+   throw(MAL, fn, Could not parse CIGAR string); \
} \
s = tmp; \
op = *s++; \
@@ -219,30 +218,26 @@ seq_length(int * ret, str * cigar)
 str
 seq_char(str * ret, int * ref_pos, str * alg_seq, int * alg_pos, str * 
alg_cigar) 
 {
-   str cigar_consumable = *alg_cigar;
+   str s = *alg_cigar;
int seq_pos = -1;
int cur_ref_pos = *alg_pos - 1;
+
+   long int cnt;
+   char op;

-   if (cigar_consumable[0] == '\0' || 
-   (cigar_consumable[0] == '*'  cigar_consumable[1] == 
'\0')) {
+   if (*s == '\0' || (*s == '*'  *(s+1) == '\0')) {
*ret = GDKstrdup(str_nil);
return MAL_SUCCEED;
}
while(TRUE) {
-   int cnt;
-   char op;
-   int nr_chars_read;
bit advance_ref_pos;
bit advance_seq_pos;
 
-   if (sscanf
-   (cigar_consumable, %d%c%n, cnt, op,
-nr_chars_read) != 2)
-   throw(MAL, seq_char,
- Error parsing CIGAR string '%s'\n, 
*alg_cigar);
+   next_cigar_op(seq_char);
+   
advance_ref_pos = (op == 'M' || op == 'D' || 
op == 'N' || op == '=' || op == 'X');
-   advance_seq_pos = (op == 'M' || op == 'I' || op == '='); // 
TODO: Find out which chars advance the seq pos
+   advance_seq_pos = (op == 'M' || op == 'I' || op == '='); // 
TODO: Find out which chars advance the seq pos exactly
if(advance_seq_pos) {
seq_pos += cnt;
}
@@ -257,8 +252,7 @@ seq_char(str * ret, int * ref_pos, str *
break;
}
}
-   cigar_consumable += nr_chars_read;
-   if(cigar_consumable[0] == '\0') {
+   if(*s == '\0') {
seq_pos = -1;
break;
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Use GDKnr_threads within bam loader

2014-11-27 Thread Robin Cijvat
Changeset: 72bd0543ee79 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=72bd0543ee79
Modified Files:
sql/backends/monet5/bam/bam_loader.c
Branch: bamloader
Log Message:

Use GDKnr_threads within bam loader


diffs (21 lines):

diff --git a/sql/backends/monet5/bam/bam_loader.c 
b/sql/backends/monet5/bam/bam_loader.c
--- a/sql/backends/monet5/bam/bam_loader.c
+++ b/sql/backends/monet5/bam/bam_loader.c
@@ -422,7 +422,7 @@ bam_loader_repos(Client cntxt, MalBlkPtr
/* For now, we hard code the number of threads, since the SQL
   level should not bother with this */
 
-   sht nr_threads = 4;
+   sht nr_threads = GDKnr_threads;
 
str *filenames = NULL;
int nr_files = 0;
@@ -543,7 +543,7 @@ bam_loader_files(Client cntxt, MalBlkPtr
sht dbschema = *getArgReference_sht(stk, pci, pci-retc + 1);
/* For now, we hard code the number of threads, since the SQL
   level should not bother with this */
-   sht nr_threads = 4;
+   sht nr_threads = GDKnr_threads;
 
str *filenames = NULL;
int nr_files = 0;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Simplified and improved some tests, added c...

2014-11-26 Thread Robin Cijvat
Changeset: 3f7305d007e1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3f7305d007e1
Added Files:
sql/backends/monet5/bam/Tests/bam_lib.reqtests
sql/backends/monet5/bam/Tests/bam_lib.sql
sql/backends/monet5/bam/Tests/coverage_usecase.reqtests
sql/backends/monet5/bam/Tests/coverage_usecase.sql
sql/backends/monet5/bam/Tests/mergetable.reqtests
sql/backends/monet5/bam/Tests/mergetable.sql
Removed Files:
sql/backends/monet5/bam/Tests/bam_lib.SQL.py
sql/backends/monet5/bam/Tests/mergetable.SQL.py
sql/backends/monet5/bam/Tests/sql/bam_lib.sql
sql/backends/monet5/bam/Tests/sql/mergetable.sql
Modified Files:
sql/backends/monet5/bam/Tests/All
Branch: bamloader
Log Message:

Simplified and improved some tests, added coverage test based on BTW-paper 
usecase; however does not work yet


diffs (truncated from 443 to 300 lines):

diff --git a/sql/backends/monet5/bam/Tests/All 
b/sql/backends/monet5/bam/Tests/All
--- a/sql/backends/monet5/bam/Tests/All
+++ b/sql/backends/monet5/bam/Tests/All
@@ -26,3 +26,4 @@ HAVE_SAMTOOLS?query2.11
 HAVE_SAMTOOLS?query2.12
 HAVE_SAMTOOLS?mergetable
 HAVE_SAMTOOLS?bam_lib
+#HAVE_SAMTOOLS?coverage_usecase
diff --git a/sql/backends/monet5/bam/Tests/bam_lib.SQL.py 
b/sql/backends/monet5/bam/Tests/bam_lib.SQL.py
deleted file mode 100644
--- a/sql/backends/monet5/bam/Tests/bam_lib.SQL.py
+++ /dev/null
@@ -1,3 +0,0 @@
-import bam
-
-bam.exec_sql_file(bam_lib.sql, {'PWD': bam.SRCDIR})
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/bam_lib.reqtests 
b/sql/backends/monet5/bam/Tests/bam_lib.reqtests
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/bam_lib.reqtests
@@ -0,0 +1,1 @@
+bam_loader_file
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/bam_lib.sql 
b/sql/backends/monet5/bam/Tests/bam_lib.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/bam_lib.sql
@@ -0,0 +1,133 @@
+SET SCHEMA bam;
+
+# BAM_FLAG
+
+# Should all give 'true'
+SELECT bam_flag(1, 'mult_segm');
+SELECT bam_flag(2, 'prop_alig');
+SELECT bam_flag(4, 'segm_unma');
+SELECT bam_flag(8, 'next_unma');
+SELECT bam_flag(16, 'segm_reve');
+SELECT bam_flag(32, 'next_reve');
+SELECT bam_flag(64, 'firs_segm');
+SELECT bam_flag(128, 'last_segm');
+SELECT bam_flag(256, 'seco_alig');
+SELECT bam_flag(512, 'qual_cont');
+SELECT bam_flag(1024, 'opti_dupl');
+SELECT bam_flag(2048, 'supp_alig');
+
+# Should all give 'false'
+SELECT bam_flag(2, 'mult_segm');
+SELECT bam_flag(4, 'prop_alig');
+SELECT bam_flag(8, 'segm_unma');
+SELECT bam_flag(16, 'next_unma');
+SELECT bam_flag(4, 'segm_reve');
+SELECT bam_flag(2, 'next_reve');
+SELECT bam_flag(4, 'firs_segm');
+SELECT bam_flag(8, 'last_segm');
+SELECT bam_flag(16, 'seco_alig');
+SELECT bam_flag(32, 'qual_cont');
+SELECT bam_flag(64, 'opti_dupl');
+SELECT bam_flag(128, 'supp_alig');
+
+# Should all give 'true'
+SELECT bam_flag(2047, 'mult_segm');
+SELECT bam_flag(2047, 'prop_alig');
+SELECT bam_flag(2047, 'segm_unma');
+SELECT bam_flag(2047, 'next_unma');
+SELECT bam_flag(2047, 'segm_reve');
+SELECT bam_flag(2047, 'next_reve');
+SELECT bam_flag(2047, 'firs_segm');
+SELECT bam_flag(2047, 'last_segm');
+SELECT bam_flag(2047, 'seco_alig');
+SELECT bam_flag(2047, 'qual_cont');
+SELECT bam_flag(2047, 'opti_dupl');
+SELECT bam_flag(4095, 'supp_alig');
+
+# Should fail
+SELECT bam_flag(111, 'Fail-hard');
+
+# Bulk
+SELECT flag,
+   bam_flag(flag, 'mult_segm') AS mult_segm,
+   bam_flag(flag, 'prop_alig') AS prop_alig,
+   bam_flag(flag, 'segm_unma') AS segm_unma,
+   bam_flag(flag, 'next_unma') AS next_unma,
+   bam_flag(flag, 'segm_reve') AS segm_reve,
+   bam_flag(flag, 'next_reve') AS next_reve,
+   bam_flag(flag, 'firs_segm') AS firs_segm,
+   bam_flag(flag, 'last_segm') AS last_segm,
+   bam_flag(flag, 'seco_alig') AS seco_alig,
+   bam_flag(flag, 'qual_cont') AS qual_cont,
+   bam_flag(flag, 'opti_dupl') AS opti_dupl,
+   bam_flag(flag, 'supp_alig') AS supp_alig
+FROM bam.alignments_1;
+
+
+
+
+# REVERSE_SEQ
+SELECT reverse_seq('A');
+SELECT reverse_seq('T');
+SELECT reverse_seq('C');
+SELECT reverse_seq('G');
+SELECT reverse_seq('R');
+SELECT reverse_seq('Y');
+SELECT reverse_seq('S');
+SELECT reverse_seq('W');
+SELECT reverse_seq('K');
+SELECT reverse_seq('M');
+SELECT reverse_seq('H');
+SELECT reverse_seq('D');
+SELECT reverse_seq('V');
+SELECT reverse_seq('B');
+SELECT reverse_seq('N');
+SELECT reverse_seq('ATCGRYSWKMHDVBN');
+SELECT reverse_seq('invalidchars');
+
+# Bulk
+SELECT seq, reverse_seq(seq) AS reverse_seq
+FROM bam.alignments_1;
+
+
+# REVERSE_QUAL
+SELECT reverse_qual('Should be reversed');
+
+# Bulk
+SELECT qual, reverse_qual(qual) AS reverse_qual
+FROM bam.alignments_1;
+
+
+
+
+# SEQ_LENGTH
+SELECT seq_length('18M');
+SELECT seq_length('3I5D6N');
+SELECT seq_length('3=1X1=1X43=1X16=1X33=');
+
+# Bulk
+SELECT cigar, seq_length(cigar) 

MonetDB: bamloader - Included bulk version of seq_char in tests

2014-11-26 Thread Robin Cijvat
Changeset: 615ce04ed0c6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=615ce04ed0c6
Modified Files:
sql/backends/monet5/bam/Tests/bam_lib.stable.out
sql/backends/monet5/bam/Tests/sql/bam_lib.sql
Branch: bamloader
Log Message:

Included bulk version of seq_char in tests


diffs (74 lines):

diff --git a/sql/backends/monet5/bam/Tests/bam_lib.stable.out 
b/sql/backends/monet5/bam/Tests/bam_lib.stable.out
--- a/sql/backends/monet5/bam/Tests/bam_lib.stable.out
+++ b/sql/backends/monet5/bam/Tests/bam_lib.stable.out
@@ -1,9 +1,9 @@
 stdout of test 'bam_lib` in directory 'sql/backends/monet5/bam` itself:
 
 
-# 10:07:17   
-# 10:07:17   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=35421 --set 
mapi_usock=/var/tmp/mtest-25140/.s.monetdb.35421 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0 --set embedded_r=yes
-# 10:07:17   
+# 08:43:53   
+# 08:43:53   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=35352 --set 
mapi_usock=/var/tmp/mtest-9342/.s.monetdb.35352 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0 --set embedded_r=yes
+# 08:43:53   
 
 # MonetDB 5 server v11.20.0
 # This is an unreleased version
@@ -13,8 +13,8 @@ stdout of test 'bam_lib` in directory 's
 # Copyright (c) 1993-July 2008 CWI.
 # Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved
 # Visit http://www.monetdb.org/ for further information
-# Listening for connection requests on mapi:monetdb://robin-xps13:35421/
-# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-25140/.s.monetdb.35421
+# Listening for connection requests on mapi:monetdb://robin-xps13:35352/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-9342/.s.monetdb.35352
 # MonetDB/SQL module loaded
 # MonetDB/R   module loaded
 
@@ -51,9 +51,9 @@ Ready.
 # loading sql script: 90_generator.sql
 # loading sql script: 99_system.sql
 
-# 10:07:18   
-# 10:07:18   /usr/bin/python2 bam_lib.SQL.py bam_lib
-# 10:07:18   
+# 08:43:54   
+# 08:43:54   /usr/bin/python2 bam_lib.SQL.py bam_lib
+# 08:43:54   
 
 #SET SCHEMA bam;
 #SELECT bam_flag(1, 'mult_segm');
@@ -771,7 +771,15 @@ Ready.
 % char # type
 % 1 # length
 [ NULL ]
+#SELECT 17922987 AS ref_pos, seq, pos, cigar, seq_char(17922987, seq, pos, 
cigar) AS seq_char
+#FROM bam.alignments_1
+#WHERE seq_char(17922987, seq, pos, cigar) IS NOT NULL;
+% .L,  bam.alignments_1,   bam.alignments_1,   bam.alignments_1,   
.L # table_name
+% ref_pos, seq,pos,cigar,  seq_char # name
+% int, clob,   int,clob,   char # type
+% 8,   100,8,  21, 1 # length
+[ 17922987,
TATACTTATAGAACAAATGAACCCCCACATAAGGTAAACAACAAAGCTACTGGTTCTTAAGCCTAACTTCAACAGTACCAGGCACCAT,
 17922987,   3=1X1=1X43=1X16=1X33=,T ]
 
-# 10:07:18   
-# 10:07:18   Done.
-# 10:07:18   
+# 08:43:54   
+# 08:43:54   Done.
+# 08:43:54   
diff --git a/sql/backends/monet5/bam/Tests/sql/bam_lib.sql 
b/sql/backends/monet5/bam/Tests/sql/bam_lib.sql
--- a/sql/backends/monet5/bam/Tests/sql/bam_lib.sql
+++ b/sql/backends/monet5/bam/Tests/sql/bam_lib.sql
@@ -131,7 +131,7 @@ SELECT seq_char(11, 'ACTAGAATGGCT', 5, '
 SELECT seq_char(16, 'ACTAGAATGGCT', 5, '3M1I3M1D5M');
 SELECT seq_char(17, 'ACTAGAATGGCT', 5, '3M1I3M1D5M');
 
-# Bulk, does not work properly yet...
-# SELECT 17922987 AS ref_pos, seq, pos, cigar, seq_char(17922987, seq, pos, 
cigar) AS seq_char
-# FROM bam.alignments_1
-# WHERE seq_char(17922987, seq, pos, cigar) IS NOT NULL;
+# Bulk
+SELECT 17922987 AS ref_pos, seq, pos, cigar, seq_char(17922987, seq, pos, 
cigar) AS seq_char
+FROM bam.alignments_1
+WHERE seq_char(17922987, seq, pos, cigar) IS NOT NULL;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Improved bam_flag bulk version by directly ...

2014-11-26 Thread Robin Cijvat
Changeset: afdccf6751c0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=afdccf6751c0
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Improved bam_flag bulk version by directly writing to BAT. Also attempted this 
for other functions, but there are still problems with those so the old 
versions are kept in tact.


diffs (195 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -229,16 +229,44 @@ seq_char(str * ret, int * ref_pos, str *
return MAL_SUCCEED;
 }
 
+#define init_props() { \
+   output-tsorted = TRUE; \
+   output-trevsorted = TRUE; \
+}
 
+#define update_props(TPE) { \
+   output-tsorted = output-tsorted  \
+   (c == 0 || *cur_out = prev_out); \
+   output-trevsorted = output-trevsorted  \
+   (c == 0 || *cur_out = prev_out); \
+   output-T-nil = output-T-nil || *cur_out == TPE##_nil; \
+   output-T-nonil = output-T-nonil  *cur_out != TPE##_nil; \
+}
 
+#define update_props_str() { \
+   output-tsorted = output-tsorted  \
+   (c == 0 || strcmp(*cur_out, prev_out) = 0); \
+   output-trevsorted = output-trevsorted  \
+   (c == 0 || strcmp(*cur_out, prev_out) = 0); \
+   output-T-nil = output-T-nil || *cur_out == str_nil; \
+   output-T-nonil = output-T-nonil  *cur_out != str_nil; \
+}
+
+#define finish_props() { \
+   BATsetcount(output, BATcount(input)); \
+   BATseqbase(output, input-hseqbase); \
+   output-tkey = FALSE; /* Tail values are not unique */ \
+}
 
 str
 bam_flag_bat(bat * ret, bat * bid, str * name)
 {
-   BAT *flags, *result;
-   BATiter li;
-   BUN p = 0, q = 0;
+   BAT *input, *output;
+   sht prev_out = 0;
+   sht *cur_in;
+   bit *cur_out;
sht k;
+   BUN c;
 
assert(ret != NULL  bid != NULL  name != NULL);
 
@@ -247,33 +275,78 @@ bam_flag_bat(bat * ret, bat * bid, str *
throw(MAL, bam_flag, Unknown flag name given: %s\n,
  *name);
 
-   if ((flags = BATdescriptor(*bid)) == NULL)
+   if ((input = BATdescriptor(*bid)) == NULL)
throw(MAL, bam_flag_bat, RUNTIME_OBJECT_MISSING);
 
/* allocate result BAT */
-   result = BATnew(TYPE_void, TYPE_bit, BATcount(flags), TRANSIENT);
-   if (result == NULL) {
+   output = BATnew(TYPE_void, TYPE_bit, BATcount(input), TRANSIENT);
+   if (output == NULL) {
throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL);
}
-   BATseqbase(result, flags-hseqbase);
-
-   li = bat_iterator(flags);
-
-   BATloop(flags, p, q) {
-   sht t = *(sht *) BUNtail(li, p);
-   bit r = kth_bit(t, k);
-
-   BUNappend(result, (ptr) r, FALSE);
+   
+   init_props();
+   cur_in = (sht *) Tloc(input, BUNfirst(input));
+   cur_out = (bit *) Tloc(output, BUNfirst(output));
+   for(c = 0; c  BATcount(input); ++c) {
+   *cur_out = kth_bit(*cur_in, k);
+   update_props(bit);
+   cur_in++;
+   prev_out = *cur_out++;
}
+   finish_props();
 
/* release input BAT-descriptor */
-   BBPreleaseref(flags-batCacheid);
+   BBPreleaseref(input-batCacheid);
 
-   BBPkeepref((*ret = result-batCacheid));
+   BBPkeepref((*ret = output-batCacheid));
 
return MAL_SUCCEED;
 }
 
+/* Commented out, since string heap addition needs to be added
+str
+reverse_seq_bat(bat * ret, bat * bid)
+{
+   BAT *input, *output;
+   str prev_out = 0;
+   str *cur_in;
+   str *cur_out;
+   BUN c;
+   str msg;
+
+   assert(ret != NULL  bid != NULL);
+
+   if ((input = BATdescriptor(*bid)) == NULL)
+   throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING);
+*/
+   /* allocate result BAT */
+/* output = BATnew(TYPE_void, TYPE_str, BATcount(input), TRANSIENT);
+   if (output == NULL) {
+   throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL);
+   }
+
+   init_props();
+   cur_in = (str *) Tloc(input, BUNfirst(input));
+   cur_out = (str *) Tloc(output, BUNfirst(output));
+   for(c = 0; c  BATcount(input); ++c) {
+   if ((msg = reverse_seq(cur_out, cur_in)) != MAL_SUCCEED) {
+   BBPreleaseref(output-batCacheid);
+   return msg;
+   }
+   update_props_str();
+   cur_in++;
+   prev_out = *cur_out++;
+   }
+   finish_props();
+*/
+   /* release input BAT-descriptor */
+/* BBPreleaseref(input-batCacheid);
+
+   BBPkeepref((*ret = output-batCacheid));
+
+   return MAL_SUCCEED;
+}*/
+
 str
 reverse_seq_bat(bat * ret, bat * bid)
 {
@@ -356,6 +429,50 @@ reverse_qual_bat(bat * ret, bat * bid)
return MAL_SUCCEED;
 }
 
+/*str Does not 

MonetDB: bamloader - Replaced linear time case-statement by cons...

2014-11-26 Thread Robin Cijvat
Changeset: 6a45c22ca31d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6a45c22ca31d
Modified Files:
sql/backends/monet5/bam/Tests/bam_lib.stable.err
sql/backends/monet5/bam/Tests/bam_lib.stable.out
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Replaced linear time case-statement by constant time dictionary lookup for 
reverse_seq udf, also improved its output


diffs (296 lines):

diff --git a/sql/backends/monet5/bam/Tests/bam_lib.stable.err 
b/sql/backends/monet5/bam/Tests/bam_lib.stable.err
--- a/sql/backends/monet5/bam/Tests/bam_lib.stable.err
+++ b/sql/backends/monet5/bam/Tests/bam_lib.stable.err
@@ -1,9 +1,9 @@
 stderr of test 'bam_lib` in directory 'sql/backends/monet5/bam` itself:
 
 
-# 09:28:35   
-# 09:28:35   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=33399 --set 
mapi_usock=/var/tmp/mtest-22883/.s.monetdb.33399 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0 --set embedded_r=yes
-# 09:28:35   
+# 14:48:31   
+# 14:48:31   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=35990 --set 
mapi_usock=/var/tmp/mtest-20228/.s.monetdb.35990 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0 --set embedded_r=yes
+# 14:48:31   
 
 # builtin opt  gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo
 # builtin opt  gdk_debug = 0
@@ -17,71 +17,23 @@ stderr of test 'bam_lib` in directory 's
 # builtin opt  sql_debug = 0
 # cmdline opt  gdk_nr_threads = 0
 # cmdline opt  mapi_open = true
-# cmdline opt  mapi_port = 33399
-# cmdline opt  mapi_usock = /var/tmp/mtest-22883/.s.monetdb.33399
+# cmdline opt  mapi_port = 35990
+# cmdline opt  mapi_usock = /var/tmp/mtest-20228/.s.monetdb.35990
 # cmdline opt  monet_prompt = 
 # cmdline opt  mal_listing = 2
 # cmdline opt  gdk_dbpath = 
/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 # cmdline opt  mal_listing = 0
 # cmdline opt  embedded_r = yes
 # cmdline opt  gdk_debug = 536870922
-# bam_loader: Loader started for 1 BAM file...
-# # bam_loader Retrieving next file id...
-# # bam_loader Initializing BAM wrapper for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
-# # bam_loader Parsing header for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
-# # bam_loader Creating alignment tables for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
-# # CREATE TABLE bam.alignments_1 ( 
-#  virtual_offset  BIGINTNOT NULL, 
-#  qnameSTRING   NOT 
NULL, 
-#  flag  SMALLINT  
NOT NULL, 
-#  rnameSTRING   NOT 
NULL, 
-#  posINT  
 NOT NULL, 
-#  mapq  SMALLINT  
NOT NULL, 
-#  cigarSTRING   NOT 
NULL, 
-#  rnextSTRING   NOT 
NULL, 
-#  pnextINT
 NOT NULL, 
-#  tlen  INT   
 NOT NULL, 
-#  seqSTRING   
  NOT NULL, 
-#  qual  STRING
  NOT NULL, 
-#  CONSTRAINT alignments_1_pkey_virtual_offset PRIMARY KEY 
(virtual_offset) 
-#  ); 
-#  
-#  CREATE TABLE bam.alignments_extra_1 ( 
-#  tagCHAR(2)  
 NOT NULL, 
-#  virtual_offset  BIGINTNOT NULL, 
-#  type  CHAR(1)   
 NOT NULL, 
-#  valueSTRING, 
-#  CONSTRAINT alignments_extra_1_pkey_tag_virtual_offset PRIMARY 
KEY (tag, virtual_offset), 
-#  CONSTRAINT alignments_extra_1_fkey_virtual_offset FOREIGN KEY 
(virtual_offset) 
-#  REFERENCES bam.alignments_1 (virtual_offset) 
-#  );
-# # bam_loader Creating reader threads...
-# # bam_loader Waiting for reader threads to finish...
-# # Thread 0 Starting on next file...
-# # Thread 0 Processing alignments of file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam' 
(file id 1)...
-# # Thread 0 All alignments in file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam' 
(file id 1) 

MonetDB: bamloader - Always clean up on exceptions

2014-11-26 Thread Robin Cijvat
Changeset: df1858231ff4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=df1858231ff4
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Always clean up on exceptions


diffs (127 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -264,6 +264,7 @@ bam_flag_bat(bat * ret, bat * bid, str *
/* allocate result BAT */
output = BATnew(TYPE_void, TYPE_bit, BATcount(input), TRANSIENT);
if (output == NULL) {
+   BBPreleaseref(input-batCacheid);
throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL);
}

@@ -345,6 +346,7 @@ reverse_seq_bat(bat * ret, bat * bid)
/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_str, BATcount(seqs), TRANSIENT);
if (result == NULL) {
+   BBPreleaseref(seqs-batCacheid);
throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL);
}
BATseqbase(result, seqs-hseqbase);
@@ -356,6 +358,7 @@ reverse_seq_bat(bat * ret, bat * bid)
str r, msg;
 
if ((msg = reverse_seq(r, t)) != MAL_SUCCEED) {
+   BBPreleaseref(seqs-batCacheid);
BBPreleaseref(result-batCacheid);
return msg;
}
@@ -386,6 +389,7 @@ reverse_qual_bat(bat * ret, bat * bid)
/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_str, BATcount(quals), TRANSIENT);
if (result == NULL) {
+   BBPreleaseref(quals-batCacheid);
throw(MAL, reverse_qual_bat, MAL_MALLOC_FAIL);
}
BATseqbase(result, quals-hseqbase);
@@ -397,6 +401,7 @@ reverse_qual_bat(bat * ret, bat * bid)
str r, msg;
 
if ((msg = reverse_qual(r, t)) != MAL_SUCCEED) {
+   BBPreleaseref(quals-batCacheid);
BBPreleaseref(result-batCacheid);
return msg;
}
@@ -471,6 +476,7 @@ seq_length_bat(bat * ret, bat * bid)
/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_int, BATcount(cigars), TRANSIENT);
if (result == NULL) {
+   BBPreleaseref(cigars-batCacheid);
throw(MAL, seq_length_bat, MAL_MALLOC_FAIL);
}
BATseqbase(result, cigars-hseqbase);
@@ -483,6 +489,7 @@ seq_length_bat(bat * ret, bat * bid)
int r;
 
if ((msg = seq_length(r, t)) != MAL_SUCCEED) {
+   BBPreleaseref(cigars-batCacheid);
BBPreleaseref(result-batCacheid);
return msg;
}
@@ -500,27 +507,33 @@ seq_length_bat(bat * ret, bat * bid)
 str
 seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * 
alg_cigar)
 {
-   BAT *seqs, *poss, *cigars, *result;
+   BAT *seqs = NULL, *poss = NULL, *cigars = NULL, *result = NULL;
BUN seq = 0, pos = 0, cigar = 0, seq_end = 0;
BATiter seq_it, pos_it, cigar_it;
 
+   str msg = MAL_SUCCEED;
+
assert(ret != NULL  ref_pos != NULL  alg_seq != NULL  alg_pos != 
NULL  alg_cigar != NULL);
 
if ((seqs = BATdescriptor(*alg_seq)) == NULL ||
(poss = BATdescriptor(*alg_pos)) == NULL ||
-   (cigars = BATdescriptor(*alg_cigar)) == NULL) 
-   throw(MAL, seq_char_bat, RUNTIME_OBJECT_MISSING);
+   (cigars = BATdescriptor(*alg_cigar)) == NULL) {
+   msg = createException(MAL, seq_char_bat, 
RUNTIME_OBJECT_MISSING);
+   goto cleanup;
+   }
 
if(BATcount(seqs) != BATcount(poss) || BATcount(seqs) != 
BATcount(cigars)) {
-   throw(MAL, seq_char_bat, 
+   msg = createException(MAL, seq_char_bat, 
Misalignment in input BATs: BUNFMT/BUNFMT/BUNFMT, 
BATcount(poss), BATcount(seqs), BATcount(cigars));
+   goto cleanup;
}

/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_str, BATcount(cigars), TRANSIENT);
if (result == NULL) {
-   throw(MAL, seq_char_bat, MAL_MALLOC_FAIL);
+   msg = createException(MAL, seq_char_bat, MAL_MALLOC_FAIL);
+   goto cleanup;
}
BATseqbase(result, seqs-hseqbase);
 
@@ -541,8 +554,7 @@ seq_char_bat(bat * ret, int * ref_pos, b
str msg;
 
if ((msg = seq_char(r, ref_pos, seq_val, pos_val, 
cigar_val)) != MAL_SUCCEED) {
-   BBPreleaseref(result-batCacheid);
-   return msg;
+   goto cleanup;
}
BUNappend(result, (ptr) r, FALSE);
++seq;
@@ -550,12 +562,13 @@ seq_char_bat(bat * ret, int * ref_pos, b
++cigar;
}

+cleanup:
/* 

MonetDB: bamloader - Improved bam_flag udfs by using lookup tabl...

2014-11-26 Thread Robin Cijvat
Changeset: 5b5cbb2c5217 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5b5cbb2c5217
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Improved bam_flag udfs by using lookup tables instead of serial string 
comparison


diffs (96 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -26,26 +26,78 @@
 #include monetdb_config.h
 #include bam_lib.h
 
-#define flag_str2sht(flag_str) \
-   (strcmp(flag_str, mult_segm) == 0 ? 0 :   \
-(strcmp(flag_str, prop_alig) == 0 ? 1 :  \
- (strcmp(flag_str, segm_unma) == 0 ? 2 : \
-  (strcmp(flag_str, next_unma) == 0 ? 3 :\
-   (strcmp(flag_str, segm_reve) == 0 ? 4 :   
\
-(strcmp(flag_str, next_reve) == 0 ? 5 :  
\
- (strcmp(flag_str, firs_segm) == 0 ? 6 : 
\
-  (strcmp(flag_str, last_segm) == 0 ? 7 :\
-   (strcmp(flag_str, seco_alig) == 0 ? 8 :   \
-(strcmp(flag_str, qual_cont) == 0 ? 9 :  \
- (strcmp(flag_str, opti_dupl) == 0 ? 10 :\
-  (strcmp(flag_str, supp_alig) == 0 ? 11 : -1
+/* Map string to integer by adding all odd and subtracting all even numbers
+ * For the strings that we want to detect, this gives the following:
+ * mult_segm - 71
+ * prop_alig - 101
+ * segm_unma - 84
+ * next_unma - 89
+ * segm_reve - 73
+ * next_reve - 78
+ * firs_segm - 83
+ * last_segm - 97
+ * seco_alig - 106
+ * qual_cont - 98
+ * opti_dupl - 118
+ * supp_alig - 102
+ *
+ * These integers fit in an array of 48 positions
+ * Define two arrays:
+ * - One that maps int position to the bit position that we
+ *   are looking for
+ * - One that maps bit position to string, so we can check if
+ *   the input really was one of the valid strings (since many)
+ *   strings will map to the same integer
+ */
+
+sht intbit_map[] = {
+0, -1,  4, -1, -1, -1, -1,  5, -1, -1,
+   -1, -1,  6,  2, -1, -1, -1, -1,  3, -1,
+   -1, -1, -1, -1, -1, -1,  7,  9, -1, -1,
+1, 11, -1, -1, -1,  8, -1, -1, -1, -1,
+   -1, -1, -1, -1, -1, -1, -1, 10
+};
+
+str posstr_map[] = {
+   mult_segm,
+   prop_alig,
+   segm_unma,
+   next_unma,
+   segm_reve,
+   next_reve,
+   firs_segm,
+   last_segm,
+   seco_alig,
+   qual_cont,
+   opti_dupl,
+   supp_alig
+};
+
+static sht 
+flag_str2sht(str flag_str) {
+   int i = 0;
+   int mult = 1;
+   int strnum = -71;
+   char c;
+   sht k = -1;
+   while((c = flag_str[i++]) != '\0') {
+   strnum += mult * (int)c;
+   mult *= -1;
+   }
+   if(strnum  0 || strnum  47 || (k = intbit_map[strnum])  0 ||
+   strcmp(posstr_map[k], flag_str) != 0) {
+   k = -1;
+   }
+   return k;
+}
 
 #define kth_bit(flag, k) ((flag  (1  k)) == (1  k))
 
 str
 bam_flag(bit * ret, sht * flag, str * name)
 {
-   sht k = flag_str2sht(*name);
+   sht k;
+   k = flag_str2sht(*name);
 
if (k  0)
throw(MAL, bam_flag, Unknown flag name given: %s\n,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Keep forward and backward pointers for reve...

2014-11-26 Thread Robin Cijvat
Changeset: 86ac986f6f30 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=86ac986f6f30
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Keep forward and backward pointers for reverse_seq and reverse_qual to improve 
performance


diffs (46 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -141,17 +141,21 @@ reverse_seq(str * ret, str * seq)
unsigned int i;
unsigned int len = strlen(*seq);
sht map_index;
+   str forward = *seq;
+   str backward;
 
result = GDKmalloc((len + 1) * sizeof(char));
if (result == NULL)
throw(MAL, reverse_seq, MAL_MALLOC_FAIL);
 
+   backward = result[len-1];
for (i = 0; i  len; ++i) {
-   map_index = (sht)((*seq)[i] - 'A');
+   map_index = (sht)(*forward++ - 'A');
if(map_index  0 || map_index  24 ||
-   (result[len - i - 1] = 
reverse_seq_map[map_index]) == 0) {
-   result[len - i - 1] = '?';
+   (*backward = reverse_seq_map[map_index]) == 0) {
+   *backward = '?';
}
+   --backward;
}
result[len] = '\0';
*ret = result;
@@ -164,12 +168,16 @@ reverse_qual(str * ret, str * qual)
str result;
unsigned int i;
unsigned int len = strlen(*qual);
+   str forward = *qual;
+   str backward;
 
result = GDKmalloc((len + 1) * sizeof(char));
if (result == NULL)
throw(MAL, reverse_qual, MAL_MALLOC_FAIL);
+
+   backward = result[len-1];
for (i = 0; i  len; ++i)
-   result[len - i - 1] = (*qual)[i];
+   *backward-- = *forward++;
result[len] = '\0';
*ret = result;
return MAL_SUCCEED;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Added thorough testing of bam_lib and added...

2014-11-25 Thread Robin Cijvat
Changeset: 91d6d8630be1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=91d6d8630be1
Added Files:
sql/backends/monet5/bam/Tests/bam.pyc
sql/backends/monet5/bam/Tests/bam_lib.SQL.py
sql/backends/monet5/bam/Tests/bam_lib.stable.err
sql/backends/monet5/bam/Tests/bam_lib.stable.out
sql/backends/monet5/bam/Tests/sql/bam_lib.sql
Removed Files:
sql/backends/monet5/bam/Tests/sql/drop_last_files.sql
Modified Files:
sql/backends/monet5/bam/Tests/All
sql/backends/monet5/bam/Tests/bam_loader_files.SQL.py
sql/backends/monet5/bam/Tests/bam_loader_files.stable.err
sql/backends/monet5/bam/Tests/bam_loader_files.stable.out
sql/backends/monet5/bam/Tests/check_files.sql
sql/backends/monet5/bam/Tests/check_files.stable.out
sql/backends/monet5/bam/Tests/check_files_exact.sql
sql/backends/monet5/bam/Tests/drop_last_files.SQL.py
sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql
sql/backends/monet5/bam/Tests/sql/sam_export.sql
Branch: bamloader
Log Message:

Added thorough testing of bam_lib and added test for just removed bug that 
occurred when loading many files in one command


diffs (truncated from 3716 to 300 lines):

diff --git a/sql/backends/monet5/bam/Tests/All 
b/sql/backends/monet5/bam/Tests/All
--- a/sql/backends/monet5/bam/Tests/All
+++ b/sql/backends/monet5/bam/Tests/All
@@ -25,3 +25,4 @@ HAVE_SAMTOOLS?query2.10
 HAVE_SAMTOOLS?query2.11
 HAVE_SAMTOOLS?query2.12
 HAVE_SAMTOOLS?mergetable
+HAVE_SAMTOOLS?bam_lib
diff --git a/sql/backends/monet5/bam/Tests/bam.pyc 
b/sql/backends/monet5/bam/Tests/bam.pyc
new file mode 100644
index 
e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..920c4a9a9889e1cc40f6d04cb102a0f9b57a828a
GIT binary patch
literal 1282
zc$}?M2G~`5dPM7e%nA*^*}4)Pqm0yo4@(+E|FQtVc#B2bmuTWZIDwYRj9oKt$|
zA$SNb+;|iofd_z@jZ?(ER(5t~clMk4W+vwEUiZhZpPrBUYtVW{tNcbHpr1etjBa-
zkyzmMh2!E+B6WGNGyyjh++BmTX|sMKOdBqW0Ifslc%Wq9y_po1+$L!y==tjSiwN
zBwe%-G%zYT0zjn4W+S)kf2()p)`7oDg_{GyFr}zAZqEfRT=`LZlkm-IvL~y7c
zuf2h4F3jCMS24{)S*r4uIOFu0xXmb(hG2Z+#k#@C){0dtb76WmLoPw-(AESwNycv
zQJO*mujiXqOjS|zV{Ik3*R%Ws!QYcQD|%_$SJjB%Jdskxr4FhN%|OQqV~alH3O
zZhzoVJm+-mPh=XDPQrK3$W9#1inS;CR2LwXS0{^5D-3HexbS)F(CRK}e_zm^12f_*
zpbSJNB4XR%LR8RPwuXz8+y_;`%Kr6T6@Km|3VS%o@1fMkVOzA!bcf4eo2zRg-s
z^pIRG-e6b}bPG|NUFHJf;wgqhnvoC3eAD3^K9X|_vo?rAw4!}m+L!HBZMU^epch%
zeU}TVcm@i)$AxBUy1H3Te6|*YC9jRbFD;K-718o(i6wLgw+moE2GH}t9qmDIKK
zvn07pLw`)RZuzuy`L)aM?9MXWSr|(Vm5lQ|Oh4Kd@awkLmZ|H2bC?r8Ehh_dgqbh
zo#*9lhk=VCucd!N-Y1;oeMa8U^UyK2MNc$oZHX;oU91WF9y8O0WEWk(LMlG3gDV0j
zdW(VP`s^inR;WQykYsu?{SpgN0be-@6)!@6tB4!Mv28~eMX%YW$SNfunj@#v5
Uu9Mgnrr02sYetXc1EXX81w@VghX4Qo

diff --git a/sql/backends/monet5/bam/Tests/bam_lib.SQL.py 
b/sql/backends/monet5/bam/Tests/bam_lib.SQL.py
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/bam_lib.SQL.py
@@ -0,0 +1,3 @@
+import bam
+
+bam.exec_sql_file(bam_lib.sql, {'PWD': bam.SRCDIR})
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/bam_lib.stable.err 
b/sql/backends/monet5/bam/Tests/bam_lib.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/bam_lib.stable.err
@@ -0,0 +1,87 @@
+stderr of test 'bam_lib` in directory 'sql/backends/monet5/bam` itself:
+
+
+# 09:28:35   
+# 09:28:35   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=33399 --set 
mapi_usock=/var/tmp/mtest-22883/.s.monetdb.33399 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0 --set embedded_r=yes
+# 09:28:35   
+
+# builtin opt  gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = 
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 33399
+# cmdline opt  mapi_usock = /var/tmp/mtest-22883/.s.monetdb.33399
+# cmdline opt  monet_prompt = 
+# cmdline opt  mal_listing = 2
+# cmdline opt  gdk_dbpath = 
/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
+# cmdline opt  mal_listing = 0
+# cmdline opt  embedded_r = yes
+# cmdline opt  gdk_debug = 536870922
+# bam_loader: Loader started for 1 BAM file...
+# # bam_loader Retrieving next file id...
+# # bam_loader Initializing BAM wrapper for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
+# # bam_loader Parsing header for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
+# # bam_loader Creating alignment tables for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
+# # CREATE TABLE bam.alignments_1 ( 
+# 

MonetDB: bamloader - Merge with default

2014-11-25 Thread Robin Cijvat
Changeset: 85bbd04c51b0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=85bbd04c51b0
Added Files:
sql/test/BugTracker-2014/Tests/case-sqrt.Bug-3627.sql
sql/test/BugTracker-2014/Tests/case-sqrt.Bug-3627.stable.err
sql/test/BugTracker-2014/Tests/case-sqrt.Bug-3627.stable.out
sql/test/BugTracker-2014/Tests/column-with-comma.Bug-3616.sql
sql/test/BugTracker-2014/Tests/column-with-comma.Bug-3616.stable.err
sql/test/BugTracker-2014/Tests/column-with-comma.Bug-3616.stable.out
Modified Files:
.hgtags
MonetDB.spec
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
clients/Tests/exports.stable.out
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
clients/python2/setup.py
clients/python3/setup.py
configure.ag
debian/changelog
gdk/gdk.h
gdk/gdk_atoms.h
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_private.h
gdk/libbat.rc
libversions
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/mtime.h
monetdb5/modules/mal/sysmon.c
monetdb5/tools/libmonetdb5.rc
sql/backends/monet5/bam/bam_lib.c
sql/backends/monet5/sql_result.c
sql/jdbc/tests/Tests/Test_PSsqldata.stable.out
sql/test/BugTracker-2014/Tests/All
sql/test/pg_regress/Tests/alter_table.stable.err
sql/test/pg_regress/Tests/arrays.stable.err
sql/test/pg_regress/Tests/boolean.stable.err
sql/test/pg_regress/Tests/case.stable.err
sql/test/pg_regress/Tests/char.stable.err
sql/test/pg_regress/Tests/comments.stable.err
sql/test/pg_regress/Tests/create_aggregate.stable.err
sql/test/pg_regress/Tests/create_misc.stable.err
sql/test/pg_regress/Tests/create_operator.stable.err
sql/test/pg_regress/Tests/date.stable.err
sql/test/pg_regress/Tests/date.stable.out
sql/test/pg_regress/Tests/drop.stable.err
sql/test/pg_regress/Tests/float4.stable.err
sql/test/pg_regress/Tests/float8.stable.err
sql/test/pg_regress/Tests/inet.stable.err
sql/test/pg_regress/Tests/inet.stable.out
sql/test/pg_regress/Tests/insert.stable.err
sql/test/pg_regress/Tests/int2.stable.err
sql/test/pg_regress/Tests/int4.stable.err
sql/test/pg_regress/Tests/int8.stable.err
sql/test/pg_regress/Tests/interval.stable.err
sql/test/pg_regress/Tests/limit.stable.err
sql/test/pg_regress/Tests/numeric.stable.err
sql/test/pg_regress/Tests/numeric_big.stable.err
sql/test/pg_regress/Tests/numerology.stable.err
sql/test/pg_regress/Tests/oid.stable.err
sql/test/pg_regress/Tests/select_into.stable.err
sql/test/pg_regress/Tests/strings.stable.err
sql/test/pg_regress/Tests/strings_cast.stable.err
sql/test/pg_regress/Tests/strings_concat.stable.err
sql/test/pg_regress/Tests/timestamp.stable.err
sql/test/pg_regress/Tests/timestamptz.stable.err
sql/test/pg_regress/Tests/varchar.stable.err
sql/test/pg_regress/Tests/without_oid.stable.err
vertoo.data
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 7109 to 300 lines):

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -583,3 +583,6 @@ 4f894ebf1a93ae895ec23d54ce2112bf21a6427e
 88a29a5a2bef5c6cef6b3bfa5ab7c4d3b855a2c8 Oct2014_release
 eb2967c408350131b63f5393704fd697c701fc49 Oct2014_5
 2da8cbcb65ed56690bad29f089d996eac347b48d Oct2014_SP1_release
+ee3d8e3af6aae6be6cfaa876a379a0751f164e97 Oct2014_7
+2da8cbcb65ed56690bad29f089d996eac347b48d Oct2014_SP1_release
+6607367487d91eb02c5eb7dfb9d96606f8f7c6f0 Oct2014_SP1_release
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -960,6 +960,9 @@ mv $RPM_BUILD_ROOT%{_datadir}/doc/MonetD
 rm -fr $RPM_BUILD_ROOT
 
 %changelog
+* Fri Nov 21 2014 Sjoerd Mullender sjo...@acm.org - 11.19.7-20141121
+- Rebuilt.
+
 * Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
 - Rebuilt.
 - BZ#3580: cosmetic change (append newline)
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1658,7 +1658,7 @@ str MTIMEtime_add_msec_interval_wrap(day
 str MTIMEtime_sub_msec_interval_wrap(daytime *ret, const daytime *t, const lng 
*msec);
 str MTIMEtimestamp(timestamp *ret, const int *sec);
 str MTIMEtimestamp2timestamp(timestamp *ret, const timestamp *src);
-str 

MonetDB: bamloader - Revert bam_lib.c and bam.mal back to their ...

2014-11-25 Thread Robin Cijvat
Changeset: 652329ec295a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=652329ec295a
Modified Files:
sql/backends/monet5/bam/bam.mal
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Revert bam_lib.c and bam.mal back to their previous bulk operator for seq_char


diffs (167 lines):

diff --git a/sql/backends/monet5/bam/bam.mal b/sql/backends/monet5/bam/bam.mal
--- a/sql/backends/monet5/bam/bam.mal
+++ b/sql/backends/monet5/bam/bam.mal
@@ -53,7 +53,9 @@ pattern bam_export(output_path:str):void
 address bam_exportf
 comment Export results in the bam.export table to a BAM file
 
+
 # BAT signatures for bam_lib
+
 module batbam;
 
 command bam_flag(flags:bat[:oid,:sht], name:str):bat[:oid,:bit]
@@ -72,6 +74,6 @@ command seq_length(cigars:bat[:oid,:str]
 address seq_length_bat
 comment Calculate the real length of a bat of DNA sequences, given their 
CIGAR string.
 
-command seq_char(ref_pos:bat[:oid,:int], alg_seq:bat[:oid,:str], 
alg_pos:bat[:oid,:int], alg_cigar:bat[:oid,:str]):bat[:oid,:str]
+command seq_char(ref_pos:int, alg_seq:bat[:oid,:str], alg_pos:bat[:oid,:int], 
alg_cigar:bat[:oid,:str]):bat[:oid,:str]
 address seq_char_bat
 comment Calculate the character in the alignment string (alg_str) that is 
aligned to position 'ref_pos', conforming to the given cigar string (bat based 
version)
diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -54,7 +54,6 @@ bam_flag(bit * ret, sht * flag, str * na
return MAL_SUCCEED;
 }
 
-// use a simple lookup table for these mappings
 str
 reverse_seq(str * ret, str * seq)
 {
@@ -254,7 +253,6 @@ bam_flag_bat(bat * ret, bat * bid, str *
/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_bit, BATcount(flags), TRANSIENT);
if (result == NULL) {
-   BBPreleaseref(flags-batCacheid);
throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL);
}
BATseqbase(result, flags-hseqbase);
@@ -291,7 +289,6 @@ reverse_seq_bat(bat * ret, bat * bid)
/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_str, BATcount(seqs), TRANSIENT);
if (result == NULL) {
-   BBPreleaseref(seqs-batCacheid);
throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL);
}
BATseqbase(result, seqs-hseqbase);
@@ -304,7 +301,6 @@ reverse_seq_bat(bat * ret, bat * bid)
 
if ((msg = reverse_seq(r, t)) != MAL_SUCCEED) {
BBPreleaseref(result-batCacheid);
-   BBPreleaseref(seqs-batCacheid);
return msg;
}
BUNappend(result, (ptr) r, FALSE);
@@ -334,7 +330,6 @@ reverse_qual_bat(bat * ret, bat * bid)
/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_str, BATcount(quals), TRANSIENT);
if (result == NULL) {
-   BBPreleaseref(quals-batCacheid);
throw(MAL, reverse_qual_bat, MAL_MALLOC_FAIL);
}
BATseqbase(result, quals-hseqbase);
@@ -346,7 +341,6 @@ reverse_qual_bat(bat * ret, bat * bid)
str r, msg;
 
if ((msg = reverse_qual(r, t)) != MAL_SUCCEED) {
-   BBPreleaseref(quals-batCacheid);
BBPreleaseref(result-batCacheid);
return msg;
}
@@ -377,7 +371,6 @@ seq_length_bat(bat * ret, bat * bid)
/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_int, BATcount(cigars), TRANSIENT);
if (result == NULL) {
-   BBPreleaseref(cigars-batCacheid);
throw(MAL, seq_length_bat, MAL_MALLOC_FAIL);
}
BATseqbase(result, cigars-hseqbase);
@@ -391,7 +384,6 @@ seq_length_bat(bat * ret, bat * bid)
 
if ((msg = seq_length(r, t)) != MAL_SUCCEED) {
BBPreleaseref(result-batCacheid);
-   BBPreleaseref(cigars-batCacheid);
return msg;
}
BUNappend(result, (ptr) r, FALSE);
@@ -409,26 +401,18 @@ seq_length_bat(bat * ret, bat * bid)
 str
 seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * 
alg_cigar)
 {
-   BAT *seqs = NULL, *poss = NULL, *refs = NULL, *cigars = NULL, *result = 
NULL;
-   BUN ref= 0, seq = 0, pos = 0, cigar = 0, seq_end = 0;
-   BATiter ref_it, seq_it, pos_it, cigar_it;
+   BAT *seqs, *poss, *cigars, *result;
+   BUN seq = 0, pos = 0, cigar = 0, seq_end = 0;
+   BATiter seq_it, pos_it, cigar_it;
 
assert(ret != NULL  ref_pos != NULL  alg_seq != NULL  alg_pos != 
NULL  alg_cigar != NULL);
 
if ((seqs = BATdescriptor(*alg_seq)) == NULL ||
(poss = BATdescriptor(*alg_pos)) == NULL ||
-   (refs = BATdescriptor(*ref_pos)) == NULL ||
-   (cigars = BATdescriptor(*alg_cigar)) == NULL) {

MonetDB: default - Add decent graphics of database schemas that ...

2014-11-25 Thread Robin Cijvat
Changeset: 7c9cec2601fa for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7c9cec2601fa
Added Files:
sql/backends/monet5/bam/default_schema.dia
sql/backends/monet5/bam/default_schema.png
sql/backends/monet5/bam/metadata.dia
sql/backends/monet5/bam/metadata.png
sql/backends/monet5/bam/paired_schema.dia
sql/backends/monet5/bam/paired_schema.png
Branch: default
Log Message:

Add decent graphics of database schemas that we can use in the documentation


diffs (truncated from 5842 to 300 lines):

diff --git a/sql/backends/monet5/bam/default_schema.dia 
b/sql/backends/monet5/bam/default_schema.dia
new file mode 100644
index 
e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..682ce8e1628447e46f8f8dd2955adcac4bcf49ef
GIT binary patch
literal 1422
zc$@)_1#$WxiwFP!21MOUGZ=*O6{@!1KsI;HD1lXiWw%e}SJ71lyMr-`-~iL
zh_8;F+9qwj+;2bgk{2K(0TSA#MrtKsn8zMJo*AZ7zI=TiV{HlN0eOZmSO4|Brf$3
z@h|m%?*IJQ)4yJweDM(YEPs3k#+rPB@bK(X9|w}+-f~NJsEh$fuPL5C@?r|{Rc1x
zEh*H}uTC^=Re=Wt2(Kkqfe;K00s%Dw#_=f0QbRXG$5WHOC@qH#gu7NfG_paVRY!N
zL`^HDXM^qu@Zo?#@KCJUlt;f`MfC)-tlIH}awJ8Y?2cx=dY%kx8Vu6e9lB2=w
z9#V2)scYRp3qiyfFdvaTjgMq4PhHS+qR`KI{jYX?6$ks+$QDW2F=3ZX5m~aELn-
z$^-*QRJ%?VhBi#Dpxk8T*XNwH1SsxgZp5`u0S{}5ol%LL-fi``as=lTqJnTJs
zqCm?N^2Eq_K9IR$?Xv@EpCXP17_J*VB4TgqXZul~U7g5#FCVYThFDu+c_3#13Trz
zgC8Id@+~w=%M@rDC1R~@R?o`qYMA^CRj%h7|b9`)ZZ+_w3aMz;C+K3N)Rct=NPP
z{)MhcCi{m}JOCVOAGLcRAJyh|EEdcd7q2)8@)fP^y!jFlVm`U6870*!{D%(dl%u
zR=F1B80p37+g0P!)*f?QZV!#qE;I)+^paSR2}xA5IVuBfIRVmNkO%1Ay9*`w7W
zl*nP_EjMoeWQEcUpxiV2vR1N1_aGL5dedRW!K1bh}w=nMjuU!!yeX`0m_WWFB
zh8QA@cYsKz*cE26g#o5vndUxLC6h3x(q);!_+$);;0{`wIF|vE`!G~hgjHMrJWLkg
zY)Xwvu^9L=TlEhOBLf3u5fLT~$qaesJiysTuyndw25UgEw+h1#H7P=6vX7l1D$-!
zrVYUu;hA*flu$kau8eA2%)-bhV_K}@o7{H@vg%ylj?Mw$3*g=Y`3J;PD3A+2-bi~@
z8$d#uvSY{(;r9Te$DsZ)Y2#o(aZRmc1_#w0GW$m809LwVOj-obzphzJumXCa?Y
z@3$5=GCp*QmZ`(3OpQBdmG@P1I_^5keY!QR|{zx{Q;H@LiK-7Q#D!CLQJcdG}K
zMW5oQL?@bD);JM9xEV`gV^Jw*1iBWNAAZ^pfOJWeSdfR-*7jdJjZg`_Tn^unwj
zVFGNc6gk{(KLvTG9H2Y0Z?gR;)fcR!i-Taay@)Md*VQdJHCyD*I5hJ~CRn$OlYi
zABxpSmQ%xLp;)b0eOx(pqpVixhobbsDJ?K0iqVSE$HQn2A64R^IDJ%{ehdIsq*kPE
zgw*RhomPc-Wq+rR^2{d+cMgRZ}eS3x%gxm_W4=Q=)L7e-`Zw-b)CL#5RlVtE)PF
z-v=xBbSZ=)$XgeO0@!YWG#`zOT96S2Wo=_C8)RFChAzLS5z;v(I|{=pX)s*zoh
z{D?%s%?zY-+o-*sj`!*r5^Ynj{pu-yYCxjb^Va=wrZ(a0nSB?z(!aUALYZf5+h5
z=pMA5I!Ce0V_9m{oYyE=$Nmvo=TY*Igx=%o2{}~T%D~ZR(fufp*6)JNP=lwwW8k
z@}9Nfo^l|AP|(%w0qxY8RBRyp2|1Be1B-Q-Qzfn}KFcSVH!(qTf0pBpqiho%k
zoy-8PsMYx?V73`42i#ai8^5wvwPC0vCXZ*!wviZe+#+^Ss?59hKSht$S-@w#rQ
zf@UXE=j-hTQX{vz?GQ+$*uY9c=`Nm{Ah=#xv@@~+k`OvfGVJmsq$~@f8e|su3
cq-@d?pGTjWx;jaHhwAFf3BbKDyvce082o?{{R30

diff --git a/sql/backends/monet5/bam/default_schema.png 
b/sql/backends/monet5/bam/default_schema.png
new file mode 100644
index 
e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f1aa49a5c4c81e2e3babd7aa127fe5df45701854
GIT binary patch
literal 54296
zc$}oX1yq%7x9`0W1yni=|(|8N;;GZb3p3q`MoXB_#xu5RgU?kZz)3F!{$7Nj|I
z`MmG9_jmTU_jkrP_wX662UgtIyypDR-+#^fHbhxb8tVq4Fm##B`YJLia?;QAP^`6
z7+2sYI}Oe%@E59sn5;Sm2FApk(k%S%bz2#22LuA=Bl16#kLF=62n01kR^pMmYtqJ)
zo2Y`C1Pi}eEUP@;Qgdt7GqcOehCKat1=J~C#kX(p)dmGllpGlf77DKNcts4+)
zhM_Gct7_KoTr)L8_5Czu#%;T9YRlWkQ{92c|U1cu;RnMSD!=5H5vrsh?b9^pF^|=
z|JE(W)do@oB8vXDEHUDr;|C|-+4r$hliuE5hNpxG1a6ttM0v8Xm(D;YLhOO-g{^T
zx@hGOrmdky1uIFw#{Ysyfqxg#tc=oCtHPm6a8lY79}{*w|QVOB6)?RSu5JnMY{V
zUH|%ko1(20e;@hkBbc}T`Qh?%68~~AAKbDeIxMC59)lPG^5tz7#!kNx+NCwv
z)2@qvTZn;V?HhkG-t6ox2hrQwa|je($Mpni|Yz2f7f8y(|_f{03xi)US0US6=e
zc{5bI^rdXMAo8^t)A$PRa(pQ0@VN$s@RZGo3LF~`H@BwynDBclcR8-r`%72JMO)AJ
z@$m5nrl%t+(6Pzyj5Q5X9cbxWJD^}PH(q_*oypGm0%nSY?S(ek1DX5Wc_t1c-1
zj`A1qF%{#!goTB1TMSoLR(hTvjGR-MDcBP8kqbLa3-QC@yyV`EX61shdhd}6|7
zYm(1#McLTc7!?)O#R#@?hPnJ3EbHNNDIn)KfXR?%~f*?Xdaz`BPF-T(+l;nSGOz
z$d6;;;eG9rmeJ8Cl9H`mUFO@b$WVw7-@xtg$W4cyv-Zd3kFqE3U4tlT{8un@r3s
zEKjTKmxA%=9334A2?;T=u=I3wBgpv%`uqDA+Lv{oQBhId2oS4fm%zx(%v6yP_B`_a
zGr0P@vXa-uXFI$=$!lZ8EZn(b20{uWn3K6xG%yO%w`(YWzLCh_{3(k}J;drZcb
zrBcaO9xfns6t?(z1UmqY)P|eIDgml%vM`5-)v(U^W)nLu6kKYr})?tUaebCLP
zk{dtn77EndXkSbbOv{8B*bSpFcBXYiVdOL}X=WJ2Q?v`1!0N51URmXi9__zJbH
zt*s?xbaZsfUI0bw#hHiURhner?ZnLm6nA?j(n=rB=XfOl)~g!or)In-G}-ZM_EX
zRlF(tv~7ABGgMV{#B=mkA#IEvms=YGlNoaa#BBr2#=13tW8oN{42^50xRe
zMP4sVP5J22qZX~%m6bcI^N4BxfB-y$)UvXPoZr)bFY9MlkBm=YExym2IKX(D+Hr
zraybmd%QAO+k_T=FN?*ZrO{C({SqYHGGbR##W^`siN{R9@YEi;9JX^~-=ueUcI
z2#1=9DLW%0BP*-z_fOA1Q!b0YJtxh*JUvG*w$qjG*xOuXubLNS@Y4OS60~X-o0B?
z!{Fib8{=t9vvSqnG!MTz_r(GINQ6artUWrT0{1%?|KtZB}?x#;29l6}HU%!4`
zSjfuf07b;FY`6XM9r{A1c5w%#`SpD`qF84#DCvj0%PiNS2nC7S;Na_`a#}b*4`J
z_z{PDlLs}(zRy8c!{3FIreBu66r58MnE*ynIg#lhXM3xVeSJhxhN-*4CW2CS~b)
zeJ%uxe=P*CaYqiqs7ilt*nw06SY-UOH95~!YQ*Fm^nE*Vp0ko{YVoJ2?GKv|LC
zQVMwMQz^YBJJ((-(BpCsd3^Rad7!oT+h9QdG2_s;|^FSbPw3=H^O90zqcpB`?u
zr0wkN1O^5|KERJ{W|U3JrCA?7ByUSxx9)b=i7t5zIO8#Zcvtd;$ENxtb@k!{KSd
zk)@?2)K?G=mCnrsX`uy%c(x@Zf;I4N}x*Yex|QwO}OUJs4|6YXGJeBUX)g=4-**
zl{{;Dot4E{YcgHy#%amGT|9pT3t;_!}Yw`-?;l-?+2dm9UU0k#0LhXvp2q0%8}i?
zd2@b3Rz`;UaDQBCZ~wapfdCO9;SLL@@)gF#Re2?EBg2E-*3+}w4buDu-qlQVYf1r
ztJZRN@gcsN+F*J4!;wv*X2^R^wlZxC6F==@c5D;j03p%Z-+vi%()GLsyYG_#5

MonetDB: bamloader - Merge with default

2014-11-25 Thread Robin Cijvat
Changeset: 6f1a94ecdfcf for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6f1a94ecdfcf
Added Files:
sql/backends/monet5/bam/default_schema.dia
sql/backends/monet5/bam/default_schema.png
sql/backends/monet5/bam/metadata.dia
sql/backends/monet5/bam/metadata.png
sql/backends/monet5/bam/paired_schema.dia
sql/backends/monet5/bam/paired_schema.png
Modified Files:
sql/server/rel_select.c
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 5854 to 300 lines):

diff --git a/sql/backends/monet5/bam/default_schema.dia 
b/sql/backends/monet5/bam/default_schema.dia
new file mode 100644
index 
e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..682ce8e1628447e46f8f8dd2955adcac4bcf49ef
GIT binary patch
literal 1422
zc$@)_1#$WxiwFP!21MOUGZ=*O6{@!1KsI;HD1lXiWw%e}SJ71lyMr-`-~iL
zh_8;F+9qwj+;2bgk{2K(0TSA#MrtKsn8zMJo*AZ7zI=TiV{HlN0eOZmSO4|Brf$3
z@h|m%?*IJQ)4yJweDM(YEPs3k#+rPB@bK(X9|w}+-f~NJsEh$fuPL5C@?r|{Rc1x
zEh*H}uTC^=Re=Wt2(Kkqfe;K00s%Dw#_=f0QbRXG$5WHOC@qH#gu7NfG_paVRY!N
zL`^HDXM^qu@Zo?#@KCJUlt;f`MfC)-tlIH}awJ8Y?2cx=dY%kx8Vu6e9lB2=w
z9#V2)scYRp3qiyfFdvaTjgMq4PhHS+qR`KI{jYX?6$ks+$QDW2F=3ZX5m~aELn-
z$^-*QRJ%?VhBi#Dpxk8T*XNwH1SsxgZp5`u0S{}5ol%LL-fi``as=lTqJnTJs
zqCm?N^2Eq_K9IR$?Xv@EpCXP17_J*VB4TgqXZul~U7g5#FCVYThFDu+c_3#13Trz
zgC8Id@+~w=%M@rDC1R~@R?o`qYMA^CRj%h7|b9`)ZZ+_w3aMz;C+K3N)Rct=NPP
z{)MhcCi{m}JOCVOAGLcRAJyh|EEdcd7q2)8@)fP^y!jFlVm`U6870*!{D%(dl%u
zR=F1B80p37+g0P!)*f?QZV!#qE;I)+^paSR2}xA5IVuBfIRVmNkO%1Ay9*`w7W
zl*nP_EjMoeWQEcUpxiV2vR1N1_aGL5dedRW!K1bh}w=nMjuU!!yeX`0m_WWFB
zh8QA@cYsKz*cE26g#o5vndUxLC6h3x(q);!_+$);;0{`wIF|vE`!G~hgjHMrJWLkg
zY)Xwvu^9L=TlEhOBLf3u5fLT~$qaesJiysTuyndw25UgEw+h1#H7P=6vX7l1D$-!
zrVYUu;hA*flu$kau8eA2%)-bhV_K}@o7{H@vg%ylj?Mw$3*g=Y`3J;PD3A+2-bi~@
z8$d#uvSY{(;r9Te$DsZ)Y2#o(aZRmc1_#w0GW$m809LwVOj-obzphzJumXCa?Y
z@3$5=GCp*QmZ`(3OpQBdmG@P1I_^5keY!QR|{zx{Q;H@LiK-7Q#D!CLQJcdG}K
zMW5oQL?@bD);JM9xEV`gV^Jw*1iBWNAAZ^pfOJWeSdfR-*7jdJjZg`_Tn^unwj
zVFGNc6gk{(KLvTG9H2Y0Z?gR;)fcR!i-Taay@)Md*VQdJHCyD*I5hJ~CRn$OlYi
zABxpSmQ%xLp;)b0eOx(pqpVixhobbsDJ?K0iqVSE$HQn2A64R^IDJ%{ehdIsq*kPE
zgw*RhomPc-Wq+rR^2{d+cMgRZ}eS3x%gxm_W4=Q=)L7e-`Zw-b)CL#5RlVtE)PF
z-v=xBbSZ=)$XgeO0@!YWG#`zOT96S2Wo=_C8)RFChAzLS5z;v(I|{=pX)s*zoh
z{D?%s%?zY-+o-*sj`!*r5^Ynj{pu-yYCxjb^Va=wrZ(a0nSB?z(!aUALYZf5+h5
z=pMA5I!Ce0V_9m{oYyE=$Nmvo=TY*Igx=%o2{}~T%D~ZR(fufp*6)JNP=lwwW8k
z@}9Nfo^l|AP|(%w0qxY8RBRyp2|1Be1B-Q-Qzfn}KFcSVH!(qTf0pBpqiho%k
zoy-8PsMYx?V73`42i#ai8^5wvwPC0vCXZ*!wviZe+#+^Ss?59hKSht$S-@w#rQ
zf@UXE=j-hTQX{vz?GQ+$*uY9c=`Nm{Ah=#xv@@~+k`OvfGVJmsq$~@f8e|su3
cq-@d?pGTjWx;jaHhwAFf3BbKDyvce082o?{{R30

diff --git a/sql/backends/monet5/bam/default_schema.png 
b/sql/backends/monet5/bam/default_schema.png
new file mode 100644
index 
e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f1aa49a5c4c81e2e3babd7aa127fe5df45701854
GIT binary patch
literal 54296
zc$}oX1yq%7x9`0W1yni=|(|8N;;GZb3p3q`MoXB_#xu5RgU?kZz)3F!{$7Nj|I
z`MmG9_jmTU_jkrP_wX662UgtIyypDR-+#^fHbhxb8tVq4Fm##B`YJLia?;QAP^`6
z7+2sYI}Oe%@E59sn5;Sm2FApk(k%S%bz2#22LuA=Bl16#kLF=62n01kR^pMmYtqJ)
zo2Y`C1Pi}eEUP@;Qgdt7GqcOehCKat1=J~C#kX(p)dmGllpGlf77DKNcts4+)
zhM_Gct7_KoTr)L8_5Czu#%;T9YRlWkQ{92c|U1cu;RnMSD!=5H5vrsh?b9^pF^|=
z|JE(W)do@oB8vXDEHUDr;|C|-+4r$hliuE5hNpxG1a6ttM0v8Xm(D;YLhOO-g{^T
zx@hGOrmdky1uIFw#{Ysyfqxg#tc=oCtHPm6a8lY79}{*w|QVOB6)?RSu5JnMY{V
zUH|%ko1(20e;@hkBbc}T`Qh?%68~~AAKbDeIxMC59)lPG^5tz7#!kNx+NCwv
z)2@qvTZn;V?HhkG-t6ox2hrQwa|je($Mpni|Yz2f7f8y(|_f{03xi)US0US6=e
zc{5bI^rdXMAo8^t)A$PRa(pQ0@VN$s@RZGo3LF~`H@BwynDBclcR8-r`%72JMO)AJ
z@$m5nrl%t+(6Pzyj5Q5X9cbxWJD^}PH(q_*oypGm0%nSY?S(ek1DX5Wc_t1c-1
zj`A1qF%{#!goTB1TMSoLR(hTvjGR-MDcBP8kqbLa3-QC@yyV`EX61shdhd}6|7
zYm(1#McLTc7!?)O#R#@?hPnJ3EbHNNDIn)KfXR?%~f*?Xdaz`BPF-T(+l;nSGOz
z$d6;;;eG9rmeJ8Cl9H`mUFO@b$WVw7-@xtg$W4cyv-Zd3kFqE3U4tlT{8un@r3s
zEKjTKmxA%=9334A2?;T=u=I3wBgpv%`uqDA+Lv{oQBhId2oS4fm%zx(%v6yP_B`_a
zGr0P@vXa-uXFI$=$!lZ8EZn(b20{uWn3K6xG%yO%w`(YWzLCh_{3(k}J;drZcb
zrBcaO9xfns6t?(z1UmqY)P|eIDgml%vM`5-)v(U^W)nLu6kKYr})?tUaebCLP
zk{dtn77EndXkSbbOv{8B*bSpFcBXYiVdOL}X=WJ2Q?v`1!0N51URmXi9__zJbH
zt*s?xbaZsfUI0bw#hHiURhner?ZnLm6nA?j(n=rB=XfOl)~g!or)In-G}-ZM_EX
zRlF(tv~7ABGgMV{#B=mkA#IEvms=YGlNoaa#BBr2#=13tW8oN{42^50xRe
zMP4sVP5J22qZX~%m6bcI^N4BxfB-y$)UvXPoZr)bFY9MlkBm=YExym2IKX(D+Hr
zraybmd%QAO+k_T=FN?*ZrO{C({SqYHGGbR##W^`siN{R9@YEi;9JX^~-=ueUcI
z2#1=9DLW%0BP*-z_fOA1Q!b0YJtxh*JUvG*w$qjG*xOuXubLNS@Y4OS60~X-o0B?
z!{Fib8{=t9vvSqnG!MTz_r(GINQ6artUWrT0{1%?|KtZB}?x#;29l6}HU%!4`
zSjfuf07b;FY`6XM9r{A1c5w%#`SpD`qF84#DCvj0%PiNS2nC7S;Na_`a#}b*4`J
z_z{PDlLs}(zRy8c!{3FIreBu66r58MnE*ynIg#lhXM3xVeSJhxhN-*4CW2CS~b)
zeJ%uxe=P*CaYqiqs7ilt*nw06SY-UOH95~!YQ*Fm^nE*Vp0ko{YVoJ2?GKv|LC
zQVMwMQz^YBJJ((-(BpCsd3^Rad7!oT+h9QdG2_s;|^FSbPw3=H^O90zqcpB`?u
zr0wkN1O^5|KERJ{W|U3JrCA?7ByUSxx9)b=i7t5zIO8#Zcvtd;$ENxtb@k!{KSd
zk)@?2)K?G=mCnrsX`uy%c(x@Zf;I4N}x*Yex|QwO}OUJs4|6YXGJeBUX)g=4-**
zl{{;Dot4E{YcgHy#%amGT|9pT3t;_!}Yw`-?;l-?+2dm9UU0k#0LhXvp2q0%8}i?
zd2@b3Rz`;UaDQBCZ~wapfdCO9;SLL@@)gF#Re2?EBg2E-*3+}w4buDu-qlQVYf1r
ztJZRN@gcsN+F*J4!;wv*X2^R^wlZxC6F==@c5D;j03p%Z-+vi%()GLsyYG_#5

MonetDB: bamloader - Treat = character in CIGAR string as the ...

2014-11-25 Thread Robin Cijvat
Changeset: 95104df7abc8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=95104df7abc8
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Treat = character in CIGAR string as the m character when calculating 
seq_char


diffs (38 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -196,7 +196,7 @@ seq_char(str * ret, int * ref_pos, str *
  Error parsing CIGAR string '%s'\n, 
*alg_cigar);
advance_ref_pos = (op == 'M' || op == 'D' || 
op == 'N' || op == '=' || op == 'X');
-   advance_seq_pos = (op == 'M' || op == 'I'); // TODO: Find out 
which chars advance the seq pos
+   advance_seq_pos = (op == 'M' || op == 'I' || op == '='); // 
TODO: Find out which chars advance the seq pos
if(advance_seq_pos) {
seq_pos += cnt;
}
@@ -397,7 +397,6 @@ seq_length_bat(bat * ret, bat * bid)
return MAL_SUCCEED;
 }
 
-
 str
 seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * 
alg_cigar)
 {
@@ -417,7 +416,7 @@ seq_char_bat(bat * ret, int * ref_pos, b
Misalignment in input BATs: BUNFMT/BUNFMT/BUNFMT, 
BATcount(poss), BATcount(seqs), BATcount(cigars));
}
-
+   
/* allocate result BAT */
result = BATnew(TYPE_void, TYPE_str, BATcount(cigars), TRANSIENT);
if (result == NULL) {
@@ -450,7 +449,7 @@ seq_char_bat(bat * ret, int * ref_pos, b
++pos;
++cigar;
}
-
+   
/* release input BAT-descriptors */
BBPreleaseref(seqs-batCacheid);
BBPreleaseref(poss-batCacheid);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merge with default

2014-11-21 Thread Robin Cijvat
Changeset: d08a2ca961f6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d08a2ca961f6
Modified Files:
.hgtags
MonetDB.spec
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
buildtools/ChangeLog-Archive
buildtools/ChangeLog.Oct2014
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
clients/python2/setup.py
clients/python3/setup.py
configure.ag
debian/changelog
gdk/ChangeLog-Archive
gdk/ChangeLog.Oct2014
gdk/gdk_batop.c
gdk/libbat.rc
java/Makefile.ag
java/build.properties
java/pom.xml
java/release.txt
libversions
monetdb5/ChangeLog-Archive
monetdb5/ChangeLog.Oct2014
monetdb5/optimizer/opt_generator.c
monetdb5/tools/libmonetdb5.rc
sql/ChangeLog-Archive
sql/ChangeLog.Oct2014
sql/backends/monet5/generator/generator.c
vertoo.data
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 799 to 300 lines):

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -581,3 +581,5 @@ 4f894ebf1a93ae895ec23d54ce2112bf21a6427e
 e92e434d19b7308c2952d50978c2cfd78e1bdb80 Oct2014_3
 4f894ebf1a93ae895ec23d54ce2112bf21a6427e Oct2014_release
 88a29a5a2bef5c6cef6b3bfa5ab7c4d3b855a2c8 Oct2014_release
+eb2967c408350131b63f5393704fd697c701fc49 Oct2014_5
+2da8cbcb65ed56690bad29f089d996eac347b48d Oct2014_SP1_release
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -91,7 +91,7 @@ Vendor: MonetDB BV i...@monetdb.org
 Group: Applications/Databases
 License: MPL - http://www.monetdb.org/Legal/MonetDBLicense
 URL: http://www.monetdb.org/
-Source: 
http://dev.monetdb.org/downloads/sources/Oct2014/%{name}-%{version}.tar.bz2
+Source: 
http://dev.monetdb.org/downloads/sources/Oct2014-SP1/%{name}-%{version}.tar.bz2
 
 BuildRequires: bison
 BuildRequires: bzip2-devel
@@ -960,6 +960,45 @@ mv $RPM_BUILD_ROOT%{_datadir}/doc/MonetD
 rm -fr $RPM_BUILD_ROOT
 
 %changelog
+* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
+- Rebuilt.
+- BZ#3580: cosmetic change (append newline)
+- BZ#3609: Incorrect use of generate_series
+- BZ#3611: quantile() and median() commands crash when used 2x on the
+  same variable on a null table
+- BZ#3612: assertion failure when deleting rows from table to which a
+  FK constraint is defined
+- BZ#3620: ORDER BY broken when using UNION ALL
+- BZ#3621: Hexadecimal literal vs decimal literal
+
+* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
+- gdk: Implemented a change to the way in which string bats are appended.
+  We now try harder to limit the growth of the string heap.
+
+* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
+- monetdb5: Fixed adding of 0 intervals to dates.
+
+* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
+- sql: Fixed sys.queue() implementation to report on other queries being
+  executed.
+
+* Fri Nov 14 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
+- sql: A number of bugs were fixed in the code to upgrade a database from
+  previous releases.  This version should fix the upgrade of a database
+  that had been upgraded to the Oct2014 release, but also properly
+  upgrade directly from Jan2014 and Feb2013 releases.
+
+* Fri Nov  7 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
+- buildtools: The libraries included in the Windows installers have been 
upgraded.
+  We now use libxml2-2.9.2, openssl-1.0.1j, pcre-8.36, and zlib-1.2.8.
+
+* Wed Nov  5 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
+- gdk: Fixed some problems with BATsample.  It was possible for BATsample to
+  return a value that was just beyond the end of the sampled BAT.  Also,
+  on some systems the range of the rand() function is rather limited
+  (0..32767) and trying to get a sample larger than this range would
+  result in an infinite loop.
+
 * Tue Oct 28 2014 Sjoerd Mullender sjo...@acm.org - 11.19.3-20141028
 - Rebuilt.
 
diff --git a/buildtools/ChangeLog-Archive b/buildtools/ChangeLog-Archive
--- a/buildtools/ChangeLog-Archive
+++ b/buildtools/ChangeLog-Archive
@@ -1,6 +1,10 @@
 # DO NOT EDIT THIS FILE -- MAINTAINED AUTOMATICALLY
 # This file contains past ChangeLog entries
 
+* Fri Nov  7 2014 Sjoerd Mullender sjo...@acm.org - 11.19.5-20141120
+- The libraries included in the Windows installers have been upgraded.
+  We now use libxml2-2.9.2, openssl-1.0.1j, pcre-8.36, and zlib-1.2.8.
+
 * Thu Apr 10 2014 Sjoerd Mullender sjo...@acm.org - 11.19.1-20141024
 - Jacqueline, the 

MonetDB: bamloader - Initialize BAT pointers to NULL, compiler d...

2014-11-21 Thread Robin Cijvat
Changeset: ecbfa7f49a2e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ecbfa7f49a2e
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Initialize BAT pointers to NULL, compiler does not like it otherwise


diffs (12 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -401,7 +401,7 @@ seq_length_bat(bat * ret, bat * bid)
 str
 seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * 
alg_cigar)
 {
-   BAT *seqs, *poss, *refs, *cigars, *result;
+   BAT *seqs = NULL, *poss = NULL, *refs = NULL, *cigars = NULL, *result = 
NULL;
BUN ref= 0, seq = 0, pos = 0, cigar = 0, seq_end = 0;
BATiter ref_it, seq_it, pos_it, cigar_it;
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merge with default

2014-11-20 Thread Robin Cijvat
Changeset: 5bb422b5d2e8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5bb422b5d2e8
Added Files:
sql/test/BugTracker-2014/Tests/typecast_error.Bug-3622.sql
Modified Files:
clients/mapiclient/tomograph.c
gdk/ChangeLog.Oct2014
gdk/gdk_atoms.c
gdk/gdk_batop.c
gdk/gdk_private.h
monetdb5/ChangeLog.Oct2014
monetdb5/mal/mal_interpreter.c
monetdb5/modules/atoms/mtime.c
sql/ChangeLog.Oct2014
sql/backends/monet5/generator/generator.c
sql/backends/monet5/sql.c
sql/server/rel_updates.c
sql/test/BugTracker-2014/Tests/All
sql/test/BugTracker-2014/Tests/queueError.Bug-3604.sql
sql/test/BugTracker-2014/Tests/queueError.Bug-3604.stable.err
sql/test/BugTracker-2014/Tests/queueError.Bug-3604.stable.out
sql/test/testdb-upgrade-chain/Tests/dump.stable.out.Windows
sql/test/testdb-upgrade/Tests/dump.stable.out.Windows
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 1012 to 300 lines):

diff --git a/clients/mapiclient/tomograph.c b/clients/mapiclient/tomograph.c
--- a/clients/mapiclient/tomograph.c
+++ b/clients/mapiclient/tomograph.c
@@ -2310,7 +2310,7 @@ main(int argc, char **argv)
 
while (1) {
int option_index = 0;
-   int c = getopt_long(argc, argv, 
d:u:p:h:?T:i:t:r:o:Db:B:A:s:m:a,
+   int c = getopt_long(argc, argv, 
d:u:p:P:h:?T:i:t:r:o:Db:B:A:s:m:a,
long_options, option_index);
if (c == -1)
break;
@@ -2339,7 +2339,7 @@ main(int argc, char **argv)
case 'u':
if (user)
free(user);
-   user = optarg;
+   user = strdup(optarg);
/* force password prompt */
if (password)
free(password);
@@ -2354,7 +2354,7 @@ main(int argc, char **argv)
case 'P':
if (password)
free(password);
-   password = optarg;
+   password = strdup(optarg);
break;
case 'p':
if (optarg)
diff --git a/gdk/ChangeLog.Oct2014 b/gdk/ChangeLog.Oct2014
--- a/gdk/ChangeLog.Oct2014
+++ b/gdk/ChangeLog.Oct2014
@@ -1,6 +1,10 @@
 # ChangeLog file for MonetDB
 # This file is updated with Maddlog
 
+* Thu Nov 20 2014 Sjoerd Mullender sjo...@acm.org
+- Implemented a change to the way in which string bats are appended.
+  We now try harder to limit the growth of the string heap.
+
 * Wed Nov  5 2014 Sjoerd Mullender sjo...@acm.org
 - Fixed some problems with BATsample.  It was possible for BATsample to
   return a value that was just beyond the end of the sampled BAT.  Also,
diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c
--- a/gdk/gdk_atoms.c
+++ b/gdk/gdk_atoms.c
@@ -1050,10 +1050,6 @@ strHash(const char *s)
return res;
 }
 
-/* if at least (2*SIZEOF_BUN), also store length (heaps are then
- * incompatible) */
-#define EXTRALEN ((SIZEOF_BUN + GDK_VARALIGN - 1)  ~(GDK_VARALIGN - 1))
-
 void
 strCleanHash(Heap *h, int rebuild)
 {
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -41,11 +41,12 @@
}   \
} while (0)
 
-/*
- * BAT insert/delete/replace
- * The content of a BAT can be appended to (removed from) another
- * using BATins (BATdel).
- */
+/* We try to be clever when appending one string bat to another.
+ * First of all, we try to actually share the string heap so that we
+ * don't need an extra copy, and if that can't be done, we see whether
+ * it makes sense to just quickly copy the whole string heap instead
+ * of inserting individual strings.  See the comments in the code for
+ * more information. */
 static BAT *
 insert_string_bat(BAT *b, BAT *n, int append, int force)
 {
@@ -54,20 +55,17 @@ insert_string_bat(BAT *b, BAT *n, int ap
size_t toff = ~(size_t) 0;  /* tail offset */
BUN p, q;   /* loop variables */
oid o = 0;  /* in case we're appending */
-   ptr hp, tp; /* head and tail value pointers */
+   const void *hp, *tp;/* head and tail value pointers */
unsigned char tbv;  /* tail value-as-bte */
-   const unsigned char *tbp = NULL;/* tail value-as-bte */
unsigned short tsv; /* tail value-as-sht */
-   const unsigned short *tsp = NULL;   /* tail value-as-sht */
 #if SIZEOF_VAR_T == 8
unsigned int tiv;   /* tail value-as-int */
-   const unsigned int *tip = NULL; /* tail value-as-int */
 #endif
var_t v;/* value */
-   const var_t *tvp = NULL;/* value */
-   int ntw, btw; 

MonetDB: bamloader - Merge with default

2014-11-19 Thread Robin Cijvat
Changeset: ec1329475b17 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec1329475b17
Added Files:
sql/backends/monet5/generator/Tests/rangejoin.sql
sql/backends/monet5/generator/Tests/rangejoin.stable.err
sql/backends/monet5/generator/Tests/rangejoin.stable.out
sql/test/BugTracker-2014/Tests/orderby_column_exp.Bug-3620.sql
sql/test/BugTracker-2014/Tests/orderby_column_exp.Bug-3620.stable.err
sql/test/BugTracker-2014/Tests/orderby_column_exp.Bug-3620.stable.out
Modified Files:
clients/Tests/MAL-signatures_all.stable.out
clients/Tests/MAL-signatures_all.stable.out.int128
clients/Tests/MAL-signatures_fits_geom.stable.out
clients/Tests/MAL-signatures_fits_geom.stable.out.int128
clients/Tests/MAL-signatures_geom.stable.out
clients/Tests/MAL-signatures_geom.stable.out.int128
clients/Tests/MAL-signatures_none.stable.out
clients/Tests/MAL-signatures_none.stable.out.int128
clients/Tests/exports.stable.out
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/algebra.h
monetdb5/modules/kernel/algebra.mal
sql/backends/monet5/generator/Tests/All
sql/backends/monet5/generator/generator.c
sql/backends/monet5/generator/generator.h
sql/backends/monet5/generator/generator.mal
sql/common/sql_types.c
sql/common/sql_types.h
sql/server/rel_psm.c
sql/server/rel_select.c
sql/server/sql_mvc.h
sql/test/BugTracker-2014/Tests/All
sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out.32bit
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 995 to 300 lines):

diff --git a/clients/Tests/MAL-signatures_all.stable.out 
b/clients/Tests/MAL-signatures_all.stable.out
--- a/clients/Tests/MAL-signatures_all.stable.out
+++ b/clients/Tests/MAL-signatures_all.stable.out
@@ -138,8 +138,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[
 address AGGRavg12_dbl;
 comment Grouped tail average on bte
 
-command aggr.avg(b:bat[:oid,:any_2]):dbl 
-address ALGavg;
+pattern aggr.avg(b:bat[:oid,:any_2]):dbl 
+address CMDcalcavg;
 comment Gives the avg of all tail values
 
 function 
aggr.corr(e1:bat[:oid,:dbl],e2:bat[:oid,:dbl],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:dbl];
diff --git a/clients/Tests/MAL-signatures_all.stable.out.int128 
b/clients/Tests/MAL-signatures_all.stable.out.int128
--- a/clients/Tests/MAL-signatures_all.stable.out.int128
+++ b/clients/Tests/MAL-signatures_all.stable.out.int128
@@ -149,8 +149,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[
 address AGGRavg12_dbl;
 comment Grouped tail average on bte
 
-command aggr.avg(b:bat[:oid,:any_2]):dbl 
-address ALGavg;
+pattern aggr.avg(b:bat[:oid,:any_2]):dbl 
+address CMDcalcavg;
 comment Gives the avg of all tail values
 
 function 
aggr.corr(e1:bat[:oid,:hge],e2:bat[:oid,:hge],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:hge];
diff --git a/clients/Tests/MAL-signatures_fits_geom.stable.out 
b/clients/Tests/MAL-signatures_fits_geom.stable.out
--- a/clients/Tests/MAL-signatures_fits_geom.stable.out
+++ b/clients/Tests/MAL-signatures_fits_geom.stable.out
@@ -138,8 +138,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[
 address AGGRavg12_dbl;
 comment Grouped tail average on bte
 
-command aggr.avg(b:bat[:oid,:any_2]):dbl 
-address ALGavg;
+pattern aggr.avg(b:bat[:oid,:any_2]):dbl 
+address CMDcalcavg;
 comment Gives the avg of all tail values
 
 function 
aggr.corr(e1:bat[:oid,:dbl],e2:bat[:oid,:dbl],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:dbl];
diff --git a/clients/Tests/MAL-signatures_fits_geom.stable.out.int128 
b/clients/Tests/MAL-signatures_fits_geom.stable.out.int128
--- a/clients/Tests/MAL-signatures_fits_geom.stable.out.int128
+++ b/clients/Tests/MAL-signatures_fits_geom.stable.out.int128
@@ -154,8 +154,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[
 address AGGRavg12_dbl;
 comment Grouped tail average on bte
 
-command aggr.avg(b:bat[:oid,:any_2]):dbl 
-address ALGavg;
+pattern aggr.avg(b:bat[:oid,:any_2]):dbl 
+address CMDcalcavg;
 comment Gives the avg of all tail values
 
 function 
aggr.corr(e1:bat[:oid,:hge],e2:bat[:oid,:hge],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:hge];
diff --git a/clients/Tests/MAL-signatures_geom.stable.out 
b/clients/Tests/MAL-signatures_geom.stable.out
--- a/clients/Tests/MAL-signatures_geom.stable.out
+++ b/clients/Tests/MAL-signatures_geom.stable.out
@@ -138,8 +138,8 @@ command aggr.avg(b:bat[:oid,:bte],e:bat[
 address AGGRavg12_dbl;
 comment Grouped tail average on bte
 
-command aggr.avg(b:bat[:oid,:any_2]):dbl 
-address ALGavg;
+pattern aggr.avg(b:bat[:oid,:any_2]):dbl 
+address CMDcalcavg;
 comment Gives the avg of all tail values
 
 function 
aggr.corr(e1:bat[:oid,:dbl],e2:bat[:oid,:dbl],g:bat[:oid,:oid],e:bat[:oid,:any_2]):bat[:oid,:dbl];
diff --git a/clients/Tests/MAL-signatures_geom.stable.out.int128 

MonetDB: bamloader - Merge with default

2014-11-19 Thread Robin Cijvat
Changeset: 04191ece031d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=04191ece031d
Modified Files:
monetdb5/optimizer/opt_generator.c
sql/backends/monet5/generator/generator.c
sql/backends/monet5/generator/generator.mal
Branch: bamloader
Log Message:

Merge with default


diffs (79 lines):

diff --git a/monetdb5/optimizer/opt_generator.c 
b/monetdb5/optimizer/opt_generator.c
--- a/monetdb5/optimizer/opt_generator.c
+++ b/monetdb5/optimizer/opt_generator.c
@@ -63,7 +63,7 @@ int
 OPTgeneratorImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr 
pci)
 {
InstrPtr p,q;
-   int i,j,k, actions=0, used= 0, cases, blocked;
+   int i,j,k, actions=0, used, cases, blocked;
 
(void) cntxt;
(void) stk;
@@ -73,6 +73,7 @@ OPTgeneratorImplementation(Client cntxt,
p = getInstrPtr(mb,i);
if ( getModuleId(p) == generatorRef  getFunctionId(p) == 
seriesRef){
/* found a target for propagation */
+   used = 0;
if ( assignedOnce(mb, getArg(p,0)) ){
cases = useCount(mb, getArg(p,0));
blocked = 0;
@@ -116,8 +117,10 @@ OPTgeneratorImplementation(Client cntxt,
else {
// check for use without 
conversion
for(k = q-retc; k  q-argc; 
k++)
-   if( getArg(q,k) == getArg(p,0))
+   if( getArg(q,k) == getArg(p,0)){
blocked++;
+   }
+   // materialize a copy and 
re-use where appropriate
}
}
// fix the original, only when all use cases 
are replaced by the overloaded function
@@ -125,8 +128,9 @@ OPTgeneratorImplementation(Client cntxt,
setModuleId(p, generatorRef);
setFunctionId(p, parametersRef);
typeChecker(cntxt-fdout, 
cntxt-nspace, mb, p, TRUE);
+   }
+   if( used)
actions++;
-   } else used = 0;
 #ifdef VLT_DEBUG
mnstr_printf(cntxt-fdout,#generator target %d 
cases %d used %d error %d\n,getArg(p,0), cases, used, p-typechk);
 #endif
diff --git a/sql/backends/monet5/generator/generator.c 
b/sql/backends/monet5/generator/generator.c
--- a/sql/backends/monet5/generator/generator.c
+++ b/sql/backends/monet5/generator/generator.c
@@ -218,7 +218,7 @@ findGeneratorDefinition(MalBlkPtr mb, In
 
for (i = 1; i  mb-stop; i++) {
q = getInstrPtr(mb, i);
-   if (q-argv[0] == target  getModuleId(q) == generatorRef  
getFunctionId(q) == parametersRef)
+   if (q-argv[0] == target  getModuleId(q) == generatorRef  
(getFunctionId(q) == parametersRef || getFunctionId(q) == seriesRef))
p = q;
if (q == pci)
return p;
diff --git a/sql/backends/monet5/generator/generator.mal 
b/sql/backends/monet5/generator/generator.mal
--- a/sql/backends/monet5/generator/generator.mal
+++ b/sql/backends/monet5/generator/generator.mal
@@ -223,15 +223,15 @@ comment Overloaded join operation;
 
 pattern join(gen:bat[:oid,:bte], low:bat[:oid,:bte], hgh:bat[:oid,:bte], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
 address VLTgenerator_rangejoin;
-pattern join(gen:bat[:oid,:sht], low:bat[:oid,:sht], hgh:bat[:oid,:bte], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
+pattern join(gen:bat[:oid,:sht], low:bat[:oid,:sht], hgh:bat[:oid,:sht], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
 address VLTgenerator_rangejoin;
-pattern join(gen:bat[:oid,:int], low:bat[:oid,:int], hgh:bat[:oid,:bte], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
+pattern join(gen:bat[:oid,:int], low:bat[:oid,:int], hgh:bat[:oid,:int], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
 address VLTgenerator_rangejoin;
-pattern join(gen:bat[:oid,:lng], low:bat[:oid,:lng], hgh:bat[:oid,:bte], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
+pattern join(gen:bat[:oid,:lng], low:bat[:oid,:lng], hgh:bat[:oid,:lng], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
 address VLTgenerator_rangejoin;
-pattern join(gen:bat[:oid,:flt], low:bat[:oid,:flt], hgh:bat[:oid,:bte], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
+pattern join(gen:bat[:oid,:flt], low:bat[:oid,:flt], hgh:bat[:oid,:flt], 
li:bit, ri:bit) (l:bat[:oid,:oid],r:bat[:oid,:oid])
 address VLTgenerator_rangejoin;

MonetDB: bamloader - Merge with default

2014-11-16 Thread Robin Cijvat
Changeset: 1901ba646704 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1901ba646704
Added Files:
clients/mapilib/ChangeLog-Archive
clients/nodejs/Tests/nodetest.js
clients/nodejs/monetdb/README.md
clients/nodejs/monetdb/mapiclient.js
clients/nodejs/monetdb/package.json
common/stream/ChangeLog-Archive
debian/libmonetdb-client7.install
debian/libmonetdb-stream6.install
debian/libmonetdb11.install
sql/backends/monet5/Tests/limithack.sql
sql/backends/monet5/Tests/limithack.stable.err
sql/backends/monet5/Tests/limithack.stable.out
sql/backends/monet5/Tests/prepare.mal
sql/backends/monet5/Tests/simpleSQL.mal
sql/backends/monet5/Tests/simpleSQL.stable.err
sql/backends/monet5/Tests/simpleSQL.stable.out
sql/backends/monet5/generator/Tests/crash.Bug-3609.sql
sql/backends/monet5/generator/Tests/crash.Bug-3609.stable.err
sql/backends/monet5/generator/Tests/crash.Bug-3609.stable.out
sql/backends/monet5/sql_assert.c
sql/backends/monet5/sql_assert.h
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_execute.h
sql/backends/monet5/sql_upgrades.c
sql/backends/monet5/sql_upgrades.h
sql/test/BugTracker-2009/Tests/bit_and.SF-2850341.stable.err.int128
sql/test/BugTracker-2009/Tests/bit_and.SF-2850341.stable.out.int128
sql/test/BugTracker-2014/Tests/empty_intermediate_quantile.Bug-3611.sql

sql/test/BugTracker-2014/Tests/empty_intermediate_quantile.Bug-3611.stable.err

sql/test/BugTracker-2014/Tests/empty_intermediate_quantile.Bug-3611.stable.out
sql/test/BugTracker-2014/Tests/fk-property-assert.Bug-3612.sql
sql/test/BugTracker-2014/Tests/fk-property-assert.Bug-3612.stable.err
sql/test/BugTracker-2014/Tests/fk-property-assert.Bug-3612.stable.out
sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.sql
sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.err

sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.err.int128
sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.out

sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.out.int128
sql/test/BugTracker-2014/Tests/nil_2dec_lng.Bug-3592.stable.out.single
sql/test/BugTracker-2014/Tests/querylog.Bug-3607.sql
sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.err
sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.out
sql/test/json/Tests/shop.sql
sql/test/json/Tests/shop.stable.err
sql/test/json/Tests/shop.stable.out
sql/test/mergetables/Tests/doubletable.sql
sql/test/mergetables/Tests/singlekeyconstraint.sql
sql/test/pg_regress/Tests/loadwisconsin.sql
sql/test/pg_regress/Tests/loadwisconsin.stable.err
sql/test/pg_regress/Tests/loadwisconsin.stable.out
Removed Files:
buildtools/ChangeLog.Jan2014
clients/ChangeLog.Jan2014
clients/python2/ChangeLog.Jan2014
clients/python3/ChangeLog.Jan2014
debian/libmonetdb-client6.install
debian/libmonetdb-stream5.install
debian/libmonetdb10.install
gdk/ChangeLog.Jan2014
geom/ChangeLog.Jan2014
java/ChangeLog.Jan2014
monetdb5/ChangeLog.Jan2014
sql/ChangeLog.Jan2014
testing/ChangeLog.Jan2014
tools/merovingian/ChangeLog.Jan2014
Modified Files:
.hgtags
MonetDB.spec
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
buildtools/ChangeLog-Archive
buildtools/ChangeLog.Oct2014
buildtools/doc/windowsbuild.rst
clients/R/MonetDB.R/NAMESPACE
clients/R/MonetDB.R/NEWS
clients/R/MonetDB.R/R/dbi.R
clients/R/MonetDB.R/R/dplyr.R
clients/R/MonetDB.R/R/mapi.R
clients/R/Tests/dplyr.R
clients/R/Tests/install-dependencies.sh
clients/R/Tests/survey.R
clients/R/Tests/survey.stable.err
clients/Tests/MAL-signatures_all.stable.err
clients/Tests/MAL-signatures_all.stable.out
clients/Tests/MAL-signatures_all.stable.out.int128
clients/Tests/MAL-signatures_fits_geom.stable.out
clients/Tests/MAL-signatures_fits_geom.stable.out.int128
clients/Tests/MAL-signatures_geom.stable.out
clients/Tests/MAL-signatures_geom.stable.out.int128
clients/Tests/MAL-signatures_none.stable.out
clients/Tests/MAL-signatures_none.stable.out.int128
clients/Tests/SQL-dump_all.stable.err

MonetDB: bamloader - Delay opening of file handles to the point ...

2014-11-11 Thread Robin Cijvat
Changeset: 2e254b0c071f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2e254b0c071f
Modified Files:
sql/backends/monet5/bam/bam_loader.c
sql/backends/monet5/bam/bam_wrapper.c
sql/backends/monet5/bam/bam_wrapper.h
Branch: bamloader
Log Message:

Delay opening of file handles to the point where they are really needed, to 
prevent opening all file handles on beginning. On my machine this starts to 
fail if trying to import  23 files at once, since for every file tens of file 
handlers to binary files are opened.


diffs (truncated from 378 to 300 lines):

diff --git a/sql/backends/monet5/bam/bam_loader.c 
b/sql/backends/monet5/bam/bam_loader.c
--- a/sql/backends/monet5/bam/bam_loader.c
+++ b/sql/backends/monet5/bam/bam_loader.c
@@ -209,10 +209,9 @@ bam_loader(Client cntxt, MalBlkPtr mb, s
goto cleanup;
}
if (nr_threads = 0) {
-   msg = createException(MAL, bam_loader,
- We can not get the work done with 
only %d threads,
- nr_threads);
-   goto cleanup;
+   nr_threads = 1;
+   } else if(nr_threads  4) {
+   nr_threads = 4;
}
 
/* Get SQL context */
@@ -381,7 +380,6 @@ bam_loader(Client cntxt, MalBlkPtr mb, s
TO_LOG(bam_loader Copying data into DB...\n);
/* All threads finished succesfully, copy all data into DB */
for (i = 0; i  nr_files; ++i) {
-   prepare_for_copy(bws + i);
if ((msg = copy_into_db(cntxt, bws + i)) != MAL_SUCCEED) {
goto cleanup;
}
diff --git a/sql/backends/monet5/bam/bam_wrapper.c 
b/sql/backends/monet5/bam/bam_wrapper.c
--- a/sql/backends/monet5/bam/bam_wrapper.c
+++ b/sql/backends/monet5/bam/bam_wrapper.c
@@ -99,6 +99,44 @@ init_bam_wrapper(bam_wrapper * bw, filet
  file_location, flushdir, strerror(errno));
}
 
+   // Construct all the file paths to the binary files
+   for (i = 0; i  6; ++i) {
+   snprintf(bw-fp_files[i], BW_FP_BUF_SIZE, %s/files_%d,
+flushdir, i);
+   }
+   for (i = 0; i  7; ++i) {
+   snprintf(bw-fp_sq[i], BW_FP_BUF_SIZE, %s/sq_%d, flushdir,
+i);
+   }
+   for (i = 0; i  13; ++i) {
+   snprintf(bw-fp_rg[i], BW_FP_BUF_SIZE, %s/rg_%d, flushdir,
+i);
+   }
+   for (i = 0; i  6; ++i) {
+   snprintf(bw-fp_pg[i], BW_FP_BUF_SIZE, %s/pg_%d, flushdir,
+i);
+   }
+   for (i = 0; i  12; ++i) {
+   snprintf(bw-fp_alignments[i], BW_FP_BUF_SIZE,
+%s/alignments_%d, flushdir, i);
+   }
+   for (i = 0; i  4; ++i) {
+   snprintf(bw-fp_alignments_extra[i], BW_FP_BUF_SIZE,
+%s/alignments_extra_%d, flushdir, i);
+   }
+   if (dbschema == 1) {
+   for (i = 0; i  23; ++i) {
+   snprintf(bw-fp_alignments_paired_primary[i],
+BW_FP_BUF_SIZE,
+%s/alignments_paired_primary_%d, flushdir,
+i);
+   snprintf(bw-fp_alignments_paired_secondary[i],
+BW_FP_BUF_SIZE,
+%s/alignments_paired_secondary_%d,
+flushdir, i);
+   }
+   }
+
if (type == BAM) {
/* Open BAM file and read its header */
if ((bw-bam.input = bam_open(file_location, r)) == NULL) {
@@ -184,105 +222,102 @@ init_bam_wrapper(bam_wrapper * bw, filet
bw-cnt_alignments_paired_secondary = 0;
bw-cnt_alignments_total = 0;
 
+
+   return MAL_SUCCEED;
+}
+
+
+static str 
+open_header_write_streams(bam_wrapper * bw)
+{
+   int i;
+
for (i = 0; i  6; ++i) {
-   snprintf(bw-fp_files[i], BW_FP_BUF_SIZE, %s/files_%d,
-flushdir, i);
if ((bw-files[i] = bsopen(bw-fp_files[i])) == NULL) {
throw(MAL, init_bam_wrapper,
  ERR_INIT_BAM_WRAPPER
  Binary file '%s' could not be opened,
- file_location, bw-fp_files[i]);
+ bw-file_location, bw-fp_files[i]);
}
}
for (i = 0; i  7; ++i) {
-   snprintf(bw-fp_sq[i], BW_FP_BUF_SIZE, %s/sq_%d, flushdir,
-i);
if ((bw-sq[i] = bsopen(bw-fp_sq[i])) == NULL) {
throw(MAL, init_bam_wrapper,
  ERR_INIT_BAM_WRAPPER
  Binary file '%s' could not be opened,
- file_location, bw-fp_sq[i]);
+   

MonetDB: default - Handle null in param array correctly

2014-11-10 Thread Robin Cijvat
Changeset: 7fbf1caaeabe for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7fbf1caaeabe
Modified Files:
clients/nodejs/monetdb/mapiclient.js
Branch: default
Log Message:

Handle null in param array correctly


diffs (13 lines):

diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -123,6 +123,9 @@ MonetDBConnection.prototype.prepare = fu
if (!error) {
var execfun = function(bindparams, ecallback) {
var quoted = bindparams.map(function(param) {
+   if(param === null) {
+   return NULL;
+   }
var type = typeof param;
switch(type) {
case 'boolean':
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - NodeJS connector: update package version

2014-11-10 Thread Robin Cijvat
Changeset: 9cc5f7d3867c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9cc5f7d3867c
Modified Files:
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

NodeJS connector: update package version


diffs (11 lines):

diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.2.1,
+  version: 0.2.2,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Cleaned up argument handling

2014-11-06 Thread Robin Cijvat
Changeset: 5facdb3d0476 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5facdb3d0476
Modified Files:
clients/nodejs/monetdb/mapiclient.js
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

Cleaned up argument handling


diffs (27 lines):

diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -80,11 +80,7 @@ MonetDBConnection.prototype.query = func
return;
}
 
-   if (Array.isArray(arguments[1])) {
-   params = arguments[1];
-   callback = arguments[2];
-   raw = arguments[3];
-
+   if (params.length  0) {
this.prepare(message, function(err, resp) {
if (err) {
if (callback != undefined) {
diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.1.8,
+  version: 0.2.0,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - NodeJS connector: Fixed bug that prevented th...

2014-11-06 Thread Robin Cijvat
Changeset: 93da657eae35 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=93da657eae35
Modified Files:
clients/nodejs/monetdb/mapiclient.js
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

NodeJS connector: Fixed bug that prevented the callback from being called when 
undefined was passed as second parameter to query


diffs (31 lines):

diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -59,7 +59,6 @@ MonetDBConnection.prototype.query = func
var params = [];
var callback = undefined;
var raw = false;
-
for (argi in arguments) {
if (typeof arguments[argi] == 'function') {
callback = arguments[argi];
@@ -103,8 +102,6 @@ MonetDBConnection.prototype.query = func
});
}
else {
-   callback = arguments[1];
-   raw = arguments[2];
if (!raw) {
message = 's' + message + ';';
}
diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.2.0,
+  version: 0.2.1,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Version 0.1.8

2014-10-31 Thread Robin Cijvat
Changeset: 24f875d842a7 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=24f875d842a7
Modified Files:
clients/nodejs/Tests/nodetest.js
clients/nodejs/monetdb/package.json
Branch: default
Log Message:

Version 0.1.8


diffs (21 lines):

diff --git a/clients/nodejs/Tests/nodetest.js b/clients/nodejs/Tests/nodetest.js
--- a/clients/nodejs/Tests/nodetest.js
+++ b/clients/nodejs/Tests/nodetest.js
@@ -109,6 +109,3 @@ conn.prepare('SELECT id from tables wher
 });
 
 conn.close();
-
-
-
diff --git a/clients/nodejs/monetdb/package.json 
b/clients/nodejs/monetdb/package.json
--- a/clients/nodejs/monetdb/package.json
+++ b/clients/nodejs/monetdb/package.json
@@ -1,6 +1,6 @@
 {
   name: monetdb,
-  version: 0.1.7,
+  version: 0.1.8,
   description: Connect MonetDB and node.js,
   main: mapiclient.js,
   author: Hannes Mühleisen han...@cwi.nl,
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merge with default

2014-10-30 Thread Robin Cijvat
Changeset: 13957da3e185 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=13957da3e185
Added Files:
sql/test/mergetables/Tests/emptymergequery.stable.err
sql/test/mergetables/Tests/emptymergequery.stable.out
sql/test/mergetables/Tests/mergeloadidx.sql
sql/test/mergetables/Tests/mergeloadidx.stable.err
sql/test/mergetables/Tests/mergeloadidx.stable.out
Modified Files:
sql/backends/monet5/bam/Tests/mergetable.stable.err
sql/backends/monet5/bam/Tests/mergetable.stable.out
sql/backends/monet5/bam/Tests/sql/mergetable.sql
sql/server/rel_schema.c
sql/server/rel_select.c
sql/storage/store.c
sql/test/mergetables/Tests/All
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 361 to 300 lines):

diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.err 
b/sql/backends/monet5/bam/Tests/mergetable.stable.err
--- a/sql/backends/monet5/bam/Tests/mergetable.stable.err
+++ b/sql/backends/monet5/bam/Tests/mergetable.stable.err
@@ -118,21 +118,13 @@ stderr of test 'mergetable` in directory
 # 08:29:32   /usr/bin/python2 mergetable.SQL.py mergetable
 # 08:29:32   
 
-MAPI  = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321
+MAPI  = (monetdb) /var/tmp/mtest-17563/.s.monetdb.34280
 QUERY = SELECT COUNT(*) FROM alignments;
-ERROR = !Unable to query empty Merge or Replica tables
-MAPI  = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321
-QUERY = ALTER TABLE alignments ADD TABLE alignments_1;
-ERROR = !ALTER MERGE TABLE: to be added table index doesn't match MERGE TABLE 
definition
-MAPI  = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321
-QUERY = SELECT COUNT(*) FROM alignments;
-ERROR = !Unable to query empty Merge or Replica tables
-MAPI  = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321
-QUERY = ALTER TABLE alignments ADD TABLE alignments_2;
-ERROR = !ALTER MERGE TABLE: to be added table index doesn't match MERGE TABLE 
definition
-MAPI  = (monetdb) /var/tmp/mtest-10986/.s.monetdb.37321
-QUERY = SELECT COUNT(*) FROM alignments;
-ERROR = !Unable to query empty Merge or Replica tables
+ERROR = !MERGE or REPLICA TABLE should have at least one table associated
+
+# 18:59:22   
+# 18:59:22   Done.
+# 18:59:22   
 
 # 08:29:33   
 # 08:29:33   Done.
diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.out 
b/sql/backends/monet5/bam/Tests/mergetable.stable.out
--- a/sql/backends/monet5/bam/Tests/mergetable.stable.out
+++ b/sql/backends/monet5/bam/Tests/mergetable.stable.out
@@ -68,7 +68,25 @@ Ready.
 #  pnext  INT   NOT NULL,
 #  tlen   INT   NOT NULL,
 #  seqSTRINGNOT NULL,
-#  qual   STRINGNOT NULL
+#  qual   STRINGNOT NULL,
+#ALTER TABLE alignments ADD TABLE alignments_1;
+#SELECT COUNT(*) FROM alignments;
+% bam.L1 # table_name
+% L1 # name
+% wrd # type
+% 2 # length
+[ 71   ]
+#ALTER TABLE alignments ADD TABLE alignments_2;
+#SELECT COUNT(*) FROM alignments;
+% .L1 # table_name
+% L1 # name
+% wrd # type
+% 3 # length
+[ 142  ]
+
+# 18:59:22   
+# 18:59:22   Done.
+# 18:59:22   
 
 # 08:29:33   
 # 08:29:33   Done.
diff --git a/sql/backends/monet5/bam/Tests/sql/mergetable.sql 
b/sql/backends/monet5/bam/Tests/sql/mergetable.sql
--- a/sql/backends/monet5/bam/Tests/sql/mergetable.sql
+++ b/sql/backends/monet5/bam/Tests/sql/mergetable.sql
@@ -16,7 +16,8 @@ CREATE MERGE TABLE alignments (
pnext  INT   NOT NULL,
tlen   INT   NOT NULL,
seqSTRINGNOT NULL,
-   qual   STRINGNOT NULL
+   qual   STRINGNOT NULL,
+   CONSTRAINT alignments_pkey_virtual_offset PRIMARY KEY 
(virtual_offset)
 );
 
 SELECT COUNT(*) FROM alignments;
diff --git a/sql/server/rel_schema.c b/sql/server/rel_schema.c
--- a/sql/server/rel_schema.c
+++ b/sql/server/rel_schema.c
@@ -1161,7 +1161,7 @@ rel_alter_table(mvc *sql, dlist *qname, 
 
/* check tables */
if (nnt) {
-   if (rel_check_tables(sql, nt, nnt)  0)
+   if (rel_check_tables(sql, t, nnt)  0)
return NULL;
cs_add(nt-tables, nnt, TR_NEW); 
}
diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c
--- a/sql/server/rel_select.c
+++ b/sql/server/rel_select.c
@@ -1756,7 +1756,7 @@ table_ref(mvc *sql, sql_rel *rel, symbol
return rel;
}
if ((isMergeTable(t) || isReplicaTable(t))  
list_empty(t-tables.set))
-   return sql_error(sql, 02, Unable to query empty Merge 
or Replica tables);
+   return sql_error(sql, 02, MERGE or REPLICA TABLE 
should have at least one table associated);
 
return rel_basetable(sql, t, 

MonetDB: bamloader - Merge with default

2014-10-30 Thread Robin Cijvat
Changeset: cd686acf3134 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cd686acf3134
Removed Files:
clients/R/MonetDB.R/src/Makevars.win
clients/R/MonetDB.R/src/mapi.c
Modified Files:
clients/R/MonetDB.R/NEWS
clients/R/MonetDB.R/R/dbi.R
clients/R/MonetDB.R/R/mapi.R
clients/R/Tests/dbi.R
clients/R/Tests/dbi.stable.out
clients/R/Tests/dplyr.R
clients/R/Tests/survey.R
clients/Tests/MAL-signatures_all.stable.out
clients/Tests/MAL-signatures_fits_geom.stable.out
clients/Tests/MAL-signatures_geom.stable.out
clients/Tests/MAL-signatures_none.stable.out
clients/Tests/exports.stable.out
gdk/gdk_firstn.c
monetdb5/modules/kernel/bat5.mal
monetdb5/modules/mal/Tests/inspect05.stable.out
monetdb5/modules/mal/Tests/pqueue.stable.out
monetdb5/modules/mal/Tests/pqueue2.stable.out
monetdb5/modules/mal/mdb.c
monetdb5/modules/mal/mdb.h
monetdb5/optimizer/opt_mergetable.c
monetdb5/tests/gdkTests/Tests/All
monetdb5/tests/gdkTests/Tests/firstn.stable.out
sql/backends/monet5/bam/bam_db_interface.h
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_scenario.h
sql/backends/monet5/sql_statistics.c
sql/backends/monet5/vaults/fits.c
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out
Branch: bamloader
Log Message:

Merge with default


diffs (truncated from 1909 to 300 lines):

diff --git a/clients/R/MonetDB.R/NEWS b/clients/R/MonetDB.R/NEWS
--- a/clients/R/MonetDB.R/NEWS
+++ b/clients/R/MonetDB.R/NEWS
@@ -1,3 +1,7 @@
+0.9.6
+- Fixed non-ASCII character handling (thanks, Roman!)
+- Fully removed C-based socket code
+
 0.9.5
 - Removed package date (Thanks, Dimitar)
 - Added sys. schema name to internal queries, so SET SCHEMA xx would not break 
things (Thanks again, Dimitar)
diff --git a/clients/R/MonetDB.R/R/dbi.R b/clients/R/MonetDB.R/R/dbi.R
--- a/clients/R/MonetDB.R/R/dbi.R
+++ b/clients/R/MonetDB.R/R/dbi.R
@@ -1,12 +1,6 @@
+# C library that contains our MAPI string splitting state machine
 C_LIBRARY - MonetDB.R
 
-.onLoad - function(lib, pkg) {
-  if (getOption(monetdb.clib, FALSE)) {
-library.dynam( C_LIBRARY, pkg, lib )
-.Call(mapiInit, PACKAGE=C_LIBRARY)
-  }
-}
-
 # Make S4 aware of S3 classes
 setOldClass(c(sockconn, connection, monetdb_mapi_conn))
 
@@ -234,6 +228,8 @@ setMethod(dbSendQuery, signature(conn=
 }  
 conn@connenv$exception - list()
 env - NULL
+# Auto-convert? 
+# statement - enc2utf8(statement)
 if (getOption(monetdb.debug.query, F))  message(QQ: ', 
statement, ')
 resp - .mapiParseResponse(.mapiRequest(conn, paste0(s, 
statement, ;), async=async))
 
@@ -539,8 +535,10 @@ setMethod(dbFetch, signature(res=Mone
 col - ct[[j]]
 if (col == .CT_NUM) 
   df[[j]] - as.numeric(parts[[j]])
-if (col == .CT_CHRR) 
+if (col == .CT_CHRR) {
   df[[j]] - parts[[j]]
+  Encoding(df[[j]]) - UTF-8
+}
 if (col == .CT_BOOL) 
   df[[j]] - parts[[j]]==true
 if (col == .CT_CHR) { 
diff --git a/clients/R/MonetDB.R/R/mapi.R b/clients/R/MonetDB.R/R/mapi.R
--- a/clients/R/MonetDB.R/R/mapi.R
+++ b/clients/R/MonetDB.R/R/mapi.R
@@ -86,20 +86,11 @@ REPLY_SIZE- 100 # Apparently, -1 me
 
 
 .mapiConnect - function(host, port, timeout) {
-  if (getOption(monetdb.clib, FALSE)) {
-return(.Call(mapiConnect, host, port, timeout, PACKAGE=C_LIBRARY))
-  } else {
-return(socketConnection(host = host, port = port, blocking = TRUE, 
open=r+b, timeout = timeout))
-  }
+  socketConnection(host = host, port = port, blocking = TRUE, open=r+b, 
timeout = timeout)
 }
 
 .mapiDisconnect - function(socket) {
-  if (getOption(monetdb.clib, FALSE)) {
-.Call(mapiDisconnect, socket, PACKAGE=C_LIBRARY)
-  } else {
-  # close, don't care about the errors...
-  tryCatch(close(socket), error=function(e){}, warning=function(w){})
-  }
+  tryCatch(close(socket), error=function(e){}, warning=function(w){})
 }
 
 .mapiCleanup - function(conObj) {
@@ -110,69 +101,48 @@ REPLY_SIZE- 100 # Apparently, -1 me
 }
 
 .mapiRead - function(con) {
-  if (getOption(monetdb.clib, FALSE)) {
-if (!identical(class(con)[[1]], externalptr))
-  stop(I can only be called with a MonetDB connection object as 
parameter.)
-respstr - .Call(mapiRead, con, PACKAGE=C_LIBRARY)
-if (getOption(monetdb.debug.mapi, F)) {
-  dstr - respstr
-  if (nchar(dstr)  300) {
-dstr - paste0(substring(dstr, 1, 200), ..., substring(dstr, 
nchar(dstr)-100, nchar(dstr))) 
-  } 
-  message(RX: ', dstr, ')
+  if (!identical(class(con)[[1]], sockconn))
+stop(I can only be called with a MonetDB connection object as parameter.)
+  resp - list()

MonetDB: default - Let close accept a callback that gets called ...

2014-10-30 Thread Robin Cijvat
Changeset: e1d4ab77171c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1d4ab77171c
Modified Files:
clients/nodejs/monetdb/mapiclient.js
Branch: default
Log Message:

Let close accept a callback that gets called when socket.destroy is called, and 
create Q based disconnect and close function


diffs (38 lines):

diff --git a/clients/nodejs/monetdb/mapiclient.js 
b/clients/nodejs/monetdb/mapiclient.js
--- a/clients/nodejs/monetdb/mapiclient.js
+++ b/clients/nodejs/monetdb/mapiclient.js
@@ -12,6 +12,7 @@ function MonetDBConnection(options, conn
this.conn_callback = conncallback;
this.mapi_blocksize = 8192;
this.do_close = false;
+   this.close_callback = null;
this.alldone = false;
 
this.queryqueue = [];
@@ -163,7 +164,8 @@ MonetDBConnection.prototype.prepare = fu
 }
 
 MonetDBConnection.prototype.disconnect =
-MonetDBConnection.prototype.close = function() {
+MonetDBConnection.prototype.close = function(callback) {
+   this.close_callback = callback;
this.do_close = true;
if (this.queryqueue.length  1) {
next_op.call(this);
@@ -238,6 +240,7 @@ function next_op() {
this.alldone = true;
if (this.do_close) {
this.socket.destroy();
+   this.close_callback  this.close_callback(null);
}
return;
}
@@ -483,7 +486,7 @@ function __get_connect_args(options) {
 }
 
 /* Q integration, robin.cij...@monetdbsolutions.com */
-['request', 'query', 'prepare'].forEach(function(funToQ) {
+['request', 'query', 'prepare', 'close', 
'disconnect'].forEach(function(funToQ) {
var funQ = funToQ + 'Q';
MonetDBConnection.prototype[funQ] = function() {
if (this.options.q == undefined) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merge with default

2014-10-21 Thread Robin Cijvat
Changeset: 7588bd33711f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7588bd33711f
Added Files:
sql/backends/monet5/bam/Tests/mergetable.SQL.py
sql/backends/monet5/bam/Tests/sql/mergetable.sql
Modified Files:
sql/backends/monet5/bam/Tests/All
sql/backends/monet5/sql.c
sql/server/rel_schema.c
sql/storage/sql_storage.h
sql/storage/store.c
sql/storage/store_dependency.c
sql/test/mergetables/Tests/All
Branch: bamloader
Log Message:

Merge with default


diffs (196 lines):

diff --git a/sql/backends/monet5/bam/Tests/All 
b/sql/backends/monet5/bam/Tests/All
--- a/sql/backends/monet5/bam/Tests/All
+++ b/sql/backends/monet5/bam/Tests/All
@@ -24,3 +24,4 @@ HAVE_SAMTOOLS?query2.9
 HAVE_SAMTOOLS?query2.10
 HAVE_SAMTOOLS?query2.11
 HAVE_SAMTOOLS?query2.12
+HAVE_SAMTOOLS?mergetable
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/mergetable.SQL.py 
b/sql/backends/monet5/bam/Tests/mergetable.SQL.py
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/mergetable.SQL.py
@@ -0,0 +1,3 @@
+import bam
+
+bam.exec_sql_file(mergetable.sql, {'PWD': bam.SRCDIR})
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/sql/mergetable.sql 
b/sql/backends/monet5/bam/Tests/sql/mergetable.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/sql/mergetable.sql
@@ -0,0 +1,30 @@
+SET SCHEMA bam;
+
+# Load files
+CALL bam_loader_repos('PWD/files', 0, 4);
+
+# Add a merge table over these files
+CREATE MERGE TABLE alignments (
+   virtual_offset BIGINTNOT NULL,
+   qname  STRINGNOT NULL,
+   flag   SMALLINT  NOT NULL,
+   rname  STRINGNOT NULL,
+   posINT   NOT NULL,
+   mapq   SMALLINT  NOT NULL,
+   cigar  STRINGNOT NULL,
+   rnext  STRINGNOT NULL,
+   pnext  INT   NOT NULL,
+   tlen   INT   NOT NULL,
+   seqSTRINGNOT NULL,
+   qual   STRINGNOT NULL
+);
+
+SELECT COUNT(*) FROM alignments;
+
+ALTER TABLE alignments ADD TABLE alignments_1;
+
+SELECT COUNT(*) FROM alignments;
+
+ALTER TABLE alignments ADD TABLE alignments_2;
+
+SELECT COUNT(*) FROM alignments;
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -564,7 +564,7 @@ alter_table(mvc *sql, char *sname, sql_t
sql_column *c = n-data;
sql_column *nc = mvc_bind_column(sql, nt, c-base.name);
 
-   if (c-null != nc-null) {
+   if (c-null != nc-null  isTable(nt)) {
mvc_null(sql, nc, c-null);
/* for non empty check for nulls */
if (c-null == 0) {
diff --git a/sql/server/rel_schema.c b/sql/server/rel_schema.c
--- a/sql/server/rel_schema.c
+++ b/sql/server/rel_schema.c
@@ -604,7 +604,7 @@ table_element(mvc *sql, symbol *s, sql_s
 {
int res = SQL_OK;
 
-   if (alter  (isView(t) || ((isMergeTable(t) || isReplicaTable(t))  
s-token != SQL_TABLE  s-token != SQL_DROP_TABLE) || (isTable(t)  
(s-token == SQL_TABLE || s-token == SQL_DROP_TABLE)) )){
+   if (alter  (isView(t) || ((isMergeTable(t) || isReplicaTable(t))  
(s-token != SQL_TABLE  s-token != SQL_DROP_TABLE  cs_size(t-tables)0)) 
|| (isTable(t)  (s-token == SQL_TABLE || s-token == SQL_DROP_TABLE)) )){
char *msg = ;
 
switch (s-token) {
@@ -1072,6 +1072,21 @@ rel_check_tables(mvc *sql, sql_table *nt
return -2;
}
}
+   if (cs_size(nt-idxs) != cs_size(nnt-idxs)) {
+   (void) sql_error(sql, 02, 3F000!ALTER MERGE TABLE: to be added 
table index doesn't match MERGE TABLE definition);
+   return -1;
+   }
+   if (cs_size(nt-idxs))
+   for (n = nt-idxs.set-h, m = nnt-idxs.set-h; n  m; n = n-next, m 
= m-next) {
+   sql_idx *ni = n-data;
+   sql_idx *mi = m-data;
+
+   /* todo check def */
+   if (ni-type != mi-type) {
+   (void) sql_error(sql, 02, 3F000!ALTER MERGE TABLE: to 
be added table index type doesn't match MERGE TABLE definition);
+   return -2;
+   }
+   }
return 0;
 }
 
@@ -1120,6 +1135,9 @@ rel_alter_table(mvc *sql, dlist *qname, 
if (t-s  !nt-s)
nt-s = t-s;
 
+   if (nt-type == tt_merge_table)
+   return sql_error(sql, 02, 42S02!ALTER TABLE: 
read only MERGE TABLES are not supported);
+
if (state == tr_readonly) {
nt = mvc_readonly(sql, nt, 1);
} else {
@@ -1162,6 +1180,9 @@ 

MonetDB: default - Newline at end of files

2014-10-21 Thread Robin Cijvat
Changeset: 37c1b2ce60e9 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=37c1b2ce60e9
Modified Files:
sql/backends/monet5/bam/Tests/mergetable.stable.err
sql/backends/monet5/bam/Tests/mergetable.stable.out
Branch: default
Log Message:

Newline at end of files


diffs (20 lines):

diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.err 
b/sql/backends/monet5/bam/Tests/mergetable.stable.err
--- a/sql/backends/monet5/bam/Tests/mergetable.stable.err
+++ b/sql/backends/monet5/bam/Tests/mergetable.stable.err
@@ -136,4 +136,4 @@ ERROR = !Unable to query empty Merge or 
 
 # 08:29:33   
 # 08:29:33   Done.
-# 08:29:33  
\ No newline at end of file
+# 08:29:33  
diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.out 
b/sql/backends/monet5/bam/Tests/mergetable.stable.out
--- a/sql/backends/monet5/bam/Tests/mergetable.stable.out
+++ b/sql/backends/monet5/bam/Tests/mergetable.stable.out
@@ -72,4 +72,4 @@ Ready.
 
 # 08:29:33   
 # 08:29:33   Done.
-# 08:29:33   
\ No newline at end of file
+# 08:29:33   
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Added key constraint on mergetable definition...

2014-10-21 Thread Robin Cijvat
Changeset: bf3858286beb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bf3858286beb
Modified Files:
sql/backends/monet5/bam/Tests/sql/mergetable.sql
Branch: default
Log Message:

Added key constraint on mergetable definition in mergetable test


diffs (13 lines):

diff --git a/sql/backends/monet5/bam/Tests/sql/mergetable.sql 
b/sql/backends/monet5/bam/Tests/sql/mergetable.sql
--- a/sql/backends/monet5/bam/Tests/sql/mergetable.sql
+++ b/sql/backends/monet5/bam/Tests/sql/mergetable.sql
@@ -16,7 +16,8 @@ CREATE MERGE TABLE alignments (
pnext  INT   NOT NULL,
tlen   INT   NOT NULL,
seqSTRINGNOT NULL,
-   qual   STRINGNOT NULL
+   qual   STRINGNOT NULL,
+   CONSTRAINT alignments_pkey_virtual_offset PRIMARY KEY 
(virtual_offset)
 );
 
 SELECT COUNT(*) FROM alignments;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merge with default

2014-10-21 Thread Robin Cijvat
Changeset: a42705485166 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a42705485166
Added Files:
sql/backends/monet5/bam/Tests/mergetable.stable.err
sql/backends/monet5/bam/Tests/mergetable.stable.out
Branch: bamloader
Log Message:

Merge with default


diffs (224 lines):

diff --git a/sql/backends/monet5/bam/Tests/mergetable.stable.err 
b/sql/backends/monet5/bam/Tests/mergetable.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/mergetable.stable.err
@@ -0,0 +1,139 @@
+stderr of test 'mergetable` in directory 'sql/backends/monet5/bam` itself:
+
+
+# 08:29:32   
+# 08:29:32   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=37321 --set 
mapi_usock=/var/tmp/mtest-10986/.s.monetdb.37321 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0 --set embedded_r=yes
+# 08:29:32   
+
+# builtin opt  gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = 
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37321
+# cmdline opt  mapi_usock = /var/tmp/mtest-10986/.s.monetdb.37321
+# cmdline opt  monet_prompt = 
+# cmdline opt  mal_listing = 2
+# cmdline opt  gdk_dbpath = 
/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
+# cmdline opt  mal_listing = 0
+# cmdline opt  embedded_r = yes
+# cmdline opt  gdk_debug = 536870922
+# bam_loader: Loader started for 2 BAM files...
+# # bam_loader Retrieving next file id...
+# # bam_loader Initializing BAM wrapper for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file2.sam'...
+# # bam_loader Initializing BAM wrapper for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
+# # bam_loader Parsing header for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file2.sam'...
+# # bam_loader Parsing header for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
+# # bam_loader Creating alignment tables for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file2.sam'...
+# # CREATE TABLE bam.alignments_1 ( 
+#  virtual_offset  BIGINTNOT NULL, 
+#  qnameSTRING   NOT 
NULL, 
+#  flag  SMALLINT  
NOT NULL, 
+#  rnameSTRING   NOT 
NULL, 
+#  posINT  
 NOT NULL, 
+#  mapq  SMALLINT  
NOT NULL, 
+#  cigarSTRING   NOT 
NULL, 
+#  rnextSTRING   NOT 
NULL, 
+#  pnextINT
 NOT NULL, 
+#  tlen  INT   
 NOT NULL, 
+#  seqSTRING   
  NOT NULL, 
+#  qual  STRING
  NOT NULL, 
+#  CONSTRAINT alignments_1_pkey_virtual_offset PRIMARY KEY 
(virtual_offset) 
+#  ); 
+#  
+#  CREATE TABLE bam.alignments_extra_1 ( 
+#  tagCHAR(2)  
 NOT NULL, 
+#  virtual_offset  BIGINTNOT NULL, 
+#  type  CHAR(1)   
 NOT NULL, 
+#  valueSTRING, 
+#  CONSTRAINT alignments_extra_1_pkey_tag_virtual_offset PRIMARY 
KEY (tag, virtual_offset), 
+#  CONSTRAINT alignments_extra_1_fkey_virtual_offset FOREIGN KEY 
(virtual_offset) 
+#  REFERENCES bam.alignments_1 (virtual_offset) 
+#  );
+# # bam_loader Creating alignment tables for file 
'/home/robin/MonetDB/SOURCE/sql/backends/monet5/bam/Tests/files/file1.bam'...
+# # CREATE TABLE bam.alignments_2 ( 
+#  virtual_offset  BIGINTNOT NULL, 
+#  qnameSTRING   NOT 
NULL, 
+#  flag  SMALLINT  
NOT NULL, 
+#  rnameSTRING   NOT 
NULL, 
+#  pos   

MonetDB: bamloader - Added UDF for calculating the character fro...

2014-10-20 Thread Robin Cijvat
Changeset: 84ebb8ecd572 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=84ebb8ecd572
Modified Files:
sql/backends/monet5/bam/85_bam.sql
sql/backends/monet5/bam/bam.mal
sql/backends/monet5/bam/bam_lib.c
sql/backends/monet5/bam/bam_lib.h
Branch: bamloader
Log Message:

Added UDF for calculating the character from an alignment that maps to some 
position in the reference string, given this position and the alignment pos, 
cigar and seq fields


diffs (182 lines):

diff --git a/sql/backends/monet5/bam/85_bam.sql 
b/sql/backends/monet5/bam/85_bam.sql
--- a/sql/backends/monet5/bam/85_bam.sql
+++ b/sql/backends/monet5/bam/85_bam.sql
@@ -25,6 +25,9 @@ RETURNS STRING EXTERNAL NAME bam.reverse
 CREATE FUNCTION bam.seq_length(cigar STRING)
 RETURNS INT EXTERNAL NAME bam.seq_length;
 
+CREATE FUNCTION bam.seq_char(ref_pos INT, alg_seq STRING, alg_pos INT, 
alg_cigar STRING)
+RETURNS CHAR(1) EXTERNAL NAME bam.seq_char;
+
 
 CREATE PROCEDURE bam.sam_export(output_path STRING)
 EXTERNAL NAME bam.sam_export;
diff --git a/sql/backends/monet5/bam/bam.mal b/sql/backends/monet5/bam/bam.mal
--- a/sql/backends/monet5/bam/bam.mal
+++ b/sql/backends/monet5/bam/bam.mal
@@ -38,6 +38,10 @@ command seq_length(cigar:str):int
 address seq_length
 comment Calculate the real length of a DNA sequence, given its CIGAR string.
 
+command seq_char(ref_pos:int, alg_seq:str, alg_pos:int, alg_cigar:str):str
+address seq_char
+comment Calculate the character in the alignment string (alg_str) that is 
aligned to position 'ref_pos', conforming to the given cigar string
+
 
 # Export signatures
 
@@ -69,3 +73,7 @@ comment Reverse a bat of DNA Quality st
 command seq_length(cigars:bat[:oid,:str]):bat[:oid,:int]
 address seq_length_bat
 comment Calculate the real length of a bat of DNA sequences, given their 
CIGAR string.
+
+command seq_char(ref_pos:int, alg_seq:bat[:oid,:str], alg_pos:bat[:oid,:int], 
alg_cigar:bat[:oid,:str]):bat[:oid,:str]
+address seq_char
+comment Calculate the character in the alignment string (alg_str) that is 
aligned to position 'ref_pos', conforming to the given cigar string (bat based 
version)
diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -141,7 +141,7 @@ reverse_qual(str * ret, str * qual)
 }
 
 str
-seq_length(int *ret, str * cigar)
+seq_length(int * ret, str * cigar)
 {
int result = 0;
str cigar_consumable = *cigar;
@@ -169,6 +169,52 @@ seq_length(int *ret, str * cigar)
return MAL_SUCCEED;
 }
 
+str
+seq_char(str * ret, int * ref_pos, str * alg_seq, int * alg_pos, str * 
alg_cigar) 
+{
+   str cigar_consumable = *alg_cigar;
+   int seq_pos = 0;
+   bit at_character = false;
+   int iterate_until = *ref_pos - *alg_pos;
+   str result;
+
+   if((result = GDKmalloc(2 * sizeof(char))) == NULL) {
+   throw(MAL, seq_char, MAL_MALLOC_FAIL);
+   }
+   result[1] = '\0';
+   
+   if (cigar_consumable[0] == '*'  cigar_consumable[1] == '\0') {
+   result[0] = '\0';
+   *ret = result;
+   return MAL_SUCCEED;
+   }
+   while(cigar_consumable[0] != '\0'  seq_pos  iterate_until) {
+   int cnt;
+   char op;
+   int nr_chars_read;
+
+   if (sscanf
+   (cigar_consumable, %d%c%n, cnt, op,
+nr_chars_read) != 2)
+   throw(MAL, seq_char,
+ Error parsing CIGAR string '%s'\n, 
*alg_cigar);
+   if (op == 'M' || op == 'D' || op == 'N' || op == '='
+   || op == 'X') {
+   seq_pos += cnt;
+   if(seq_pos  iterate_until) 
+   seq_pos = iterate_until;
+   at_character = true;
+   } else {
+   at_character = false;
+   }
+   cigar_consumable += nr_chars_read;
+   }
+   result[0] = at_character ? (*alg_seq)[seq_pos] : '\0';
+   *ret = result;
+   return MAL_SUCCEED;
+}
+
+
 
 
 str
@@ -335,3 +381,65 @@ seq_length_bat(bat * ret, bat * bid)
 
return MAL_SUCCEED;
 }
+
+
+str
+seq_char_bat(bat * ret, int * ref_pos, bat * alg_seq, bat * alg_pos, bat * 
alg_cigar)
+{
+   BAT *seqs, *poss, *cigars, *result;
+   BUN seq = 0, pos = 0, cigar = 0, seq_end = 0;
+   BATiter seq_it, pos_it, cigar_it;
+
+   assert(ret != NULL  ref_pos != NULL  alg_seq != NULL  alg_pos != 
NULL  alg_cigar != NULL);
+
+   if ((seqs = BATdescriptor(*alg_seq)) == NULL ||
+   (poss = BATdescriptor(*alg_pos)) == NULL ||
+   (cigars = BATdescriptor(*alg_cigar)) == NULL) 
+   throw(MAL, seq_char_bat, RUNTIME_OBJECT_MISSING);
+
+   if(BATcount(seqs) != BATcount(poss) || BATcount(seqs) != 
BATcount(cigars)) {

MonetDB: bamloader - Fixed erroneous function reference

2014-10-20 Thread Robin Cijvat
Changeset: cd4153c81e49 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cd4153c81e49
Modified Files:
sql/backends/monet5/bam/bam.mal
Branch: bamloader
Log Message:

Fixed erroneous function reference


diffs (10 lines):

diff --git a/sql/backends/monet5/bam/bam.mal b/sql/backends/monet5/bam/bam.mal
--- a/sql/backends/monet5/bam/bam.mal
+++ b/sql/backends/monet5/bam/bam.mal
@@ -75,5 +75,5 @@ address seq_length_bat
 comment Calculate the real length of a bat of DNA sequences, given their 
CIGAR string.
 
 command seq_char(ref_pos:int, alg_seq:bat[:oid,:str], alg_pos:bat[:oid,:int], 
alg_cigar:bat[:oid,:str]):bat[:oid,:str]
-address seq_char
+address seq_char_bat
 comment Calculate the character in the alignment string (alg_str) that is 
aligned to position 'ref_pos', conforming to the given cigar string (bat based 
version)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Make seq_char also work properly on inserti...

2014-10-20 Thread Robin Cijvat
Changeset: 3ed973da6129 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3ed973da6129
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Make seq_char also work properly on insertions/deletions that occur in the 
CIGAR string. Furthermore, add some extra error checks


diffs (114 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -146,7 +146,8 @@ seq_length(int * ret, str * cigar)
int result = 0;
str cigar_consumable = *cigar;
 
-   if (cigar_consumable[0] == '*'  cigar_consumable[1] == '\0') {
+   if (cigar_consumable[0] == '\0' || 
+   (cigar_consumable[0] == '*'  cigar_consumable[1] == 
'\0')) {
*ret = -1;
return MAL_SUCCEED;
}
@@ -173,44 +174,58 @@ str
 seq_char(str * ret, int * ref_pos, str * alg_seq, int * alg_pos, str * 
alg_cigar) 
 {
str cigar_consumable = *alg_cigar;
-   int seq_pos = 0;
-   bit at_character = false;
-   int iterate_until = *ref_pos - *alg_pos;
-   str result;
-
-   if((result = GDKmalloc(2 * sizeof(char))) == NULL) {
-   throw(MAL, seq_char, MAL_MALLOC_FAIL);
-   }
-   result[1] = '\0';
+   int seq_pos = -1;
+   int cur_ref_pos = *alg_pos - 1;

-   if (cigar_consumable[0] == '*'  cigar_consumable[1] == '\0') {
-   result[0] = '\0';
-   *ret = result;
+   if (cigar_consumable[0] == '\0' || 
+   (cigar_consumable[0] == '*'  cigar_consumable[1] == 
'\0')) {
+   *ret = GDKstrdup(str_nil);
return MAL_SUCCEED;
}
-   while(cigar_consumable[0] != '\0'  seq_pos  iterate_until) {
+   while(TRUE) {
int cnt;
char op;
int nr_chars_read;
+   bit advance_ref_pos;
+   bit advance_seq_pos;
 
if (sscanf
(cigar_consumable, %d%c%n, cnt, op,
 nr_chars_read) != 2)
throw(MAL, seq_char,
  Error parsing CIGAR string '%s'\n, 
*alg_cigar);
-   if (op == 'M' || op == 'D' || op == 'N' || op == '='
-   || op == 'X') {
+   advance_ref_pos = (op == 'M' || op == 'D' || 
+   op == 'N' || op == '=' || op == 'X');
+   advance_seq_pos = (op == 'M' || op == 'I'); // TODO: Find out 
which chars advance the seq pos
+   if(advance_seq_pos) {
seq_pos += cnt;
-   if(seq_pos  iterate_until) 
-   seq_pos = iterate_until;
-   at_character = true;
-   } else {
-   at_character = false;
+   }
+   if (advance_ref_pos) {
+   cur_ref_pos += cnt;
+   if(cur_ref_pos = *ref_pos) {
+   if(!advance_seq_pos) {
+   seq_pos = -1;
+   } else {
+   seq_pos -= (cur_ref_pos - *ref_pos);
+   }
+   break;
+   }
}
cigar_consumable += nr_chars_read;
+   if(cigar_consumable[0] == '\0') {
+   seq_pos = -1;
+   break;
+   }
}
-   result[0] = at_character ? (*alg_seq)[seq_pos] : '\0';
-   *ret = result;
+   if(seq_pos  0 || seq_pos = (int)strlen(*alg_seq)) {
+   *ret = GDKstrdup(str_nil);
+   return MAL_SUCCEED;
+   }
+   if(((*ret) = GDKmalloc(2 * sizeof(char))) == NULL) {
+   throw(MAL, seq_char, MAL_MALLOC_FAIL);
+   }
+   (*ret)[0] = (*alg_seq)[seq_pos];
+   (*ret)[1] = '\0';
return MAL_SUCCEED;
 }
 
@@ -398,7 +413,9 @@ seq_char_bat(bat * ret, int * ref_pos, b
throw(MAL, seq_char_bat, RUNTIME_OBJECT_MISSING);
 
if(BATcount(seqs) != BATcount(poss) || BATcount(seqs) != 
BATcount(cigars)) {
-   throw(MAL, seq_char_bat, Misalignment in input BATs);
+   throw(MAL, seq_char_bat, 
+   Misalignment in input BATs: BUNFMT/BUNFMT/BUNFMT, 
+   BATcount(poss), BATcount(seqs), BATcount(cigars));
}
 
/* allocate result BAT */
@@ -428,7 +445,7 @@ seq_char_bat(bat * ret, int * ref_pos, b
BBPreleaseref(result-batCacheid);
return msg;
}
-   BUNappend(result, (ptr) r, FALSE);
+   BUNappend(result, (ptr) r, FALSE);
++seq;
++pos;
++cigar;

MonetDB: bamloader - Merge with default

2014-10-20 Thread Robin Cijvat
Changeset: 0eef8c674c73 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eef8c674c73
Added Files:
sql/test/BugTracker-2014/Tests/queueError.Bug-3604.sql
sql/test/BugTracker-2014/Tests/queueError.Bug-3604.stable.err
sql/test/BugTracker-2014/Tests/queueError.Bug-3604.stable.out
sql/test/mergetables/Tests/alter.sql
sql/test/mergetables/Tests/emptymergequery.sql
sql/test/mergetables/Tests/mergekey.sql
sql/test/mergetables/Tests/types.sql
Modified Files:
clients/R/Tests/install-dependencies.sh
clients/Tests/MAL-signatures_all.stable.out
clients/Tests/MAL-signatures_all.stable.out.int128
clients/Tests/MAL-signatures_fits_geom.stable.out
clients/Tests/MAL-signatures_fits_geom.stable.out.int128
clients/Tests/MAL-signatures_geom.stable.out
clients/Tests/MAL-signatures_geom.stable.out.int128
clients/Tests/MAL-signatures_none.stable.out
clients/Tests/MAL-signatures_none.stable.out.int128
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_bbp.c
gdk/gdk_cross.c
gdk/gdk_group.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_join.c
gdk/gdk_private.h
gdk/gdk_search.c
gdk/gdk_select.c
gdk/gdk_storage.c
gdk/gdk_unique.c
java/src/nl/cwi/monetdb/jdbc/MonetDatabaseMetaData.java
monetdb5/extras/rdf/rdf.h
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_linker.c
monetdb5/mal/mal_linker.h
monetdb5/mal/mal_private.h
monetdb5/modules/atoms/blob.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/mtime.h
monetdb5/modules/atoms/str.c
monetdb5/modules/atoms/str.h
monetdb5/modules/kernel/aggr.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/algebra.h
monetdb5/modules/kernel/bat5.c
monetdb5/modules/kernel/bat5.h
monetdb5/modules/kernel/batcolor.mal
monetdb5/modules/kernel/batstr.c
monetdb5/modules/kernel/counters.c
monetdb5/modules/kernel/counters.mal
monetdb5/modules/mal/batExtensions.c
monetdb5/modules/mal/batExtensions.h
monetdb5/modules/mal/batcalc.c
monetdb5/modules/mal/bbp.h
monetdb5/modules/mal/factories.c
monetdb5/modules/mal/factories.h
monetdb5/modules/mal/inspect.h
monetdb5/modules/mal/language.c
monetdb5/modules/mal/language.h
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/mal_mapi.h
monetdb5/modules/mal/mat.c
monetdb5/modules/mal/mat.h
monetdb5/modules/mal/mdb.c
monetdb5/modules/mal/mdb.h
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/pcre.mal
monetdb5/modules/mal/profiler.c
monetdb5/modules/mal/profiler.h
monetdb5/modules/mal/transaction.c
sql/backends/monet5/Tests/All
sql/backends/monet5/datacell/basket.h
sql/backends/monet5/datacell/petrinet.c
sql/backends/monet5/generator/generator.c
sql/backends/monet5/rest/rest_jsonstore_handle_get.h
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
sql/backends/monet5/sql.mal
sql/backends/monet5/sql_cast.c
sql/backends/monet5/sql_cast.h
sql/backends/monet5/sql_cast_impl_down_from_flt.h
sql/backends/monet5/sql_cast_impl_down_from_int.h
sql/backends/monet5/sql_cast_impl_up_to_flt.h
sql/backends/monet5/sql_cast_impl_up_to_int.h
sql/backends/monet5/sql_gencode.h
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_scenario.h
sql/benchmarks/ssbm/Tests/01-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/02-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/03-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/04-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/05-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/06-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/07-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/08-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/09-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/10-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/11-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/12-explain.stable.out.int128
sql/benchmarks/ssbm/Tests/13-explain.stable.out.int128
sql/server/rel_planner.c
sql/server/rel_schema.c
sql/test/BugTracker-2014/Tests/All
sql/test/leaks/Tests/check0_nogeom.stable.out
sql/test/leaks/Tests/check3_nogeom.stable.out
sql/test/leaks/Tests/check4_nogeom.stable.out
sql/test/leaks/Tests/check5_nogeom.stable.out
sql/test/leaks/Tests/select1_nogeom.stable.out

MonetDB: default - Add test case for creating a merge table over...

2014-10-20 Thread Robin Cijvat
Changeset: 6cb7f375f65d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6cb7f375f65d
Added Files:
sql/backends/monet5/bam/Tests/mergetable.SQL.py
sql/backends/monet5/bam/Tests/sql/mergetable.sql
Modified Files:
sql/backends/monet5/bam/Tests/All
Branch: default
Log Message:

Add test case for creating a merge table over two BAM files, which causes a 
server crash


diffs (53 lines):

diff --git a/sql/backends/monet5/bam/Tests/All 
b/sql/backends/monet5/bam/Tests/All
--- a/sql/backends/monet5/bam/Tests/All
+++ b/sql/backends/monet5/bam/Tests/All
@@ -24,3 +24,4 @@ HAVE_SAMTOOLS?query2.9
 HAVE_SAMTOOLS?query2.10
 HAVE_SAMTOOLS?query2.11
 HAVE_SAMTOOLS?query2.12
+HAVE_SAMTOOLS?mergetable
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/mergetable.SQL.py 
b/sql/backends/monet5/bam/Tests/mergetable.SQL.py
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/mergetable.SQL.py
@@ -0,0 +1,3 @@
+import bam
+
+bam.exec_sql_file(mergetable.sql, {'PWD': bam.SRCDIR})
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/sql/mergetable.sql 
b/sql/backends/monet5/bam/Tests/sql/mergetable.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/bam/Tests/sql/mergetable.sql
@@ -0,0 +1,30 @@
+SET SCHEMA bam;
+
+# Load files
+CALL bam_loader_repos('PWD/files', 0, 4);
+
+# Add a merge table over these files
+CREATE MERGE TABLE alignments (
+   virtual_offset BIGINTNOT NULL,
+   qname  STRINGNOT NULL,
+   flag   SMALLINT  NOT NULL,
+   rname  STRINGNOT NULL,
+   posINT   NOT NULL,
+   mapq   SMALLINT  NOT NULL,
+   cigar  STRINGNOT NULL,
+   rnext  STRINGNOT NULL,
+   pnext  INT   NOT NULL,
+   tlen   INT   NOT NULL,
+   seqSTRINGNOT NULL,
+   qual   STRINGNOT NULL
+);
+
+SELECT COUNT(*) FROM alignments;
+
+ALTER TABLE alignments ADD TABLE alignments_1;
+
+SELECT COUNT(*) FROM alignments;
+
+ALTER TABLE alignments ADD TABLE alignments_2;
+
+SELECT COUNT(*) FROM alignments;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: default - Fixed test query 2.4

2014-08-11 Thread Robin Cijvat
Changeset: 8228693a6130 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8228693a6130
Modified Files:
sql/backends/monet5/bam/Tests/query2.4.sql
sql/backends/monet5/bam/Tests/query2.4.stable.out
Branch: default
Log Message:

Fixed test query 2.4


diffs (200 lines):

diff --git a/sql/backends/monet5/bam/Tests/query2.4.sql 
b/sql/backends/monet5/bam/Tests/query2.4.sql
--- a/sql/backends/monet5/bam/Tests/query2.4.sql
+++ b/sql/backends/monet5/bam/Tests/query2.4.sql
@@ -3,29 +3,36 @@ FROM bam.alignments_1
 WHERE qname IN (
 SELECT qname
 FROM (
-SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
-bam.bam_flag(flag, 'firs_segm') AS firs_segm
-FROM bam.alignments_1
-WHERE bam.bam_flag(flag, 'firs_segm')  bam.bam_flag(flag, 
'last_segm')
+SELECT qname, SUM(segm_unma), SUM(seco_alig), COUNT(*)
+FROM (
+SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
+bam.bam_flag(flag, 'firs_segm') AS firs_segm
+FROM bam.alignments_1
+WHERE bam.bam_flag(flag, 'firs_segm')  bam.bam_flag(flag, 
'last_segm')
+) AS sub
+GROUP BY qname, firs_segm
+HAVING SUM(segm_unma)  COUNT(*)
+   AND (COUNT(*) - SUM(seco_alig))  1
 ) AS qnames
-GROUP BY qname, firs_segm
-HAVING SUM(segm_unma)  COUNT(*)
-   AND (COUNT(*) - SUM(seco_alig))  1
-)
-ORDER BY qname;
+);
+
 
 SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
 FROM bam.unpaired_all_alignments_3
 WHERE qname IN (
 SELECT qname
 FROM (
-SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
-bam.bam_flag(flag, 'firs_segm') AS firs_segm
-FROM bam.unpaired_all_alignments_3
-WHERE bam.bam_flag(flag, 'firs_segm')  bam.bam_flag(flag, 
'last_segm')
+SELECT qname, SUM(segm_unma), SUM(seco_alig), COUNT(*)
+FROM (
+SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
+bam.bam_flag(flag, 'firs_segm') AS firs_segm
+FROM bam.unpaired_all_alignments_3
+WHERE bam.bam_flag(flag, 'firs_segm')  bam.bam_flag(flag, 
'last_segm')
+) AS sub
+GROUP BY qname, firs_segm
+HAVING SUM(segm_unma)  COUNT(*)
+   AND (COUNT(*) - SUM(seco_alig))  1
 ) AS qnames
-GROUP BY qname, firs_segm
-HAVING SUM(segm_unma)  COUNT(*)
-   AND (COUNT(*) - SUM(seco_alig))  1
-)
-ORDER BY qname;
+);
+
+
diff --git a/sql/backends/monet5/bam/Tests/query2.4.stable.out 
b/sql/backends/monet5/bam/Tests/query2.4.stable.out
--- a/sql/backends/monet5/bam/Tests/query2.4.stable.out
+++ b/sql/backends/monet5/bam/Tests/query2.4.stable.out
@@ -1,9 +1,9 @@
 stdout of test 'query2.4` in directory 'sql/backends/monet5/bam` itself:
 
 
-# 10:22:42   
-# 10:22:42   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=32986 --set 
mapi_usock=/var/tmp/mtest-16074/.s.monetdb.32986 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
-# 10:22:42   
+# 17:36:06   
+# 17:36:06   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=33676 --set 
mapi_usock=/var/tmp/mtest-11149/.s.monetdb.33676 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
+# 17:36:06   
 
 # MonetDB 5 server v11.18.0
 # This is an unreleased version
@@ -13,29 +13,27 @@ stdout of test 'query2.4` in directory '
 # Copyright (c) 1993-July 2008 CWI.
 # Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved
 # Visit http://www.monetdb.org/ for further information
-# Listening for connection requests on mapi:monetdb://robin-xps13:32986/
-# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-16074/.s.monetdb.32986
+# Listening for connection requests on mapi:monetdb://robin-xps13:33676/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-11149/.s.monetdb.33676
 # MonetDB/SQL module loaded
 
 Ready.
 
-# 10:22:42   
-# 10:22:42   mclient -lsql -ftest -Eutf-8 -i -e 
--host=/var/tmp/mtest-16074 --port=32986
-# 10:22:42   
+# 17:36:06   
+# 17:36:06   mclient -lsql -ftest -Eutf-8 -i -e 
--host=/var/tmp/mtest-11149 --port=33676
+# 17:36:06   
 
 #SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
 #FROM bam.alignments_1
 #WHERE qname IN (
 #SELECT qname
 #FROM (
-#SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
-#bam.bam_flag(flag, 'firs_segm') AS firs_segm
-#FROM 

MonetDB: Oct2014 - Fixed test query 2.4

2014-08-11 Thread Robin Cijvat
Changeset: bcc095a70a73 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bcc095a70a73
Modified Files:
sql/backends/monet5/bam/Tests/query2.4.sql
sql/backends/monet5/bam/Tests/query2.4.stable.out
Branch: Oct2014
Log Message:

Fixed test query 2.4
(grafted from 8228693a61305d8bdf68082a366ab5c7cd2ada10)


diffs (200 lines):

diff --git a/sql/backends/monet5/bam/Tests/query2.4.sql 
b/sql/backends/monet5/bam/Tests/query2.4.sql
--- a/sql/backends/monet5/bam/Tests/query2.4.sql
+++ b/sql/backends/monet5/bam/Tests/query2.4.sql
@@ -3,29 +3,36 @@ FROM bam.alignments_1
 WHERE qname IN (
 SELECT qname
 FROM (
-SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
-bam.bam_flag(flag, 'firs_segm') AS firs_segm
-FROM bam.alignments_1
-WHERE bam.bam_flag(flag, 'firs_segm')  bam.bam_flag(flag, 
'last_segm')
+SELECT qname, SUM(segm_unma), SUM(seco_alig), COUNT(*)
+FROM (
+SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
+bam.bam_flag(flag, 'firs_segm') AS firs_segm
+FROM bam.alignments_1
+WHERE bam.bam_flag(flag, 'firs_segm')  bam.bam_flag(flag, 
'last_segm')
+) AS sub
+GROUP BY qname, firs_segm
+HAVING SUM(segm_unma)  COUNT(*)
+   AND (COUNT(*) - SUM(seco_alig))  1
 ) AS qnames
-GROUP BY qname, firs_segm
-HAVING SUM(segm_unma)  COUNT(*)
-   AND (COUNT(*) - SUM(seco_alig))  1
-)
-ORDER BY qname;
+);
+
 
 SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
 FROM bam.unpaired_all_alignments_3
 WHERE qname IN (
 SELECT qname
 FROM (
-SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
-bam.bam_flag(flag, 'firs_segm') AS firs_segm
-FROM bam.unpaired_all_alignments_3
-WHERE bam.bam_flag(flag, 'firs_segm')  bam.bam_flag(flag, 
'last_segm')
+SELECT qname, SUM(segm_unma), SUM(seco_alig), COUNT(*)
+FROM (
+SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
+bam.bam_flag(flag, 'firs_segm') AS firs_segm
+FROM bam.unpaired_all_alignments_3
+WHERE bam.bam_flag(flag, 'firs_segm')  bam.bam_flag(flag, 
'last_segm')
+) AS sub
+GROUP BY qname, firs_segm
+HAVING SUM(segm_unma)  COUNT(*)
+   AND (COUNT(*) - SUM(seco_alig))  1
 ) AS qnames
-GROUP BY qname, firs_segm
-HAVING SUM(segm_unma)  COUNT(*)
-   AND (COUNT(*) - SUM(seco_alig))  1
-)
-ORDER BY qname;
+);
+
+
diff --git a/sql/backends/monet5/bam/Tests/query2.4.stable.out 
b/sql/backends/monet5/bam/Tests/query2.4.stable.out
--- a/sql/backends/monet5/bam/Tests/query2.4.stable.out
+++ b/sql/backends/monet5/bam/Tests/query2.4.stable.out
@@ -1,9 +1,9 @@
 stdout of test 'query2.4` in directory 'sql/backends/monet5/bam` itself:
 
 
-# 10:22:42   
-# 10:22:42   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=32986 --set 
mapi_usock=/var/tmp/mtest-16074/.s.monetdb.32986 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
-# 10:22:42   
+# 17:36:06   
+# 17:36:06   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=33676 --set 
mapi_usock=/var/tmp/mtest-11149/.s.monetdb.33676 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
+# 17:36:06   
 
 # MonetDB 5 server v11.18.0
 # This is an unreleased version
@@ -13,29 +13,27 @@ stdout of test 'query2.4` in directory '
 # Copyright (c) 1993-July 2008 CWI.
 # Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved
 # Visit http://www.monetdb.org/ for further information
-# Listening for connection requests on mapi:monetdb://robin-xps13:32986/
-# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-16074/.s.monetdb.32986
+# Listening for connection requests on mapi:monetdb://robin-xps13:33676/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-11149/.s.monetdb.33676
 # MonetDB/SQL module loaded
 
 Ready.
 
-# 10:22:42   
-# 10:22:42   mclient -lsql -ftest -Eutf-8 -i -e 
--host=/var/tmp/mtest-16074 --port=32986
-# 10:22:42   
+# 17:36:06   
+# 17:36:06   mclient -lsql -ftest -Eutf-8 -i -e 
--host=/var/tmp/mtest-11149 --port=33676
+# 17:36:06   
 
 #SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
 #FROM bam.alignments_1
 #WHERE qname IN (
 #SELECT qname
 #FROM (
-#SELECT qname, bam.bam_flag(flag, 'seco_alig') AS seco_alig, 
bam.bam_flag(flag, 'segm_unma') AS segm_unma, 
-#

MonetDB: bamloader - Fixed most tests in sql/test, the only ones...

2014-07-22 Thread Robin Cijvat
Changeset: b95d9e5134d0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b95d9e5134d0
Added Files:
sql/test/BugConstraints/Tests/add_constraint_1643522_bam.sql.src
sql/test/BugConstraints/Tests/add_constraint_1643522_bam.stable.err
sql/test/BugConstraints/Tests/add_constraint_1643522_bam.stable.out

sql/test/BugDay_2005-10-06_2.8/Tests/MapiClient-dump.SF-905851_bam.SQL.py.src

sql/test/BugDay_2005-10-06_2.8/Tests/MapiClient-dump.SF-905851_bam.stable.err

sql/test/BugDay_2005-10-06_2.8/Tests/MapiClient-dump.SF-905851_bam.stable.out

sql/test/BugDay_2005-11-09_2.8/Tests/ORDER_BY_evaluation_error.SF-1023658_bam.sql.src

sql/test/BugDay_2005-11-09_2.8/Tests/ORDER_BY_evaluation_error.SF-1023658_bam.stable.err

sql/test/BugDay_2005-11-09_2.8/Tests/ORDER_BY_evaluation_error.SF-1023658_bam.stable.out

sql/test/BugTracker-2008/Tests/sql_command_kills_db.SF-2233677_bam.sql.src

sql/test/BugTracker-2008/Tests/sql_command_kills_db.SF-2233677_bam.stable.err

sql/test/BugTracker-2008/Tests/sql_command_kills_db.SF-2233677_bam.stable.out
sql/test/BugTracker-2009/Tests/mclient-lsql-D_bam.SQL.sh.src
sql/test/BugTracker-2009/Tests/mclient-lsql-D_bam.stable.err
sql/test/BugTracker-2009/Tests/mclient-lsql-D_bam.stable.out

sql/test/BugTracker-2009/Tests/name_clash_with_dump.SF-2780395_bam.sql.src

sql/test/BugTracker-2009/Tests/name_clash_with_dump.SF-2780395_bam.stable.err

sql/test/BugTracker-2009/Tests/name_clash_with_dump.SF-2780395_bam.stable.out
sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552_bam.sql.src
sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552_bam.stable.err
sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552_bam.stable.out

sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552_bam.stable.out.32bit
sql/test/BugTracker-2011/Tests/count-count-distinct.Bug-2808_bam.sql.src

sql/test/BugTracker-2011/Tests/count-count-distinct.Bug-2808_bam.stable.err

sql/test/BugTracker-2011/Tests/count-count-distinct.Bug-2808_bam.stable.out
sql/test/BugTracker-2011/Tests/exists-select.Bug-2933_bam.sql.src
sql/test/BugTracker-2011/Tests/exists-select.Bug-2933_bam.stable.err
sql/test/BugTracker-2011/Tests/exists-select.Bug-2933_bam.stable.out

sql/test/BugTracker-2011/Tests/interrupted-initialization.Bug-2875_bam.py.src

sql/test/BugTracker-2011/Tests/interrupted-initialization.Bug-2875_bam.stable.err

sql/test/BugTracker-2011/Tests/interrupted-initialization.Bug-2875_bam.stable.out

sql/test/BugTracker-2012/Tests/aggregate_vs_positional_column_crash.Bug-3085_bam.sql.src

sql/test/BugTracker-2012/Tests/aggregate_vs_positional_column_crash.Bug-3085_bam.stable.err

sql/test/BugTracker-2012/Tests/aggregate_vs_positional_column_crash.Bug-3085_bam.stable.out
sql/test/BugTracker-2013/Tests/qualified_aggrname.Bug-3332_bam.sql.src

sql/test/BugTracker-2013/Tests/qualified_aggrname.Bug-3332_bam.stable.err

sql/test/BugTracker-2013/Tests/qualified_aggrname.Bug-3332_bam.stable.out
sql/test/BugTracker-2014/Tests/select-having.Bug-3458_bam.sql.src
sql/test/BugTracker-2014/Tests/select-having.Bug-3458_bam.stable.err
sql/test/BugTracker-2014/Tests/select-having.Bug-3458_bam.stable.out
sql/test/BugTracker/Tests/multi-column-constraint.SF-1964587_bam.sql.src

sql/test/BugTracker/Tests/multi-column-constraint.SF-1964587_bam.stable.err

sql/test/BugTracker/Tests/multi-column-constraint.SF-1964587_bam.stable.out
sql/test/BugTracker/Tests/rank_over.SF-1691098_bam.sql.src
sql/test/BugTracker/Tests/rank_over.SF-1691098_bam.stable.err
sql/test/BugTracker/Tests/rank_over.SF-1691098_bam.stable.out
sql/test/BugTracker/Tests/with_only_once.SF-1720293_bam.sql.src
sql/test/BugTracker/Tests/with_only_once.SF-1720293_bam.stable.err
sql/test/BugTracker/Tests/with_only_once.SF-1720293_bam.stable.out
sql/test/DropCascade/Tests/drop_columns_cascade_bam.sql.src
sql/test/DropCascade/Tests/drop_columns_cascade_bam.stable.err
sql/test/DropCascade/Tests/drop_columns_cascade_bam.stable.out
sql/test/DropCascade/Tests/drop_pkey_cascade_bam.sql.src
sql/test/DropCascade/Tests/drop_pkey_cascade_bam.stable.err
sql/test/DropCascade/Tests/drop_pkey_cascade_bam.stable.out
sql/test/Dump/Tests/dump-empty_bam.SQL.py.src
sql/test/Dump/Tests/dump-empty_bam.stable.err
sql/test/Dump/Tests/dump-empty_bam.stable.out
sql/test/Dump/Tests/dump_bam.SQL.py.src
sql/test/Dump/Tests/dump_bam.stable.err
sql/test/Dump/Tests/dump_bam.stable.out
sql/test/UserDump/Tests/create_bam.SQL.py.src
sql/test/UserDump/Tests/create_bam.stable.err

MonetDB: bamloader - Merged local repos with branch

2014-07-22 Thread Robin Cijvat
Changeset: 4194703b0d57 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4194703b0d57
Added Files:
sql/test/BugTracker-2014/Tests/oid-table-assert.Bug-3514.sql
sql/test/BugTracker-2014/Tests/oid-table-assert.Bug-3514.stable.err
sql/test/BugTracker-2014/Tests/oid-table-assert.Bug-3514.stable.out
sql/test/BugTracker-2014/Tests/round-properties.Bug-3515.sql
sql/test/BugTracker-2014/Tests/round-properties.Bug-3515.stable.err
sql/test/BugTracker-2014/Tests/round-properties.Bug-3515.stable.out
Modified Files:
.hgtags
MonetDB.spec
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
buildtools/ChangeLog-Archive
buildtools/ChangeLog.Jan2014
clients/R/MonetDB.R/DESCRIPTION
clients/Tests/exports.stable.out
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
clients/python2/setup.py
clients/python3/setup.py
configure.ag
debian/changelog
gdk/gdk.h
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_imprints.c
gdk/gdk_join.c
gdk/gdk_join_legacy.c
gdk/gdk_logger.c
gdk/gdk_sample.c
gdk/gdk_select.c
gdk/gdk_unique.c
gdk/gdk_utils.c
gdk/libbat.rc
libversions
monetdb5/extras/sphinx/sphinx.c
monetdb5/mal/mal_authorize.c
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_profiler.c
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/array.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/kernel/batmmath.c
monetdb5/modules/kernel/batstr.c
monetdb5/modules/kernel/microbenchmark.c
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/cluster.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/mat.c
monetdb5/modules/mal/mdb.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/sample.c
monetdb5/modules/mal/sample.h
monetdb5/modules/mal/zorder.c
monetdb5/optimizer/opt_wrapper.c
monetdb5/tools/libmonetdb5.rc
sql/backends/monet5/datacell/basket.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_round_impl.h
sql/benchmarks/ssbm/Tests/01-13.reqtests
sql/benchmarks/ssbm/Tests/01-explain.reqtests
sql/benchmarks/ssbm/Tests/01-plan.reqtests
sql/benchmarks/ssbm/Tests/01.reqtests
sql/benchmarks/ssbm/Tests/02-explain.reqtests
sql/benchmarks/ssbm/Tests/02-plan.reqtests
sql/benchmarks/ssbm/Tests/02.reqtests
sql/benchmarks/ssbm/Tests/03-explain.reqtests
sql/benchmarks/ssbm/Tests/03-plan.reqtests
sql/benchmarks/ssbm/Tests/03.reqtests
sql/benchmarks/ssbm/Tests/04-explain.reqtests
sql/benchmarks/ssbm/Tests/04-plan.reqtests
sql/benchmarks/ssbm/Tests/04.reqtests
sql/benchmarks/ssbm/Tests/05-explain.reqtests
sql/benchmarks/ssbm/Tests/05-plan.reqtests
sql/benchmarks/ssbm/Tests/05.reqtests
sql/benchmarks/ssbm/Tests/06-explain.reqtests
sql/benchmarks/ssbm/Tests/06-plan.reqtests
sql/benchmarks/ssbm/Tests/06.reqtests
sql/benchmarks/ssbm/Tests/07-explain.reqtests
sql/benchmarks/ssbm/Tests/07-plan.reqtests
sql/benchmarks/ssbm/Tests/07.reqtests
sql/benchmarks/ssbm/Tests/08-explain.reqtests
sql/benchmarks/ssbm/Tests/08-plan.reqtests
sql/benchmarks/ssbm/Tests/08.reqtests
sql/benchmarks/ssbm/Tests/09-explain.reqtests
sql/benchmarks/ssbm/Tests/09-plan.reqtests
sql/benchmarks/ssbm/Tests/09.reqtests
sql/benchmarks/ssbm/Tests/10-explain.reqtests
sql/benchmarks/ssbm/Tests/10-plan.reqtests
sql/benchmarks/ssbm/Tests/10.reqtests
sql/benchmarks/ssbm/Tests/11-explain.reqtests
sql/benchmarks/ssbm/Tests/11-plan.reqtests
sql/benchmarks/ssbm/Tests/11.reqtests
sql/benchmarks/ssbm/Tests/12-explain.reqtests
sql/benchmarks/ssbm/Tests/12-plan.reqtests
sql/benchmarks/ssbm/Tests/12.reqtests
sql/benchmarks/ssbm/Tests/13-explain.reqtests
sql/benchmarks/ssbm/Tests/13-plan.reqtests
sql/benchmarks/ssbm/Tests/13.reqtests
sql/benchmarks/ssbm/Tests/check1.reqtests
sql/benchmarks/ssbm/Tests/load.reqtests
sql/storage/bat/bat_logger.c
sql/storage/bat/bat_utils.c

MonetDB: bamloader - Merge with default

2014-07-22 Thread Robin Cijvat
Changeset: bc4ce7f2abb9 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bc4ce7f2abb9
Modified Files:
sql/test/BugTracker/Tests/rank_over.SF-1691098.sql
sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out
sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql
sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out
testing/Mtest.py.in
Branch: bamloader
Log Message:

Merge with default


diffs (167 lines):

diff --git a/sql/test/BugTracker/Tests/rank_over.SF-1691098.sql 
b/sql/test/BugTracker/Tests/rank_over.SF-1691098.sql
--- a/sql/test/BugTracker/Tests/rank_over.SF-1691098.sql
+++ b/sql/test/BugTracker/Tests/rank_over.SF-1691098.sql
@@ -1,1 +1,9 @@
-select RANK () OVER () as foo from tables;
+create table rankbug (id int);
+insert into rankbug values (42);
+insert into rankbug select * from rankbug;
+insert into rankbug select * from rankbug;
+insert into rankbug select * from rankbug;
+insert into rankbug select * from rankbug;
+insert into rankbug select * from rankbug;
+select RANK () OVER () as foo from rankbug;
+drop table rankbug;
diff --git a/sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out 
b/sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out
--- a/sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out
+++ b/sql/test/BugTracker/Tests/rank_over.SF-1691098.stable.out
@@ -24,8 +24,21 @@ Ready.
 # 16:03:46   Mtimeout -timeout 60 MapiClient -lsql -umonetdb -Pmonetdb 
--host=localhost --port=38533 
 # 16:03:46   
 
-#select RANK () OVER () as foo from tables;
-% .L # table_name
+#create table rankbug (id int);
+#insert into rankbug values (42);
+[ 1]
+#insert into rankbug select * from rankbug;
+[ 1]
+#insert into rankbug select * from rankbug;
+[ 2]
+#insert into rankbug select * from rankbug;
+[ 4]
+#insert into rankbug select * from rankbug;
+[ 8]
+#insert into rankbug select * from rankbug;
+[ 16   ]
+#select RANK () OVER () as foo from rankbug;
+% sys.L # table_name
 % foo # name
 % int # type
 % 2 # length
@@ -61,14 +74,7 @@ Ready.
 [ 30   ]
 [ 31   ]
 [ 32   ]
-[ 33   ]
-[ 34   ]
-[ 35   ]
-[ 36   ]
-[ 37   ]
-[ 38   ]
-[ 39   ]
-[ 40   ]
+#drop table rankbug;
 
 # 17:29:50   
 # 17:29:50   Done.
diff --git a/sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql 
b/sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql
--- a/sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql
+++ b/sql/test/BugTracker/Tests/with_only_once.SF-1720293.sql
@@ -1,2 +1,10 @@
-WITH t AS ( SELECT count(*) FROM tables ) SELECT * FROM t;
-WITH t AS ( SELECT count(*) FROM tables ) SELECT * FROM t;
+create table withonlybug (id int);
+insert into withonlybug values (42);
+insert into withonlybug select * from withonlybug;
+insert into withonlybug select * from withonlybug;
+insert into withonlybug select * from withonlybug;
+insert into withonlybug select * from withonlybug;
+insert into withonlybug select * from withonlybug;
+WITH t AS ( SELECT count(*) FROM withonlybug ) SELECT * FROM t;
+WITH t AS ( SELECT count(*) FROM withonlybug ) SELECT * FROM t;
+drop table withonlybug;
diff --git a/sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out 
b/sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out
--- a/sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out
+++ b/sql/test/BugTracker/Tests/with_only_once.SF-1720293.stable.out
@@ -14,27 +14,63 @@ stdout of test 'with_only_once.SF-172029
 # MonetDB/SQL module v2.25.0 loaded
 
 Ready.
-#function user.main():void;
-#clients.quit();
-#end main;
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_compress.sql
+# loading sql script: 18_dictionary.sql
+# loading sql script: 19_cluster.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 24_zorder.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 39_analytics.sql
+# loading sql script: 40_geom.sql
+# loading sql script: 46_gsl.sql
+# loading sql script: 75_storagemodel.sql
+# loading sql script: 80_statistics.sql
+# loading sql script: 80_udf.sql
+# loading sql script: 99_system.sql
 
+# 17:55:39   
+# 17:55:39   mclient -lsql -ftest -Eutf-8 -i -e 
--host=/var/tmp/mtest-30978 --port=32166
+# 17:55:39   
 
-# 22:41:45   
-# 22:41:45   Mtimeout -timeout 60 MapiClient -lsql -umonetdb -Pmonetdb 
--host=localhost --port=32869 
-# 22:41:45   
-
-#WITH t AS ( SELECT count(*) FROM tables ) SELECT * FROM t;
-% .t # table_name
+#create table withonlybug (id int);
+#insert into withonlybug values (42);

MonetDB: bamloader - Adjusted jdbc tests to comply with bamloade...

2014-07-21 Thread Robin Cijvat
Changeset: f4280bc65958 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f4280bc65958
Added Files:
sql/jdbc/tests/Tests/Test_Dobjects_bam.SQL.sh
sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.err
sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.out
sql/jdbc/tests/Tests/Test_JdbcClient_bam.SQL.sh
sql/jdbc/tests/Tests/Test_JdbcClient_bam.sh
sql/jdbc/tests/Tests/Test_JdbcClient_bam.sh.src
sql/jdbc/tests/Tests/Test_JdbcClient_bam.stable.err
sql/jdbc/tests/Tests/Test_JdbcClient_bam.stable.out
Modified Files:
sql/jdbc/tests/Tests/All
Branch: bamloader
Log Message:

Adjusted jdbc tests to comply with bamloader branch


diffs (truncated from 590 to 300 lines):

diff --git a/sql/jdbc/tests/Tests/All b/sql/jdbc/tests/Tests/All
--- a/sql/jdbc/tests/Tests/All
+++ b/sql/jdbc/tests/Tests/All
@@ -5,7 +5,8 @@ HAVE_JDBCTESTS?Test_Creplysize
 HAVE_JDBCTESTS?Test_Csavepoints
 HAVE_JDBCTESTS?Test_Ctransaction
 #HAVE_JDBCTESTS?Test_Csendthread  # unfortunately has runtime-dependant output
-HAVE_JDBCTESTS?Test_Dobjects
+HAVE_JDBCTESTS!HAVE_SAMTOOLS?Test_Dobjects
+HAVE_JDBCTESTSHAVE_SAMTOOLS?Test_Dobjects_bam
 HAVE_JDBCTESTS?Test_PSgeneratedkeys
 HAVE_JDBCTESTS?Test_PSsomeamount
 HAVE_JDBCTESTS?Test_PSlargebatchval
@@ -21,7 +22,8 @@ HAVE_JDBCTESTS?Test_Rpositioning
 HAVE_JDBCTESTS?Test_Rtimedate
 HAVE_JDBCTESTS?Test_Rsqldata
 HAVE_JDBCTESTS?Test_Sbatching
-HAVE_JDBCCLIENT_JAR?Test_JdbcClient
+HAVE_JDBCCLIENT_JAR!HAVE_SAMTOOLS?Test_JdbcClient
+HAVE_JDBCCLIENT_JARHAVE_SAMTOOLS?Test_JdbcClient_bam
 HAVE_JDBCTESTS?BugConcurrent_clients_SF_1504657
 HAVE_JDBCTESTS?BugConcurrent_sequences
 HAVE_JDBCTESTS?BugDatabaseMetaData_Bug_3356
diff --git a/sql/jdbc/tests/Tests/Test_Dobjects_bam.SQL.sh 
b/sql/jdbc/tests/Tests/Test_Dobjects_bam.SQL.sh
new file mode 100755
--- /dev/null
+++ b/sql/jdbc/tests/Tests/Test_Dobjects_bam.SQL.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+TST=${TST/_bam}
+$TSTSRCDIR/Test.SQL.sh ${*/_bam}
diff --git a/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.err 
b/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.err
@@ -0,0 +1,40 @@
+stderr of test 'Test_Dobjects_bam` in directory 'sql/jdbc/tests` itself:
+
+
+# 15:28:55   
+# 15:28:55   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=37247 --set 
mapi_usock=/var/tmp/mtest-21432/.s.monetdb.37247 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_jdbc_tests --set 
mal_listing=0
+# 15:28:55   
+
+# builtin opt  gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = 
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37247
+# cmdline opt  mapi_usock = /var/tmp/mtest-21432/.s.monetdb.37247
+# cmdline opt  monet_prompt = 
+# cmdline opt  mal_listing = 2
+# cmdline opt  gdk_dbpath = 
/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_jdbc_tests
+# cmdline opt  mal_listing = 0
+# cmdline opt  gdk_debug = 536870922
+
+# 15:28:56   
+# 15:28:56   ./Test_Dobjects_bam.SQL.sh Test_Dobjects_bam
+# 15:28:56   
+
+
+# 15:28:56   
+# 15:28:56   java Test_Dobjects 
jdbc:monetdb://robin-xps13:37247/mTests_sql_jdbc_tests?user=monetdbpassword=monetdb
+# 15:28:56   
+
+
+# 15:28:56   
+# 15:28:56   Done.
+# 15:28:56   
diff --git a/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.out 
b/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/jdbc/tests/Tests/Test_Dobjects_bam.stable.out
@@ -0,0 +1,133 @@
+stdout of test 'Test_Dobjects_bam` in directory 'sql/jdbc/tests` itself:
+
+
+# 15:28:55   
+# 15:28:55   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=37247 --set 
mapi_usock=/var/tmp/mtest-21432/.s.monetdb.37247 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_jdbc_tests --set 
mal_listing=0
+# 15:28:55   
+
+# MonetDB 5 server v11.18.0
+# This is an unreleased version
+# Serving database 'mTests_sql_jdbc_tests', using 4 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically 
linked
+# Found 7.333 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://robin-xps13:37247/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-21432/.s.monetdb.37247
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL 

MonetDB: bamloader - Extended some more tests

2014-07-21 Thread Robin Cijvat
Changeset: 5d30a783ecdd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5d30a783ecdd
Added Files:

sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.sql.src

sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.err

sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.out

sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-1.Bug-3063_bam.sql.src

sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-1.Bug-3063_bam.stable.err

sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-1.Bug-3063_bam.stable.out
sql/test/Dependencies/Tests/Dependencies_bam.SQL.py.src
sql/test/Dependencies/Tests/Dependencies_bam.stable.err
sql/test/Dependencies/Tests/Dependencies_bam.stable.out
sql/test/Tests/keys_bam.sql.src
sql/test/Tests/keys_bam.stable.err
sql/test/Tests/keys_bam.stable.out
Modified Files:
sql/test/BugTracker-2012/Tests/All
sql/test/Dependencies/Tests/All
sql/test/Tests/All
sql/test/bugs/Tests/All
Branch: bamloader
Log Message:

Extended some more tests


diffs (truncated from 1327 to 300 lines):

diff --git a/sql/test/BugTracker-2012/Tests/All 
b/sql/test/BugTracker-2012/Tests/All
--- a/sql/test/BugTracker-2012/Tests/All
+++ b/sql/test/BugTracker-2012/Tests/All
@@ -29,8 +29,10 @@ name_resolution_in_proc.Bug-3074
 segfault_incorrect_head.Bug-3028
 cast_varchar_to_double.Bug-3071
 table_returning_func_returns_too_many_columns.Bug-3077
-table_functions_fail_after_restart-0.Bug-3063
-table_functions_fail_after_restart-1.Bug-3063
+!HAVE_SAMTOOLS?table_functions_fail_after_restart-0.Bug-3063
+HAVE_SAMTOOLS?table_functions_fail_after_restart-0.Bug-3063_bam
+!HAVE_SAMTOOLS?table_functions_fail_after_restart-1.Bug-3063
+HAVE_SAMTOOLS?table_functions_fail_after_restart-1.Bug-3063_bam
 user_defined_decimal_function.Bug-2992
 non_keyword_no.Bug-3079
 round-type-inference.Bug-2987
diff --git 
a/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.sql.src
 
b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.sql.src
new file mode 100644
--- /dev/null
+++ 
b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.sql.src
@@ -0,0 +1,1 @@
+$RELSRCDIR/table_functions_fail_after_restart-0.Bug-3063.sql
diff --git 
a/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.err
 
b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.err
new file mode 100644
--- /dev/null
+++ 
b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.err
@@ -0,0 +1,35 @@
+stderr of test 'table_functions_fail_after_restart-0.Bug-3063_bam` in 
directory 'sql/test/BugTracker-2012` itself:
+
+
+# 09:28:52   
+# 09:28:52   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=36187 --set 
mapi_usock=/var/tmp/mtest-1669/.s.monetdb.36187 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_test_BugTracker-2012
 --set mal_listing=0
+# 09:28:52   
+
+# builtin opt  gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = 
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 36187
+# cmdline opt  mapi_usock = /var/tmp/mtest-1669/.s.monetdb.36187
+# cmdline opt  monet_prompt = 
+# cmdline opt  mal_listing = 2
+# cmdline opt  gdk_dbpath = 
/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_test_BugTracker-2012
+# cmdline opt  mal_listing = 0
+# cmdline opt  gdk_debug = 536870922
+
+# 09:28:52   
+# 09:28:52   mclient -lsql -ftest -Eutf-8 -i -e 
--host=/var/tmp/mtest-1669 --port=36187
+# 09:28:52   
+
+
+# 09:28:53   
+# 09:28:53   Done.
+# 09:28:53   
diff --git 
a/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.out
 
b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.out
new file mode 100644
--- /dev/null
+++ 
b/sql/test/BugTracker-2012/Tests/table_functions_fail_after_restart-0.Bug-3063_bam.stable.out
@@ -0,0 +1,43 @@
+stdout of test 'table_functions_fail_after_restart-0.Bug-3063_bam` in 
directory 'sql/test/BugTracker-2012` itself:
+
+
+# 09:28:52   
+# 09:28:52   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=36187 --set 
mapi_usock=/var/tmp/mtest-1669/.s.monetdb.36187 --set monet_prompt= 
--forcemito --set mal_listing=2 

MonetDB: bamloader - Moved creation of static SQL tables from c ...

2014-07-16 Thread Robin Cijvat
Changeset: 99e071e2ea6d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=99e071e2ea6d
Modified Files:
sql/backends/monet5/bam/85_bam.sql
sql/backends/monet5/bam/bam_db_interface.c
sql/backends/monet5/bam/bam_db_interface.h
sql/backends/monet5/bam/bam_loader.c
Branch: bamloader
Log Message:

Moved creation of static SQL tables from c code to 85_bam.sql


diffs (truncated from 304 to 300 lines):

diff --git a/sql/backends/monet5/bam/85_bam.sql 
b/sql/backends/monet5/bam/85_bam.sql
--- a/sql/backends/monet5/bam/85_bam.sql
+++ b/sql/backends/monet5/bam/85_bam.sql
@@ -31,3 +31,70 @@ EXTERNAL NAME bam.sam_export;
 
 CREATE PROCEDURE bam.bam_export(output_path STRING)
 EXTERNAL NAME bam.bam_export;
+
+
+
+CREATE TABLE bam.files ( 
+   file_id BIGINT  
NOT NULL, 
+   file_location   STRING  NOT 
NULL, 
+   dbschemaSMALLINT
NOT NULL, 
+   format_version  VARCHAR(7), 
+   sorting_order   VARCHAR(10), 
+   commentsSTRING, 
+   CONSTRAINT files_pkey_file_id PRIMARY KEY (file_id) 
+);
+
+CREATE TABLE bam.sq ( 
+   sn  STRING  
NOT NULL, 
+   file_id BIGINT  
NOT NULL, 
+   ln  INT, 
+   asINT, 
+   m5  STRING, 
+   sp  STRING, 
+   ur  STRING, 
+   CONSTRAINT sq_pkey_sn_file_id PRIMARY KEY (sn, file_id), 
+   CONSTRAINT sq_fkey_file_id FOREIGN KEY (file_id) REFERENCES bam.files 
(file_id) 
+);
+
+CREATE TABLE bam.rg ( 
+   id  STRING  
NOT NULL, 
+   file_id BIGINT  
NOT NULL, 
+   cn  STRING, 
+   ds  STRING, 
+   dt  TIMESTAMP, 
+   fo  STRING, 
+   ks  STRING, 
+   lb  STRING, 
+   pg  STRING, 
+   pi  INT, 
+   pl  STRING, 
+   pu  STRING, 
+   sm  STRING, 
+   CONSTRAINT rg_pkey_id_file_id PRIMARY KEY (id, file_id), 
+   CONSTRAINT rg_fkey_file_id FOREIGN KEY (file_id) REFERENCES bam.files 
(file_id) 
+);
+
+CREATE TABLE bam.pg ( 
+   id  STRING  
NOT NULL, 
+   file_id BIGINT  
NOT NULL, 
+   pn  STRING, 
+   cl  STRING, 
+   pp  STRING, 
+   vn  STRING, 
+   CONSTRAINT pg_pkey_id_file_id PRIMARY KEY (id, file_id), 
+   CONSTRAINT pg_fkey_file_id FOREIGN KEY (file_id) REFERENCES bam.files 
(file_id) 
+);
+
+CREATE TABLE bam.export ( 
+   qname   STRING  
NOT NULL, 
+   flagSMALLINT
NOT NULL, 
+   rname   STRING  
NOT NULL, 
+   pos INT 
NOT NULL, 
+   mapqSMALLINT
NOT NULL, 
+   cigar   STRING  
NOT NULL, 
+   rnext   STRING  
NOT NULL, 
+   pnext   INT 
NOT NULL, 
+   tlenINT 
NOT NULL, 
+   seq STRING  
NOT NULL, 
+   qualSTRING  
NOT NULL  
+);
diff --git 

MonetDB: bamloader - Fix system absolute paths in test outputs

2014-07-15 Thread Robin Cijvat
Changeset: 9cb7c7c1cf07 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9cb7c7c1cf07
Modified Files:
sql/backends/monet5/bam/Tests/bam_loader_file.stable.err
sql/backends/monet5/bam/Tests/bam_loader_file.stable.out
sql/backends/monet5/bam/Tests/bam_loader_files.stable.err
sql/backends/monet5/bam/Tests/bam_loader_files.stable.out
sql/backends/monet5/bam/Tests/check_files.sql
sql/backends/monet5/bam/Tests/check_files.stable.err
sql/backends/monet5/bam/Tests/check_files.stable.out
sql/backends/monet5/bam/Tests/check_files_exact.sql
sql/backends/monet5/bam/Tests/check_files_exact.stable.err
sql/backends/monet5/bam/Tests/check_files_exact.stable.out
sql/backends/monet5/bam/Tests/drop_last_files.stable.err
sql/backends/monet5/bam/Tests/drop_last_files.stable.out
sql/backends/monet5/bam/Tests/query1.1.stable.err
sql/backends/monet5/bam/Tests/query1.1.stable.out
sql/backends/monet5/bam/Tests/query1.2.stable.err
sql/backends/monet5/bam/Tests/query1.2.stable.out
sql/backends/monet5/bam/Tests/query1.3.stable.err
sql/backends/monet5/bam/Tests/query1.3.stable.out
sql/backends/monet5/bam/Tests/query1.4.stable.err
sql/backends/monet5/bam/Tests/query1.4.stable.out
sql/backends/monet5/bam/Tests/query1.5.stable.err
sql/backends/monet5/bam/Tests/query1.5.stable.out
sql/backends/monet5/bam/Tests/query2.1.stable.err
sql/backends/monet5/bam/Tests/query2.1.stable.out
sql/backends/monet5/bam/Tests/query2.10.stable.err
sql/backends/monet5/bam/Tests/query2.10.stable.out
sql/backends/monet5/bam/Tests/query2.11.stable.err
sql/backends/monet5/bam/Tests/query2.11.stable.out
sql/backends/monet5/bam/Tests/query2.12.stable.err
sql/backends/monet5/bam/Tests/query2.12.stable.out
sql/backends/monet5/bam/Tests/query2.2.stable.err
sql/backends/monet5/bam/Tests/query2.2.stable.out
sql/backends/monet5/bam/Tests/query2.3.stable.err
sql/backends/monet5/bam/Tests/query2.3.stable.out
sql/backends/monet5/bam/Tests/query2.4.stable.err
sql/backends/monet5/bam/Tests/query2.4.stable.out
sql/backends/monet5/bam/Tests/query2.5.stable.err
sql/backends/monet5/bam/Tests/query2.5.stable.out
sql/backends/monet5/bam/Tests/query2.6.stable.err
sql/backends/monet5/bam/Tests/query2.6.stable.out
sql/backends/monet5/bam/Tests/query2.7.stable.err
sql/backends/monet5/bam/Tests/query2.7.stable.out
sql/backends/monet5/bam/Tests/query2.8.stable.err
sql/backends/monet5/bam/Tests/query2.8.stable.out
sql/backends/monet5/bam/Tests/query2.9.stable.err
sql/backends/monet5/bam/Tests/query2.9.stable.out
sql/backends/monet5/bam/Tests/sam_export.stable.err
sql/backends/monet5/bam/Tests/sam_export.stable.out
sql/backends/monet5/bam/Tests/verify_load.stable.err
sql/backends/monet5/bam/Tests/verify_load.stable.out
Branch: bamloader
Log Message:

Fix system absolute paths in test outputs


diffs (truncated from 2583 to 300 lines):

diff --git a/sql/backends/monet5/bam/Tests/bam_loader_file.stable.err 
b/sql/backends/monet5/bam/Tests/bam_loader_file.stable.err
--- a/sql/backends/monet5/bam/Tests/bam_loader_file.stable.err
+++ b/sql/backends/monet5/bam/Tests/bam_loader_file.stable.err
@@ -1,9 +1,9 @@
 stderr of test 'bam_loader_file` in directory 'sql/backends/monet5/bam` itself:
 
 
-# 13:29:48   
-# 13:29:48   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=37021 --set 
mapi_usock=/var/tmp/mtest-21062/.s.monetdb.37021 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
-# 13:29:48   
+# 10:22:37   
+# 10:22:37   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=32986 --set 
mapi_usock=/var/tmp/mtest-16074/.s.monetdb.32986 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
+# 10:22:37   
 
 # builtin opt  gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo
 # builtin opt  gdk_debug = 0
@@ -17,16 +17,14 @@ stderr of test 'bam_loader_file` in dire
 # builtin opt  sql_debug = 0
 # cmdline opt  gdk_nr_threads = 0
 # cmdline opt  mapi_open = true
-# cmdline opt  mapi_port = 37021
-# cmdline opt  mapi_usock = /var/tmp/mtest-21062/.s.monetdb.37021
+# cmdline opt  mapi_port = 32986
+# cmdline opt  mapi_usock = /var/tmp/mtest-16074/.s.monetdb.32986
 # cmdline opt  monet_prompt = 
 # cmdline opt  mal_listing = 2
 # cmdline opt  gdk_dbpath = 
/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 # cmdline opt  mal_listing = 0
 # cmdline opt  gdk_debug = 536870922
 # bam_loader: Loader 

MonetDB: bamloader - Fix file permissions

2014-07-15 Thread Robin Cijvat
Changeset: 5b99954e0ebb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5b99954e0ebb
Modified Files:
sql/backends/monet5/bam/Tests/All
sql/backends/monet5/bam/Tests/query1.1.sql
sql/backends/monet5/bam/Tests/query1.2.sql
sql/backends/monet5/bam/Tests/query1.3.sql
sql/backends/monet5/bam/Tests/query1.4.sql
sql/backends/monet5/bam/Tests/query1.5.sql
sql/backends/monet5/bam/Tests/query2.1.sql
sql/backends/monet5/bam/Tests/query2.10.sql
sql/backends/monet5/bam/Tests/query2.11.sql
sql/backends/monet5/bam/Tests/query2.12.sql
sql/backends/monet5/bam/Tests/query2.2.sql
sql/backends/monet5/bam/Tests/query2.3.sql
sql/backends/monet5/bam/Tests/query2.4.sql
sql/backends/monet5/bam/Tests/query2.5.sql
sql/backends/monet5/bam/Tests/query2.6.sql
sql/backends/monet5/bam/Tests/query2.7.sql
sql/backends/monet5/bam/Tests/query2.8.sql
sql/backends/monet5/bam/Tests/query2.9.sql
sql/backends/monet5/bam/Tests/test.modules
Branch: bamloader
Log Message:

Fix file permissions


diffs (57 lines):

diff --git a/sql/backends/monet5/bam/Tests/All 
b/sql/backends/monet5/bam/Tests/All
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query1.1.sql 
b/sql/backends/monet5/bam/Tests/query1.1.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query1.2.sql 
b/sql/backends/monet5/bam/Tests/query1.2.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query1.3.sql 
b/sql/backends/monet5/bam/Tests/query1.3.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query1.4.sql 
b/sql/backends/monet5/bam/Tests/query1.4.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query1.5.sql 
b/sql/backends/monet5/bam/Tests/query1.5.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.1.sql 
b/sql/backends/monet5/bam/Tests/query2.1.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.10.sql 
b/sql/backends/monet5/bam/Tests/query2.10.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.11.sql 
b/sql/backends/monet5/bam/Tests/query2.11.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.12.sql 
b/sql/backends/monet5/bam/Tests/query2.12.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.2.sql 
b/sql/backends/monet5/bam/Tests/query2.2.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.3.sql 
b/sql/backends/monet5/bam/Tests/query2.3.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.4.sql 
b/sql/backends/monet5/bam/Tests/query2.4.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.5.sql 
b/sql/backends/monet5/bam/Tests/query2.5.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.6.sql 
b/sql/backends/monet5/bam/Tests/query2.6.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.7.sql 
b/sql/backends/monet5/bam/Tests/query2.7.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.8.sql 
b/sql/backends/monet5/bam/Tests/query2.8.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/query2.9.sql 
b/sql/backends/monet5/bam/Tests/query2.9.sql
old mode 100755
new mode 100644
diff --git a/sql/backends/monet5/bam/Tests/test.modules 
b/sql/backends/monet5/bam/Tests/test.modules
old mode 100755
new mode 100644
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Added required role parameter to calls to B...

2014-07-15 Thread Robin Cijvat
Changeset: c8c0548c31cc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c8c0548c31cc
Modified Files:
sql/backends/monet5/bam/bam_lib.c
Branch: bamloader
Log Message:

Added required role parameter to calls to BATnew


diffs (39 lines):

diff --git a/sql/backends/monet5/bam/bam_lib.c 
b/sql/backends/monet5/bam/bam_lib.c
--- a/sql/backends/monet5/bam/bam_lib.c
+++ b/sql/backends/monet5/bam/bam_lib.c
@@ -190,7 +190,7 @@ bam_flag_bat(bat * ret, bat * bid, str *
throw(MAL, bam_flag_bat, RUNTIME_OBJECT_MISSING);
 
/* allocate result BAT */
-   result = BATnew(TYPE_void, TYPE_bit, BATcount(flags));
+   result = BATnew(TYPE_void, TYPE_bit, BATcount(flags), TRANSIENT);
if (result == NULL) {
throw(MAL, bam_flag_bat, MAL_MALLOC_FAIL);
}
@@ -226,7 +226,7 @@ reverse_seq_bat(bat * ret, bat * bid)
throw(MAL, reverse_seq_bat, RUNTIME_OBJECT_MISSING);
 
/* allocate result BAT */
-   result = BATnew(TYPE_void, TYPE_str, BATcount(seqs));
+   result = BATnew(TYPE_void, TYPE_str, BATcount(seqs), TRANSIENT);
if (result == NULL) {
throw(MAL, reverse_seq_bat, MAL_MALLOC_FAIL);
}
@@ -267,7 +267,7 @@ reverse_qual_bat(bat * ret, bat * bid)
throw(MAL, reverse_qual_bat, RUNTIME_OBJECT_MISSING);
 
/* allocate result BAT */
-   result = BATnew(TYPE_void, TYPE_str, BATcount(quals));
+   result = BATnew(TYPE_void, TYPE_str, BATcount(quals), TRANSIENT);
if (result == NULL) {
throw(MAL, reverse_qual_bat, MAL_MALLOC_FAIL);
}
@@ -308,7 +308,7 @@ seq_length_bat(bat * ret, bat * bid)
throw(MAL, seq_length_bat, RUNTIME_OBJECT_MISSING);
 
/* allocate result BAT */
-   result = BATnew(TYPE_void, TYPE_int, BATcount(cigars));
+   result = BATnew(TYPE_void, TYPE_int, BATcount(cigars), TRANSIENT);
if (result == NULL) {
throw(MAL, seq_length_bat, MAL_MALLOC_FAIL);
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Fix stack smashing error by allocating a bi...

2014-07-15 Thread Robin Cijvat
Changeset: 6c764974f9f0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6c764974f9f0
Modified Files:
sql/backends/monet5/bam/bam_loader.c
Branch: bamloader
Log Message:

Fix stack smashing error by allocating a big enough buffer


diffs (30 lines):

diff --git a/sql/backends/monet5/bam/bam_loader.c 
b/sql/backends/monet5/bam/bam_loader.c
--- a/sql/backends/monet5/bam/bam_loader.c
+++ b/sql/backends/monet5/bam/bam_loader.c
@@ -187,7 +187,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s
sql_schema *s = NULL;
sql_table *files_table = NULL;
lng cur_file_id;
-   char buf_threads_msg[] = There were reader threads that contained 
errors:\n;
+   char buf_threads_msg[4096] = There were reader threads that contained 
errors:\n;
int threads_msg_len = strlen(buf_threads_msg);
int i, errnr;
str msg = MAL_SUCCEED;
@@ -355,7 +355,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s
TO_LOG(bam_loader Waiting for reader threads to finish...\n);
/* Wait until all threads finish and collect their
 * messages. Though it is not very likely, it could be the
-* case that more than 1 thread generate an error message (not
+* case that more than 1 thread generates an error message (not
 * likely because threads exit once they notice that another
 * thread has failed).  Therefore, we collect all error
 * messages in one big error string
@@ -382,7 +382,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s
 * use MAX to make sure we don't add a
 * negative amount to threads_msg_len */
step = snprintf(msg + threads_msg_len,
-   2048 - threads_msg_len, * %s\n,
+   4096 - threads_msg_len, * %s\n,
r_thread_data[i].msg);
threads_msg_len += MAX(0, step);
GDKfree(r_thread_data[i].msg);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - test.modules is useless after changes in All

2014-07-15 Thread Robin Cijvat
Changeset: 0b8e5a569aa3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0b8e5a569aa3
Removed Files:
sql/backends/monet5/bam/Tests/test.modules
Branch: bamloader
Log Message:

test.modules is useless after changes in All


diffs (6 lines):

diff --git a/sql/backends/monet5/bam/Tests/test.modules 
b/sql/backends/monet5/bam/Tests/test.modules
deleted file mode 100644
--- a/sql/backends/monet5/bam/Tests/test.modules
+++ /dev/null
@@ -1,1 +0,0 @@
-bam
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Splitted test into many small tests

2014-07-14 Thread Robin Cijvat
Changeset: b3b973d8c3da for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b3b973d8c3da
Added Files:
sql/backends/monet5/bam/Tests/bam.py
sql/backends/monet5/bam/Tests/bam_export.SQL.py
sql/backends/monet5/bam/Tests/bam_loader_file.SQL.py
sql/backends/monet5/bam/Tests/bam_loader_file.stable.err
sql/backends/monet5/bam/Tests/bam_loader_file.stable.out
sql/backends/monet5/bam/Tests/bam_loader_files.SQL.py
sql/backends/monet5/bam/Tests/bam_loader_files.stable.err
sql/backends/monet5/bam/Tests/bam_loader_files.stable.out
sql/backends/monet5/bam/Tests/bam_loader_repos.SQL.py
sql/backends/monet5/bam/Tests/bam_loader_repos.stable.err
sql/backends/monet5/bam/Tests/bam_loader_repos.stable.out
sql/backends/monet5/bam/Tests/check_files.sql
sql/backends/monet5/bam/Tests/check_files.stable.err
sql/backends/monet5/bam/Tests/check_files.stable.out
sql/backends/monet5/bam/Tests/check_files_exact.sql
sql/backends/monet5/bam/Tests/check_files_exact.stable.err
sql/backends/monet5/bam/Tests/check_files_exact.stable.out
sql/backends/monet5/bam/Tests/drop_last_files.SQL.py
sql/backends/monet5/bam/Tests/drop_last_files.stable.err
sql/backends/monet5/bam/Tests/drop_last_files.stable.out
sql/backends/monet5/bam/Tests/query1.1.sql
sql/backends/monet5/bam/Tests/query1.1.stable.err
sql/backends/monet5/bam/Tests/query1.1.stable.out
sql/backends/monet5/bam/Tests/query1.2.sql
sql/backends/monet5/bam/Tests/query1.2.stable.err
sql/backends/monet5/bam/Tests/query1.2.stable.out
sql/backends/monet5/bam/Tests/query1.3.sql
sql/backends/monet5/bam/Tests/query1.3.stable.err
sql/backends/monet5/bam/Tests/query1.3.stable.out
sql/backends/monet5/bam/Tests/query1.4.sql
sql/backends/monet5/bam/Tests/query1.4.stable.err
sql/backends/monet5/bam/Tests/query1.4.stable.out
sql/backends/monet5/bam/Tests/query1.5.sql
sql/backends/monet5/bam/Tests/query1.5.stable.err
sql/backends/monet5/bam/Tests/query1.5.stable.out
sql/backends/monet5/bam/Tests/query1.6.sql
sql/backends/monet5/bam/Tests/query2.1.sql
sql/backends/monet5/bam/Tests/query2.1.stable.err
sql/backends/monet5/bam/Tests/query2.1.stable.out
sql/backends/monet5/bam/Tests/query2.10.sql
sql/backends/monet5/bam/Tests/query2.10.stable.err
sql/backends/monet5/bam/Tests/query2.10.stable.out
sql/backends/monet5/bam/Tests/query2.11.sql
sql/backends/monet5/bam/Tests/query2.11.stable.err
sql/backends/monet5/bam/Tests/query2.11.stable.out
sql/backends/monet5/bam/Tests/query2.12.sql
sql/backends/monet5/bam/Tests/query2.12.stable.err
sql/backends/monet5/bam/Tests/query2.12.stable.out
sql/backends/monet5/bam/Tests/query2.2.sql
sql/backends/monet5/bam/Tests/query2.2.stable.err
sql/backends/monet5/bam/Tests/query2.2.stable.out
sql/backends/monet5/bam/Tests/query2.3.sql
sql/backends/monet5/bam/Tests/query2.3.stable.err
sql/backends/monet5/bam/Tests/query2.3.stable.out
sql/backends/monet5/bam/Tests/query2.4.sql
sql/backends/monet5/bam/Tests/query2.4.stable.err
sql/backends/monet5/bam/Tests/query2.4.stable.out
sql/backends/monet5/bam/Tests/query2.5.sql
sql/backends/monet5/bam/Tests/query2.5.stable.err
sql/backends/monet5/bam/Tests/query2.5.stable.out
sql/backends/monet5/bam/Tests/query2.6.sql
sql/backends/monet5/bam/Tests/query2.6.stable.err
sql/backends/monet5/bam/Tests/query2.6.stable.out
sql/backends/monet5/bam/Tests/query2.7.sql
sql/backends/monet5/bam/Tests/query2.7.stable.err
sql/backends/monet5/bam/Tests/query2.7.stable.out
sql/backends/monet5/bam/Tests/query2.8.sql
sql/backends/monet5/bam/Tests/query2.8.stable.err
sql/backends/monet5/bam/Tests/query2.8.stable.out
sql/backends/monet5/bam/Tests/query2.9.sql
sql/backends/monet5/bam/Tests/query2.9.stable.err
sql/backends/monet5/bam/Tests/query2.9.stable.out
sql/backends/monet5/bam/Tests/sam_export.SQL.py
sql/backends/monet5/bam/Tests/sam_export.stable.err
sql/backends/monet5/bam/Tests/sam_export.stable.out
sql/backends/monet5/bam/Tests/sql/bam_export.sql
sql/backends/monet5/bam/Tests/sql/bam_loader_file.sql
sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql
sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql
sql/backends/monet5/bam/Tests/sql/drop_last_files.sql
sql/backends/monet5/bam/Tests/sql/sam_export.sql
sql/backends/monet5/bam/Tests/verify_load.sql
sql/backends/monet5/bam/Tests/verify_load.stable.err
sql/backends/monet5/bam/Tests/verify_load.stable.out
Removed Files:

MonetDB: bamloader - Attach all BAM related SQL functions and pr...

2014-07-14 Thread Robin Cijvat
Changeset: c11e529234ec for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c11e529234ec
Modified Files:
sql/backends/monet5/bam/85_bam.sql
sql/backends/monet5/bam/Tests/query1.1.sql
sql/backends/monet5/bam/Tests/query1.2.sql
sql/backends/monet5/bam/Tests/query2.1.sql
sql/backends/monet5/bam/Tests/query2.10.sql
sql/backends/monet5/bam/Tests/query2.11.sql
sql/backends/monet5/bam/Tests/query2.12.sql
sql/backends/monet5/bam/Tests/query2.2.sql
sql/backends/monet5/bam/Tests/query2.3.sql
sql/backends/monet5/bam/Tests/query2.4.sql
sql/backends/monet5/bam/Tests/query2.8.sql
sql/backends/monet5/bam/Tests/query2.9.sql
sql/backends/monet5/bam/Tests/sql/bam_export.sql
sql/backends/monet5/bam/Tests/sql/bam_loader_file.sql
sql/backends/monet5/bam/Tests/sql/bam_loader_files.sql
sql/backends/monet5/bam/Tests/sql/bam_loader_repos.sql
sql/backends/monet5/bam/Tests/sql/drop_last_files.sql
sql/backends/monet5/bam/Tests/sql/sam_export.sql
Branch: bamloader
Log Message:

Attach all BAM related SQL functions and procedures to the bam schema instead 
of the sys schema


diffs (truncated from 573 to 300 lines):

diff --git a/sql/backends/monet5/bam/85_bam.sql 
b/sql/backends/monet5/bam/85_bam.sql
--- a/sql/backends/monet5/bam/85_bam.sql
+++ b/sql/backends/monet5/bam/85_bam.sql
@@ -1,31 +1,33 @@
-CREATE PROCEDURE bam_loader_repos(bam_repos STRING, dbschema SMALLINT, 
nr_threads SMALLINT)
+CREATE SCHEMA bam;
+
+CREATE PROCEDURE bam.bam_loader_repos(bam_repos STRING, dbschema SMALLINT, 
nr_threads SMALLINT)
 EXTERNAL NAME bam.bam_loader_repos;
 
-CREATE PROCEDURE bam_loader_files(bam_files STRING, dbschema SMALLINT, 
nr_threads SMALLINT)
+CREATE PROCEDURE bam.bam_loader_files(bam_files STRING, dbschema SMALLINT, 
nr_threads SMALLINT)
 EXTERNAL NAME bam.bam_loader_files;
 
-CREATE PROCEDURE bam_loader_file(bam_file STRING, dbschema SMALLINT)
+CREATE PROCEDURE bam.bam_loader_file(bam_file STRING, dbschema SMALLINT)
 EXTERNAL NAME bam.bam_loader_file;
 
-CREATE PROCEDURE bam_drop_file(file_id BIGINT, dbschema SMALLINT)
+CREATE PROCEDURE bam.bam_drop_file(file_id BIGINT, dbschema SMALLINT)
 EXTERNAL NAME bam.bam_drop_file;
 
 
-CREATE FUNCTION bam_flag(flag SMALLINT, name STRING)
+CREATE FUNCTION bam.bam_flag(flag SMALLINT, name STRING)
 RETURNS BOOLEAN EXTERNAL NAME bam.bam_flag;
 
-CREATE FUNCTION reverse_seq(seq STRING)
+CREATE FUNCTION bam.reverse_seq(seq STRING)
 RETURNS STRING EXTERNAL NAME bam.reverse_seq;
 
-CREATE FUNCTION reverse_qual(qual STRING)
+CREATE FUNCTION bam.reverse_qual(qual STRING)
 RETURNS STRING EXTERNAL NAME bam.reverse_qual;
 
-CREATE FUNCTION seq_length(cigar STRING)
+CREATE FUNCTION bam.seq_length(cigar STRING)
 RETURNS INT EXTERNAL NAME bam.seq_length;
 
 
-CREATE PROCEDURE sam_export(output_path STRING)
+CREATE PROCEDURE bam.sam_export(output_path STRING)
 EXTERNAL NAME bam.sam_export;
 
-CREATE PROCEDURE bam_export(output_path STRING)
+CREATE PROCEDURE bam.bam_export(output_path STRING)
 EXTERNAL NAME bam.bam_export;
diff --git a/sql/backends/monet5/bam/Tests/query1.1.sql 
b/sql/backends/monet5/bam/Tests/query1.1.sql
--- a/sql/backends/monet5/bam/Tests/query1.1.sql
+++ b/sql/backends/monet5/bam/Tests/query1.1.sql
@@ -1,6 +1,6 @@
 SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
 FROM bam.alignments_1
-WHERE bam_flag(flag, 'seco_alig') = False
+WHERE bam.bam_flag(flag, 'seco_alig') = False
 ORDER BY qname;
 
 SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
@@ -8,5 +8,5 @@ FROM bam.unpaired_primary_alignments_3
 UNION
 SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
 FROM bam.unpaired_alignments_3
-WHERE bam_flag(flag, 'seco_alig') = False
+WHERE bam.bam_flag(flag, 'seco_alig') = False
 ORDER BY qname;
diff --git a/sql/backends/monet5/bam/Tests/query1.2.sql 
b/sql/backends/monet5/bam/Tests/query1.2.sql
--- a/sql/backends/monet5/bam/Tests/query1.2.sql
+++ b/sql/backends/monet5/bam/Tests/query1.2.sql
@@ -1,6 +1,6 @@
 SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
 FROM bam.alignments_1
-WHERE bam_flag(flag, 'seco_alig') = False
+WHERE bam.bam_flag(flag, 'seco_alig') = False
 ORDER BY rname, pos;
 
 SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
@@ -8,5 +8,5 @@ FROM bam.unpaired_primary_alignments_3
 UNION
 SELECT qname, flag, rname, pos, mapq, cigar, rnext, pnext, tlen, seq, qual
 FROM bam.unpaired_alignments_3
-WHERE bam_flag(flag, 'seco_alig') = False
+WHERE bam.bam_flag(flag, 'seco_alig') = False
 ORDER BY rname, pos;
diff --git a/sql/backends/monet5/bam/Tests/query2.1.sql 
b/sql/backends/monet5/bam/Tests/query2.1.sql
--- a/sql/backends/monet5/bam/Tests/query2.1.sql
+++ b/sql/backends/monet5/bam/Tests/query2.1.sql
@@ -1,41 +1,41 @@
 WITH alig AS (
 SELECT qname, flag, seq, qual
 FROM bam.alignments_1
-

MonetDB: bamloader - Merge with default branch

2014-07-13 Thread Robin Cijvat
Changeset: 1ce852d1a673 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1ce852d1a673
Added Files:
gdk/gdk_firstn.c
monetdb5/tests/gdkTests/Tests/firstn.mal
monetdb5/tests/gdkTests/Tests/firstn.stable.err
monetdb5/tests/gdkTests/Tests/firstn.stable.out
Removed Files:
monetdb5/modules/mal/pqueue.c
monetdb5/modules/mal/pqueue.h
monetdb5/modules/mal/pqueue.mal
Modified Files:
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures_gsl.stable.out
clients/Tests/MAL-signatures_nocfitsio.stable.out
clients/Tests/MAL-signatures_sphinxclient.stable.out
clients/Tests/exports.stable.out
gdk/Makefile.ag
gdk/gdk.h
java/src/nl/cwi/monetdb/jdbc/MonetDatabaseMetaData.java
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/algebra.h
monetdb5/modules/kernel/algebra.mal
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/Tests/pqueue.mal
monetdb5/modules/mal/Tests/pqueue.stable.out
monetdb5/modules/mal/Tests/pqueue2.mal
monetdb5/modules/mal/Tests/pqueue2.stable.out
monetdb5/modules/mal/Tests/pqueue3.mal
monetdb5/modules/mal/Tests/pqueue3.stable.out
monetdb5/modules/mal/mal_init.mal
monetdb5/optimizer/opt_centipede.c
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
monetdb5/tests/gdkTests/Tests/All
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_gencode.c
sql/benchmarks/tpch/Tests/02-explain.stable.out
sql/benchmarks/tpch/Tests/03-explain.stable.out
sql/benchmarks/tpch/Tests/10-explain.stable.out
sql/benchmarks/tpch/Tests/18-explain.stable.out
sql/benchmarks/tpch/Tests/21-explain.stable.out
sql/benchmarks/tpch/Tests/21-explain.stable.out.32bit
sql/server/rel_select.c
sql/server/sql_parser.y

sql/test/BugTracker-2010/Tests/LIMIT_OFFSET_big-endian.Bug-2622.stable.out
sql/test/pg_regress/Tests/numeric.sql
sql/test/pg_regress/Tests/numeric.stable.err
sql/test/pg_regress/Tests/numeric.stable.out
sql/test/pg_regress/Tests/oid.sql
sql/test/pg_regress/Tests/oid.stable.err
sql/test/pg_regress/Tests/oid.stable.out
sql/test/pg_regress/postgresql2sql99.sh
Branch: bamloader
Log Message:

Merge with default branch


diffs (truncated from 11401 to 300 lines):

diff --git a/NT/installer32/MonetDB5-SQL-Installer.vdproj 
b/NT/installer32/MonetDB5-SQL-Installer.vdproj
--- a/NT/installer32/MonetDB5-SQL-Installer.vdproj
+++ b/NT/installer32/MonetDB5-SQL-Installer.vdproj
@@ -927,12 +927,6 @@
 }
 Entry
 {
-MsmKey = 8:_F4DDA57B13344198B902859413BF2B9D
-OwnerKey = 8:_UNDEFINED
-MsmSig = 8:_UNDEFINED
-}
-Entry
-{
 MsmKey = 8:_F671CE9CE9A64006AFFFAFBCBDA8EDCE
 OwnerKey = 8:_UNDEFINED
 MsmSig = 8:_UNDEFINED
@@ -4052,26 +4046,6 @@
 IsDependency = 11:FALSE
 IsolateTo = 8:
 }
-
{1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_F4DDA57B13344198B902859413BF2B9D
-{
-SourcePath = 8:lib\\monetdb5\\pqueue.mal
-TargetName = 8:pqueue.mal
-Tag = 8:
-Folder = 8:_B593DAA0787744D6A44D17884E0C3951
-Condition = 8:
-Transitive = 11:FALSE
-Vital = 11:TRUE
-ReadOnly = 11:FALSE
-Hidden = 11:FALSE
-System = 11:FALSE
-Permanent = 11:FALSE
-SharedLegacy = 11:FALSE
-PackageAs = 3:1
-Register = 3:1
-Exclude = 11:FALSE
-IsDependency = 11:FALSE
-IsolateTo = 8:
-}
 
{1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_F671CE9CE9A64006AFFFAFBCBDA8EDCE
 {
 SourcePath = 8:lib\\monetdb5\\createdb\\23_skyserver.sql
diff --git a/NT/installer64/MonetDB5-SQL-Installer.vdproj 
b/NT/installer64/MonetDB5-SQL-Installer.vdproj
--- a/NT/installer64/MonetDB5-SQL-Installer.vdproj
+++ b/NT/installer64/MonetDB5-SQL-Installer.vdproj
@@ -927,12 +927,6 @@
 }
 Entry
 {
-MsmKey = 8:_F4DDA57B13344198B902859413BF2B9D
-OwnerKey = 8:_UNDEFINED
-MsmSig = 8:_UNDEFINED
-}
-Entry
-{
 MsmKey = 8:_F671CE9CE9A64006AFFFAFBCBDA8EDCE
 OwnerKey = 8:_UNDEFINED
 MsmSig = 8:_UNDEFINED
@@ -4052,26 +4046,6 @@
 IsDependency = 11:FALSE
 IsolateTo = 8:
 }
-
{1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_F4DDA57B13344198B902859413BF2B9D
-{
-SourcePath = 

MonetDB: bamloader - Fixed some memory issues

2014-07-10 Thread Robin Cijvat
Changeset: f123faa305d8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f123faa305d8
Modified Files:
sql/backends/monet5/bam/bam_loader.c
sql/backends/monet5/bam/bam_wrapper.h
Branch: bamloader
Log Message:

Fixed some memory issues


diffs (100 lines):

diff --git a/sql/backends/monet5/bam/bam_loader.c 
b/sql/backends/monet5/bam/bam_loader.c
--- a/sql/backends/monet5/bam/bam_loader.c
+++ b/sql/backends/monet5/bam/bam_loader.c
@@ -571,7 +571,8 @@ bam_loader_files(Client cntxt, MalBlkPtr
 
char cur;
int line_size;
-   size_t line_buf_size = 128;
+   str line = NULL;
+   size_t line_buf_size = 0;
int nr_lines = 1;
 
str msg = MAL_SUCCEED;
@@ -607,39 +608,35 @@ bam_loader_files(Client cntxt, MalBlkPtr
goto cleanup;
}
/* Enables cleanup to check individual files */
-   memset(filenames, 0, sizeof(nr_lines * sizeof(str)));
-
-   /* malloc 128 chars for every file; this is safe since call to
-* getline automatically reallocs when buffer is too small */
-   for (i = 0; i  nr_lines; ++i) {
-   if ((filenames[i] =
-GDKmalloc(line_buf_size * sizeof(char))) == NULL) {
-   msg = createException(MAL, bam_loader_files,
- MAL_MALLOC_FAIL);
-   goto cleanup;
-   }
-   }
+   memset(filenames, 0, nr_lines * sizeof(str));
 
/* Reset file pointer to beginning of file */
rewind(f);
 
/* Read BAM filenames into files array */
while ((line_size =
-   getline(filenames[nr_files], line_buf_size, f)) != -1) {
-   str l = filenames[nr_files];
+   getline(line, line_buf_size, f)) = 0) {
 
-   if (l[0] == '\n' || l[0] == '#') {
+   if (line_size == 0 || line[0] == '\n' || line[0] == '\r' || 
line[0] == '#') {
/* Skip empty lines or lines that start with a hash */
+   free(line);
+   line = NULL;
+   /* We need to reset buf size to 0, since getline will 
have updated it */
+   line_buf_size = 0;
continue;
}
-   if (l[line_size - 1] == '\n') {
+   if (line[line_size - 1] == '\n') {
/* Newline character is not part of the filepath */
-   l[line_size - 1] = '\0';
+   line[line_size - 1] = '\0';
}
-   ++nr_files;
-   /* We need to reset it to 128, since getline might
-* have changed its value */
-   line_buf_size = 128;
+   if(line_size  1  line[line_size - 2] == '\r') {
+   /* Return character is also not a part of the filepath 
*/
+   line[line_size - 2] = '\0';
+   }
+   filenames[nr_files++] = line;
+   line = NULL;
+   /* We need to reset it to 0, since getline will have updated 
this val */
+   line_buf_size = 0;
}
 
if (ferror(f)) {
@@ -661,10 +658,16 @@ bam_loader_files(Client cntxt, MalBlkPtr
 nr_threads);
 
  cleanup:
+   if(line) {
+   //if line still contains anything, this is something that was 
put there
+   //by getline on the last, failed attempt to read a line
+   free(line);
+   }
+
if (filenames) {
for (i = 0; i  nr_files; ++i) {
if (filenames[i]) {
-   GDKfree(filenames[i]);
+   free(filenames[i]);
}
}
GDKfree(filenames);
diff --git a/sql/backends/monet5/bam/bam_wrapper.h 
b/sql/backends/monet5/bam/bam_wrapper.h
--- a/sql/backends/monet5/bam/bam_wrapper.h
+++ b/sql/backends/monet5/bam/bam_wrapper.h
@@ -31,7 +31,7 @@
 
 #include samtools/sam.h
 
-#define BW_FP_BUF_SIZE 128
+#define BW_FP_BUF_SIZE 1024
 
 typedef sht ordering;
 typedef enum {SAM, BAM} filetype;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Merge with default branch

2014-07-10 Thread Robin Cijvat
Changeset: c0cf13acf65a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c0cf13acf65a
Modified Files:
MonetDB.spec
clients/Tests/SQL-dump.stable.out.32bit
configure.ag
gdk/gdk_logger.c
gdk/gdk_posix.c
gdk/gdk_private.h
gdk/gdk_storage.c
sql/server/sql_atom.c
sql/server/sql_parser.y
sql/test/pg_regress/Tests/char.sql
sql/test/pg_regress/Tests/char.stable.err
sql/test/pg_regress/Tests/char.stable.out
sql/test/pg_regress/Tests/float4.sql
sql/test/pg_regress/Tests/float4.stable.err
sql/test/pg_regress/Tests/float4.stable.out
sql/test/pg_regress/Tests/float8.sql
sql/test/pg_regress/Tests/float8.stable.err
sql/test/pg_regress/Tests/float8.stable.out
sql/test/pg_regress/Tests/int2.sql
sql/test/pg_regress/Tests/int2.stable.err
sql/test/pg_regress/Tests/int2.stable.out
sql/test/pg_regress/Tests/int4.sql
sql/test/pg_regress/Tests/int4.stable.err
sql/test/pg_regress/Tests/int4.stable.out
sql/test/pg_regress/Tests/int8.sql
sql/test/pg_regress/Tests/int8.stable.err
sql/test/pg_regress/Tests/int8.stable.out
sql/test/pg_regress/Tests/numerology.sql
sql/test/pg_regress/Tests/polygon.sql
sql/test/pg_regress/postgresql2sql99.sh
testing/Mtest.py.in
Branch: bamloader
Log Message:

Merge with default branch


diffs (truncated from 3786 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -597,9 +597,9 @@ used from the MAL level.
 Summary: MonetDB5 SQL server modules
 Group: Applications/Databases
 Requires: MonetDB5-server = %{version}-%{release}
-%if %{?rhel:0}%{!?rhel:1}
-# for systemd-tmpfiles
-Requires: systemd-units
+%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} = 7
+# RHEL = 7, and all current Fedora
+Requires: %{_bindir}/systemd-tmpfiles
 %endif
 Obsoletes: MonetDB-SQL-devel
 Obsoletes: %{name}-SQL
@@ -613,7 +613,7 @@ accelerators.  It also has an SQL fronte
 This package contains the SQL frontend for MonetDB.  If you want to
 use SQL with MonetDB, you will need to install this package.
 
-%if %{?rhel:0}%{!?rhel:1}
+%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} = 7
 %post SQL-server5
 systemd-tmpfiles --create %{_sysconfdir}/tmpfiles.d/monetdbd.conf
 %endif
@@ -623,11 +623,11 @@ systemd-tmpfiles --create %{_sysconfdir}
 %{_bindir}/monetdb
 %{_bindir}/monetdbd
 %dir %attr(775,monetdb,monetdb) %{_localstatedir}/log/monetdb
-%if %{?rhel:0}%{!?rhel:1}
-# Fedora 15 and newer
+%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} = 7
+# RHEL = 7, and all current Fedora
 %{_sysconfdir}/tmpfiles.d/monetdbd.conf
 %else
-# RedHat Enterprise Linux
+# RedHat Enterprise Linux  7
 %dir %attr(775,monetdb,monetdb) %{_localstatedir}/run/monetdb
 %exclude %{_sysconfdir}/tmpfiles.d/monetdbd.conf
 %endif
diff --git a/clients/Tests/SQL-dump.stable.out.32bit 
b/clients/Tests/SQL-dump.stable.out.32bit
--- a/clients/Tests/SQL-dump.stable.out.32bit
+++ b/clients/Tests/SQL-dump.stable.out.32bit
@@ -6998,8 +6998,8 @@ 6302  text  create function json.text(j
 6306   text  create function json.text(js string)\nreturns string external 
name json.text; json  true1   false   false   false   6230
 6310   text  create function json.text(js int)\nreturns string external 
name json.text;json  true1   false   false   false   6230
 6314   output-- The remainder awaits the implementation \n\ncreate 
aggregate json.output(js json)\nreturns string external name json.output;  
 json  true3   false   false   false   6230
-6318   tojsonarray   -- create function json.object(*) returns json 
external name json.objectrender;\n\n-- create function json.array(*) returns 
json external name json.arrayrender;\n\n-- unnesting the JSON structure\n\n-- 
create function json.unnest(js json)\n-- returns table( id integer, k string, v 
string) external name json.unnest;\n\n-- create function json.unnest(js 
json)\n-- returns table( k string, v string) external name json.unnest;\n\n-- 
create function json.unnest(js json)\n-- returns table( v string) external name 
json.unnest;\n\n-- create function json.nest table( id integer, k string, v 
string)\n-- returns json external name json.nest;\n\ncreate aggregate 
json.tojsonarray( x string ) returns string external name json.aggr;
json  true3   false   false   false   6230
-6322   tojsonarray   create aggregate json.tojsonarray( x double ) returns 
string external name json.aggr; json  true3   false   false   false  
 6230
+6318   tojsonarray   -- create function json.object(*) returns json 
external name json.objectrender;\n\n-- create function json.array(*) returns 
json external name json.arrayrender;\n\n-- unnesting the JSON structure\n\n-- 
create function json.unnest(js json)\n-- returns table( id integer, k string, v 
string) external name json.unnest;\n\n-- create function 

MonetDB: bamloader - Merge with changes Sjoerd

2014-07-10 Thread Robin Cijvat
Changeset: 350c99a6a6f8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=350c99a6a6f8
Modified Files:
configure.ag
Branch: bamloader
Log Message:

Merge with changes Sjoerd

___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Fixed test

2014-07-10 Thread Robin Cijvat
Changeset: e43b1e7b7612 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e43b1e7b7612
Modified Files:
sql/backends/monet5/bam/Tests/test.SQL.py
sql/backends/monet5/bam/Tests/test.stable.err
sql/backends/monet5/bam/Tests/test.stable.out
Branch: bamloader
Log Message:

Fixed test


diffs (truncated from 2855 to 300 lines):

diff --git a/sql/backends/monet5/bam/Tests/test.SQL.py 
b/sql/backends/monet5/bam/Tests/test.SQL.py
--- a/sql/backends/monet5/bam/Tests/test.SQL.py
+++ b/sql/backends/monet5/bam/Tests/test.SQL.py
@@ -177,7 +177,7 @@ for (file_id, dbschema) in to_query:
 # Insert both the SAM and BAM files into the database. Note that we always 
insert into
 # dbschema 0, since the outputted files aren't necessarily ordered by 
qname.
 c.stdin.write(CALL bam_loader_file('%s', 0);\n% tmp_output(file_id))
-c.stdin.write(CALL bam_loader_file('%s', 0);\n% tmp_output(file_id, 
sam=False))
+#c.stdin.write(CALL bam_loader_file('%s', 0);\n% tmp_output(file_id, 
sam=False))
 
 # We now have the alignment data for this file in three different 
alignment tables. The ultimate
 # export/import test is now to see if the data in them is exactly the same 
(except for virtual_offset)
@@ -206,4 +206,4 @@ sys.stderr.write(err)
 # All that is left to do is delete the temporary SAM/BAM files
 for (file_id, dbschema) in to_query:
 os.remove(tmp_output(file_id))
-os.remove(tmp_output(file_id, sam=False))
\ No newline at end of file
+#os.remove(tmp_output(file_id, sam=False))
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/test.stable.err 
b/sql/backends/monet5/bam/Tests/test.stable.err
--- a/sql/backends/monet5/bam/Tests/test.stable.err
+++ b/sql/backends/monet5/bam/Tests/test.stable.err
@@ -1,1596 +1,18 @@
 stderr of test 'test` in directory 'sql/backends/monet5/bam` itself:
 
 
-# 16:03:13   
-# 16:03:13   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=31808 --set 
mapi_usock=/var/tmp/mtest-26892/.s.monetdb.31808 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
-# 16:03:13   
 
-# builtin opt  gdk_dbpath = /home/robin/MonetDB/PREFIX/var/monetdb5/dbfarm/demo
-# builtin opt  gdk_debug = 0
-# builtin opt  gdk_vmtrim = no
-# builtin opt  monet_prompt = 
-# builtin opt  monet_daemon = no
-# builtin opt  mapi_port = 5
-# builtin opt  mapi_open = false
-# builtin opt  mapi_autosense = false
-# builtin opt  sql_optimizer = default_pipe
-# builtin opt  sql_debug = 0
-# cmdline opt  gdk_nr_threads = 0
-# cmdline opt  mapi_open = true
-# cmdline opt  mapi_port = 31808
-# cmdline opt  mapi_usock = /var/tmp/mtest-26892/.s.monetdb.31808
-# cmdline opt  monet_prompt = 
-# cmdline opt  mal_listing = 2
-# cmdline opt  gdk_dbpath = 
/home/robin/MonetDB/PREFIX/var/MonetDB/mTests_sql_backends_monet5_bam
-# cmdline opt  mal_listing = 0
-# cmdline opt  gdk_debug = 536870922
-# bam_loader: Loader started for 2 BAM files...
-# # bam_loader Creating schema 'bam'...
-# # CREATE SCHEMA bam;
-# # bam_loader Creating table 'files'...
-# # CREATE TABLE bam.files ( 
-#  file_id BIGINTNOT 
NULL, 
-#  file_location STRING  NOT 
NULL, 
-#  dbschema   SMALLINT 
NOT NULL, 
-#  format_version   VARCHAR(7), 
-#  sorting_order VARCHAR(10), 
-#  comments   STRING, 
-#  CONSTRAINT files_pkey_file_id PRIMARY KEY (file_id) 
-#  );
-# # bam_loader Creating table 'sq'...
-# # CREATE TABLE bam.sq ( 
-#  sn   STRING 
  NOT NULL, 
-#  file_id BIGINTNOT 
NULL, 
-#  ln   INT, 
-#  as INT, 
-#  m5   
STRING, 
-#  sp   
STRING, 
-#  ur   
STRING, 
-#  CONSTRAINT sq_pkey_sn_file_id PRIMARY KEY (sn, file_id), 
-#  CONSTRAINT sq_fkey_file_id FOREIGN KEY (file_id) REFERENCES 
bam.files (file_id) 
-#  );
-# # bam_loader Creating table 'rg'...
-# # CREATE TABLE bam.rg ( 
-#  id   STRING 
  NOT NULL, 
-#  file_id BIGINTNOT 
NULL, 
-#  cn   
STRING, 
-#  ds  

MonetDB: bamloader - Improved test greatly

2014-07-03 Thread Robin Cijvat
Changeset: e1dd2e16216e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1dd2e16216e
Removed Files:
sql/backends/monet5/bam/Tests/load.sql
Modified Files:
sql/backends/monet5/bam/Tests/test.SQL.py
sql/backends/monet5/bam/Tests/test.stable.err
sql/backends/monet5/bam/Tests/test.stable.out
Branch: bamloader
Log Message:

Improved test greatly


diffs (truncated from 4768 to 300 lines):

diff --git a/sql/backends/monet5/bam/Tests/load.sql 
b/sql/backends/monet5/bam/Tests/load.sql
deleted file mode 100644
--- a/sql/backends/monet5/bam/Tests/load.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-CALL bam_loader_repos('PWD/files', 0, 32);
-CALL bam_loader_repos('PWD/files/queryname', 1, 32);
diff --git a/sql/backends/monet5/bam/Tests/test.SQL.py 
b/sql/backends/monet5/bam/Tests/test.SQL.py
--- a/sql/backends/monet5/bam/Tests/test.SQL.py
+++ b/sql/backends/monet5/bam/Tests/test.SQL.py
@@ -13,38 +13,119 @@ def new_client():
 
 def set_var(c, name, value, numeric=True):
 if numeric:
-c.stdin.write('DECLARE %s integer;'% name)
-c.stdin.write('SET %s=%s;'%(name, value))
+c.stdin.write('DECLARE %s integer;\n'% name)
+c.stdin.write('SET %s=%s;\n'%(name, value))
 else:
-c.stdin.write('DECLARE %s varchar(32);'% name)
-c.stdin.write('SET %s=\'%s\';'% (name, value))
+c.stdin.write('DECLARE %s varchar(32);\n'% name)
+c.stdin.write('SET %s=\'%s\';\n'% (name, value))
 
-# start with loading bam files into the database
+
+
+
+
+# start with loading bam files into the database with all loader functions we 
have built
+# we assume that all bam or sam files are stored into the files directory.
+# these files will all be loaded into the database with all loading methods 
that we have.
+# files in the directory files/queryname are assumed to be ordered by 
queryname and are
+# therefore also loaded into the pairwise storage schema
+
 c = new_client()
-for ln in open(os.path.join(SRCDIR,load.sql)):
-c.stdin.write(ln.replace('PWD', SRCDIR))
+
+# get files
+files_path = os.path.join(SRCDIR, files)
+files_qname_path = os.path.join(files_path, queryname)
+files = [ f for f in os.listdir(files_path) if 
os.path.isfile(os.path.join(files_path, f)) ]
+files_qname = [ f for f in os.listdir(files_qname_path) if 
os.path.isfile(os.path.join(files_qname_path, f)) ]
+
+# use bam_loader_repos
+c.stdin.write(CALL bam_loader_repos('%s', 0, 8);\n% files_path)
+c.stdin.write(CALL bam_loader_repos('%s', 1, 8);\n% files_qname_path)
+
+# use bam_loader_files by first writing temporary files and then invoking 
these loaders
+f_tmp_path = os.path.join(SRCDIR, tmp_list.txt)
+f_tmp_qname_path = os.path.join(SRCDIR, tmp_list_qname.txt)
+f_tmp = open(f_tmp_path, w)
+f_tmp_qname = open(f_tmp_qname_path, w)
+
+for file in files:
+f_tmp.write(%s\n% os.path.join(files_path, file))
+
+for file in files_qname:
+f_tmp_qname.write(%s\n% os.path.join(files_qname_path, file))
+
+f_tmp.close()
+f_tmp_qname.close()
+
+c.stdin.write(CALL bam_loader_files('%s', 0, 8);\n% f_tmp_path)
+c.stdin.write(CALL bam_loader_files('%s', 1, 8);\n% f_tmp_qname_path)
+
+# use bam_loader_file on all separate files
+for file in files:
+c.stdin.write(CALL bam_loader_file('%s', 0);\n% os.path.join(files_path, 
file))
+
+for file in files_qname:
+c.stdin.write(CALL bam_loader_file('%s', 1);\n% 
os.path.join(files_qname_path, file))
 
 out, err = c.communicate()
 sys.stdout.write(out)
 sys.stderr.write(err)
 
+# clean up the temporary files
+os.remove(f_tmp_path)
+os.remove(f_tmp_qname_path)
 
-# now retrieve the file ids that have been inserted
-# (should be 1,2,..,nr_files, but still we extract it from the
-# db to be flexible to logical changes in the bam loader as
-# much as possible)
+
+# extract the files that have been inserted into the database
+# gives us a list of (file_id, dbschema) tuples
 c = new_client()
-c.stdin.write(SELECT file_id, dbschema FROM bam.files;)
+c.stdin.write(SELECT file_id, dbschema FROM bam.files;\n)
 out, err = c.communicate()
-p = re.compile('^\s*\[\s*(\d)\s*,\s*(\d)\s*\]\s*$', re.MULTILINE)
-files_to_test = []
+sys.stdout.write(out)
+sys.stderr.write(err)
+
+p = re.compile('^\s*\[\s*(\d+)\s*,\s*(\d)\s*\]\s*$', re.MULTILINE) # Parses 
raw DB output
+inserted = []
 for match in p.finditer(out):
-files_to_test.append((int(match.group(1)), int(match.group(2
+inserted.append((int(match.group(1)), int(match.group(2
 
-# Now we will execute all benchmark queries on all BAM files in the bam.files 
table
-# and output all data contained in the aux table.
-# Furthermore, we transfer every file to the export table, use sam_export to 
write
-# the contents to a SAM file and then print the raw contents of this SAM file
+
+
+# now we will test the drop functionality by removing all duplicate bam/sam 
files from the database
+# we know that only the first len(files) + len(files_qname) files are unique, 
so we remove
+# 

MonetDB: bamloader - finished bam_export. For this, switched who...

2014-07-02 Thread Robin Cijvat
Changeset: d149e885d77d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d149e885d77d
Modified Files:
sql/backends/monet5/bam/Makefile.ag
sql/backends/monet5/bam/bam_export.c
sql/backends/monet5/bam/bam_wrapper.c
sql/backends/monet5/bam/bam_wrapper.h
Branch: bamloader
Log Message:

finished bam_export. For this, switched whole bam library code from libbam to 
libhts since it provides better functionality for our purpose.


diffs (truncated from 612 to 300 lines):

diff --git a/sql/backends/monet5/bam/Makefile.ag 
b/sql/backends/monet5/bam/Makefile.ag
--- a/sql/backends/monet5/bam/Makefile.ag
+++ b/sql/backends/monet5/bam/Makefile.ag
@@ -30,8 +30,7 @@ INCLUDES = .. \
../../../../common/stream \
../../../../gdk \
../../../../tools/merovingian \
-   ../../../../tools/merovingian/daemon \
-   $(SAMTOOLS_CFLAGS)
+   ../../../../tools/merovingian/daemon
 
 lib__bam = {
MODULE
@@ -44,7 +43,7 @@ lib__bam = {
  bam_export.c bam_export.h
LIBS = ../../../../monetdb5/tools/libmonetdb5 \
   ../../../../gdk/libbat \
-  $(SAMTOOLS_LIBS)
+  -lhts
 }
 
 headers_mal = {
diff --git a/sql/backends/monet5/bam/bam_export.c 
b/sql/backends/monet5/bam/bam_export.c
--- a/sql/backends/monet5/bam/bam_export.c
+++ b/sql/backends/monet5/bam/bam_export.c
@@ -27,10 +27,11 @@
 
 #include monetdb_config.h
 
-#include samtools/bam.h
+#include htslib/sam.h
+#include htslib/kstring.h
 
 #include bam_globals.h
-#include bam_db_interface.h
+//#include bam_db_interface.h
 #include bam_export.h
 
 
@@ -43,189 +44,6 @@ typedef struct bam_field {
 } bam_field;
 
 
-/**
- * Copied directly from bam.h/bam_import.c for use by fill_bam_alig
- * Can not change the calls to realloc to GDKrealloc, since
- * bam_destroy1 does not use GDKfree...
- */
-#ifndef kroundup32
-/*! @function
-  @abstract  Round an integer to the next closest power-2 integer.
-  @param  x  integer to be rounded (in place)
-  @discussion x will be modified.
- */
-#define kroundup32(x) (--(x), (x)|=(x)1, (x)|=(x)2, (x)|=(x)4, 
(x)|=(x)8, (x)|=(x)16, ++(x))
-#endif
-static inline uint8_t *
-alloc_data(bam1_t *b, int size)
-{
-   if (b-m_data  size) {
-   b-m_data = size;
-   kroundup32(b-m_data);
-   b-data = (uint8_t*)realloc(b-data, b-m_data);
-   }
-   return b-data;
-}
-
-/* Error macro */
-#define FILL_BAM_ALIG_ERR Error processing alignment '%d': 
-
-/**
- * I had to write this function, which contains much low level
- * function calls to the Samtools library, since the sam_read1 function
- * contained in bam.h/bam_import.c only works on files. This function is
- * an adjusted variant of the sam_read1 function that gets its fields
- * directly instead of reads it from a tamFile.
- * I wrote this function since I did not feel like first writing everything
- * to a SAM file and then applying sam_read1 since that is just a waste of I/O.
- * Note that I did do a write to/read from SAM for the BAM header, but headers 
are
- * insignificant compared to alignment data.
- *
- * I also added realloc checking, since this was not done in the Samtools code
- * and i properly throw a MAL_MALLOC_FAIL whenever applicable.
- */
-static str
-fill_bam_alig(str qname, sht flag, str rname, int pos,
-   sht mapq, str cigar, str rnext, int pnext,
-   int tlen, str seq, str qual,
-   bam_header_t *header, bam1_t *b, int alignment_nr)
-{
-   int doff = 0;
-   bam1_core_t *c = b-core;
-
-   /* Note: in sam_read1, str-s is often used. This string is not NULL 
terminated! */
-   { // qname
-   c-l_qname = strlen(qname) + 1;
-   if(alloc_data(b, doff + c-l_qname) == NULL) {
-   throw(MAL, fill_bam_alig,
-   FILL_BAM_ALIG_ERR MAL_MALLOC_FAIL, 
alignment_nr);
-   }
-   memcpy(b-data + doff, qname, c-l_qname);
-   doff += c-l_qname;
-   }
-   { // flag
-   c-flag = flag;
-   }
-   { // rname, pos, mapq
-   c-tid = bam_get_tid(header, rname);
-   if (c-tid  0  strcmp(rname, *) != 0) {
-   /* Should not happen, since we built the header 
structure ourselves */
-   throw(MAL, fill_bam_alig,
-   FILL_BAM_ALIG_ERR SQ entry '%s' not found in 
header table,
-   alignment_nr, rname);
-   }
-   c-pos = pos - 1;
-   c-qual = mapq;
-   }
-   { // cigar
-   str s, t;
-   int i, op;
-   long x;
-   c-n_cigar = 0;
-   if (cigar[0] != '*') {
-   uint32_t *cigar_enc;
-   for (s = cigar; *s != '\0'; ++s) {
-   if ((isalpha(*s)) || (*s=='=')) {
-

MonetDB: bamloader - Merge with default branch

2014-07-02 Thread Robin Cijvat
Changeset: 805d4f32d3b6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=805d4f32d3b6
Added Files:
geom/BugTracker/Tests/copy_into_mbr.3492.modules
geom/BugTracker/Tests/copy_into_mbr.3492.sql
geom/BugTracker/Tests/copy_into_mbr.3492.stable.err
geom/BugTracker/Tests/copy_into_mbr.3492.stable.out
Modified Files:
MonetDB.spec
clients/R/MonetDB.R/DESCRIPTION
clients/R/MonetDB.R/NEWS
clients/R/MonetDB.R/R/control.R
clients/R/MonetDB.R/R/monetdb.R
clients/R/MonetDB.R/src/mapi.c
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures_gsl.stable.out
clients/Tests/MAL-signatures_nocfitsio.stable.out
clients/Tests/MAL-signatures_sphinxclient.stable.out
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.64bit.oid32
clients/Tests/SQL-dump_gsl.stable.out
clients/Tests/SQL-dump_gsl.stable.out.oid32
clients/Tests/SQL-dump_nogeom.stable.out
clients/Tests/exports.stable.out
clients/mapiclient/dump.c
clients/mapiclient/mclient.c
clients/odbc/driver/ODBCConvert.c
clients/odbc/driver/ODBCUtil.h
clients/python2/monetdb/sql/monetize.py
clients/python2/test/test_monetize.py
clients/python3/monetdb/sql/monetize.py
clients/python3/test/test_monetize.py
gdk/gdk_logger.c
gdk/gdk_logger.h
gdk/gdk_sample.c
gdk/gdk_unique.c
geom/BugTracker/Tests/All
geom/monetdb5/geom.c
monetdb5/mal/mal_import.c
monetdb5/modules/atoms/batxml.c
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/json.mal
monetdb5/optimizer/opt_centipede.c
monetdb5/optimizer/opt_pushselect.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_statistics.c
sql/benchmarks/tpch/Tests/17-plan.stable.out
sql/benchmarks/tpch/Tests/22-plan.stable.out
sql/common/sql_types.c
sql/scripts/40_json.sql
sql/storage/bat/bat_logger.c
sql/storage/store.c
sql/test/BugTracker/Tests/between_with_column.SF-1959410.stable.err
sql/test/Tests/systemfunctions.stable.out
sql/test/Tests/systemfunctions_gsl.stable.out
sql/test/Tests/systemfunctions_nogeom.stable.out
sql/test/centipede/Tests/olap.stable.out
sql/test/json/Tests/aggregate01.sql
sql/test/json/Tests/aggregate01.stable.out
sql/test/pg_regress/Tests/All
sql/test/pg_regress/Tests/aggregates.stable.err
sql/test/pg_regress/Tests/aggregates.stable.out
sql/test/pg_regress/Tests/alter_table.sql
sql/test/pg_regress/Tests/alter_table.stable.err
sql/test/pg_regress/Tests/alter_table.stable.out
sql/test/pg_regress/Tests/arrays.sql
sql/test/pg_regress/Tests/arrays.stable.err
sql/test/pg_regress/Tests/arrays.stable.out
sql/test/pg_regress/Tests/bit.stable.err
sql/test/pg_regress/Tests/bit.stable.out
sql/test/pg_regress/Tests/boolean.sql
sql/test/pg_regress/Tests/boolean.stable.err
sql/test/pg_regress/Tests/boolean.stable.out
sql/test/pg_regress/Tests/box.stable.err
sql/test/pg_regress/Tests/box.stable.out
sql/test/pg_regress/Tests/btree_index.stable.err
sql/test/pg_regress/Tests/btree_index.stable.out
sql/test/pg_regress/Tests/case.stable.err
sql/test/pg_regress/Tests/case.stable.out
sql/test/pg_regress/Tests/char.stable.err
sql/test/pg_regress/Tests/char.stable.out
sql/test/pg_regress/Tests/circle.stable.err
sql/test/pg_regress/Tests/circle.stable.out
sql/test/pg_regress/Tests/cluster.sql
sql/test/pg_regress/Tests/cluster.stable.err
sql/test/pg_regress/Tests/cluster.stable.out
sql/test/pg_regress/Tests/comments.stable.err
sql/test/pg_regress/Tests/comments.stable.out
sql/test/pg_regress/Tests/conversion.stable.err
sql/test/pg_regress/Tests/conversion.stable.out
sql/test/pg_regress/Tests/copy2.stable.err
sql/test/pg_regress/Tests/copy2.stable.out
sql/test/pg_regress/Tests/create_aggregate.stable.err
sql/test/pg_regress/Tests/create_aggregate.stable.out
sql/test/pg_regress/Tests/create_index.stable.err
sql/test/pg_regress/Tests/create_index.stable.out
sql/test/pg_regress/Tests/create_misc.stable.err
sql/test/pg_regress/Tests/create_misc.stable.out
sql/test/pg_regress/Tests/create_operator.stable.err
sql/test/pg_regress/Tests/create_operator.stable.out
sql/test/pg_regress/Tests/create_table.sql
sql/test/pg_regress/Tests/create_table.stable.err
sql/test/pg_regress/Tests/create_table.stable.out
sql/test/pg_regress/Tests/create_type.stable.err
sql/test/pg_regress/Tests/create_type.stable.out

MonetDB: bamloader - Adapted .ag files, MonetDB.spec and debian/...

2014-07-02 Thread Robin Cijvat
Changeset: 43f84e370374 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=43f84e370374
Modified Files:
MonetDB.spec
configure.ag
debian/rules
sql/backends/monet5/Makefile.ag
sql/backends/monet5/bam/Makefile.ag
Branch: bamloader
Log Message:

Adapted .ag files, MonetDB.spec and debian/rules to look for htslib instead of 
samtools


diffs (202 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -36,8 +36,8 @@
 %define with_geos 1
 %endif
 
-%if %{?_with_samtools:1}%{!?_with_samtools:0}
-%define with_samtools 1
+%if %{?_with_hts:1}%{!?_with_hts:0}
+%define with_hts 1
 %endif
 
 Name: %{name}
@@ -82,8 +82,8 @@ BuildRequires: rubygems-devel
 %endif
 BuildRequires: unixODBC-devel
 BuildRequires: zlib-devel
-%if %{?with_samtools:1}%{!?with_samtools:0}
-BuildRequires: samtools-devel
+%if %{?with_hts:1}%{!?with_hts:0}
+BuildRequires: hts-devel
 %endif
 
 # need to define python_sitelib on RHEL 5 and older
@@ -456,7 +456,7 @@ numerical analysis (gsl).
 %{_libdir}/monetdb5/gsl.mal
 %{_libdir}/monetdb5/lib_gsl.so
 
-%if %{?_with_samtools:1}%{!?_with_samtools:0}
+%if %{?_with_hts:1}%{!?_with_hts:0}
 %package bam-MonetDB5
 Summary: MonetDB5 SQL interface to the bam library
 Group: Applications/Databases
@@ -542,7 +542,7 @@ fi
 %exclude %{_libdir}/monetdb5/lib_geom.so
 %endif
 %exclude %{_libdir}/monetdb5/lib_gsl.so
-%if %{?_with_samtools:1}%{!?_with_samtools:0}
+%if %{?_with_hts:1}%{!?_with_hts:0}
 %exclude %{_libdir}/monetdb5/bam.mal
 %exclude %{_libdir}/monetdb5/autoload/*_bam.mal
 %exclude %{_libdir}/monetdb5/lib_bam.so
@@ -640,7 +640,7 @@ systemd-tmpfiles --create %{_sysconfdir}
 %exclude %{_libdir}/monetdb5/createdb/*_geom.sql
 %endif
 %exclude %{_libdir}/monetdb5/createdb/*_gsl.sql
-%if %{?_with_samtools:1}%{!?_with_samtools:0}
+%if %{?_with_hts:1}%{!?_with_hts:0}
 %exclude %{_libdir}/monetdb5/createdb/*_bam.sql
 %endif
 # %exclude %{_libdir}/monetdb5/createdb/*_rdf.sql
@@ -800,7 +800,7 @@ developer, but if you do want to test, t
--with-readline=yes \
--with-rubygem=%{?rhel:no}%{!?rhel:yes} \
--with-rubygem-dir=%{?rhel:no}%{!?rhel:%{gem_dir}} \
-   --with-samtools=%{?with_samtools:yes}%{!?with_samtools:no} \
+   --with-hts=%{?with_hts:yes}%{!?with_hts:no} \
--with-sphinxclient=no \
--with-unixodbc=yes \
--with-valgrind=no \
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -2362,49 +2362,51 @@ AC_SUBST(MSEED_CFLAGS, $MSEED_CFLAGS)
 AC_SUBST(MSEED_LIBS, $MSEED_LIBS)
 AM_CONDITIONAL(HAVE_MSEED, test x$have_mseed != xno)
 
-# samtools, only used by sql
-org_have_samtools=auto
-have_samtools=$org_have_samtools
-AC_ARG_WITH(samtools,
-   AS_HELP_STRING([--with-samtools=DIR],
-   [samtools (providing bam library) are installed in DIR]),
-   [have_samtools=$withval], [have_samtools=auto])
-if test x$have_samtools != xno; then
-   case $have_samtools in
+# hts (required by BAM library), only used by sql
+org_have_bam=auto
+have_bam=$org_have_bam
+AC_ARG_WITH(hts,
+   AS_HELP_STRING([--with-hts=DIR],
+   [hts (providing bam library) are installed in DIR]),
+   [have_bam=$withval], [have_bam=auto])
+if test x$have_bam != xno; then
+   case $have_bam in
auto|yes)
;;
*)
-   SAMTOOLS_CFLAGS=-I$have_samtools/include
-   SAMTOOLS_LIBS=-L$have_samtools/lib -lbam
+   BAM_CFLAGS=-I$have_bam/include
+   BAM_LIBS=-L$have_bam/lib -lhts
;;
esac
 
save_CPPFLAGS=$CPPFLAGS
save_LDFLAGS=$LDFLAGS
-   CPPFLAGS=$CPPFLAGS $SAMTOOLS_CFLAGS
-   LDFLAGS=$LDFLAGS $SAMTOOLS_LIBS
-   AC_CHECK_HEADER(samtools/bam.h,
-   AC_CHECK_HEADER(samtools/bgzf.h,
-   AC_CHECK_LIB(bam, bam_header_read,
-   AC_DEFINE(HAVE_SAMTOOLS, 1, [Define if you have 
the samtools (providing the bam library)]),
-   [ if test x$have_samtools != xauto; then 
AC_MSG_ERROR([-lbam library not found]); fi
- have_samtools=no; why_have_samtools=(bam 
library not found) ],
-   [-lm -lpthread -lz]),
-   [ if test x$have_samtools != xauto; then 
AC_MSG_ERROR([samtools/bgzf.h header not found]); fi
- have_samtools=no; why_have_samtools=(samtools/bgzf.h 
header not found) ]),
-   [ if test x$have_samtools != xauto; then 
AC_MSG_ERROR([samtools/bam.h header not found]); fi
- have_samtools=no; why_have_samtools=(samtools/bam.h header 
not found) ])
+   CPPFLAGS=$CPPFLAGS $BAM_CFLAGS
+   LDFLAGS=$LDFLAGS $BAM_LIBS
+   AC_CHECK_HEADER(htslib/sam.h,
+   AC_CHECK_HEADER(htslib/bgzf.h,
+   AC_CHECK_HEADER(htslib/kstring.h,
+   AC_CHECK_LIB(hts, 

MonetDB: bamloader - Fixed indentation (spaces to tabs) and remo...

2014-07-01 Thread Robin Cijvat
Changeset: aeed76d1d22e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=aeed76d1d22e
Modified Files:
sql/backends/monet5/bam/Makefile.ag
sql/backends/monet5/bam/bam_db_interface.c
sql/backends/monet5/bam/bam_db_interface.h
sql/backends/monet5/bam/bam_export.c
sql/backends/monet5/bam/bam_export.h
sql/backends/monet5/bam/bam_globals.h
sql/backends/monet5/bam/bam_lib.c
sql/backends/monet5/bam/bam_loader.c
sql/backends/monet5/bam/bam_loader.h
sql/backends/monet5/bam/bam_wrapper.c
sql/backends/monet5/bam/bam_wrapper.h
Branch: bamloader
Log Message:

Fixed indentation (spaces to tabs) and removed whitespace at the end of lines


diffs (truncated from 4809 to 300 lines):

diff --git a/sql/backends/monet5/bam/Makefile.ag 
b/sql/backends/monet5/bam/Makefile.ag
--- a/sql/backends/monet5/bam/Makefile.ag
+++ b/sql/backends/monet5/bam/Makefile.ag
@@ -37,14 +37,14 @@ lib__bam = {
MODULE
DIR = libdir/monetdb5
SOURCES = bam_loader.c bam_loader.h \
- bam_wrapper.c bam_wrapper.h \
- bam_db_interface.c bam_db_interface.h \
- bam_globals.c bam_globals.h \
- bam_lib.c bam_lib.h \
- bam_export.c bam_export.h
+ bam_wrapper.c bam_wrapper.h \
+ bam_db_interface.c bam_db_interface.h \
+ bam_globals.c bam_globals.h \
+ bam_lib.c bam_lib.h \
+ bam_export.c bam_export.h
LIBS = ../../../../monetdb5/tools/libmonetdb5 \
-  ../../../../gdk/libbat \
-  $(SAMTOOLS_LIBS)
+  ../../../../gdk/libbat \
+  $(SAMTOOLS_LIBS)
 }
 
 headers_mal = {
diff --git a/sql/backends/monet5/bam/bam_db_interface.c 
b/sql/backends/monet5/bam/bam_db_interface.c
--- a/sql/backends/monet5/bam/bam_db_interface.c
+++ b/sql/backends/monet5/bam/bam_db_interface.c
@@ -28,172 +28,172 @@
 #include bam_db_interface.h
 
 #define SQL_CREATE_STORAGE_0 \
-CREATE TABLE bam.alignments_LLFMT ( \n\
-virtual_offsetBIGINT  NOT NULL, \n\
-qname STRING  NOT NULL, \n\
-flag  SMALLINTNOT NULL, \n\
-rname STRING  NOT NULL, \n\
-pos   INT NOT NULL, \n\
-mapq  SMALLINTNOT NULL, \n\
-cigar STRING  NOT NULL, \n\
-rnext STRING  NOT NULL, \n\
-pnext INT NOT NULL, \n\
-tlen  INT NOT NULL, \n\
-seq   STRING  NOT NULL, \n\
-qual  STRING  NOT NULL, \n\
-CONSTRAINT alignments_LLFMT_pkey_virtual_offset PRIMARY KEY 
(virtual_offset) \n\
-); \n\
-\n\
-CREATE TABLE bam.alignments_extra_LLFMT ( \n\
-tag   CHAR(2) NOT NULL, \n\
-virtual_offsetBIGINT  NOT NULL, \n\
-type  CHAR(1) NOT NULL, \n\
-value STRING, \n\
-CONSTRAINT alignments_extra_LLFMT_pkey_tag_virtual_offset PRIMARY 
KEY (tag, virtual_offset), \n\
-CONSTRAINT alignments_extra_LLFMT_fkey_virtual_offset FOREIGN KEY 
(virtual_offset) \n\
-REFERENCES bam.alignments_LLFMT (virtual_offset) \n\
-);
+   CREATE TABLE bam.alignments_LLFMT ( \n\
+   virtual_offset  BIGINTNOT NULL, \n\
+   qnameSTRING   NOT 
NULL, \n\
+   flag  SMALLINT  
NOT NULL, \n\
+   rnameSTRING   NOT 
NULL, \n\
+   posINT  
 NOT NULL, \n\
+   mapq  SMALLINT  
NOT NULL, \n\
+   cigarSTRING   NOT 
NULL, \n\
+   rnextSTRING   NOT 
NULL, \n\
+   pnextINT
 NOT NULL, \n\
+   tlen  INT   
 NOT NULL, \n\
+   seqSTRING   
  NOT NULL, \n\
+   qual  STRING
  NOT NULL, \n\
+   CONSTRAINT alignments_LLFMT_pkey_virtual_offset PRIMARY KEY 
(virtual_offset) \n\
+   ); \n\
+   \n\
+   CREATE TABLE bam.alignments_extra_LLFMT ( \n\
+   tag  

MonetDB: bamloader - Finished SAM loader functionality. Turned o...

2014-04-09 Thread Robin Cijvat
Changeset: 19aee3b937e8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=19aee3b937e8
Modified Files:
sql/backends/monet5/bam/bam_loader.c
sql/backends/monet5/bam/bam_wrapper.c
sql/backends/monet5/bam/bam_wrapper.h
Branch: bamloader
Log Message:

Finished SAM loader functionality. Turned out to be a lot of work, since 
Samtools wouldn't cooperate. Therefore, a SAM file is now entirely parsed by 
this BAM library, without any help of Samtools. Test update will follow soon.


diffs (truncated from 1146 to 300 lines):

diff --git a/sql/backends/monet5/bam/bam_loader.c 
b/sql/backends/monet5/bam/bam_loader.c
--- a/sql/backends/monet5/bam/bam_loader.c
+++ b/sql/backends/monet5/bam/bam_loader.c
@@ -283,7 +283,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s
 
/* Parse all headers */
for (i = 0; i  nr_files; ++i) {
-   TO_LOG(bam_loader Parsing BAM header for file '%s'...\n,
+   TO_LOG(bam_loader Parsing header for file '%s'...\n,
   filenames[i]);
if ((msg = process_header(bws + i)) != MAL_SUCCEED) {
goto cleanup;
@@ -310,7 +310,7 @@ bam_loader(Client cntxt, MalBlkPtr mb, s
 
/* Create alignment storage */
for (i = 0; i  nr_files; ++i) {
-   TO_LOG(bam_loader Creating alignment tables for BAM file 
'%s'...\n, filenames[i]);
+   TO_LOG(bam_loader Creating alignment tables for file 
'%s'...\n, filenames[i]);
if ((dbschema == 0
  create_alignment_storage_0(cntxt,
   bam.create_storage_0,
diff --git a/sql/backends/monet5/bam/bam_wrapper.c 
b/sql/backends/monet5/bam/bam_wrapper.c
--- a/sql/backends/monet5/bam/bam_wrapper.c
+++ b/sql/backends/monet5/bam/bam_wrapper.c
@@ -106,22 +106,63 @@ init_bam_wrapper(bam_wrapper * bw, filet
  ERR_INIT_BAM_WRAPPER BAM file could not be opened,
  file_location);
}
-   if ((bw-header = bam_header_read(bw-bam.input)) == NULL) {
+   if ((bw-bam.header = bam_header_read(bw-bam.input)) == NULL) {
throw(MAL, init_bam_wrapper,
ERR_INIT_BAM_WRAPPER Unable to read header from file,
file_location);
 }
 } else {
 /* Open SAM file and read its header */
-if ((bw-sam.input = samopen(file_location, r, NULL)) == NULL) {
+int bufsize = 4096;
+lng header_len = 0;
+if ((bw-sam.input = open_rastream(file_location)) == NULL) {
throw(MAL, init_bam_wrapper,
  ERR_INIT_BAM_WRAPPER SAM file could not be opened,
  file_location);
}
-   if ((bw-header = bw-sam.input-header) == NULL) {
-   throw(MAL, init_bam_wrapper,
-   ERR_INIT_BAM_WRAPPER Unable to read header from file,
-   file_location);
+if ((bw-sam.header = (str)GDKmalloc(bufsize * sizeof(char))) == NULL) 
{
+throw(MAL, init_bam_wrapper, 
+  ERR_INIT_BAM_WRAPPER MAL_MALLOC_FAIL, file_location);
+}
+while (TRUE) {
+int read = mnstr_readline(bw-sam.input, bw-sam.header + 
header_len, bufsize - header_len);
+
+if (read = 0) {
+throw(MAL, init_bam_wrapper, 
+  ERR_INIT_BAM_WRAPPER Could not read line of SAM 
header, 
+  file_location);
+}
+
+if (bw-sam.header[header_len] != '@') {
+/* This is not a header line, we assume that the header is 
finished.
+ * Rewind stream to start of line and stop reading */
+if (mnstr_fsetpos(bw-sam.input, header_len)  0) {
+throw(MAL, init_bam_wrapper, 
+  ERR_INIT_BAM_WRAPPER Could not read last line of SAM 
header, 
+  file_location);
+}
+bw-sam.header[header_len] = '\0'; /* Truncate alignment data 
*/
+break;
+}
+
+if (bw-sam.header[header_len+read-1] != '\n') {
+/* This line was not completed. Increase buffer size, rewind 
stream
+ * and try again */
+bufsize *= 2;
+if ((bw-sam.header = (str)GDKrealloc(bw-sam.header, bufsize 
* sizeof(char))) == NULL) {
+throw(MAL, init_bam_wrapper, 
+  ERR_INIT_BAM_WRAPPER MAL_MALLOC_FAIL, file_location);
+}
+if (mnstr_fsetpos(bw-sam.input, header_len)  0) {
+throw(MAL, init_bam_wrapper, 
+  ERR_INIT_BAM_WRAPPER Could not read last line of SAM 
header, 
+  file_location);
+}
+continue;
+}
+
+ 

MonetDB: bamloader - Merged

2014-04-09 Thread Robin Cijvat
Changeset: 793acf412199 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=793acf412199
Added Files:
clients/mapilib/ChangeLog
common/stream/ChangeLog
geom/sql/Tests/T1.modules
geom/sql/Tests/T10.modules
geom/sql/Tests/T11.modules
geom/sql/Tests/T12.modules
geom/sql/Tests/T13.modules
geom/sql/Tests/T14.modules
geom/sql/Tests/T15.modules
geom/sql/Tests/T16.modules
geom/sql/Tests/T17.modules
geom/sql/Tests/T18.modules
geom/sql/Tests/T19.modules
geom/sql/Tests/T2.modules
geom/sql/Tests/T20.modules
geom/sql/Tests/T21.modules
geom/sql/Tests/T22.modules
geom/sql/Tests/T23.modules
geom/sql/Tests/T24.modules
geom/sql/Tests/T25.modules
geom/sql/Tests/T26.modules
geom/sql/Tests/T27.modules
geom/sql/Tests/T28.modules
geom/sql/Tests/T29.modules
geom/sql/Tests/T3.modules
geom/sql/Tests/T30.modules
geom/sql/Tests/T31.modules
geom/sql/Tests/T32.modules
geom/sql/Tests/T33.modules
geom/sql/Tests/T34.modules
geom/sql/Tests/T35.modules
geom/sql/Tests/T36.modules
geom/sql/Tests/T37.modules
geom/sql/Tests/T38.modules
geom/sql/Tests/T39.modules
geom/sql/Tests/T4.modules
geom/sql/Tests/T40.modules
geom/sql/Tests/T41.modules
geom/sql/Tests/T42.modules
geom/sql/Tests/T43.modules
geom/sql/Tests/T44.modules
geom/sql/Tests/T45.modules
geom/sql/Tests/T46.modules
geom/sql/Tests/T47.modules
geom/sql/Tests/T48.modules
geom/sql/Tests/T49.modules
geom/sql/Tests/T5.modules
geom/sql/Tests/T50.modules
geom/sql/Tests/T51.modules
geom/sql/Tests/T52.modules
geom/sql/Tests/T6.modules
geom/sql/Tests/T7.modules
geom/sql/Tests/T8.modules
geom/sql/Tests/T9.modules
geom/sql/Tests/conformance_drop.modules
geom/sql/Tests/conformance_load.modules
monetdb5/modules/mal/Tests/pqueue2.mal
monetdb5/modules/mal/Tests/pqueueElim.mal
monetdb5/modules/mal/Tests/pqueueElim.stable.err
monetdb5/modules/mal/Tests/pqueueElim.stable.out
sql/backends/monet5/bam/Tests/test.modules
sql/backends/monet5/datacell/Tests/datacell99.sql
sql/backends/monet5/datacell/Tests/datacell99.stable.err
sql/backends/monet5/datacell/Tests/datacell99.stable.out
sql/backends/monet5/datacell/Tests/emili.sql
sql/backends/monet5/datacell/Tests/emili.txt

sql/test/BugTracker-2014/Tests/crash_on_groupby_distinct_serial.Bug-3463.sql

sql/test/BugTracker-2014/Tests/crash_on_groupby_distinct_serial.Bug-3463.stable.err

sql/test/BugTracker-2014/Tests/crash_on_groupby_distinct_serial.Bug-3463.stable.out
sql/test/BugTracker-2014/Tests/crash_on_or_with_in.Bug-3461.sql
sql/test/BugTracker-2014/Tests/crash_on_or_with_in.Bug-3461.stable.err
sql/test/BugTracker-2014/Tests/crash_on_or_with_in.Bug-3461.stable.out
sql/test/BugTracker-2014/Tests/in_incorrect_multi.Bug-3462.sql
sql/test/BugTracker-2014/Tests/in_incorrect_multi.Bug-3462.stable.err
sql/test/BugTracker-2014/Tests/in_incorrect_multi.Bug-3462.stable.out
sql/test/BugTracker-2014/Tests/select-having.Bug-3458.sql
sql/test/BugTracker-2014/Tests/select-having.Bug-3458.stable.err
sql/test/BugTracker-2014/Tests/select-having.Bug-3458.stable.out
sql/test/BugTracker-2014/Tests/setreadonly_forgets_changes.Bug-3446.sql

sql/test/BugTracker-2014/Tests/setreadonly_forgets_changes.Bug-3446.stable.err

sql/test/BugTracker-2014/Tests/setreadonly_forgets_changes.Bug-3446.stable.out
Removed Files:
sql/backends/monet5/datacell/Tests/emili/README
sql/backends/monet5/datacell/Tests/emili/emili.sql
sql/backends/monet5/datacell/Tests/emili/sensor.log.gz
sql/backends/monet5/datacell/Tests/pair00.mal
sql/backends/monet5/datacell/Tests/pair00.stable.err
sql/backends/monet5/datacell/Tests/pair00.stable.out
sql/backends/monet5/datacell/Tests/petrinet00.mal
sql/backends/monet5/datacell/Tests/petrinet00.stable.err
sql/backends/monet5/datacell/Tests/petrinet00.stable.out
Modified Files:
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
buildtools/ChangeLog.Jan2014
buildtools/doc/windowsbuild.rst
clients/ChangeLog.Jan2014
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures_gsl.stable.out
clients/Tests/MAL-signatures_nocfitsio.stable.out
clients/Tests/MAL-signatures_sphinxclient.stable.out
clients/Tests/SQL-dump.stable.out.32bit
clients/Tests/SQL-dump.stable.out.64bit.oid32

MonetDB: bamloader - Small bug fix

2014-04-09 Thread Robin Cijvat
Changeset: fb17f0c487cc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fb17f0c487cc
Modified Files:
sql/backends/monet5/bam/bam_wrapper.c
sql/backends/monet5/bam/bam_wrapper.h
Branch: bamloader
Log Message:

Small bug fix


diffs (40 lines):

diff --git a/sql/backends/monet5/bam/bam_wrapper.c 
b/sql/backends/monet5/bam/bam_wrapper.c
--- a/sql/backends/monet5/bam/bam_wrapper.c
+++ b/sql/backends/monet5/bam/bam_wrapper.c
@@ -182,6 +182,7 @@ init_bam_wrapper(bam_wrapper * bw, filet
bw-cnt_alignments_extra = 0;
bw-cnt_alignments_paired_primary = 0;
bw-cnt_alignments_paired_secondary = 0;
+bw-cnt_alignments_total = 0;
 
for (i = 0; i  6; ++i) {
snprintf(bw-fp_files[i], BW_FP_BUF_SIZE, %s/files_%d,
@@ -1968,7 +1969,7 @@ complete_qname_group(alignment ** alignm
return MAL_SUCCEED;
 }
 
-#define BAMSAM_TELL(bw) (bw-type == BAM ? bam_tell(bw-bam.input) : 
(bw-cnt_alignments + 1))
+#define BAMSAM_TELL(bw) (bw-type == BAM ? bam_tell(bw-bam.input) : 
(bw-cnt_alignments_total + 1))
 
 str
 process_alignments(bam_wrapper * bw, bit * some_thread_failed)
@@ -2048,6 +2049,8 @@ process_alignments(bam_wrapper * bw, bit
 break;
 }
}
+
+++bw-cnt_alignments_total;
 
if (bw-dbschema == 1  alig_index  0
 strcmp((bw-type == BAM ? bam1_qname(alig) : a-qname),
diff --git a/sql/backends/monet5/bam/bam_wrapper.h 
b/sql/backends/monet5/bam/bam_wrapper.h
--- a/sql/backends/monet5/bam/bam_wrapper.h
+++ b/sql/backends/monet5/bam/bam_wrapper.h
@@ -69,6 +69,7 @@ typedef struct bam_wrapper {
lng cnt_alignments_extra;
lng cnt_alignments_paired_primary;
lng cnt_alignments_paired_secondary;
+lng cnt_alignments_total;
 
/* File paths to binary files */
char fp_files[6][BW_FP_BUF_SIZE];
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


MonetDB: bamloader - Included tests for sam_export function

2014-04-09 Thread Robin Cijvat
Changeset: 0a3325139576 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0a3325139576
Modified Files:
sql/backends/monet5/bam/Tests/test.SQL.py
sql/backends/monet5/bam/Tests/test.stable.out
Branch: bamloader
Log Message:

Included tests for sam_export function


diffs (truncated from 508 to 300 lines):

diff --git a/sql/backends/monet5/bam/Tests/test.SQL.py 
b/sql/backends/monet5/bam/Tests/test.SQL.py
--- a/sql/backends/monet5/bam/Tests/test.SQL.py
+++ b/sql/backends/monet5/bam/Tests/test.SQL.py
@@ -43,6 +43,8 @@ for match in p.finditer(out):
 
 # Now we will execute all benchmark queries on all BAM files in the bam.files 
table
 # and output all data contained in the aux table.
+# Furthermore, we transfer every file to the export table, use sam_export to 
write
+# the contents to a SAM file and then print the raw contents of this SAM file
 c = new_client()
 set_var(c, 'rname_1_3', 'chr22', numeric=False)
 set_var(c, 'pos_1_3_1', 100)
@@ -56,6 +58,7 @@ set_var(c, 'rname_2_10', 'chr22', numeri
 set_var(c, 'pos_2_10', 8000)
 set_var(c, 'distance_2_12', 1000)
 
+output_files = []
 for f in files_to_test:
 # benchmark 1
 for uc in range(1, 6):
@@ -71,6 +74,15 @@ for f in files_to_test:
 #write all aux data
 c.stdin.write(SELECT * FROM bam.alignments_extra_%d;% f[0]);
 
+#load into export table
+c.stdin.write(INSERT INTO bam.export (SELECT qname, flag, rname, pos, 
mapq, cigar, rnext, pnext, tlen, seq, qual FROM bam.%salignments_%d);\
+% (('unpaired_all_' if f[1] == 1 else ''), f[0]))
+output_files.append(os.path.join(SRCDIR, (output_%d.sam% f[0])))
+c.stdin.write(CALL sam_export('%s');% output_files[-1])
+
+
+
+
 out, err = c.communicate()
 
 # The output will contain explicit file_ids in the table names, making the 
table names
@@ -83,3 +95,11 @@ out = (re.subn(p, replace, out))[0]
 
 sys.stdout.write(out)
 sys.stderr.write(err)
+
+
+# All that is left to do is write the contents of the exported SAM files and 
delete them
+for f in output_files:
+sys.stdout.write(\n\nContents of exported SAM file '%s':\n% f)
+for ln in open(f):
+sys.stdout.write(ln)
+os.remove(f)
\ No newline at end of file
diff --git a/sql/backends/monet5/bam/Tests/test.stable.out 
b/sql/backends/monet5/bam/Tests/test.stable.out
--- a/sql/backends/monet5/bam/Tests/test.stable.out
+++ b/sql/backends/monet5/bam/Tests/test.stable.out
@@ -1,9 +1,9 @@
 stdout of test 'test` in directory 'sql/backends/monet5/bam` itself:
 
 
-# 21:48:24   
-# 21:48:24   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=30475 --set 
mapi_usock=/var/tmp/mtest-32164/.s.monetdb.30475 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/export/scratch2/robin/Monet/PREFIX/DFT/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
-# 21:48:24   
+# 23:17:55   
+# 23:17:55   mserver5 --debug=10 --set gdk_nr_threads=0 --set 
mapi_open=true --set mapi_port=35794 --set 
mapi_usock=/var/tmp/mtest-20301/.s.monetdb.35794 --set monet_prompt= 
--forcemito --set mal_listing=2 
--dbpath=/export/scratch2/robin/Monet/PREFIX/DFT/var/MonetDB/mTests_sql_backends_monet5_bam
 --set mal_listing=0
+# 23:17:55   
 
 # MonetDB 5 server v11.18.0
 # This is an unreleased version
@@ -13,8 +13,8 @@ stdout of test 'test` in directory 'sql/
 # Copyright (c) 1993-July 2008 CWI.
 # Copyright (c) August 2008-2014 MonetDB B.V., all rights reserved
 # Visit http://www.monetdb.org/ for further information
-# Listening for connection requests on mapi:monetdb://essen.ins.cwi.nl:30475/
-# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-32164/.s.monetdb.30475
+# Listening for connection requests on mapi:monetdb://essen.ins.cwi.nl:35794/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-20301/.s.monetdb.35794
 # MonetDB/GIS module loaded
 # MonetDB/JAQL module loaded
 # MonetDB/SQL module loaded
@@ -61,9 +61,9 @@ Ready.
 #WARNING To speedup calc.min a bulk operator implementation is needed
 #Interpreting lng as wrd.
 
-# 21:48:24   
-# 21:48:24   /usr/bin/python2 test.SQL.py test
-# 21:48:24   
+# 23:17:55   
+# 23:17:55   /usr/bin/python2 test.SQL.py test
+# 23:17:55   
 
 #DECLARE rname_1_3 varchar(32);SET rname_1_3='chr22';DECLARE pos_1_3_1 
integer;SET pos_1_3_1=100;DECLARE pos_1_3_2 integer;SET 
pos_1_3_2=2000;DECLARE qname_1_4 varchar(32);SET 
qname_1_4='sim_22_1_1';DECLARE mapq_1_5 integer;SET mapq_1_5=200;DECLARE 
mapq_2_1 integer;SET mapq_2_1=100;DECLARE rname_2_9 varchar(32);SET 
rname_2_9='chr22';DECLARE pos_2_9 integer;SET pos_2_9=39996433;DECLARE 
rname_2_10 varchar(32);SET rname_2_10='chr22';DECLARE pos_2_10 integer;SET 
pos_2_10=8000;DECLARE distance_2_12 integer;SET 
distance_2_12=1000;SELECT qname, flag, rname, pos, mapq, cigar, rnext, 
pnext, tlen, seq, qual
 #DECLARE rname_1_3 varchar(32);SET rname_1_3='chr22';DECLARE pos_1_3_1 
integer;SET 

MonetDB: bamloader - Merged

2014-04-09 Thread Robin Cijvat
Changeset: ba874198a299 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ba874198a299
Added Files:
monetdb5/modules/atoms/Tests/strpad.mal
monetdb5/modules/atoms/Tests/strpad.stable.err
monetdb5/modules/atoms/Tests/strpad.stable.out
monetdb5/modules/atoms/Tests/strtrim.mal
monetdb5/modules/atoms/Tests/strtrim.stable.err
monetdb5/modules/atoms/Tests/strtrim.stable.out
sql/backends/monet5/datacell/Tests/emilifinish.stable.err
sql/backends/monet5/datacell/Tests/emilifinish.stable.out
sql/backends/monet5/datacell/Tests/emilistart.sql
sql/backends/monet5/datacell/Tests/emilistart.stable.err
sql/backends/monet5/datacell/Tests/emilistart.stable.out
sql/test/Tests/str-pad.sql
sql/test/Tests/str-pad.stable.err
sql/test/Tests/str-pad.stable.out
sql/test/Tests/str-trim.sql
sql/test/Tests/str-trim.stable.err
sql/test/Tests/str-trim.stable.out
Removed Files:
monetdb5/modules/mal/Tests/pqueueElim.mal
monetdb5/modules/mal/Tests/pqueueElim.stable.err
monetdb5/modules/mal/Tests/pqueueElim.stable.out
sql/backends/monet5/datacell/Tests/emili.sql
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures_gsl.stable.out
clients/Tests/MAL-signatures_nocfitsio.stable.out
clients/Tests/MAL-signatures_sphinxclient.stable.out
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.64bit.oid32
clients/Tests/SQL-dump_gsl.stable.out
clients/Tests/SQL-dump_gsl.stable.out.oid32
clients/Tests/SQL-dump_nogeom.stable.out
clients/Tests/exports.stable.out
clients/mapiclient/dump.c
clients/mapiclient/mclient.c
clients/mapiclient/tomograph.c
clients/odbc/driver/SQLColAttribute.c
clients/odbc/driver/SQLColumnPrivileges.c
clients/odbc/driver/SQLColumns.c
clients/odbc/driver/SQLForeignKeys.c
clients/odbc/driver/SQLPrimaryKeys.c
clients/odbc/driver/SQLProcedures.c
clients/odbc/driver/SQLSpecialColumns.c
clients/odbc/driver/SQLStatistics.c
clients/odbc/driver/SQLTablePrivileges.c
clients/odbc/driver/SQLTables.c
common/stream/stream.c
gdk/gdk.h
gdk/gdk_imprints.c
gdk/gdk_storage.c
gdk/gdk_system.c
gdk/gdk_utils.c
monetdb5/extras/sphinx/sphinx.c
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_profiler.h
monetdb5/mal/mal_readline.c
monetdb5/mal/mal_recycle.c
monetdb5/mal/mal_resolve.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/str.c
monetdb5/modules/atoms/str.h
monetdb5/modules/atoms/str.mal
monetdb5/modules/atoms/url.c
monetdb5/modules/kernel/aggr.c
monetdb5/modules/kernel/batstr.c
monetdb5/modules/kernel/batstr.mal
monetdb5/modules/mal/Tests/All
monetdb5/modules/mal/Tests/pqueue.mal
monetdb5/modules/mal/Tests/pqueue.stable.out
monetdb5/modules/mal/Tests/pqueue2.mal
monetdb5/modules/mal/bbp.c
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/pqueue.c
monetdb5/modules/mal/pqueue.h
monetdb5/modules/mal/pqueue.mal
monetdb5/modules/mal/profiler.c
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/sysmon.c
monetdb5/optimizer/opt_cluster.c
monetdb5/optimizer/opt_dataflow.c
monetdb5/optimizer/opt_macro.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_multiplex.c
monetdb5/optimizer/opt_pushselect.c
monetdb5/optimizer/opt_remap.c
monetdb5/optimizer/opt_remoteQueries.c
sql/ChangeLog
sql/backends/monet5/datacell/Tests/All
sql/backends/monet5/datacell/Tests/dcload.sql
sql/backends/monet5/datacell/Tests/dcload.stable.err
sql/backends/monet5/datacell/opt_datacell.c
sql/backends/monet5/datacell/opt_datacell.h
sql/backends/monet5/datacell/petrinet.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
sql/backends/monet5/sql.mal
sql/backends/monet5/sql_cast.c
sql/backends/monet5/sql_optimizer.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_statistics.c
sql/backends/monet5/vaults/fits.c
sql/common/sql_types.c
sql/server/rel_exp.c
sql/server/rel_exp.h
sql/server/rel_optimizer.c
sql/server/rel_planner.c
sql/server/rel_select.c
sql/server/sql_privileges.c
sql/test/Tests/50ways.stable.err
sql/test/Tests/coalesce.stable.err
sql/test/Tests/create_in_schema.stable.err
sql/test/Tests/decimal.stable.err

MonetDB: bamloader - Finished sam_export and properly catch the ...

2014-04-09 Thread Robin Cijvat
Changeset: 19686dfee569 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=19686dfee569
Modified Files:
sql/backends/monet5/bam/bam_export.c
Branch: bamloader
Log Message:

Finished sam_export and properly catch the not yet working bam_export


diffs (75 lines):

diff --git a/sql/backends/monet5/bam/bam_export.c 
b/sql/backends/monet5/bam/bam_export.c
--- a/sql/backends/monet5/bam/bam_export.c
+++ b/sql/backends/monet5/bam/bam_export.c
@@ -30,6 +30,7 @@
 #include samtools/bam.h
 
 #include bam_globals.h
+#include bam_db_interface.h
 #include bam_export.h
 
 
@@ -44,7 +45,7 @@ typedef struct bam_field {
 
 /**
  * Copied directly from bam.h/bam_import.c for use by fill_bam_alig
- * Can not change the call to realloc to GDKrealloc, since
+ * Can not change the calls to realloc to GDKrealloc, since
  * bam_destroy1 does not use GDKfree...
  */
 #ifndef kroundup32
@@ -393,6 +394,7 @@ cleanup_fields(bam_field fields[11]) {
 }
 }
 
+
 #define CUR_STR(field, i) ((str) BUNtail(field.iter, (field.cur+i)))
 #define CUR_SHT(field, i) (*(sht *) BUNtail(field.iter, (field.cur+i)))
 #define CUR_INT(field, i) (*(int *) BUNtail(field.iter, (field.cur+i)))
@@ -409,6 +411,7 @@ sam_export(Client cntxt, MalBlkPtr mb, M
 int tuple_count = 0;
 
 int i;
+str sql;
 str msg = MAL_SUCCEED;
 
 memset(fields, 0, 11 * sizeof(bam_field));
@@ -433,6 +436,13 @@ sam_export(Client cntxt, MalBlkPtr mb, M
 CUR_STR(fields[9], i), CUR_STR(fields[10], i));
 }
 
+/* If we got here, we succesfully exported the result. Drop all data in 
export table */
+sql = DELETE FROM bam.export;;
+RUN_SQL(cntxt, sql, bam.drop_export, msg);
+if (msg != MAL_SUCCEED) {
+REUSE_EXCEPTION(msg, MAL, sam_export, Could not clear the export 
table after exporting: %s, msg);
+}
+
 (void)stk;
 (void)pci;
 
@@ -465,7 +475,10 @@ bam_export(Client cntxt, MalBlkPtr mb, M
 bam1_t *alig = NULL;
 
 int i;
+str sql;
 str msg = MAL_SUCCEED;
+
+throw(MAL, bam_export, Exporting to BAM files is not implemented yet. 
This is our first priority for the next release of the BAM library.);
 
 if ((output = bam_open(output_path, wb)) == NULL) {
 msg = createException(MAL, bam_export, Could not open output file 
'%s' for writing, output_path);
@@ -518,6 +531,13 @@ bam_export(Client cntxt, MalBlkPtr mb, M
 bam_write1(output, alig);
 }
 
+/* If we got here, we succesfully exported the result. Drop all data in 
export table */
+sql = DELETE FROM bam.export;;
+RUN_SQL(cntxt, sql, bam.drop_export, msg);
+if (msg != MAL_SUCCEED) {
+REUSE_EXCEPTION(msg, MAL, sam_export, Could not clear the export 
table after exporting: %s, msg);
+}
+
 (void)stk;
 (void)pci;
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list


  1   2   >