date:20170612

MonetDB: jitudf - When mprotect triggers, check if the address a...

2017-06-12 Thread Mark Raasveldt

Changeset: c5b67bdd8bcd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c5b67bdd8bcd
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

When mprotect triggers, check if the address actually falls in the area we 
protected.


diffs (192 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -27,8 +27,24 @@ typedef struct _allocated_region {
struct _allocated_region *next;
 } allocated_region;
 
-static __thread allocated_region *allocated_regions;
-static __thread jmp_buf jump_buffer;
+struct _mprotected_region;
+typedef struct _mprotected_region {
+   void *addr;
+   size_t len;
+
+   void* actual_addr;
+   size_t actual_len;
+
+   struct _mprotected_region *next;
+} mprotected_region;
+
+static char *mprotect_region(void *addr, size_t len, int flags,
+mprotected_region 
**regions);
+static char *clear_mprotect(void *addr, size_t len);
+
+static allocated_region *allocated_regions[THREADS];
+static mprotected_region **actual_mprotected_regions[THREADS];
+static jmp_buf jump_buffer[THREADS];
 
 typedef char *(*jitted_function)(void **inputs, void **outputs,
 
malloc_function_ptr malloc);
@@ -81,19 +97,49 @@ static bool WriteTextToFile(FILE *f, con
 
 static void handler(int sig, siginfo_t *si, void *unused)
 {
+   int actually_protected_area = false;
+   mprotected_region* found_region = NULL;
+   int tid = THRgettid();
+
(void)sig;
-   (void)si;
(void)unused;
// we caught a segfault or bus error
-   longjmp(jump_buffer, 1);
-}
+   // this can be either because 
+   // (1) the function accessed a protected piece of memory
+   // (2) the function caused a segfault by e.g. dereferencing a NULL 
pointer
+   // in the first case, this *might* be a valid memory access
+   // this is because we are required to align our mprotects on page 
boundaries
+   // thus sometimes we mprotect a page where only part of the page
+   // should actually be protected. Thus for this case we check if the 
access
+   // was actually an error
 
-struct _mprotected_region;
-typedef struct _mprotected_region {
-   void *addr;
-   size_t len;
-   struct _mprotected_region *next;
-} mprotected_region;
+   if (actual_mprotected_regions[tid]) {
+   mprotected_region* region = *actual_mprotected_regions[tid];
+   while(region) {
+   if (si->si_addr >= region->addr && (char*) si->si_addr 
<= (char*) region->addr + region->len) {
+   // the address belongs to this mprotected region
+   found_region = region;
+   if (si->si_addr >= region->actual_addr && 
+   (char*) si->si_addr <= (char*) 
region->actual_addr + region->actual_len) {
+   // and the address is actually supposed 
to be protected
+   actually_protected_area = true;
+   break;
+   }
+   }
+   region = region->next;
+   }
+   }
+   if (found_region && !actually_protected_area) {
+   // this is NOT an actually protected area
+   // thus the segfault/bus error is invalid
+   // the nasty part here is that we have to unprotect the entire 
page now
+   // thus opening us up to future modifications of the data
+   clear_mprotect(found_region->addr, found_region->len);
+   found_region->addr = NULL;
+   } else {
+   longjmp(jump_buffer[tid], 1);
+   }
+}
 
 static char *mprotect_region(void *addr, size_t len, int flags,
 mprotected_region 
**regions)
@@ -101,6 +147,8 @@ static char *mprotect_region(void *addr,
mprotected_region *region;
int pagesize;
void *page_begin;
+   void* actual_addr = addr;
+   size_t actual_len = len;
if (len == 0)
return NULL;
// check if the region is page-aligned
@@ -126,12 +174,16 @@ static char *mprotect_region(void *addr,
region->addr = addr;
region->len = len;
region->next = *regions;
+   region->actual_addr = actual_addr;
+   region->actual_len = actual_len;
*regions = region;
return NULL;
 }
 
 static char *clear_mprotect(void *addr, size_t len)
 {
+   if (!addr) return NULL;
+
if (mprotect(addr, len, PROT_READ | PROT_WRITE) < 0) {
return strerror(errno);
}
@@ -156,7 +208,7 @@ static

MonetDB: default - Merge with Jul2017 branch.

2017-06-12 Thread Sjoerd Mullender

Changeset: 5733cdb6fa53 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5733cdb6fa53
Modified Files:
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
Branch: default
Log Message:

Merge with Jul2017 branch.


diffs (truncated from 18042 to 300 lines):

diff --git a/sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 
b/sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
--- a/sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
+++ b/sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
@@ -54,7 +54,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,
 language_name VARCHAR(20) NOT NULL UNIQUE);
 INSERT INTO sys.function_languages (language_id, language_name) VALUES
-(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (4, 'C'), (5, 'Java'), 
(6, 'Python'), (7, 'Python Mapped');
+(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (6, 'Python'), (7, 
'Python Mapped'), (8, 'Python2'), (9, 'Python2 Mapped'), (10, 'Python3'), (11, 
'Python3 Mapped');
 ALTER TABLE sys.function_languages SET READ ONLY;
 CREATE TABLE sys.key_types (
 key_type_id   SMALLINT NOT NULL PRIMARY KEY,
diff --git a/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out 
b/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
--- a/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
+++ b/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
@@ -54,7 +54,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,
 language_name VARCHAR(20) NOT NULL UNIQUE);
 INSERT INTO sys.function_languages (language_id, language_name) VALUES
-(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (4, 'C'), (5, 'Java'), 
(6, 'Python'), (7, 'Python Mapped');
+(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (6, 'Python'), (7, 
'Python Mapped'), (8, 'Python2'), (9, 'Python2 Mapped'), (10, 'Python3'), (11, 
'Python3 Mapped');
 ALTER TABLE sys.function_languages SET READ ONLY;
 CREATE TABLE sys.key_types (
 key_type_id   SMALLINT NOT NULL PRIMARY KEY,
diff --git a/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128 
b/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
--- a/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
+++ b/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
@@ -5282,7 +5282,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,
 language_name VARCHAR(20) NOT NULL UNIQUE);
 INSERT INTO sys.function_languages (language_id, language_name) VALUES
-(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (4, 'C'), (5, 'Java'), 
(6, 'Python'), (7, 'Python Mapped');
+(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (6, 'Python'), (7, 
'Python Mapped'), (8, 'Python2'), (9, 'Python2 Mapped'), (10, 'Python3'), (11, 
'Python3 Mapped');
 ALTER TABLE sys.function_languages SET READ ONLY;
 CREATE TABLE sys.key_types (
 key_type_id   SMALLINT NOT NULL PRIMARY KEY,
diff --git a/sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128 
b/sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
--- a/sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
+++ b/sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
@@ -53,7 +53,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,
 language_name VARCHAR(20) NOT NULL UNIQUE);
 INSERT INTO sys.function_languages (language_id, language_name) VALUES
-(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (4, 'C'), (5, 'Java'), 
(6, 'Python'), (7, 'Python Mapped');
+(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (6, 'Python'), (7, 
'Python Mapped'), (8, 'Python2'), (9, 'Python2 Mapped'), (10, 'Python3'), (11, 
'Python3 Mapped');
 ALTER TABLE sys.function_languages SET READ ONLY;
 CREATE TABLE sys.key_types (
 key_type_id   SMALLINT NOT NULL PRIMARY KEY,
diff --git a/sql/test/emptydb-upgrade/Tests/upgrade.stable.out 
b/sql/test/emptydb-upgrade/Tests/upgrade.stable.out
--- a/sql/test/emptydb-upgrade/Tests/upgrade.stable.out
+++ b/sql/test/emptydb-upgrade/Tests/upgrade.stable.out
@@ -53,7 +53,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,

MonetDB: subjoin - merged with default

2017-06-12 Thread Niels Nes

Changeset: 21bcc73d9cc4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=21bcc73d9cc4
Added Files:
sql/test/BugTracker-2017/Tests/crash-dce.Bug-6330.sql
sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.sql

sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.err

sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.out
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.sql
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.stable.err
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.stable.out
sql/test/Tests/ifexists-views.sql
sql/test/Tests/ifexists-views.stable.err
sql/test/Tests/ifexists-views.stable.out
Modified Files:
MonetDB.spec
clients/ChangeLog.Dec2016
common/stream/stream.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_pushselect.c
sql/backends/monet5/sql_cat.c
sql/backends/monet5/sql_upgrades.c
sql/include/sql_catalog.h
sql/scripts/51_sys_schema_extension.sql
sql/server/rel_optimizer.c
sql/server/rel_schema.c
sql/server/rel_select.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/test/BugTracker-2017/Tests/All
sql/test/Tests/All
sql/test/Tests/copy-into-fwf.sql.in
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out
tools/merovingian/daemon/controlrunner.c
tools/merovingian/utils/control.c
Branch: subjoin
Log Message:

merged with default


diffs (truncated from 1135 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -311,6 +311,11 @@ MonetDB, you will very likely need this 
 Summary: MonetDB - Monet Database Management System Client Programs
 Group: Applications/Databases
 Requires: %{name}-client%{?_isa} = %{version}-%{release}
+%if (0%{?fedora} >= 22)
+# tomograph executes these two:
+Recommends: /usr/bin/gs
+Recommends: /usr/bin/gnuplot
+%endif
 
 %description client-tools
 MonetDB is a database management system that is developed from a
diff --git a/clients/ChangeLog.Dec2016 b/clients/ChangeLog.Dec2016
--- a/clients/ChangeLog.Dec2016
+++ b/clients/ChangeLog.Dec2016
@@ -1,3 +1,7 @@
 # ChangeLog file for clients
 # This file is updated with Maddlog
 
+* Mon Jun 12 2017 Sjoerd Mullender 
+- A bug was fixed in the implementation of the functions SQLTables in
+  the ODBC driver where not enough memory was allocated.
+
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -5444,11 +5444,11 @@ typedef struct {
char filler;
// state
size_t line_len;
-   char* in_buf;
-   char* out_buf;
+   char *in_buf;
+   char *out_buf;
size_t out_buf_start;
size_t out_buf_remaining;
-   char* nl_buf;
+   char nl_buf[1];
 } stream_fwf_data;
 
 
@@ -5458,10 +5458,11 @@ stream_fwf_read(stream *s, void *buf, si
stream_fwf_data *fsd;
size_t to_write = cnt;
size_t buf_written = 0;
-   if (strcmp(s->name, STREAM_FWF_NAME) != 0 || elmsize != 1) {
+
+   fsd = (stream_fwf_data*) s->stream_data.p;
+   if (fsd == NULL || elmsize != 1) {
return -1;
}
-   fsd = (stream_fwf_data*) s->stream_data.p;
 
while (to_write > 0) {
// input conversion
@@ -5499,12 +5500,12 @@ stream_fwf_read(stream *s, void *buf, si
 
// now we know something is in output_buf so deliver it
if (fsd->out_buf_remaining <= to_write) {
-   memcpy((char*)buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
to_write -= fsd->out_buf_remaining;
buf_written += fsd->out_buf_remaining;
fsd->out_buf_remaining = 0;
} else {
-   memcpy((char*) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
fsd->out_buf_start += to_write;
fsd->out_buf_remaining -= to_write;
buf_written += to_write;
@@ -5518,25 +5519,35 @@ stream_fwf_read(stream *s, void *buf, si
 static void
 stream_fwf_close(stream *s)
 {
-   if (strcmp(s->name, STREAM_FWF_NAME) == 0) {
+   stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
+
+   if (fsd != NULL) {
stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
mnstr_close(fsd->s);
+

MonetDB: Jul2017 - Approve upgrade after changeset 3307d3f0424a ...

2017-06-12 Thread Sjoerd Mullender

Changeset: 68e5985eba42 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=68e5985eba42
Modified Files:
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
Branch: Jul2017
Log Message:

Approve upgrade after changeset 3307d3f0424a and release of Dec2016-SP5.


diffs (truncated from 18042 to 300 lines):

diff --git a/sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 
b/sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
--- a/sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
+++ b/sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
@@ -54,7 +54,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,
 language_name VARCHAR(20) NOT NULL UNIQUE);
 INSERT INTO sys.function_languages (language_id, language_name) VALUES
-(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (4, 'C'), (5, 'Java'), 
(6, 'Python'), (7, 'Python Mapped');
+(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (6, 'Python'), (7, 
'Python Mapped'), (8, 'Python2'), (9, 'Python2 Mapped'), (10, 'Python3'), (11, 
'Python3 Mapped');
 ALTER TABLE sys.function_languages SET READ ONLY;
 CREATE TABLE sys.key_types (
 key_type_id   SMALLINT NOT NULL PRIMARY KEY,
diff --git a/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out 
b/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
--- a/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
+++ b/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
@@ -54,7 +54,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,
 language_name VARCHAR(20) NOT NULL UNIQUE);
 INSERT INTO sys.function_languages (language_id, language_name) VALUES
-(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (4, 'C'), (5, 'Java'), 
(6, 'Python'), (7, 'Python Mapped');
+(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (6, 'Python'), (7, 
'Python Mapped'), (8, 'Python2'), (9, 'Python2 Mapped'), (10, 'Python3'), (11, 
'Python3 Mapped');
 ALTER TABLE sys.function_languages SET READ ONLY;
 CREATE TABLE sys.key_types (
 key_type_id   SMALLINT NOT NULL PRIMARY KEY,
diff --git a/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128 
b/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
--- a/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
+++ b/sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
@@ -5282,7 +5282,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,
 language_name VARCHAR(20) NOT NULL UNIQUE);
 INSERT INTO sys.function_languages (language_id, language_name) VALUES
-(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (4, 'C'), (5, 'Java'), 
(6, 'Python'), (7, 'Python Mapped');
+(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (6, 'Python'), (7, 
'Python Mapped'), (8, 'Python2'), (9, 'Python2 Mapped'), (10, 'Python3'), (11, 
'Python3 Mapped');
 ALTER TABLE sys.function_languages SET READ ONLY;
 CREATE TABLE sys.key_types (
 key_type_id   SMALLINT NOT NULL PRIMARY KEY,
diff --git a/sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128 
b/sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
--- a/sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
+++ b/sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
@@ -53,7 +53,7 @@ CREATE TABLE sys.function_languages (
 language_id   SMALLINT NOT NULL PRIMARY KEY,
 language_name VARCHAR(20) NOT NULL UNIQUE);
 INSERT INTO sys.function_languages (language_id, language_name) VALUES
-(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (4, 'C'), (5, 'Java'), 
(6, 'Python'), (7, 'Python Mapped');
+(0, 'Internal C'), (1, 'MAL'), (2, 'SQL'), (3, 'R'), (6, 'Python'), (7, 
'Python Mapped'), (8, 'Python2'), (9, 'Python2 Mapped'), (10, 'Python3'), (11, 
'Python3 Mapped');
 ALTER TABLE sys.function_languages SET READ ONLY;
 CREATE TABLE sys.key_types (
 key_type_id   SMALLINT NOT NULL PRIMARY KEY,
diff --git a/sql/test/emptydb-upgrade/Tests/upgrade.stable.out 
b/sql/test/emptydb-upgrade/Tests/upgrade.stable.out
--- a/sql/test/emptydb-upgrade/Tests/upgrade.stable.out
+++ b/sql/test/emptydb-upgrade/Tests/upgrade.stable.out
@@ -53,7 +53,7 @@ CREATE TABLE sys.function_languages (

MonetDB: jitudf - Merge with default.

2017-06-12 Thread Mark Raasveldt

Changeset: c4a6d8f49f54 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c4a6d8f49f54
Added Files:
sql/test/BugTracker-2017/Tests/crash-dce.Bug-6330.sql
sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.sql

sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.err

sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.out
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.sql
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.stable.err
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.stable.out
sql/test/Tests/ifexists-views.sql
sql/test/Tests/ifexists-views.stable.err
sql/test/Tests/ifexists-views.stable.out
Modified Files:
MonetDB.spec
clients/ChangeLog.Dec2016
common/stream/stream.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_pushselect.c
sql/backends/monet5/sql_cat.c
sql/backends/monet5/sql_upgrades.c
sql/include/sql_catalog.h
sql/scripts/51_sys_schema_extension.sql
sql/server/rel_optimizer.c
sql/server/rel_schema.c
sql/server/rel_select.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/test/BugTracker-2017/Tests/All
sql/test/Tests/All
sql/test/Tests/copy-into-fwf.sql.in
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out
tools/merovingian/daemon/controlrunner.c
tools/merovingian/utils/control.c
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 1128 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -311,6 +311,11 @@ MonetDB, you will very likely need this 
 Summary: MonetDB - Monet Database Management System Client Programs
 Group: Applications/Databases
 Requires: %{name}-client%{?_isa} = %{version}-%{release}
+%if (0%{?fedora} >= 22)
+# tomograph executes these two:
+Recommends: /usr/bin/gs
+Recommends: /usr/bin/gnuplot
+%endif
 
 %description client-tools
 MonetDB is a database management system that is developed from a
diff --git a/clients/ChangeLog.Dec2016 b/clients/ChangeLog.Dec2016
--- a/clients/ChangeLog.Dec2016
+++ b/clients/ChangeLog.Dec2016
@@ -1,3 +1,7 @@
 # ChangeLog file for clients
 # This file is updated with Maddlog
 
+* Mon Jun 12 2017 Sjoerd Mullender 
+- A bug was fixed in the implementation of the functions SQLTables in
+  the ODBC driver where not enough memory was allocated.
+
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -5444,11 +5444,11 @@ typedef struct {
char filler;
// state
size_t line_len;
-   char* in_buf;
-   char* out_buf;
+   char *in_buf;
+   char *out_buf;
size_t out_buf_start;
size_t out_buf_remaining;
-   char* nl_buf;
+   char nl_buf[1];
 } stream_fwf_data;
 
 
@@ -5458,10 +5458,11 @@ stream_fwf_read(stream *s, void *buf, si
stream_fwf_data *fsd;
size_t to_write = cnt;
size_t buf_written = 0;
-   if (strcmp(s->name, STREAM_FWF_NAME) != 0 || elmsize != 1) {
+
+   fsd = (stream_fwf_data*) s->stream_data.p;
+   if (fsd == NULL || elmsize != 1) {
return -1;
}
-   fsd = (stream_fwf_data*) s->stream_data.p;
 
while (to_write > 0) {
// input conversion
@@ -5499,12 +5500,12 @@ stream_fwf_read(stream *s, void *buf, si
 
// now we know something is in output_buf so deliver it
if (fsd->out_buf_remaining <= to_write) {
-   memcpy((char*)buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
to_write -= fsd->out_buf_remaining;
buf_written += fsd->out_buf_remaining;
fsd->out_buf_remaining = 0;
} else {
-   memcpy((char*) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
fsd->out_buf_start += to_write;
fsd->out_buf_remaining -= to_write;
buf_written += to_write;
@@ -5518,25 +5519,35 @@ stream_fwf_read(stream *s, void *buf, si
 static void
 stream_fwf_close(stream *s)
 {
-   if (strcmp(s->name, STREAM_FWF_NAME) == 0) {
+   stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
+
+   if (fsd != NULL) {
stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
mnstr_close(fsd->s);
+

MonetDB: jitudf - Linux compilation fixes.

2017-06-12 Thread Mark Raasveldt

Changeset: ec71c22bf9bc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec71c22bf9bc
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi06.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Linux compilation fixes.


diffs (49 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi06.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi06.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi06.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi06.sql
@@ -5,15 +5,16 @@ START TRANSACTION;
 
 CREATE FUNCTION capi06(inp INTEGER) RETURNS INTEGER LANGUAGE CPP {
 #include 
-   std::vector elements;
-   for(size_t i = 0; i < inp.count; i++) {
-   elements.push_back(inp.data[i]);
-   }
-   std::sort(elements.begin(), elements.end());
-   result->initialize(result, inp.count);
-   for(size_t i = 0; i < inp.count; i++) {
-   result->data[i] = elements[i];
-   }
+#include 
+std::vector elements;
+for(size_t i = 0; i < inp.count; i++) {
+elements.push_back(inp.data[i]);
+}
+std::sort(elements.begin(), elements.end());
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+result->data[i] = elements[i];
+}
 };
 
 CREATE TABLE integers(i INTEGER);
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -169,7 +169,7 @@ static void *wrapped_GDK_malloc(size_t s
region->next = allocated_regions;
allocated_regions = region;
 
-   return ptr + sizeof(allocated_region);
+   return (char*)ptr + sizeof(allocated_region);
 }
 
 #define GENERATE_BASE_HEADERS(type, tpename)   
\
@@ -571,6 +571,7 @@ static str CUDFeval(Client cntxt, MalBlk
ATTEMPT_TO_WRITE_DATA_TO_FILE(f,

  exprStr + preprocessor_start,

  i - preprocessor_start);
+   ATTEMPT_TO_WRITE_TO_FILE(f, 
"\n");
for (j = preprocessor_start; j 
< i; j++) {
// now overwrite the 
preprocessor directive in the
// expression string 
with spaces
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Add function cache to avoid recompiling functi...

2017-06-12 Thread Mark Raasveldt

Changeset: 63378d373390 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=63378d373390
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi08.sql
sql/backends/monet5/UDF/capi/Tests/capi08.stable.err
sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Add function cache to avoid recompiling functions constantly.


diffs (truncated from 780 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -7,3 +7,4 @@ capi04
 capi05
 capi06
 capi07
+capi08
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi08.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi08.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi08.sql
@@ -0,0 +1,34 @@
+# test caching behavior
+
+START TRANSACTION;
+
+CREATE FUNCTION capi08(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+result->data[i] = inp.data[i] * 2;
+}
+};
+
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+
+SELECT capi08(i) FROM integers;
+# this function should be loaded from the cache
+SELECT capi08(i) FROM integers;
+
+DROP FUNCTION capi08;
+
+CREATE FUNCTION capi08(inp DOUBLE) RETURNS DOUBLE LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+result->data[i] = inp.data[i] * 2;
+}
+};
+
+# same code and same function name, but different parameters
+# this needs to be recompiled now
+SELECT capi08(i) FROM integers;
+SELECT capi08(i) FROM integers;
+
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi08.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi08` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 16:28:17 >  
+# 16:28:17 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37555" "--set" 
"mapi_usock=/var/tmp/mtest-59671/.s.monetdb.37555" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 16:28:17 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37555
+# cmdline opt  mapi_usock = /var/tmp/mtest-59671/.s.monetdb.37555
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 16:28:17 >  
+# 16:28:17 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-59671" "--port=37555"
+# 16:28:17 >  
+
+
+# 16:28:18 >  
+# 16:28:18 >  "Done."
+# 16:28:18 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
@@ -0,0 +1,124 @@
+stdout of test 'capi08` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 16:28:17 >  
+# 16:28:17 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37555" "--set" 
"mapi_usock=/var/tmp/mtest-59671/.s.monetdb.37555" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 16:28:17 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:37555/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-59671/.s.monetdb.37555
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+#

MonetDB: jitudf - Correctly support aggregates and add testcases.

2017-06-12 Thread Mark Raasveldt

Changeset: 579afc3ecc82 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=579afc3ecc82
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi06.sql
sql/backends/monet5/UDF/capi/Tests/capi06.stable.err
sql/backends/monet5/UDF/capi/Tests/capi06.stable.out
sql/backends/monet5/UDF/capi/Tests/capi07.sql
sql/backends/monet5/UDF/capi/Tests/capi07.stable.err
sql/backends/monet5/UDF/capi/Tests/capi07.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Correctly support aggregates and add testcases.


diffs (truncated from 331 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -5,3 +5,5 @@ capi02
 capi03
 capi04
 capi05
+capi06
+capi07
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi06.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi06.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi06.sql
@@ -0,0 +1,24 @@
+
+# test CPP UDFs
+
+START TRANSACTION;
+
+CREATE FUNCTION capi06(inp INTEGER) RETURNS INTEGER LANGUAGE CPP {
+#include 
+   std::vector elements;
+   for(size_t i = 0; i < inp.count; i++) {
+   elements.push_back(inp.data[i]);
+   }
+   std::sort(elements.begin(), elements.end());
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   result->data[i] = elements[i];
+   }
+};
+
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (3), (4), (1), (2), (5);
+
+SELECT capi06(i) FROM integers;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi06.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi06.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi06.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi06` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 14:53:38 >  
+# 14:53:38 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=38582" "--set" 
"mapi_usock=/var/tmp/mtest-20999/.s.monetdb.38582" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 14:53:38 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 38582
+# cmdline opt  mapi_usock = /var/tmp/mtest-20999/.s.monetdb.38582
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 14:53:38 >  
+# 14:53:38 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-20999" "--port=38582"
+# 14:53:38 >  
+
+
+# 14:53:39 >  
+# 14:53:39 >  "Done."
+# 14:53:39 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi06.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi06.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi06.stable.out
@@ -0,0 +1,93 @@
+stdout of test 'capi06` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 14:53:38 >  
+# 14:53:38 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=38582" "--set" 
"mapi_usock=/var/tmp/mtest-20999/.s.monetdb.38582" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 14:53:38 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:38582/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-20999/.s.monetdb.38582
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script:

MonetDB: jitudf - Formatting.

2017-06-12 Thread Mark Raasveldt

Changeset: 86e62d4bfe87 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=86e62d4bfe87
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/capi/capi.h
sql/backends/monet5/UDF/capi/capi.mal
sql/backends/monet5/UDF/capi/cheader.h
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_statement.c
sql/include/sql_catalog.h
sql/server/rel_psm.c
sql/server/sql_parser.y
Branch: jitudf
Log Message:

Formatting.


diffs (truncated from 1538 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -24,67 +24,73 @@
 
 struct _allocated_region;
 typedef struct _allocated_region {
-   struct _allocated_region* next;
+   struct _allocated_region *next;
 } allocated_region;
 
-
-static __thread allocated_region* allocated_regions;
+static __thread allocated_region *allocated_regions;
 static __thread jmp_buf jump_buffer;
 
-static str
-CUDFeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped);
+static str CUDFeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
+   bit grouped);
 
-str CUDFevalStd(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) {
+str CUDFevalStd(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
return CUDFeval(cntxt, mb, stk, pci, 0);
 }
 
-str CUDFevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) {
+str CUDFevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
return CUDFeval(cntxt, mb, stk, pci, 1);
 }
 
-str CUDFprelude(void *ret) {
-   (void) ret;
+str CUDFprelude(void *ret)
+{
+   (void)ret;
return MAL_SUCCEED;
 }
 
-typedef char* (*jitted_function)(void** inputs, void** outputs, 
malloc_function_ptr malloc);
+typedef char *(*jitted_function)(void **inputs, void **outputs,
+
malloc_function_ptr malloc);
 
-static bool
-WriteDataToFile(FILE* f, const void* data, size_t data_size) {
+static bool WriteDataToFile(FILE *f, const void *data, size_t data_size)
+{
fwrite(data, data_size, 1, f);
return (!ferror(f));
 }
 
-static bool
-WriteTextToFile(FILE* f, const char* data) {
+static bool WriteTextToFile(FILE *f, const char *data)
+{
return WriteDataToFile(f, data, strlen(data));
 }
 
-static void handler(int sig, siginfo_t *si, void *unused) {
-   (void) sig;
-   (void) si;
-   (void) unused;
+static void handler(int sig, siginfo_t *si, void *unused)
+{
+   (void)sig;
+   (void)si;
+   (void)unused;
// we caught a segfault or bus error
longjmp(jump_buffer, 1);
 }
 
 struct _mprotected_region;
 typedef struct _mprotected_region {
-   void* addr;
+   void *addr;
size_t len;
-   struct _mprotected_region* next;
+   struct _mprotected_region *next;
 } mprotected_region;
 
-static char*
-mprotect_region(void* addr, size_t len, int flags, mprotected_region** 
regions) {
-   mprotected_region* region;
+static char *mprotect_region(void *addr, size_t len, int flags,
+mprotected_region 
**regions)
+{
+   mprotected_region *region;
int pagesize;
-   void* page_begin;
-   if (len == 0) return NULL;
+   void *page_begin;
+   if (len == 0)
+   return NULL;
// check if the region is page-aligned
-   
+
pagesize = getpagesize();
-   page_begin = (void*)((size_t)addr - (size_t)addr % pagesize);
+   page_begin = (void *)((size_t)addr - (size_t)addr % pagesize);
if (page_begin != addr) {
// data is not page-aligned
len += ((size_t)addr - (size_t)page_begin);
@@ -108,60 +114,60 @@ mprotect_region(void* addr, size_t len, 
return NULL;
 }
 
-static char*
-clear_mprotect(void* addr, size_t len) {
+static char *clear_mprotect(void *addr, size_t len)
+{
if (mprotect(addr, len, PROT_READ | PROT_WRITE) < 0) {
return strerror(errno);
}
return NULL;
 }
 
-#define ATTEMPT_TO_WRITE_TO_FILE(f, data) \
-   if (!WriteTextToFile(f, data)) { \
-   errno = 0; \
-   msg = createException(MAL, "cudf.eval", "Write error."); \
-   goto wrapup; \
+#define ATTEMPT_TO_WRITE_TO_FILE(f, data)  
\
+   if (!WriteTextToFile(f, data)) {
   \
+   errno = 0;  
   \
+   msg = createException(MAL, "cudf.eval", "Write error.");
   \
+   goto wrapup;
   \
}
 
-
-#define ATTEMPT_TO_WRITE_DATA_TO_FILE(f, data,

MonetDB: default - Merge with Jul2017 branch.

2017-06-12 Thread Sjoerd Mullender

Changeset: e75ebf8fbbdd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e75ebf8fbbdd
Added Files:
sql/test/BugTracker-2017/Tests/crash-dce.Bug-6330.sql
sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.sql

sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.err

sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.out
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.sql
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.stable.err
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.stable.out
sql/test/Tests/ifexists-views.sql
sql/test/Tests/ifexists-views.stable.err
sql/test/Tests/ifexists-views.stable.out
Modified Files:
MonetDB.spec
clients/ChangeLog.Dec2016
common/stream/stream.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_pushselect.c
sql/backends/monet5/sql_cat.c
sql/backends/monet5/sql_upgrades.c
sql/include/sql_catalog.h
sql/scripts/51_sys_schema_extension.sql
sql/server/rel_optimizer.c
sql/server/rel_schema.c
sql/server/rel_select.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/test/BugTracker-2017/Tests/All
sql/test/Tests/All
sql/test/Tests/copy-into-fwf.sql.in
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out
tools/merovingian/daemon/controlrunner.c
tools/merovingian/utils/control.c
Branch: default
Log Message:

Merge with Jul2017 branch.


diffs (truncated from 1135 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -311,6 +311,11 @@ MonetDB, you will very likely need this 
 Summary: MonetDB - Monet Database Management System Client Programs
 Group: Applications/Databases
 Requires: %{name}-client%{?_isa} = %{version}-%{release}
+%if (0%{?fedora} >= 22)
+# tomograph executes these two:
+Recommends: /usr/bin/gs
+Recommends: /usr/bin/gnuplot
+%endif
 
 %description client-tools
 MonetDB is a database management system that is developed from a
diff --git a/clients/ChangeLog.Dec2016 b/clients/ChangeLog.Dec2016
--- a/clients/ChangeLog.Dec2016
+++ b/clients/ChangeLog.Dec2016
@@ -1,3 +1,7 @@
 # ChangeLog file for clients
 # This file is updated with Maddlog
 
+* Mon Jun 12 2017 Sjoerd Mullender 
+- A bug was fixed in the implementation of the functions SQLTables in
+  the ODBC driver where not enough memory was allocated.
+
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -5444,11 +5444,11 @@ typedef struct {
char filler;
// state
size_t line_len;
-   char* in_buf;
-   char* out_buf;
+   char *in_buf;
+   char *out_buf;
size_t out_buf_start;
size_t out_buf_remaining;
-   char* nl_buf;
+   char nl_buf[1];
 } stream_fwf_data;
 
 
@@ -5458,10 +5458,11 @@ stream_fwf_read(stream *s, void *buf, si
stream_fwf_data *fsd;
size_t to_write = cnt;
size_t buf_written = 0;
-   if (strcmp(s->name, STREAM_FWF_NAME) != 0 || elmsize != 1) {
+
+   fsd = (stream_fwf_data*) s->stream_data.p;
+   if (fsd == NULL || elmsize != 1) {
return -1;
}
-   fsd = (stream_fwf_data*) s->stream_data.p;
 
while (to_write > 0) {
// input conversion
@@ -5499,12 +5500,12 @@ stream_fwf_read(stream *s, void *buf, si
 
// now we know something is in output_buf so deliver it
if (fsd->out_buf_remaining <= to_write) {
-   memcpy((char*)buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
to_write -= fsd->out_buf_remaining;
buf_written += fsd->out_buf_remaining;
fsd->out_buf_remaining = 0;
} else {
-   memcpy((char*) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
fsd->out_buf_start += to_write;
fsd->out_buf_remaining -= to_write;
buf_written += to_write;
@@ -5518,25 +5519,35 @@ stream_fwf_read(stream *s, void *buf, si
 static void
 stream_fwf_close(stream *s)
 {
-   if (strcmp(s->name, STREAM_FWF_NAME) == 0) {
+   stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
+
+   if (fsd != NULL) {
stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
mnstr_close(fsd->s);
+

MonetDB: Jul2017 - Merge with Dec2016 branch.

2017-06-12 Thread Sjoerd Mullender

Changeset: 8e4dbdde17af for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8e4dbdde17af
Added Files:
sql/test/BugTracker-2017/Tests/crash-dce.Bug-6330.sql
Modified Files:
clients/ChangeLog.Dec2016
common/stream/stream.c
sql/server/rel_optimizer.c
sql/server/rel_updates.c
sql/test/BugTracker-2017/Tests/All
sql/test/Tests/copy-into-fwf.sql.in
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out
tools/merovingian/utils/control.c
Branch: Jul2017
Log Message:

Merge with Dec2016 branch.


diffs (273 lines):

diff --git a/clients/ChangeLog.Dec2016 b/clients/ChangeLog.Dec2016
--- a/clients/ChangeLog.Dec2016
+++ b/clients/ChangeLog.Dec2016
@@ -1,3 +1,7 @@
 # ChangeLog file for clients
 # This file is updated with Maddlog
 
+* Mon Jun 12 2017 Sjoerd Mullender 
+- A bug was fixed in the implementation of the functions SQLTables in
+  the ODBC driver where not enough memory was allocated.
+
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -5444,11 +5444,11 @@ typedef struct {
char filler;
// state
size_t line_len;
-   char* in_buf;
-   char* out_buf;
+   char *in_buf;
+   char *out_buf;
size_t out_buf_start;
size_t out_buf_remaining;
-   char* nl_buf;
+   char nl_buf[1];
 } stream_fwf_data;
 
 
@@ -5458,10 +5458,11 @@ stream_fwf_read(stream *s, void *buf, si
stream_fwf_data *fsd;
size_t to_write = cnt;
size_t buf_written = 0;
-   if (strcmp(s->name, STREAM_FWF_NAME) != 0 || elmsize != 1) {
+
+   fsd = (stream_fwf_data*) s->stream_data.p;
+   if (fsd == NULL || elmsize != 1) {
return -1;
}
-   fsd = (stream_fwf_data*) s->stream_data.p;
 
while (to_write > 0) {
// input conversion
@@ -5499,12 +5500,12 @@ stream_fwf_read(stream *s, void *buf, si
 
// now we know something is in output_buf so deliver it
if (fsd->out_buf_remaining <= to_write) {
-   memcpy((char*)buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
to_write -= fsd->out_buf_remaining;
buf_written += fsd->out_buf_remaining;
fsd->out_buf_remaining = 0;
} else {
-   memcpy((char*) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
fsd->out_buf_start += to_write;
fsd->out_buf_remaining -= to_write;
buf_written += to_write;
@@ -5518,25 +5519,35 @@ stream_fwf_read(stream *s, void *buf, si
 static void
 stream_fwf_close(stream *s)
 {
-   if (strcmp(s->name, STREAM_FWF_NAME) == 0) {
+   stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
+
+   if (fsd != NULL) {
stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
mnstr_close(fsd->s);
+   mnstr_destroy(fsd->s);
free(fsd->widths);
free(fsd->in_buf);
free(fsd->out_buf);
-   free(fsd->nl_buf);
free(fsd);
-   }
-   // FIXME destroy(s);
-}
-
-stream*
+   s->stream_data.p = NULL;
+   }
+}
+
+static void
+stream_fwf_destroy(stream *s)
+{
+   stream_fwf_close(s);
+   destroy(s);
+}
+
+stream *
 stream_fwf_create (stream *s, size_t num_fields, size_t *widths, char filler)
 {
stream *ns;
stream_fwf_data *fsd = malloc(sizeof(stream_fwf_data));
size_t i, out_buf_len;
-   if (!fsd) {
+
+   if (fsd == NULL) {
return NULL;
}
fsd->s = s;
@@ -5548,34 +5559,27 @@ stream_fwf_create (stream *s, size_t num
fsd->line_len += widths[i];
}
fsd->in_buf = malloc(fsd->line_len);
-   if (!fsd->in_buf) {
+   if (fsd->in_buf == NULL) {
free(fsd);
return NULL;
}
out_buf_len = fsd->line_len * 3;
fsd->out_buf = malloc(out_buf_len);
-   if (!fsd->out_buf) {
+   if (fsd->out_buf == NULL) {
free(fsd->in_buf);
free(fsd);
return NULL;
}
fsd->out_buf_remaining = 0;
-   fsd->nl_buf = malloc(1);
-   if (!fsd->nl_buf) {
+   if ((ns = create_stream(STREAM_FWF_NAME)) == NULL) {
free(fsd->in_buf);
free(fsd->out_buf);
free(fsd);
return NULL;
}
-   if ((ns =

MonetDB: Jul2017 - Recommend ghostscript (gs) and gnuplot for to...

2017-06-12 Thread Sjoerd Mullender

Changeset: 22fc7046aa54 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=22fc7046aa54
Modified Files:
MonetDB.spec
Branch: Jul2017
Log Message:

Recommend ghostscript (gs) and gnuplot for tomograph.


diffs (15 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -311,6 +311,11 @@ MonetDB, you will very likely need this 
 Summary: MonetDB - Monet Database Management System Client Programs
 Group: Applications/Databases
 Requires: %{name}-client%{?_isa} = %{version}-%{release}
+%if (0%{?fedora} >= 22)
+# tomograph executes these two:
+Recommends: /usr/bin/gs
+Recommends: /usr/bin/gnuplot
+%endif
 
 %description client-tools
 MonetDB is a database management system that is developed from a
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - Fix test.

2017-06-12 Thread Sjoerd Mullender

Changeset: b0ee92754265 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b0ee92754265
Modified Files:
sql/test/Tests/copy-into-fwf.sql.in
Branch: Dec2016
Log Message:

Fix test.


diffs (12 lines):

diff --git a/sql/test/Tests/copy-into-fwf.sql.in 
b/sql/test/Tests/copy-into-fwf.sql.in
--- a/sql/test/Tests/copy-into-fwf.sql.in
+++ b/sql/test/Tests/copy-into-fwf.sql.in
@@ -12,7 +12,7 @@ create table mtcars (
 carb DOUBLE PRECISION
 );
 
-copy into mtcars from '$TSTSRCDIR/mtcars' using delimiters 'a','b','c' null as 
'' fwf (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2);
+copy into mtcars from '$QTSTSRCDIR/mtcars' using delimiters 'a','b','c' null 
as '' fwf (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2);
 
 select * from mtcars;
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - Plug a few memory leaks.

2017-06-12 Thread Sjoerd Mullender

Changeset: 041e2f015b9a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=041e2f015b9a
Modified Files:
common/stream/stream.c
sql/server/rel_updates.c
Branch: Dec2016
Log Message:

Plug a few memory leaks.


diffs (148 lines):

diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -4712,11 +4712,11 @@ typedef struct {
char filler;
// state
size_t line_len;
-   char* in_buf;
-   char* out_buf;
+   char *in_buf;
+   char *out_buf;
size_t out_buf_start;
size_t out_buf_remaining;
-   char* nl_buf;
+   char nl_buf[1];
 } stream_fwf_data;
 
 
@@ -4726,10 +4726,11 @@ stream_fwf_read(stream *s, void *buf, si
stream_fwf_data *fsd;
size_t to_write = cnt;
size_t buf_written = 0;
-   if (strcmp(s->name, STREAM_FWF_NAME) != 0 || elmsize != 1) {
+
+   fsd = (stream_fwf_data*) s->stream_data.p;
+   if (fsd == NULL || elmsize != 1) {
return -1;
}
-   fsd = (stream_fwf_data*) s->stream_data.p;
 
while (to_write > 0) {
// input conversion
@@ -4767,12 +4768,12 @@ stream_fwf_read(stream *s, void *buf, si
 
// now we know something is in output_buf so deliver it
if (fsd->out_buf_remaining <= to_write) {
-   memcpy((char*)buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
to_write -= fsd->out_buf_remaining;
buf_written += fsd->out_buf_remaining;
fsd->out_buf_remaining = 0;
} else {
-   memcpy((char*) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
fsd->out_buf_start += to_write;
fsd->out_buf_remaining -= to_write;
buf_written += to_write;
@@ -4786,25 +4787,35 @@ stream_fwf_read(stream *s, void *buf, si
 static void
 stream_fwf_close(stream *s)
 {
-   if (strcmp(s->name, STREAM_FWF_NAME) == 0) {
+   stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
+
+   if (fsd != NULL) {
stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
mnstr_close(fsd->s);
+   mnstr_destroy(fsd->s);
free(fsd->widths);
free(fsd->in_buf);
free(fsd->out_buf);
-   free(fsd->nl_buf);
free(fsd);
+   s->stream_data.p = NULL;
}
-   // FIXME destroy(s);
 }
 
-stream*
+static void
+stream_fwf_destroy(stream *s)
+{
+   stream_fwf_close(s);
+   destroy(s);
+}
+
+stream *
 stream_fwf_create (stream *s, size_t num_fields, size_t *widths, char filler)
 {
stream *ns;
stream_fwf_data *fsd = malloc(sizeof(stream_fwf_data));
size_t i, out_buf_len;
-   if (!fsd) {
+
+   if (fsd == NULL) {
return NULL;
}
fsd->s = s;
@@ -4816,34 +4827,27 @@ stream_fwf_create (stream *s, size_t num
fsd->line_len += widths[i];
}
fsd->in_buf = malloc(fsd->line_len);
-   if (!fsd->in_buf) {
+   if (fsd->in_buf == NULL) {
free(fsd);
return NULL;
}
out_buf_len = fsd->line_len * 3;
fsd->out_buf = malloc(out_buf_len);
-   if (!fsd->out_buf) {
+   if (fsd->out_buf == NULL) {
free(fsd->in_buf);
free(fsd);
return NULL;
}
fsd->out_buf_remaining = 0;
-   fsd->nl_buf = malloc(1);
-   if (!fsd->nl_buf) {
+   if ((ns = create_stream(STREAM_FWF_NAME)) == NULL) {
free(fsd->in_buf);
free(fsd->out_buf);
free(fsd);
return NULL;
}
-   if ((ns = create_stream(STREAM_FWF_NAME)) == NULL) {
-   free(fsd->in_buf);
-   free(fsd->out_buf);
-   free(fsd->nl_buf);
-   free(fsd);
-   return NULL;
-   }
ns->read = stream_fwf_read;
ns->close = stream_fwf_close;
+   ns->destroy = stream_fwf_destroy;
ns->write = NULL;
ns->flush = NULL;
ns->access = ST_READ;
diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c
--- a/sql/server/rel_updates.c
+++ b/sql/server/rel_updates.c
@@ -1173,7 +1173,7 @@ rel_import(mvc *sql, sql_table *t, char 
sql_exp *import;
sql_schema *sys = mvc_bind_schema(sql, "sys");
sql_subfunc *f = sql_find_func(sql->sa, sys, "copyfrom", 10, F_UNION, 
NULL);
-   char* fwf_string = NULL;
+   char *fwf_string = NULL;

MonetDB: Dec2016 - Added changelog message for changeset b6ed5ac...

2017-06-12 Thread Sjoerd Mullender

Changeset: 30e815b5acef for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=30e815b5acef
Modified Files:
clients/ChangeLog.Dec2016
Branch: Dec2016
Log Message:

Added changelog message for changeset b6ed5ac2f6f5.


diffs (11 lines):

diff --git a/clients/ChangeLog.Dec2016 b/clients/ChangeLog.Dec2016
--- a/clients/ChangeLog.Dec2016
+++ b/clients/ChangeLog.Dec2016
@@ -1,3 +1,7 @@
 # ChangeLog file for clients
 # This file is updated with Maddlog
 
+* Mon Jun 12 2017 Sjoerd Mullender 
+- A bug was fixed in the implementation of the functions SQLTables in
+  the ODBC driver where not enough memory was allocated.
+
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Add test for automatically freeing stray mallocs.

2017-06-12 Thread Mark Raasveldt

Changeset: 718daabfbf43 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=718daabfbf43
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi05.sql
sql/backends/monet5/UDF/capi/Tests/capi05.stable.err
sql/backends/monet5/UDF/capi/Tests/capi05.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
Branch: jitudf
Log Message:

Add test for automatically freeing stray mallocs.


diffs (165 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -4,3 +4,4 @@ capi01
 capi02
 capi03
 capi04
+capi05
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi05.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi05.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi05.sql
@@ -0,0 +1,20 @@
+# memory leak test
+# randomly allocate some buffers
+
+
+START TRANSACTION;
+
+CREATE FUNCTION capi05(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+   char* mydata = malloc(1000);
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   result->data[i] = inp.data[i] * 2;
+   }
+};
+
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+
+SELECT capi05(i) FROM integers;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi05.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi05.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi05.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi05` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 12:58:52 >  
+# 12:58:52 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=39823" "--set" 
"mapi_usock=/var/tmp/mtest-37374/.s.monetdb.39823" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 12:58:52 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 39823
+# cmdline opt  mapi_usock = /var/tmp/mtest-37374/.s.monetdb.39823
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 12:58:52 >  
+# 12:58:52 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-37374" "--port=39823"
+# 12:58:52 >  
+
+
+# 12:58:53 >  
+# 12:58:53 >  "Done."
+# 12:58:53 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi05.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi05.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi05.stable.out
@@ -0,0 +1,88 @@
+stdout of test 'capi05` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 12:58:52 >  
+# 12:58:52 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=39823" "--set" 
"mapi_usock=/var/tmp/mtest-37374/.s.monetdb.39823" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 12:58:52 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:39823/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-37374/.s.monetdb.39823
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 27_rejects.sql
+# loading sql script: 39_analytics.sql
+# loading sql

MonetDB: jitudf - Automatically clean up any mallocs performed i...

2017-06-12 Thread Mark Raasveldt

Changeset: dce7fa42e62a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dce7fa42e62a
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Automatically clean up any mallocs performed in the jitted function.


diffs (190 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
@@ -10,8 +10,6 @@ CREATE FUNCTION capi04(inp STRING) RETUR
if (inp.is_null(inp.data[i])) {
result->data[i] = result->null_value;
} else {
-   // the contract says we must use "malloc" to allocate 
for the result strings
-   // "malloc" is a function pointer that actually points 
to GDKmalloc
result->data[i] = malloc(strlen(inp.data[i]) + 2);
strcpy(result->data[i] + 1, inp.data[i]);
result->data[i][0] = 'H';
@@ -35,8 +33,28 @@ CREATE FUNCTION capi04(inp STRING) RETUR
 };
 
 SELECT capi04(i) FROM strings;
+ROLLBACK;
+
+START TRANSACTION;
+# return constant strings, instead of allocated strings
+CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
+#include 
+
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   if (inp.is_null(inp.data[i])) {
+   result->data[i] = result->null_value;
+   } else {
+   result->data[i] = malloc(strlen(inp.data[i]) + 2);
+   strcpy(result->data[i] + 1, inp.data[i]);
+   result->data[i] = "hello";
+   }
+   }
+};
+
+CREATE TABLE strings(i STRING);
+INSERT INTO strings VALUES ('ello'), ('ow'), (NULL), ('onestly?'), ('annes');
+
+SELECT capi04(i) FROM strings;
 
 ROLLBACK;
-
-
-
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
@@ -54,14 +54,10 @@ Ready.
 # loading sql script: 90_generator.sql
 # loading sql script: 90_generator_hge.sql
 # loading sql script: 99_system.sql
-Hello
-How
-Honestly?
-Hannes
 
-# 15:50:30 >  
-# 15:50:30 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-20003" "--port=32751"
-# 15:50:30 >  
+# 12:56:26 >  
+# 12:56:26 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-37150" "--port=31836"
+# 12:56:26 >  
 
 #START TRANSACTION;
 #CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
@@ -98,6 +94,35 @@ Hannes
 #  }
 #};
 #ROLLBACK;
+#START TRANSACTION;
+#CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
+##include 
+#
+#  result->initialize(result, inp.count);
+#  for(size_t i = 0; i < inp.count; i++) {
+#  if (inp.is_null(inp.data[i])) {
+#  result->data[i] = result->null_value;
+#  } else {
+#  result->data[i] = malloc(strlen(inp.data[i]) + 2);
+#  strcpy(result->data[i] + 1, inp.data[i]);
+#  result->data[i] = "hello";
+#  }
+#  }
+#};
+#CREATE TABLE strings(i STRING);
+#INSERT INTO strings VALUES ('ello'), ('ow'), (NULL), ('onestly?'), ('annes');
+[ 5]
+#SELECT capi04(i) FROM strings;
+% sys.L2 # table_name
+% L2 # name
+% clob # type
+% 5 # length
+[ "hello"  ]
+[ "hello"  ]
+[ NULL ]
+[ "hello"  ]
+[ "hello"  ]
+#ROLLBACK;
 
 # 15:50:31 >  
 # 15:50:31 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -22,6 +22,13 @@
 #include 
 #include 
 
+struct _allocated_region;
+typedef struct _allocated_region {
+   struct _allocated_region* next;
+} allocated_region;
+
+
+static __thread allocated_region* allocated_regions;
 static __thread jmp_buf jump_buffer;
 
 static str
@@ -125,7 +132,7 @@ clear_mprotect(void* addr, size_t len) {
}
 
 
-static void* wrapped_GDK_malloc(size_t size) {
+static void* jump_GDK_malloc(size_t size) {
void* ptr = GDKmalloc(size);
if (!ptr) {
longjmp(jump_buffer, 2);
@@ -133,11 +140,21 @@ static void* wrapped_GDK_malloc(size_t s
return ptr;
 }
 
+static void* wrapped_GDK_malloc(size_t size) {
+   allocated_region* region;
+   void* ptr = jump_GDK_malloc(size + sizeof(allocated_region));
+   region = (allocated_region*)ptr;
+   region->next = allocated_regions;
+   allocated_regions = region;
+   
+   return ptr + sizeof(allocated_region);
+}
+
 #define

MonetDB: jitudf - Create temporary files in the LEFTOVERS direct...

2017-06-12 Thread Mark Raasveldt

Changeset: 1b9f39d82528 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1b9f39d82528
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Create temporary files in the LEFTOVERS directory of the BBP farm.


diffs (151 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
--- a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
+++ b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
@@ -36,6 +36,12 @@ ERROR = !No data returned.
 MAPI  = (monetdb) /var/tmp/mtest-2836/.s.monetdb.35915
 QUERY = SELECT capi03(i) FROM integers;
 ERROR = !Something went wrong!
+MAPI  = (monetdb) /var/tmp/mtest-25067/.s.monetdb.39212
+QUERY = SELECT capi03(i) FROM integers;
+ERROR = !Attempting to write to the input or triggered a segfault/bus error
+MAPI  = (monetdb) /var/tmp/mtest-25067/.s.monetdb.39212
+QUERY = SELECT capi03(i) FROM integers;
+ERROR = !Attempting to write to the input or triggered a segfault/bus error
 
 # 17:24:55 >  
 # 17:24:55 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
--- a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
+++ b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
@@ -87,6 +87,26 @@ Ready.
 #INSERT INTO integers VALUES (1), (2), (3), (4), (5);
 [ 5]
 #ROLLBACK;
+#START TRANSACTION;
+#CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+#  inp.data[0] = 10;
+#  result->initialize(result, inp.count);
+#  for(size_t i = 0; i < inp.count; i++) {
+#  result->data[i] = inp.data[i] * 2;
+#  }
+#};
+#CREATE TABLE integers(i INTEGER);
+#INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+[ 5]
+#ROLLBACK;
+#START TRANSACTION;
+#CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+#  int x = *((int*)NULL);
+#};
+#CREATE TABLE integers(i INTEGER);
+#INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+[ 5]
+#ROLLBACK;
 
 # 17:24:55 >  
 # 17:24:55 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
@@ -15,7 +15,6 @@ CREATE FUNCTION capi04(inp STRING) RETUR
result->data[i] = malloc(strlen(inp.data[i]) + 2);
strcpy(result->data[i] + 1, inp.data[i]);
result->data[i][0] = 'H';
-   printf("%s\n", result->data[i]);
}
}
 };
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -223,6 +223,8 @@ static daytime time_from_data(cudf_data_
 static void data_from_timestamp(timestamp d, cudf_data_timestamp* ptr);
 static timestamp timestamp_from_data(cudf_data_timestamp* ptr);
 
+static char valid_path_characters[] = "abcdefghijklmnopqrstuvwxyz";
+
 static str
 CUDFeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped) 
{
sql_func * sqlfun = NULL;
@@ -232,6 +234,7 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal
char argbuf[64];
char buf[BUFSIZ];
char fname[BUFSIZ];
+   char oname[BUFSIZ];
char libname[BUFSIZ];
char error_buf[BUFSIZ];
char total_error_buf[8192];
@@ -358,9 +361,52 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal
}
}
 
-   // name of the source file and lib file that will be generated
-   snprintf(fname, BUFSIZ, "%s.c", funcname);
-   snprintf(libname, BUFSIZ, "lib%s%s", funcname, SO_EXT);
+   {
+   const int RANDOM_NAME_SIZE = 32;
+   char* path = NULL;
+   const char* prefix = "LEFTOVERS"DIR_SEP_STR;
+   size_t prefix_size = strlen(prefix);
+
+   memcpy(buf, prefix, sizeof(char) * strlen(prefix));
+   // generate a random 32-character name for the temporary files
+   for(i = prefix_size; i < prefix_size + RANDOM_NAME_SIZE; i++) {
+   buf[i] = valid_path_characters[rand() % 
(sizeof(valid_path_characters) - 1)];
+   }
+   buf[i] = '\0';
+   path = GDKfilepath(0, BATDIR, buf, "c");
+   if (!path) {
+   msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
+   goto wrapup;
+   }
+   strcpy(fname, path);
+   strcpy(oname, fname);
+   oname[strlen(oname) - 1] = 'o';
+   GDKfree(path);
+
+   memmove(buf + strlen(SO_PREFIX) + prefix_size, buf + 
prefix_size, i

MonetDB: subjoin - only set the need_left when the order is real...

2017-06-12 Thread Niels Nes

Changeset: 969e826b209b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=969e826b209b
Modified Files:
sql/backends/monet5/rel_bin.c
Branch: subjoin
Log Message:

only set the need_left when the order is realy needed.


diffs (18 lines):

diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -1695,12 +1695,12 @@ rel2bin_join(backend *be, sql_rel *rel, 
node *en = NULL, *n;
stmt *left = NULL, *right = NULL, *join = NULL, *jl, *jr, *lsel = NULL, 
*rsel = NULL, *oleft = NULL, *oright = NULL;
stmt *ld = NULL, *rd = NULL;
-   int need_left = (rel->flag == LEFT_JOIN || rel->op == op_left || 
rel->op == op_full);
+   int need_left = (rel->flag == LEFT_JOIN);
 
if (rel->l) { /* first construct the left sub relation */
left = subrel_bin(be, rel->l, refs);
if (left && left->type == st_project) {
-   if (need_left) {
+   if (need_left || rel->op == op_left || rel->op == 
op_full) {
left = rel2bin_post_project(be, left);
} else {
printf("# JOIN Left post_project \n");
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - When mprotect triggers, check if the address a...

MonetDB: default - Merge with Jul2017 branch.

MonetDB: subjoin - merged with default

MonetDB: Jul2017 - Approve upgrade after changeset 3307d3f0424a ...

MonetDB: jitudf - Merge with default.

MonetDB: jitudf - Linux compilation fixes.

MonetDB: jitudf - Add function cache to avoid recompiling functi...

MonetDB: jitudf - Correctly support aggregates and add testcases.

MonetDB: jitudf - Formatting.

MonetDB: default - Merge with Jul2017 branch.

MonetDB: Jul2017 - Merge with Dec2016 branch.

MonetDB: Jul2017 - Recommend ghostscript (gs) and gnuplot for to...

MonetDB: Dec2016 - Fix test.

MonetDB: Dec2016 - Plug a few memory leaks.

MonetDB: Dec2016 - Added changelog message for changeset b6ed5ac...

MonetDB: jitudf - Add test for automatically freeing stray mallocs.

MonetDB: jitudf - Automatically clean up any mallocs performed i...

MonetDB: jitudf - Create temporary files in the LEFTOVERS direct...

MonetDB: subjoin - only set the need_left when the order is real...

19 matches

Site Navigation

Mail list logo

Footer information