MonetDB: Mar2018 - In-place updates using JIT C UDFs testcase.

[Mark Raasveldt]

Changeset: b0eba40185d7 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b0eba40185d7
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi18.sql
sql/backends/monet5/UDF/capi/Tests/capi18.stable.err
sql/backends/monet5/UDF/capi/Tests/capi18.stable.out
Branch: Mar2018
Log Message:

In-place updates using JIT C UDFs testcase.


diffs (179 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi18.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi18.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi18.sql
@@ -0,0 +1,24 @@
+
+START TRANSACTION;
+
+CREATE FUNCTION capi00(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+size_t i;
+result->initialize(result, inp.count);
+for(i = 0; i < inp.count; i++) {
+result->data[i] = inp.data[i] * 2;
+}
+};
+
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+
+SELECT * FROM integers;
+
+UPDATE integers SET i=capi00(i);
+
+SELECT * FROM integers;
+
+DROP FUNCTION capi00;
+DROP TABLE integers;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi18.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi18.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi18.stable.err
@@ -0,0 +1,36 @@
+stderr of test 'capi18` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 15:58:24 >  
+# 15:58:24 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37255" "--set" 
"mapi_usock=/var/tmp/mtest-15164/.s.monetdb.37255" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_py=true" "--set" "embedded_c=true"
+# 15:58:24 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37255
+# cmdline opt  mapi_usock = /var/tmp/mtest-15164/.s.monetdb.37255
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  embedded_py = true
+# cmdline opt  embedded_c = true
+# cmdline opt  gdk_debug = 553648138
+
+# 15:58:25 >  
+# 15:58:25 >  "mclient" "-lsql" "-ftest" "-tnone" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-15164" "--port=37255"
+# 15:58:25 >  
+
+
+# 15:58:25 >  
+# 15:58:25 >  "Done."
+# 15:58:25 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi18.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi18.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi18.stable.out
@@ -0,0 +1,104 @@
+stdout of test 'capi18` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 15:58:24 >  
+# 15:58:24 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37255" "--set" 
"mapi_usock=/var/tmp/mtest-15164/.s.monetdb.37255" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_py=true" "--set" "embedded_c=true"
+# 15:58:24 >  
+
+# MonetDB 5 server v11.29.4
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993 - July 2008 CWI.
+# Copyright (c) August 2008 - 2018 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-30.eduroam.cwi.nl:37255/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-15164/.s.monetdb.37255
+# MonetDB/SQL module loaded
+# MonetDB/Python2 module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 21_dependency_views.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 27_rejects.sql
+# loading sql script: 39_analytics.sql
+# loading sql script: 39_analytics_hge.sql
+# loading sql script: 

MonetDB: Mar2018 - Don't crash on use of free().

[Mark Raasveldt]

Changeset: d1028e028629 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d1028e028629
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi17.sql
sql/backends/monet5/UDF/capi/Tests/capi17.stable.err
sql/backends/monet5/UDF/capi/Tests/capi17.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/capi/cheader.h
Branch: Mar2018
Log Message:

Don't crash on use of free().


diffs (235 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -16,4 +16,4 @@ NOT_WIN32?capi13
 NOT_WIN32?capi14
 NOT_WIN32?capi15
 NOT_WIN32?capi16
-
+NOT_WIN32?capi17
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi17.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi17.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi17.sql
@@ -0,0 +1,27 @@
+
+
+
+START TRANSACTION;
+
+CREATE FUNCTION capi17(inp OID) RETURNS BOOLEAN LANGUAGE C {
+size_t i;
+result->initialize(result, inp.count);
+char* value = malloc(100);
+for(i = 0; i < inp.count; i++) {
+if (inp.data[i] == inp.null_value) {
+result->data[i] = 0;
+} else {
+result->data[i] = 1;
+}
+}
+free(value);
+};
+CREATE TABLE oids(i OID);
+INSERT INTO oids(i) VALUES (100), (NULL), (200);
+
+SELECT * FROM oids WHERE capi17(i);
+
+DROP FUNCTION capi17;
+DROP TABLE oids;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi17.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi17.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi17.stable.err
@@ -0,0 +1,36 @@
+stderr of test 'capi17` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 15:53:02 >  
+# 15:53:02 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37852" "--set" 
"mapi_usock=/var/tmp/mtest-14326/.s.monetdb.37852" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_py=true" "--set" "embedded_c=true"
+# 15:53:02 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37852
+# cmdline opt  mapi_usock = /var/tmp/mtest-14326/.s.monetdb.37852
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  embedded_py = true
+# cmdline opt  embedded_c = true
+# cmdline opt  gdk_debug = 553648138
+
+# 15:53:03 >  
+# 15:53:03 >  "mclient" "-lsql" "-ftest" "-tnone" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-14326" "--port=37852"
+# 15:53:03 >  
+
+
+# 15:53:05 >  
+# 15:53:05 >  "Done."
+# 15:53:05 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi17.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi17.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi17.stable.out
@@ -0,0 +1,95 @@
+stdout of test 'capi17` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 15:53:02 >  
+# 15:53:02 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37852" "--set" 
"mapi_usock=/var/tmp/mtest-14326/.s.monetdb.37852" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_py=true" "--set" "embedded_c=true"
+# 15:53:02 >  
+
+# MonetDB 5 server v11.29.4
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993 - July 2008 CWI.
+# Copyright (c) August 2008 - 2018 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-30.eduroam.cwi.nl:37852/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-14326/.s.monetdb.37852
+# MonetDB/SQL module loaded
+# MonetDB/Python2 module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 

MonetDB: Mar2018 - Fix scalar value inputs/outputs in JIT C UDFs...

[Mark Raasveldt]

Changeset: 90c0bd2e2aef for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=90c0bd2e2aef
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi16.sql
sql/backends/monet5/UDF/capi/Tests/capi16.stable.err
sql/backends/monet5/UDF/capi/Tests/capi16.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: Mar2018
Log Message:

Fix scalar value inputs/outputs in JIT C UDFs and add testcase.


diffs (truncated from 399 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -15,4 +15,5 @@ NOT_WIN32?capi12
 NOT_WIN32?capi13
 NOT_WIN32?capi14
 NOT_WIN32?capi15
+NOT_WIN32?capi16
 
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi16.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi16.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi16.sql
@@ -0,0 +1,112 @@
+
+START TRANSACTION;
+CREATE FUNCTION string_reverse(input STRING) RETURNS STRING
+LANGUAGE C {
+#include 
+size_t i, j;
+result->initialize(result, input.count);
+for(i = 0; i < input.count; i++) {
+char* input_string = input.data[i];
+size_t len = strlen(input_string);
+result->data[i] = malloc(len + 1);
+for(j = 0; j < len; j++) {
+result->data[i][j] = input_string[len - j - 1];
+}
+result->data[i][len] = '\0';
+}
+};
+
+SELECT 'hello', string_reverse('hello');
+
+
+CREATE FUNCTION capi10(inp BLOB) RETURNS BLOB LANGUAGE C {
+size_t i;
+result->initialize(result, inp.count);
+for(i = 0; i < inp.count; i++) {
+if (inp.is_null(inp.data[i])) {
+result->data[i] = result->null_value;
+} else {
+result->data[i].data = malloc(inp.data[i].size);
+memcpy(result->data[i].data, inp.data[i].data, inp.data[i].size);
+result->data[i].size = inp.data[i].size;
+}
+}
+};
+
+SELECT BLOB '0000', capi10(BLOB '0000');
+
+
+CREATE FUNCTION capi00(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+size_t i;
+result->initialize(result, inp.count);
+for(i = 0; i < inp.count; i++) {
+result->data[i] = inp.data[i] * 2;
+}
+};
+
+SELECT 1, capi00(1);
+
+CREATE FUNCTION capi02_increment_year(d DATE) RETURNS DATE
+language C
+{
+size_t i;
+result->initialize(result, d.count);
+for(i = 0; i < result->count; i++) {
+if (d.is_null(d.data[i])) {
+result->data[i] = result->null_value;
+} else {
+result->data[i].year = d.data[i].year + 1;
+result->data[i].month = d.data[i].month;
+result->data[i].day = d.data[i].day;
+}
+}
+};
+
+
+SELECT capi02_increment_year('1992-09-20');
+
+
+CREATE FUNCTION capi02_randomize_time(d TIME) RETURNS TIME
+language C
+{
+size_t i;
+result->initialize(result, d.count);
+for(i = 0; i < result->count; i++) {
+if (d.is_null(d.data[i])) {
+result->data[i] = result->null_value;
+} else {
+result->data[i].hours = (i + 1234) % 24;
+result->data[i].minutes = (i + 1234) % 60;
+result->data[i].seconds = (i + 1234) % 60;
+result->data[i].ms = (i + 1234) % 1000;
+}
+}
+};
+
+SELECT capi02_randomize_time('03:03:02.0101');
+
+
+CREATE FUNCTION capi02_increment_timestamp(d TIMESTAMP) RETURNS TIMESTAMP
+language C
+{
+size_t i;
+result->initialize(result, d.count);
+for(i = 0; i < result->count; i++) {
+if (d.is_null(d.data[i])) {
+result->data[i] = result->null_value;
+} else {
+result->data[i].date.year = d.data[i].date.year + 1;
+result->data[i].date.month = d.data[i].date.month;
+result->data[i].date.day = d.data[i].date.day;
+
+result->data[i].time.hours = (i + 1234) % 24;
+result->data[i].time.minutes = (i + 1234) % 60;
+result->data[i].time.seconds = (i + 1234) % 60;
+result->data[i].time.ms = (i + 1234) % 1000;
+}
+}
+};
+
+SELECT capi02_increment_timestamp('1992-09-20 03:03:02.0101');
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi16.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi16.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi16.stable.err
@@ -0,0 +1,35 @@
+stderr of test 'capi16` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 15:47:18 >  
+# 15:47:18 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=36686" "--set" 
"mapi_usock=/var/tmp/mtest-6494/.s.monetdb.36686" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_c=true"
+# 15:47:18 >  
+
+# builtin opt  gdk_dbpath = 

MonetDB: Mar2018 - Fix for zero length input columns in JIT C API.

[Mark Raasveldt]

Changeset: 783204486bc3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=783204486bc3
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi15.sql
sql/backends/monet5/UDF/capi/Tests/capi15.stable.err
sql/backends/monet5/UDF/capi/Tests/capi15.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: Mar2018
Log Message:

Fix for zero length input columns in JIT C API.


diffs (truncated from 366 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -14,4 +14,5 @@ NOT_WIN32?capi11
 NOT_WIN32?capi12
 NOT_WIN32?capi13
 NOT_WIN32?capi14
+NOT_WIN32?capi15
 
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi15.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi15.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi15.sql
@@ -0,0 +1,19 @@
+
+START TRANSACTION;
+
+CREATE FUNCTION capi16(i INTEGER, j STRING, k BLOB, l DATE, m TIMESTAMP, n 
TIME)
+RETURNS INTEGER
+LANGUAGE C {
+result->initialize(result, i.count);
+for(size_t it = 0; it < i.count; it++) {
+result->data[it] = i.data[it] * 2;
+}
+};
+
+CREATE TABLE capi16table(i INTEGER, j STRING, k BLOB, l DATE, m TIMESTAMP, n 
TIME);
+SELECT i, capi16(i, j, k, l, m, n) FROM capi16table;
+
+DROP TABLE capi16table;
+DROP FUNCTION capi16;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi15.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi15.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi15.stable.err
@@ -0,0 +1,35 @@
+stderr of test 'capi15` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 14:47:44 >  
+# 14:47:44 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=32682" "--set" 
"mapi_usock=/var/tmp/mtest-72472/.s.monetdb.32682" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_c=true"
+# 14:47:44 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 32682
+# cmdline opt  mapi_usock = /var/tmp/mtest-72472/.s.monetdb.32682
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  embedded_c = true
+# cmdline opt  gdk_debug = 553648138
+
+# 14:47:44 >  
+# 14:47:44 >  "mclient" "-lsql" "-ftest" "-tnone" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-72472" "--port=32682"
+# 14:47:44 >  
+
+
+# 14:47:45 >  
+# 14:47:45 >  "Done."
+# 14:47:45 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi15.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi15.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi15.stable.out
@@ -0,0 +1,85 @@
+stdout of test 'capi15` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 14:47:44 >  
+# 14:47:44 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=32682" "--set" 
"mapi_usock=/var/tmp/mtest-72472/.s.monetdb.32682" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_c=true"
+# 14:47:44 >  
+
+# MonetDB 5 server v11.29.4
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993 - July 2008 CWI.
+# Copyright (c) August 2008 - 2018 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-30.eduroam.cwi.nl:32682/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-72472/.s.monetdb.32682
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 

MonetDB: Mar2018 - Fix JIT UDF is_null function for new flt_nil ...

[Mark Raasveldt]

Changeset: 0361ebdfd02b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0361ebdfd02b
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi14.sql
sql/backends/monet5/UDF/capi/Tests/capi14.stable.err
sql/backends/monet5/UDF/capi/Tests/capi14.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: Mar2018
Log Message:

Fix JIT UDF is_null function for new flt_nil and dbl_nil and add new testcase.


diffs (257 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -12,3 +12,6 @@ NOT_WIN32?capi09
 NOT_WIN32?capi10
 NOT_WIN32?capi11
 NOT_WIN32?capi12
+NOT_WIN32?capi13
+NOT_WIN32?capi14
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi14.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi14.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi14.sql
@@ -0,0 +1,35 @@
+# NULL values
+START TRANSACTION;
+
+CREATE FUNCTION capi14(i INTEGER, j REAL, k DOUBLE) RETURNS DOUBLE LANGUAGE C {
+size_t it;
+result->initialize(result, i.count);
+for(it = 0; it < i.count; it++) {
+result->data[it] = 0;
+if (!i.is_null(i.data[it])) {
+result->data[it] += i.data[it];
+} else {
+result->data[it] -= 1;
+}
+if (!j.is_null(j.data[it])) {
+result->data[it] += j.data[it];
+} else {
+result->data[it] -= 1;
+}
+if (!k.is_null(k.data[it])) {
+result->data[it] += k.data[it];
+} else {
+result->data[it] -= 1;
+}
+}
+};
+
+CREATE TABLE vals(i INTEGER, j REAL, k DOUBLE);
+INSERT INTO vals VALUES (1, NULL, 1), (NULL, 2, 2), (3, 3, NULL), (NULL, NULL, 
NULL), (5, 5, 5);
+
+SELECT i, j, k, capi14(i, j, k) FROM vals;
+
+DROP FUNCTION capi14;
+DROP TABLE vals;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi14.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi14.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi14.stable.err
@@ -0,0 +1,36 @@
+stderr of test 'capi14` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 09:44:31 >  
+# 09:44:31 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=32793" "--set" 
"mapi_usock=/var/tmp/mtest-48691/.s.monetdb.32793" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_py=true" "--set" "embedded_c=true"
+# 09:44:31 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 32793
+# cmdline opt  mapi_usock = /var/tmp/mtest-48691/.s.monetdb.32793
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  embedded_py = true
+# cmdline opt  embedded_c = true
+# cmdline opt  gdk_debug = 553648138
+
+# 09:44:31 >  
+# 09:44:31 >  "mclient" "-lsql" "-ftest" "-tnone" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-48691" "--port=32793"
+# 09:44:31 >  
+
+
+# 09:44:32 >  
+# 09:44:32 >  "Done."
+# 09:44:32 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi14.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi14.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi14.stable.out
@@ -0,0 +1,101 @@
+stdout of test 'capi14` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 09:44:31 >  
+# 09:44:31 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=32793" "--set" 
"mapi_usock=/var/tmp/mtest-48691/.s.monetdb.32793" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" 
"--set" "embedded_py=true" "--set" "embedded_c=true"
+# 09:44:31 >  
+
+# MonetDB 5 server v11.29.2
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993 - July 2008 CWI.
+# Copyright (c) August 2008 - 2018 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://Marks-MacBook-Pro.local:32793/
+# Listening for UNIX domain connection requests on 

MonetDB: Mar2018 - Make mprotect and longjmp on malloc failure o...

[Mark Raasveldt]

Changeset: 85fc1cd7c486 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=85fc1cd7c486
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: Mar2018
Log Message:

Make mprotect and longjmp on malloc failure optional in CUDFs.

If mprotect is disabled (default), input columns are always copied instead of 
protected from writing, allowing the user to safely write to them.
If longjmp on malloc failure is disabled (default), mallocs must be manually 
checked inside the CUDF.


diffs (252 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -23,6 +23,11 @@
 #pragma GCC diagnostic ignored "-Wclobbered"
 #endif
 
+const char *mprotect_enableflag = "enable_mprotect";
+static bool option_enable_mprotect = false;
+const char *longjmp_enableflag = "enable_longjmp";
+static bool option_enable_longjmp = false;
+
 struct _allocated_region;
 typedef struct _allocated_region {
struct _allocated_region *next;
@@ -80,6 +85,8 @@ str CUDFprelude(void *ret)
if (!cudf_initialized) {
MT_lock_init(_lock, "cache_lock");
cudf_initialized = true;
+   option_enable_mprotect = GDKgetenv_istrue(mprotect_enableflag) 
|| GDKgetenv_isyes(mprotect_enableflag);
+   option_enable_longjmp = GDKgetenv_istrue(longjmp_enableflag) || 
GDKgetenv_isyes(longjmp_enableflag);
}
return MAL_SUCCEED;
 }
@@ -107,6 +114,7 @@ static void handler(int sig, siginfo_t *
 }
 
 static bool can_mprotect_region(void* addr) {
+   if (!option_enable_mprotect) return false;
int pagesize = getpagesize();
void* page_begin = (void *)((size_t)addr - (size_t)addr % pagesize);
return page_begin == addr;
@@ -160,7 +168,7 @@ static char *clear_mprotect(void *addr, 
 static void *jump_GDK_malloc(size_t size)
 {
void *ptr = GDKmalloc(size);
-   if (!ptr) {
+   if (!ptr && option_enable_longjmp) {
longjmp(jump_buffer[THRgettid()], 2);
}
return ptr;
@@ -211,7 +219,8 @@ static void *wrapped_GDK_zalloc_nojump(s
}   
   \
b = COLnew(0, TYPE_##tpename, count, TRANSIENT);
   \
if (!b) {   
   \
-   longjmp(jump_buffer[THRgettid()], 2);   
   \
+   if (option_enable_longjmp) 
longjmp(jump_buffer[THRgettid()], 2);   \
+   else return;
   \
}   
   \
self->bat = (void*) b;  
   \
self->count = count;
   \
@@ -444,12 +453,14 @@ static str CUDFeval(Client cntxt, MalBlk
// the input data
 
// we remove them from the pthread_sigmask
-   (void)sigemptyset(_set);
-   (void)sigaddset(_set, SIGSEGV);
-   (void)sigaddset(_set, SIGBUS);
-   (void)pthread_sigmask(SIG_UNBLOCK, _set, NULL);
+   if (option_enable_mprotect) {
+   (void)sigemptyset(_set);
+   (void)sigaddset(_set, SIGSEGV);
+   (void)sigaddset(_set, SIGBUS);
+   (void)pthread_sigmask(SIG_UNBLOCK, _set, NULL);
 
-   memset(, 0, sizeof(sa));
+   memset(, 0, sizeof(sa));
+   }
 
if (!grouped) {
sql_subfunc *sqlmorefun =
@@ -1216,75 +1227,79 @@ static str CUDFeval(Client cntxt, MalBlk
// set up a longjmp point
// this longjmp point is used for some error handling in the C function
// such as failed mallocs
-   ret = setjmp(jump_buffer[tid]);
-   if (ret < 0) {
-   // error value
-   msg = createException(MAL, "cudf.eval", "Failed setjmp: %s",
- strerror(errno));
-   errno = 0;
-   goto wrapup;
-   } else if (ret > 0) {
-   if (ret == 1) {
-   msg = createException(MAL, "cudf.eval", "Attempting to 
write to "
-   
"the input or triggered a "
-   
"segfault/bus error");
-   } else if (ret == 2) {
-   msg = createException(MAL, "cudf.eval",
- "Malloc 
failure in internal function!");
-   } else {
-   // we jumped here
-   msg = createException(MAL, "cudf.eval", "We 

MonetDB: default - Don't allow threading when doing a loopback q...

[Mark Raasveldt]

Changeset: 7106bb2eafae for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7106bb2eafae
Modified Files:
sql/backends/monet5/UDF/pyapi/connection.c
Branch: default
Log Message:

Don't allow threading when doing a loopback query.


diffs (15 lines):

diff --git a/sql/backends/monet5/UDF/pyapi/connection.c 
b/sql/backends/monet5/UDF/pyapi/connection.c
--- a/sql/backends/monet5/UDF/pyapi/connection.c
+++ b/sql/backends/monet5/UDF/pyapi/connection.c
@@ -54,9 +54,9 @@ static PyObject *_connection_execute(Py_
PyObject *result;
res_table *output = NULL;
char *res = NULL;
-Py_BEGIN_ALLOW_THREADS;
+//Py_BEGIN_ALLOW_THREADS;
res = _connection_query(self->cntxt, query, );
-Py_END_ALLOW_THREADS;
+//Py_END_ALLOW_THREADS;
GDKfree(query);
if (res != MAL_SUCCEED) {
PyErr_Format(PyExc_Exception, "SQL Query Failed: %s",
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: mtestplusplus - Merge with default.

[Mark Raasveldt]

Changeset: 13256f288ffc for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=13256f288ffc
Modified Files:
monetdb5/optimizer/opt_wrapper.c
sql/backends/monet5/sql_upgrades.c
Branch: mtestplusplus
Log Message:

Merge with default.


diffs (76 lines):

diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c
--- a/monetdb5/optimizer/opt_wrapper.c
+++ b/monetdb5/optimizer/opt_wrapper.c
@@ -104,8 +104,8 @@ str OPTwrapper (Client cntxt, MalBlkPtr 
str msg = MAL_SUCCEED;
lng clk;
 
-if (cntxt->mode == FINISHCLIENT)
-throw(MAL, "optimizer", "prematurely stopped client");
+   if (cntxt->mode == FINISHCLIENT)
+   throw(MAL, "optimizer", "prematurely stopped client");
 
if( p == NULL)
throw(MAL, "opt_wrapper", "missing optimizer statement");
@@ -154,8 +154,7 @@ str OPTwrapper (Client cntxt, MalBlkPtr 
if (codes[i].nme == 0)
throw(MAL, optimizer, "Optimizer implementation '%s' missing", 
fcnnme);
 
-   //OPTIMIZERDEBUG {
-   {
+   OPTIMIZERDEBUG {
fprintf(stderr,"=FINISHED %s  %d\n",optimizer, actions);
fprintFunction(stderr,mb,0,LIST_MAL_DEBUG );
}
diff --git a/sql/backends/monet5/sql_upgrades.c 
b/sql/backends/monet5/sql_upgrades.c
--- a/sql/backends/monet5/sql_upgrades.c
+++ b/sql/backends/monet5/sql_upgrades.c
@@ -1036,25 +1036,32 @@ sql_update_default_geom(Client c, mvc *s
 }
 
 static str
-sql_remove_environment_func(Client c)
+sql_remove_environment_func(Client c, mvc *sql)
 {
+   sql_schema *s = NULL;
+   sql_table *t = NULL;
size_t bufsize = 1000, pos = 0;
char *buf = GDKmalloc(bufsize), *err = NULL;
if (buf== NULL)
throw(SQL, "sql_remove_environment_func", SQLSTATE(HY001) 
MAL_MALLOC_FAIL);
 
+   /* because issueing sql: "drop view sys.environment cascade;" fails 
with with error: !SQLException:sql.drop_view:42000!DROP VIEW: cannot drop 
system view 'environment'
+  we need to drop the system view using internal function: 
mvc_drop_table() */
+   s = mvc_bind_schema(sql, "sys");
+   t = mvc_bind_table(sql, s, "environment");
+   mvc_drop_table(sql, s, t, 1);   // drop the system view: 
sys.environment cascade
+
pos += snprintf(buf + pos, bufsize - pos,
-   "drop view sys.environment cascade;\n"
-   "drop function sys.environment() cascade\n"
+   "drop function sys.environment() cascade;\n"
"create view sys.environment as select * from 
sys.env();\n"
"GRANT SELECT ON sys.environment TO PUBLIC;\n"
-   "update sys._tables set system = true where name = 
'environment' and schema_id in (select id from schemas where name = 'sys');\n");
+   "update sys._tables set system = true where system = 
false and name = 'environment' and schema_id in (select id from schemas where 
name = 'sys');\n");
 
pos += snprintf(buf + pos, bufsize - pos,
"delete from sys.systemfunctions where function_id not 
in (select id from sys.functions);\n");
 
assert(pos < bufsize);
-   printf("Running database upgrade commands:\n%s\n", buf);
+   printf("Running database upgrade commands:\ndrop view sys.environment 
cascade;\n%s\n", buf);
err = SQLstatementIntern(c, , "update", 1, 0, NULL);
GDKfree(buf);
return err; /* usually MAL_SUCCEED */
@@ -1176,8 +1183,8 @@ SQLupgrades(Client c, mvc *m)
}
}
 
-   if (sql_bind_func(m->sa, s, "environment", NULL, NULL, F_FUNC)) {
-   if ((err = sql_remove_environment_func(c)) != NULL) {
+   if (sql_bind_func_(m->sa, s, "environment", NULL, F_UNION)) {
+   if ((err = sql_remove_environment_func(c, m)) != NULL) {
fprintf(stderr, "!%s\n", err);
freeException(err);
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: mtestplusplus - Remove debug code.

[Mark Raasveldt]

Changeset: e30ea2f0ab4f for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e30ea2f0ab4f
Modified Files:
testing/Mtest.py.in
Branch: mtestplusplus
Log Message:

Remove debug code.


diffs (11 lines):

diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
@@ -4386,7 +4386,6 @@ def main(argv) :
 if verbose:
 print("\nRunning all tests in directories %s.\n" % 
str(testdirs))
 
-parallel_mode = True
 if not parallel_mode:
 for d in testdirs:
 perform_dir((d, 0, env, BusyPorts))
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: mtestplusplus - Added --parallel mode to mtest.

[Mark Raasveldt]

Changeset: 3c06b3f71fe6 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3c06b3f71fe6
Modified Files:
testing/Mtest.py.in
testing/process.py
Branch: mtestplusplus
Log Message:

Added --parallel mode to mtest.


diffs (170 lines):

diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
@@ -37,6 +37,7 @@ procdebug = False
 verbose = False
 quiet = False
 fast_mode = False
+parallel_mode = False
 
 releaserun = False
 
@@ -1207,6 +1208,34 @@ def find_test_dirs(thisdir) :
 return testdirs
 ### find_test_dirs(thisdir) #
 
+def perform_dir(tpl):
+d = tpl[0]
+fid = tpl[1]
+env = tpl[2]
+BusyPorts = tpl[3]
+body_good = []
+body_bad = []
+t, elem, diff, interrupted = PerformDir(env, d, [], BusyPorts, True)
+#t_ = t_ + t
+if elem is not None:
+if diff <= F_OK:
+body_good.append(elem)
+else:
+body_bad.append(elem)
+if interrupted:
+return
+if global_timeout and start_time + global_timeout < time.time():
+print('\nGlobal testing timeout reached\n')
+return
+if not testweb:
+# after a directory has been tested, create
+# the index file so that we can look at test
+# results while the tests are running
+env['TSTDIR'] = ""
+env['TSTTRGDIR'] = os.path.join(TSTTRGBASE, TSTPREF)
+body = body_bad + body_good
+CreateHtmlIndex(env, *body)
+
 def PerformDir(env, testdir, testlist, BusyPorts, all_tests = False) :
 interrupted = False
 td = 0
@@ -2407,7 +2436,7 @@ def RunTest(env, TST, BusyPorts, COND, o
 cmd = ['diff']
 if ACCURACYout >= 0:
 cmd.append('-d')
-cmd.extend(['-Bb', '-F^#', '-I%s' % par['IGNORE'],
+cmd.extend(['-B', '-b', '-F^#', '-I%s' % par['IGNORE'],
 '-U%s' % par['CONTEXT'],
 '%s%s.FILTERED' % (TST, STABLEout),
 '%s.test.out.FILTERED' % TST])
@@ -2712,7 +2741,7 @@ def killProc(proc, outfile = None, cmd =
 except OSError:
 pass
 
-def LaunchIt(cmd, TestInput, TestOut, TestErr, TimeOut, RunningServer, SrvrOut 
= None) :
+def LaunchIt(cmd, TestInput, TestOut, TestErr, TimeOut, RunningServer, 
ServerType, SrvrOut = None) :
 global setpgrp
 if not SrvrOut:
 SrvrOut = process.PIPE
@@ -2725,7 +2754,7 @@ def LaunchIt(cmd, TestInput, TestOut, Te
 if procdebug:
 print('LaunchIt: starting process "%s" (inpipe)\n' % '" "'.join(cmd))
 setpgrp = True
-if 'SERVER_PROCESS' not in RunningServer or RunningServer['CONFIG'] != cmd 
or RunningServer['SERVER_PROCESS'].killed or 
RunningServer['SERVER_PROCESS'].stdin.closed:
+if ServerType == "MAL" or 'SERVER_PROCESS' not in RunningServer or 
RunningServer['CONFIG'] != cmd or RunningServer['SERVER_PROCESS'].killed or 
RunningServer['SERVER_PROCESS'].stdin.closed:
 # launch a new server, configuration changed
 if 'SERVER_PROCESS' in RunningServer and not 
RunningServer['SERVER_PROCESS'].killed:
 # if there is a running server, kill it
@@ -2741,9 +2770,9 @@ def LaunchIt(cmd, TestInput, TestOut, Te
 proc.stderr = TestErr
 
 # maybe buffer output as it comes to avoid deadlock
-if SrvrOut == process.PIPE:
+if SrvrOut == process.PIPE and not isinstance(proc.stdout, 
process._BufferedPipe):
 proc.stdout = process._BufferedPipe(proc.stdout)
-if TestErr == process.PIPE:
+if TestErr == process.PIPE and not isinstance(proc.stderr, 
process._BufferedPipe):
 proc.stderr = process._BufferedPipe(proc.stderr)
 proc.killed = False
 t = Timer(TimeOut, killProc, args = [proc, TestErr, cmd])
@@ -2936,7 +2965,7 @@ def DoIt(env, SERVER, CALL, TST, EXT, PR
 # enable Python integration in server
 Srvr.extend(['--set', 'embedded_py=true'])
 
-pSrvr, pSrvrTimer = LaunchIt(Srvr, 
'\nio.printf("\\nReady.\\n");\n', SrvrOut, SrvrErr, TIMEOUT, RUNNING_SERVER)
+pSrvr, pSrvrTimer = LaunchIt(Srvr, 
'\nio.printf("\\nReady.\\n");\n', SrvrOut, SrvrErr, TIMEOUT, RUNNING_SERVER, 
SERVER)
 
 
 ln="dummy"
@@ -3120,9 +3149,6 @@ def DoIt(env, SERVER, CALL, TST, EXT, PR
 else:
 CollectIt(pSrvr.stdout, SrvrOut, False)
 
-
-
-
 pSrvrTimer.cancel()
 if procdebug:
 print('DoIt: process exited "%s" (%s)\n' % ('" "'.join(Srvr), 
pSrvr.returncode))
@@ -4359,27 +4385,23 @@ def main(argv) :
 else:
 if verbose:
 print("\nRunning all tests in directories %s.\n" % 
str(testdirs))
-for d in testdirs:
-t, elem, diff, interrupted = PerformDir(env, d, [], 
BusyPorts, True)
-t_ = t_ + t
-if elem is not None:
-if diff <= F_OK:
-

MonetDB: mtestplusplus - Add --fast mode to Mtest.py, keep serve...

[Mark Raasveldt]

Changeset: f0d1b166367c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f0d1b166367c
Modified Files:
testing/Mtest.py.in
testing/process.py
Branch: mtestplusplus
Log Message:

Add --fast mode to Mtest.py, keep server open between different test runs.


diffs (189 lines):

diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
@@ -36,6 +36,7 @@ import glob
 procdebug = False
 verbose = False
 quiet = False
+fast_mode = False
 
 releaserun = False
 
@@ -1313,10 +1314,11 @@ def PerformDir(env, testdir, testlist, B
 if not verbose and not quiet:
 print('\nRunning in %s' % TSTDIR)
 alllinks = []
+RUNNING_SERVER = {}
 try:
 for TST,COND in testlist:
 os.environ['TST'] = TST
-tt, FtOut, FtErr, bodyline, reason, links = RunTest(env, TST, 
BusyPorts, COND, oktests, length, all_tests)
+tt, FtOut, FtErr, bodyline, reason, links = RunTest(env, TST, 
BusyPorts, COND, oktests, length, all_tests, RUNNING_SERVER)
 alllinks.extend(links)
 if tt:
 t = "%7.3f" % tt
@@ -1639,7 +1641,7 @@ def returnCode(proc, f = None):
 f.write('\nInterrupt\n')
 f.flush()
 return 'interrupt'  # Interrupt
-if proc.returncode < 0:
+if proc.returncode < 0 and proc.returncode:
 if f is not None:
 f.write('\nSignal %d\n' % -proc.returncode)
 f.flush()
@@ -1859,7 +1861,7 @@ def CategorizeResult(TST, SockTime):
 e = SockTime
 return o, e
 
-def RunTest(env, TST, BusyPorts, COND, oktests, length, all_tests) :
+def RunTest(env, TST, BusyPorts, COND, oktests, length, all_tests, 
RUNNING_SERVER) :
 global setpgrp
 Failed = F_SKIP
 FailedOut = F_SKIP
@@ -2187,7 +2189,7 @@ def RunTest(env, TST, BusyPorts, COND, o
 TestErr.close()
 
 t0 = time.time()
-tres = DoIt(env, SERVER, CALL, TST, EXT, PRELUDE, TestOutFile, 
TestErrFile, STIMEOUT, CTIMEOUT, TIMEOUT, ME, MAPIsockets, length, nomito, 
threads, COND, all_tests)
+tres = DoIt(env, SERVER, CALL, TST, EXT, PRELUDE, TestOutFile, 
TestErrFile, STIMEOUT, CTIMEOUT, TIMEOUT, ME, MAPIsockets, length, nomito, 
threads, COND, all_tests, RUNNING_SERVER)
 if tres == 'segfault':
 # rename core file, if any -- might have to check
 # /proc/sys/kernel/core_pattern in the future but hopefully
@@ -2710,7 +2712,7 @@ def killProc(proc, outfile = None, cmd =
 except OSError:
 pass
 
-def LaunchIt(cmd, TestInput, TestOut, TestErr, TimeOut, SrvrOut = None) :
+def LaunchIt(cmd, TestInput, TestOut, TestErr, TimeOut, RunningServer, SrvrOut 
= None) :
 global setpgrp
 if not SrvrOut:
 SrvrOut = process.PIPE
@@ -2723,8 +2725,21 @@ def LaunchIt(cmd, TestInput, TestOut, Te
 if procdebug:
 print('LaunchIt: starting process "%s" (inpipe)\n' % '" "'.join(cmd))
 setpgrp = True
-proc = process.Popen(cmd, stdin = process.PIPE, stdout = SrvrOut,
+if 'SERVER_PROCESS' not in RunningServer or RunningServer['CONFIG'] != cmd 
or RunningServer['SERVER_PROCESS'].killed or 
RunningServer['SERVER_PROCESS'].stdin.closed:
+# launch a new server, configuration changed
+if 'SERVER_PROCESS' in RunningServer and not 
RunningServer['SERVER_PROCESS'].killed:
+# if there is a running server, kill it
+killProc(RunningServer['SERVER_PROCESS'])
+proc = process.Popen(cmd, stdin = process.PIPE, stdout = SrvrOut,
  stderr = TestErr, universal_newlines = True)
+if fast_mode:
+# for fast mode, keep the server cached and don't kill it
+RunningServer['CONFIG'] = cmd
+RunningServer['SERVER_PROCESS'] = proc
+else:
+proc = RunningServer['SERVER_PROCESS']
+proc.stderr = TestErr
+
 # maybe buffer output as it comes to avoid deadlock
 if SrvrOut == process.PIPE:
 proc.stdout = process._BufferedPipe(proc.stdout)
@@ -2744,10 +2759,10 @@ def LaunchIt(cmd, TestInput, TestOut, Te
 return proc, t
 ### LaunchIt(cmd, TestIn, TestOut, TestErr, TimeOut, SrvrOut) #
 
-def CollectIt(pOut, TestOut) :
+def CollectIt(pOut, TestOut, blocking=True) :
 if pOut:
 while True:
-buf = pOut.read(8192)
+buf = pOut.read(8192, blocking)
 if not buf:
 break
 TestOut.write(buf)
@@ -2816,7 +2831,7 @@ def mapi_ping(port,lang) :
 return False
 ### mapi_ping() #
 
-def DoIt(env, SERVER, CALL, TST, EXT, PRELUDE, TestOutFile, TestErrFile, 
STIMEOUT, CTIMEOUT, TIMEOUT, ME, MAPIsockets, length, nomito, threads, COND, 
all_tests) :
+def DoIt(env, SERVER, CALL, TST, EXT, PRELUDE, TestOutFile, TestErrFile, 
STIMEOUT, CTIMEOUT, TIMEOUT, ME, MAPIsockets, length, nomito, threads, COND, 
all_tests, 

MonetDB: jitudf - Close branch after merge with default.

[Mark Raasveldt]

Changeset: e0d0579c6b97 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e0d0579c6b97
Branch: jitudf
Log Message:

Close branch after merge with default.

___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Merge with jitudf.

[Mark Raasveldt]

Changeset: 0665312ee4d1 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0665312ee4d1
Added Files:
sql/backends/monet5/UDF/capi/50_capi.mal
sql/backends/monet5/UDF/capi/Makefile.ag
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/Tests/capi00.sql
sql/backends/monet5/UDF/capi/Tests/capi00.stable.err
sql/backends/monet5/UDF/capi/Tests/capi00.stable.out
sql/backends/monet5/UDF/capi/Tests/capi01.sql
sql/backends/monet5/UDF/capi/Tests/capi01.stable.err
sql/backends/monet5/UDF/capi/Tests/capi01.stable.out
sql/backends/monet5/UDF/capi/Tests/capi02.sql
sql/backends/monet5/UDF/capi/Tests/capi02.stable.err
sql/backends/monet5/UDF/capi/Tests/capi02.stable.out
sql/backends/monet5/UDF/capi/Tests/capi03.sql
sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/Tests/capi04.stable.err
sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
sql/backends/monet5/UDF/capi/Tests/capi05.sql
sql/backends/monet5/UDF/capi/Tests/capi05.stable.err
sql/backends/monet5/UDF/capi/Tests/capi05.stable.out
sql/backends/monet5/UDF/capi/Tests/capi06.sql
sql/backends/monet5/UDF/capi/Tests/capi06.stable.err
sql/backends/monet5/UDF/capi/Tests/capi06.stable.out
sql/backends/monet5/UDF/capi/Tests/capi07.sql
sql/backends/monet5/UDF/capi/Tests/capi07.stable.err
sql/backends/monet5/UDF/capi/Tests/capi07.stable.out
sql/backends/monet5/UDF/capi/Tests/capi08.sql
sql/backends/monet5/UDF/capi/Tests/capi08.stable.err
sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
sql/backends/monet5/UDF/capi/Tests/capi09.sql
sql/backends/monet5/UDF/capi/Tests/capi09.stable.err
sql/backends/monet5/UDF/capi/Tests/capi09.stable.out
sql/backends/monet5/UDF/capi/Tests/capi10.sql
sql/backends/monet5/UDF/capi/Tests/capi10.stable.err
sql/backends/monet5/UDF/capi/Tests/capi10.stable.out
sql/backends/monet5/UDF/capi/Tests/capi11.sql
sql/backends/monet5/UDF/capi/Tests/capi11.stable.err
sql/backends/monet5/UDF/capi/Tests/capi11.stable.out
sql/backends/monet5/UDF/capi/Tests/capi12.sql
sql/backends/monet5/UDF/capi/Tests/capi12.stable.err
sql/backends/monet5/UDF/capi/Tests/capi12.stable.out
sql/backends/monet5/UDF/capi/Tests/capi13.sql
sql/backends/monet5/UDF/capi/Tests/capi13.stable.err
sql/backends/monet5/UDF/capi/Tests/capi13.stable.out
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/capi/capi.h
sql/backends/monet5/UDF/capi/capi.mal
sql/backends/monet5/UDF/capi/cheader.h
sql/backends/monet5/UDF/udf/80_udf.mal
sql/backends/monet5/UDF/udf/80_udf.sql
sql/backends/monet5/UDF/udf/80_udf_hge.mal
sql/backends/monet5/UDF/udf/80_udf_hge.sql
sql/backends/monet5/UDF/udf/Makefile.ag
sql/backends/monet5/UDF/udf/README
sql/backends/monet5/UDF/udf/Tests/All
sql/backends/monet5/UDF/udf/Tests/udf-fuse.sql
sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.err
sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.out
sql/backends/monet5/UDF/udf/Tests/udf-reverse.sql
sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.err
sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.out
sql/backends/monet5/UDF/udf/udf.c
sql/backends/monet5/UDF/udf/udf.h
sql/backends/monet5/UDF/udf/udf.mal
sql/backends/monet5/UDF/udf/udf_hge.mal
sql/backends/monet5/UDF/udf/udf_impl.h
Removed Files:
sql/backends/monet5/UDF/cudf/80_udf.mal
sql/backends/monet5/UDF/cudf/80_udf.sql
sql/backends/monet5/UDF/cudf/80_udf_hge.mal
sql/backends/monet5/UDF/cudf/80_udf_hge.sql
sql/backends/monet5/UDF/cudf/Makefile.ag
sql/backends/monet5/UDF/cudf/README
sql/backends/monet5/UDF/cudf/Tests/All
sql/backends/monet5/UDF/cudf/Tests/udf-fuse.sql
sql/backends/monet5/UDF/cudf/Tests/udf-fuse.stable.err
sql/backends/monet5/UDF/cudf/Tests/udf-fuse.stable.out
sql/backends/monet5/UDF/cudf/Tests/udf-reverse.sql
sql/backends/monet5/UDF/cudf/Tests/udf-reverse.stable.err
sql/backends/monet5/UDF/cudf/Tests/udf-reverse.stable.out
sql/backends/monet5/UDF/cudf/udf.c
sql/backends/monet5/UDF/cudf/udf.h
sql/backends/monet5/UDF/cudf/udf.mal
sql/backends/monet5/UDF/cudf/udf_hge.mal
sql/backends/monet5/UDF/cudf/udf_impl.h
Modified Files:
.hgignore
buildtools/autogen/autogen.py
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_bbp.c
gdk/gdk_utils.c
gdk/gdk_utils.h

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: 584074062c90 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=584074062c90
Added Files:
sql/backends/monet5/Tests/pyloader06.stable.out.32bit
sql/backends/monet5/Tests/pyloader07.stable.out.32bit
sql/test/BugTracker-2017/Tests/groupby_assertion.Bug-6338.stable.err
sql/test/BugTracker-2017/Tests/groupby_assertion.Bug-6338.stable.out
sql/test/BugTracker-2017/Tests/parsererror.Bug-6381.stable.err
sql/test/BugTracker-2017/Tests/parsererror.Bug-6381.stable.out
Modified Files:
clients/Tests/mclient-uri.SQL.sh
clients/mapiclient/Tests/mclient--help.stable.err
clients/mapiclient/mclient.1
clients/mapiclient/mclient.c
gdk/gdk_aggr.c
monetdb5/optimizer/Tests/manifold2.stable.out.single
sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
sql/benchmarks/tpch/LOCKED/Tests/12.stable.out
sql/benchmarks/tpch/LOCKED/Tests/16.stable.out
sql/benchmarks/tpch/LOCKED/Tests/18.stable.out
sql/benchmarks/tpch/LOCKED/Tests/19.stable.out
sql/benchmarks/tpch/LOCKED/Tests/22.stable.out
sql/benchmarks/tpch/Tests/01-22.stable.out
sql/benchmarks/tpch/Tests/12.stable.out
sql/benchmarks/tpch/Tests/16.stable.out
sql/benchmarks/tpch/Tests/18.stable.out
sql/benchmarks/tpch/Tests/19.stable.out
sql/benchmarks/tpch/Tests/22.stable.out
sql/server/rel_optimizer.c
sql/server/rel_select.c
sql/server/rel_updates.c
sql/test/BugTracker-2017/Tests/cleanup_statistics.Bug-6439.stable.out
sql/test/BugTracker-2017/Tests/lateral.Bug-6310.stable.err
sql/test/BugTracker-2017/Tests/lateral.Bug-6310.stable.out
sql/test/BugTracker-2017/Tests/parsererror.Bug-6381.sql
sql/test/BugTracker-2017/Tests/spurious_error.Bug-6344.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6477.stable.err
sql/test/emptydb/Tests/check.stable.out.32bit
testing/Mtest.py.in
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 1742 to 300 lines):

diff --git a/clients/Tests/mclient-uri.SQL.sh b/clients/Tests/mclient-uri.SQL.sh
--- a/clients/Tests/mclient-uri.SQL.sh
+++ b/clients/Tests/mclient-uri.SQL.sh
@@ -2,8 +2,8 @@
 
 # test the URI parsing capabilities of the MAPI library
 
-Mlog "mclient -d 
mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb -f test -E 
utf-8 -s select 1"
+Mlog "mclient -d 
mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb -f test -t 
none-E utf-8 -s select 1"
 mclient -d "mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql=monetdb" 
-f test -E utf-8 -s 'select 1'
 
-Mlog "mclient -d 
mapi:monetdb://$MAPIHOST/.s.monetdb.$MAPIPORT?database=$TSTDB=sql=monetdb
 -f test -E utf-8 -s select 1"
+Mlog "mclient -d 
mapi:monetdb://$MAPIHOST/.s.monetdb.$MAPIPORT?database=$TSTDB=sql=monetdb
 -f test -t none -E utf-8 -s select 1"
 mclient -d 
"mapi:monetdb://$MAPIHOST/.s.monetdb.$MAPIPORT?database=$TSTDB=sql=monetdb"
 -f test -E utf-8 -s 'select 1'
diff --git a/clients/mapiclient/Tests/mclient--help.stable.err 
b/clients/mapiclient/Tests/mclient--help.stable.err
--- a/clients/mapiclient/Tests/mclient--help.stable.err
+++ b/clients/mapiclient/Tests/mclient--help.stable.err
@@ -19,9 +19,10 @@ Options are:
  -d database | --database=database  database to connect to (may be URI)
  -e  | --echoecho the query
  -E charset  | --encoding=charset specify encoding (character set) of the 
terminal
- -f kind | --format=kind  specify output format {csv,tab,raw,sql,xml}
+ -f kind | --format=kind  specify output format 
{csv,tab,raw,sql,xml,trash}
  -H  | --history load/save cmdline history (default off)
- -i  | --interactive[=tm] interpret `\' commands on stdin, use time 
formatting {ms,s,m}
+ -i  | --interactive  interpret `\' commands on stdin
+ -t  | --timer=format use time formatting 
{clock,minutes,seconds,milliseconds,microseconds,none}
  -l language | --language=lang{sql,mal}
  -L logfile  | --log=logfile save client/server interaction
  -s stmt | --statement=stmt  run single statement
diff --git a/clients/mapiclient/mclient.1 b/clients/mapiclient/mclient.1
--- a/clients/mapiclient/mclient.1
+++ b/clients/mapiclient/mclient.1
@@ -148,22 +148,17 @@ these are supported.
 Specify the portnumber of the server (default:
 .BR 5 ).
 .TP
-\fB\-\-interactive\fP[\fB=\fP\fItimermode\fP] (\fB\-i\fP[\fItimermode\fP])
+\fB\-\-interactive\fP
 When reading from standard input, interpret lines starting with
 .B \e
 (backslash) specially.
 See the section BACKSLASH COMMANDS below.
-This is the default if standard input is a terminal.
-The optional \fItimermode\fP argument controls the
-format of the time reported for queries.
-Note that no space is allowed between
-.B \-i
-and
-.IR timermode .
-The default mode is
-\fBhuman\fP which adjusts the time precision 

MonetDB: jitudf - Compilation fix.

[Mark Raasveldt]

Changeset: aedc5eb276e8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=aedc5eb276e8
Modified Files:
sql/backends/monet5/UDF/capi/cheader.h
Branch: jitudf
Log Message:

Compilation fix.


diffs (37 lines):

diff --git a/sql/backends/monet5/UDF/capi/cheader.h 
b/sql/backends/monet5/UDF/capi/cheader.h
--- a/sql/backends/monet5/UDF/capi/cheader.h
+++ b/sql/backends/monet5/UDF/capi/cheader.h
@@ -38,25 +38,16 @@ typedef struct {
void *bat;  
   \
}
 
-typedef int8_t bit;
-typedef int8_t bte;
-typedef int16_t sht;
-typedef int64_t lng;
-typedef size_t oid;
-typedef float flt;
-typedef double dbl;
-typedef char *str;
-
-DEFAULT_STRUCT_DEFINITION(bit, bit);
-DEFAULT_STRUCT_DEFINITION(bte, bte);
-DEFAULT_STRUCT_DEFINITION(sht, sht);
+DEFAULT_STRUCT_DEFINITION(int8_t, bit);
+DEFAULT_STRUCT_DEFINITION(int8_t, bte);
+DEFAULT_STRUCT_DEFINITION(int16_t, sht);
 DEFAULT_STRUCT_DEFINITION(int, int);
-DEFAULT_STRUCT_DEFINITION(lng, lng);
-DEFAULT_STRUCT_DEFINITION(flt, flt);
-DEFAULT_STRUCT_DEFINITION(dbl, dbl);
-DEFAULT_STRUCT_DEFINITION(str, str);
+DEFAULT_STRUCT_DEFINITION(int64_t, lng);
+DEFAULT_STRUCT_DEFINITION(float, flt);
+DEFAULT_STRUCT_DEFINITION(double, dbl);
+DEFAULT_STRUCT_DEFINITION(char*, str);
 DEFAULT_STRUCT_DEFINITION(cudf_data_date, date);
 DEFAULT_STRUCT_DEFINITION(cudf_data_time, time);
 DEFAULT_STRUCT_DEFINITION(cudf_data_timestamp, timestamp);
 DEFAULT_STRUCT_DEFINITION(cudf_data_blob, blob);
-DEFAULT_STRUCT_DEFINITION(oid, oid);
+DEFAULT_STRUCT_DEFINITION(size_t, oid);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: f8b81dc77e96 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f8b81dc77e96
Added Files:
sql/test/BugTracker-2017/Tests/crash-in-topn.Bug-6478.sql
sql/test/BugTracker-2017/Tests/crash-in-topn.Bug-6478.stable.err
sql/test/BugTracker-2017/Tests/crash-in-topn.Bug-6478.stable.out
sql/test/BugTracker-2017/Tests/function-resolution.Bug-6436.sql
sql/test/BugTracker-2017/Tests/function-resolution.Bug-6436.stable.err
sql/test/BugTracker-2017/Tests/function-resolution.Bug-6436.stable.out
sql/test/BugTracker-2017/Tests/generate-resolution.Bug-6471.sql
sql/test/BugTracker-2017/Tests/generate-resolution.Bug-6471.stable.err
sql/test/BugTracker-2017/Tests/generate-resolution.Bug-6471.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6477.sql
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6477.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6477.stable.out
sql/test/BugTracker-2017/Tests/update_on_procedure.Bug-6479.sql
sql/test/BugTracker-2017/Tests/update_on_procedure.Bug-6479.stable.err
sql/test/BugTracker-2017/Tests/update_on_procedure.Bug-6479.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.32bit
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.32bit
sql/test/pg_regress/Tests/numeric.stable.out.FreeBSD.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.32bit
sql/test/testdb-upgrade/Tests/upgrade.stable.out.32bit
Removed Files:
sql/test/pg_regress/Tests/float8.stable.err.Windows
Modified Files:
.hgtags
MonetDB.spec
NT/monetdb_config.h.in
NT/rules.msc
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
configure.ag
debian/changelog
gdk/ChangeLog-Archive
gdk/ChangeLog.Jul2017
gdk/gdk_firstn.c
gdk/libbat.rc
libversions
monetdb5/mal/mal_linker.c
monetdb5/modules/kernel/aggr.c
monetdb5/optimizer/opt_mitosis.c
monetdb5/tools/libmonetdb5.rc
sql/backends/monet5/sql_result.c
sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out
sql/benchmarks/tpch/LOCKED/Tests/01-22.stable.out.int128
sql/benchmarks/tpch/LOCKED/Tests/22.stable.out.int128
sql/benchmarks/tpch/Tests/01-22.stable.out
sql/benchmarks/tpch/Tests/01-22.stable.out.int128
sql/benchmarks/tpch/Tests/22.stable.out.int128
sql/common/sql_types.c
sql/common/sql_types.h
sql/server/rel_optimizer.c
sql/server/rel_select.c
sql/server/rel_semantic.c

sql/test/BugDay_2005-10-06_2.9.3/Tests/variable_arg_substring.SF_942981.stable.out

sql/test/BugTracker-2010/Tests/substring_display_width.Bug-2683.stable.out
sql/test/BugTracker-2014/Tests/round-properties.Bug-3515.stable.out
sql/test/BugTracker-2014/Tests/round.Bug-3542.stable.out.int128
sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out
sql/test/BugTracker-2017/Tests/All
sql/test/BugTracker-2017/Tests/splitpart.Bug-6194.stable.out
sql/test/Tests/round.stable.out
sql/test/Tests/string.stable.out
sql/test/pg_regress/Tests/float8.stable.err
sql/test/pg_regress/Tests/float8.stable.err.int128
sql/test/pg_regress/Tests/numeric.stable.err
sql/test/pg_regress/Tests/numeric.stable.out.int128
sql/test/pg_regress/Tests/strings.stable.out
vertoo.data
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 23464 to 300 lines):

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -685,3 +685,5 @@ fce449d5af88de715fde36624815a56ef34c56bb
 3d26c58579a18157659b590899a8f73f45954ec6 Jul2017_9
 fce449d5af88de715fde36624815a56ef34c56bb Jul2017_SP2_release
 3d26c58579a18157659b590899a8f73f45954ec6 Jul2017_SP2_release
+b9276cb02ced7d98bcb27b5e6df600b2791f1fcd Jul2017_11
+b9276cb02ced7d98bcb27b5e6df600b2791f1fcd Jul2017_SP3_release
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -132,7 +132,7 @@ Vendor: MonetDB BV 
 Group: Applications/Databases
 License: MPLv2.0
 URL: https://www.monetdb.org/
-Source: 
https://www.monetdb.org/downloads/sources/Jul2017-SP2/%{name}-%{version}.tar.bz2
+Source: 
https://www.monetdb.org/downloads/sources/Jul2017-SP3/%{name}-%{version}.tar.bz2
 
 # we need systemd for the _unitdir macro to exist
 # we need checkpolicy and selinux-policy-devel for the SELinux policy
@@ -1046,6 +1046,48 @@ done
 %postun -p /sbin/ldconfig
 
 %changelog
+* Mon Dec 04 2017 Sjoerd Mullender  - 11.27.11-20171204
+- Rebuilt.
+- BZ#3898: Deadlock on insertion
+- BZ#6429: ROUND produces wrong data type
+- BZ#6436: Query sequence with 2x ifthenelse() and next 

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: 805f78a823d8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=805f78a823d8
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk_batop.c
gdk/gdk_orderidx.c
monetdb5/modules/mal/orderidx.c
sql/common/sql_mem.c
sql/include/sql_mem.h
sql/test/BugTracker-2012/Tests/currenttime.Bug-2781.SQL.py
sql/test/BugTracker-2017/Tests/oidx-on-strings.Bug-6202.stable.err
sql/test/orderidx/Tests/oidx_all_types.stable.err
sql/test/orderidx/Tests/oidx_all_types.stable.out
testing/Mtest.py.in
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 481 to 300 lines):

diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -1353,6 +1353,11 @@ BATsort(BAT **sorted, BAT **order, BAT *
GDKerror("BATsort: b must exist\n");
return GDK_FAIL;
}
+   if (!ATOMlinear(b->ttype)) {
+   GDKerror("BATsort: type %s cannot be sorted\n",
+ATOMname(b->ttype));
+   return GDK_FAIL;
+   }
if (o != NULL &&
(ATOMtype(o->ttype) != TYPE_oid || /* oid tail */
 BATcount(o) != BATcount(b) || /* same size as b */
diff --git a/gdk/gdk_orderidx.c b/gdk/gdk_orderidx.c
--- a/gdk/gdk_orderidx.c
+++ b/gdk/gdk_orderidx.c
@@ -155,72 +155,41 @@ persistOIDX(BAT *b)
 gdk_return
 BATorderidx(BAT *b, int stable)
 {
-   Heap *m;
-   oid *restrict mv;
-   oid seq;
-   BUN p, q;
-   BAT *bn = NULL;
-
if (BATcheckorderidx(b))
return GDK_SUCCEED;
-   MT_lock_set((b->batCacheid));
-   if (b->torderidx) {
-   MT_lock_unset((b->batCacheid));
-   return GDK_SUCCEED;
-   }
-   if ((m = createOIDXheap(b, stable)) == NULL) {
-   MT_lock_unset((b->batCacheid));
-   return GDK_FAIL;
-   }
-
-   mv = (oid *) m->base + ORDERIDXOFF;
-
-   seq = b->hseqbase;
-   for (p = 0, q = BATcount(b); p < q; p++)
-   mv[p] = seq + p;
-
if (!BATtdense(b)) {
-   /* we need to sort a copy of the column so as not to
-* change the original */
-   bn = COLcopy(b, b->ttype, TRUE, TRANSIENT);
-   if (bn == NULL) {
-   HEAPfree(m, 1);
-   GDKfree(m);
-   MT_lock_unset((b->batCacheid));
+   BAT *on;
+   if (BATsort(NULL, , NULL, b, NULL, NULL, 0, stable) != 
GDK_SUCCEED)
return GDK_FAIL;
-   }
-   if (stable) {
-   if (GDKssort(Tloc(bn, 0), mv,
-bn->tvheap ? bn->tvheap->base : NULL,
-BATcount(bn), Tsize(bn), SIZEOF_OID,
-bn->ttype) != GDK_SUCCEED) {
-   HEAPfree(m, 1);
-   GDKfree(m);
-   MT_lock_unset((b->batCacheid));
-   BBPunfix(bn->batCacheid);
-   return GDK_FAIL;
+   assert(BATcount(b) == BATcount(on));
+   if (on->tdense) {
+   /* if the order bat is dense, the input was
+* sorted and we don't need an order index */
+   assert(b->tnosorted == 0);
+   if (!b->tsorted) {
+   b->tsorted = 1;
+   b->tnosorted = 0;
+   b->batDirtydesc = 1;
}
} else {
-   GDKqsort(Tloc(bn, 0), mv,
-bn->tvheap ? bn->tvheap->base : NULL,
-BATcount(bn), Tsize(bn), SIZEOF_OID,
-bn->ttype);
+   /* BATsort quite possibly already created the
+* order index, but just to be sure... */
+   MT_lock_set((b->batCacheid));
+   if (b->torderidx == NULL) {
+   Heap *m;
+   if ((m = createOIDXheap(b, stable)) == NULL) {
+   
MT_lock_unset((b->batCacheid));
+   return GDK_FAIL;
+   }
+   memcpy((oid *) m->base + ORDERIDXOFF, Tloc(on, 
0), BATcount(on) * sizeof(oid));
+   b->torderidx = m;
+   b->batDirtydesc = 1;
+   persistOIDX(b);
+   }
+   MT_lock_unset((b->batCacheid));
}
-   /* we must unfix after releasing the lock since we
-* might get deadlock otherwise (we're holding a 

MonetDB: jitudf - Use int8_t for bit/bte and approve exports.

[Mark Raasveldt]

Changeset: 9534324d7e0a for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9534324d7e0a
Modified Files:
clients/Tests/exports.stable.out
sql/backends/monet5/UDF/capi/cheader.h
Branch: jitudf
Log Message:

Use int8_t for bit/bte and approve exports.


diffs (48 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -527,6 +527,7 @@ char *mapi_get_host(Mapi mid);
 char *mapi_get_lang(Mapi mid);
 int64_t mapi_get_last_id(MapiHdl hdl);
 int mapi_get_len(MapiHdl hdl, int fnr);
+int64_t mapi_get_maloptimizertime(MapiHdl hdl);
 char *mapi_get_mapi_version(Mapi mid);
 char *mapi_get_monet_version(Mapi mid);
 char *mapi_get_motd(Mapi mid);
diff --git a/sql/backends/monet5/UDF/capi/cheader.h 
b/sql/backends/monet5/UDF/capi/cheader.h
--- a/sql/backends/monet5/UDF/capi/cheader.h
+++ b/sql/backends/monet5/UDF/capi/cheader.h
@@ -38,16 +38,25 @@ typedef struct {
void *bat;  
   \
}
 
-DEFAULT_STRUCT_DEFINITION(signed char, bit);
-DEFAULT_STRUCT_DEFINITION(signed char, bte);
-DEFAULT_STRUCT_DEFINITION(short, sht);
+typedef int8_t bit;
+typedef int8_t bte;
+typedef int16_t sht;
+typedef int64_t lng;
+typedef size_t oid;
+typedef float flt;
+typedef double dbl;
+typedef char *str;
+
+DEFAULT_STRUCT_DEFINITION(bit, bit);
+DEFAULT_STRUCT_DEFINITION(bte, bte);
+DEFAULT_STRUCT_DEFINITION(sht, sht);
 DEFAULT_STRUCT_DEFINITION(int, int);
-DEFAULT_STRUCT_DEFINITION(int64_t, lng);
-DEFAULT_STRUCT_DEFINITION(float, flt);
-DEFAULT_STRUCT_DEFINITION(double, dbl);
-DEFAULT_STRUCT_DEFINITION(char *, str);
+DEFAULT_STRUCT_DEFINITION(lng, lng);
+DEFAULT_STRUCT_DEFINITION(flt, flt);
+DEFAULT_STRUCT_DEFINITION(dbl, dbl);
+DEFAULT_STRUCT_DEFINITION(str, str);
 DEFAULT_STRUCT_DEFINITION(cudf_data_date, date);
 DEFAULT_STRUCT_DEFINITION(cudf_data_time, time);
 DEFAULT_STRUCT_DEFINITION(cudf_data_timestamp, timestamp);
 DEFAULT_STRUCT_DEFINITION(cudf_data_blob, blob);
-DEFAULT_STRUCT_DEFINITION(size_t, oid);
+DEFAULT_STRUCT_DEFINITION(oid, oid);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: d0c8200da692 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d0c8200da692
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
clients/mapilib/mapi.h
gdk/gdk.h
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_group.c
gdk/gdk_hash.c
gdk/gdk_hash.h
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_private.h
gdk/gdk_unique.c
monetdb5/mal/mal_resource.h
monetdb5/modules/kernel/status.c
monetdb5/optimizer/opt_support.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_result.h
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 1311 to 300 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -1301,11 +1301,11 @@ RAWrenderer(MapiHdl hdl)
 }
 
 static void
-TIMERrenderer(MapiHdl hdl, int64_t querytime)
+TIMERrenderer(MapiHdl hdl, int64_t querytime, int64_t maloptimizertime)
 {
SQLqueryEcho(hdl);
mapi_next_result(hdl);
-   printf("%" PRId64 " %s\n", querytime, timerHuman());
+   printf("%s sql:0 opt:%" PRId64 " run:%" PRId64 "\n", timerHuman(),  
maloptimizertime, querytime);
 }
 
 
@@ -1827,6 +1827,7 @@ format_result(Mapi mid, MapiHdl hdl, cha
int64_t aff, lid;
char *reply;
int64_t querytime;
+   int64_t maloptimizer;
 #ifdef HAVE_POPEN
stream *saveFD;
 
@@ -1860,12 +1861,13 @@ format_result(Mapi mid, MapiHdl hdl, cha
continue;
case Q_UPDATE:
querytime = mapi_get_querytime(hdl);
+   maloptimizer = mapi_get_maloptimizertime(hdl);
SQLqueryEcho(hdl);
if (formatter == RAWformatter ||
formatter == TESTformatter)
mnstr_printf(toConsole, "[ %" PRId64 "\t]\n", 
mapi_rows_affected(hdl));
else if (formatter == TIMERformatter)
-   printf("%" PRId64 " %s\n", querytime, 
timerHuman());
+   TIMERrenderer(hdl, querytime, maloptimizer);
else {
aff = mapi_rows_affected(hdl);
lid = mapi_get_last_id(hdl);
@@ -1887,6 +1889,7 @@ format_result(Mapi mid, MapiHdl hdl, cha
continue;
case Q_SCHEMA:
querytime = mapi_get_querytime(hdl);
+   maloptimizer = mapi_get_maloptimizertime(hdl);
SQLqueryEcho(hdl);
if (formatter == TABLEformatter) {
mnstr_printf(toConsole, "operation successful");
@@ -1895,7 +1898,7 @@ format_result(Mapi mid, MapiHdl hdl, cha
 timerHuman());
mnstr_printf(toConsole, "\n");
} else if (formatter == TIMERformatter)
-   printf("%" PRId64 " %s\n", querytime, 
timerHuman());
+   TIMERrenderer(hdl, querytime, maloptimizer);
continue;
case Q_TRANS:
SQLqueryEcho(hdl);
@@ -1914,6 +1917,7 @@ format_result(Mapi mid, MapiHdl hdl, cha
break;
case Q_TABLE:
querytime = mapi_get_querytime(hdl);
+   maloptimizer = mapi_get_maloptimizertime(hdl);
break;
default:
if (formatter == TABLEformatter && specials != 
DEBUGmodifier) {
@@ -1974,7 +1978,7 @@ format_result(Mapi mid, MapiHdl hdl, cha
}
break;
case TIMERformatter:
-   TIMERrenderer(hdl, querytime);
+   TIMERrenderer(hdl, querytime, maloptimizer);
break;
case SAMformatter:
SAMrenderer(hdl);
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -885,6 +885,7 @@ struct MapiResultSet {
int64_t row_count;
int64_t last_id;
int64_t querytime;
+   int64_t maloptimizertime;
int fieldcnt;
int maxfields;
char *errorstr; /* error from server */
@@ -1450,6 +1451,7 @@ new_result(MapiHdl hdl)
result->querytype = -1;
result->errorstr = NULL;
result->querytime = 0;
+   result->maloptimizertime = 0;
memset(result->sqlstate, 0, sizeof(result->sqlstate));
 
result->tuple_count = 0;
@@ -3759,6 +3761,7 @@ 

MonetDB: jitudf - Use int64_t for lng instead of long long.

[Mark Raasveldt]

Changeset: 4aa28c764de0 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4aa28c764de0
Modified Files:
sql/backends/monet5/UDF/capi/cheader.h
Branch: jitudf
Log Message:

Use int64_t for lng instead of long long.


diffs (18 lines):

diff --git a/sql/backends/monet5/UDF/capi/cheader.h 
b/sql/backends/monet5/UDF/capi/cheader.h
--- a/sql/backends/monet5/UDF/capi/cheader.h
+++ b/sql/backends/monet5/UDF/capi/cheader.h
@@ -1,3 +1,5 @@
+
+#include 
 
 typedef void *(*malloc_function_ptr)(size_t);
 
@@ -40,7 +42,7 @@ DEFAULT_STRUCT_DEFINITION(signed char, b
 DEFAULT_STRUCT_DEFINITION(signed char, bte);
 DEFAULT_STRUCT_DEFINITION(short, sht);
 DEFAULT_STRUCT_DEFINITION(int, int);
-DEFAULT_STRUCT_DEFINITION(long long, lng);
+DEFAULT_STRUCT_DEFINITION(int64_t, lng);
 DEFAULT_STRUCT_DEFINITION(float, flt);
 DEFAULT_STRUCT_DEFINITION(double, dbl);
 DEFAULT_STRUCT_DEFINITION(char *, str);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: 07a50617009f for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=07a50617009f
Added Files:
sql/test/BugTracker-2017/Tests/cleanup_statistics.Bug-6439.sql
sql/test/BugTracker-2017/Tests/cleanup_statistics.Bug-6439.stable.err
sql/test/BugTracker-2017/Tests/cleanup_statistics.Bug-6439.stable.out
sql/test/mergetables/Tests/sqlsmith.Bug-6472.sql
sql/test/mergetables/Tests/sqlsmith.Bug-6472.stable.err
sql/test/mergetables/Tests/sqlsmith.Bug-6472.stable.out
Modified Files:
.hgignore
NT/monetdb_config.h.in
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
clients/mapiclient/dotmonetdb.c
clients/mapiclient/dump.c
clients/mapiclient/mclient.c
clients/mapiclient/mhelp.c
clients/mapiclient/mhelp.h
clients/mapiclient/msqldump.h
clients/mapilib/mapi.c
clients/mapilib/mapi.h
clients/odbc/driver/ODBCConvert.c
clients/odbc/driver/SQLBrowseConnect.c
clients/odbc/driver/SQLConnect.c
clients/odbc/driver/SQLFetch.c
clients/odbc/setup/drvcfg.c
clients/odbc/winsetup/install.c
common/stream/ChangeLog
common/stream/stream.c
common/stream/stream.h
common/utils/mcrypt.c
common/utils/msabaoth.c
common/utils/mutils.c
configure.ag
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_atoms.c
gdk/gdk_atoms.h
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_group.c
gdk/gdk_hash.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_logger.c
gdk/gdk_orderidx.c
gdk/gdk_posix.h
gdk/gdk_project.c
gdk/gdk_storage.c
gdk/gdk_unique.c
gdk/gdk_utils.c
monetdb5/mal/mal_atom.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_readline.c
monetdb5/mal/mal_session.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/url.c
monetdb5/modules/kernel/alarm.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/mal/bbp.c
monetdb5/modules/mal/calc.c
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/inspect.c
monetdb5/modules/mal/language.c
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/mdb.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/profiler.mal
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/remote.h
monetdb5/modules/mal/sabaoth.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/txtsim.h
monetdb5/modules/mal/wlc.c
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_01.stable.out
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_result.h
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_upgrades.c
sql/backends/monet5/sql_user.c
sql/backends/monet5/vaults/bam/Tests/query2.1.stable.out
sql/backends/monet5/vaults/bam/Tests/query2.10.stable.out
sql/backends/monet5/vaults/bam/Tests/query2.2.stable.out
sql/backends/monet5/vaults/bam/Tests/query2.2.stable.out.int128
sql/backends/monet5/vaults/bam/bam_export.c
sql/backends/monet5/vaults/bam/bam_wrapper.c
sql/backends/monet5/vaults/lidar/lidar.c
sql/backends/monet5/vaults/vault.c
sql/backends/monet5/wlr.c
sql/common/sql_types.h
sql/server/rel_exp.c
sql/server/rel_exp.h
sql/server/rel_optimizer.c
sql/server/rel_rel.c
sql/server/rel_select.c
sql/server/sql_atom.c
sql/server/sql_atom.h
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/storage/store.c

sql/test/BugTracker-2012/Tests/scalar_subquery_with_alias.Bug-3093.stable.out

sql/test/BugTracker-2016/Tests/nested-subquery-in-select.Bug-6125.stable.out
sql/test/BugTracker-2017/Tests/All

sql/test/BugTracker-2017/Tests/crash_on_count_div_count.Bug-6201.stable.out

sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.err

sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.out
sql/test/Tests/marcin1.stable.out
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out

MonetDB: jitudf - Merge with default

[Mark Raasveldt]

Changeset: c6dada6109ea for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c6dada6109ea
Added Files:
sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.sql
sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.err
sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.out
sql/test/BugTracker-2017/Tests/limit.Bug-6322.sql
sql/test/BugTracker-2017/Tests/limit.Bug-6322.stable.err
sql/test/BugTracker-2017/Tests/limit.Bug-6322.stable.out

sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.sql

sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.err

sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.out
sql/test/BugTracker-2017/Tests/skip_problem_best_effort.Bug-6442.sql

sql/test/BugTracker-2017/Tests/skip_problem_best_effort.Bug-6442.stable.err

sql/test/BugTracker-2017/Tests/skip_problem_best_effort.Bug-6442.stable.out
sql/test/mergetables/Tests/mergedb.Bug-6820.reqtests
sql/test/mergetables/Tests/mergedb_drop.reqtests
sql/test/mergetables/Tests/qlsmith-exist-lateral.reqtests
sql/test/mergetables/Tests/sqlsmith-apply-outer-join-or.sql
sql/test/mergetables/Tests/sqlsmith-apply-outer-join-or.stable.err
sql/test/mergetables/Tests/sqlsmith-apply-outer-join-or.stable.out
sql/test/mergetables/Tests/sqlsmith-exist-lateral.reqtests
sql/test/mergetables/Tests/sqlsmith-exist-lateral.sql
sql/test/mergetables/Tests/sqlsmith-exist-lateral.stable.err
sql/test/mergetables/Tests/sqlsmith-exist-lateral.stable.out
sql/test/mergetables/Tests/sqlsmith.Bug-6426.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6451.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6453.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6455.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6459.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6459.stable.out.int128
Removed Files:
sql/benchmarks/tpch/fileleak/Tests/All
sql/benchmarks/tpch/fileleak/Tests/check0.reqtests
sql/benchmarks/tpch/fileleak/Tests/check0.sql
sql/benchmarks/tpch/fileleak/Tests/check0.stable.err
sql/benchmarks/tpch/fileleak/Tests/check0.stable.out
sql/benchmarks/tpch/fileleak/Tests/check1.reqtests
sql/benchmarks/tpch/fileleak/Tests/check1.sql
sql/benchmarks/tpch/fileleak/Tests/check1.stable.err
sql/benchmarks/tpch/fileleak/Tests/check1.stable.out
sql/benchmarks/tpch/fileleak/Tests/create.sql.src
sql/benchmarks/tpch/fileleak/Tests/create.stable.err
sql/benchmarks/tpch/fileleak/Tests/create.stable.out
sql/benchmarks/tpch/fileleak/Tests/delete_all.SQL.py
sql/benchmarks/tpch/fileleak/Tests/delete_all.stable.err
sql/benchmarks/tpch/fileleak/Tests/delete_all.stable.out
sql/benchmarks/tpch/fileleak/Tests/leaks.SQL.py
sql/benchmarks/tpch/fileleak/Tests/leaks.stable.err
sql/benchmarks/tpch/fileleak/Tests/leaks.stable.out
sql/benchmarks/tpch/fileleak/Tests/leaks.stable.out.Windows
sql/benchmarks/tpch/fileleak/Tests/load.SQL.py
sql/benchmarks/tpch/fileleak/Tests/load.reqtests
sql/benchmarks/tpch/fileleak/Tests/load.stable.err
sql/benchmarks/tpch/fileleak/Tests/load.stable.out
sql/benchmarks/tpch/fileleak/Tests/test.py
sql/benchmarks/tpch/fileleak/Tests/test.sql
sql/test/BugTracker-2017/Tests/limit.bug-6322.sql
sql/test/leaks/Tests/All
sql/test/leaks/Tests/check1.reqtests
sql/test/leaks/Tests/check1.sql
sql/test/leaks/Tests/check1.stable.err
sql/test/leaks/Tests/check1.stable.out
sql/test/leaks/Tests/check1.stable.out.int128
sql/test/leaks/Tests/check2.reqtests
sql/test/leaks/Tests/check2.sql.src
sql/test/leaks/Tests/check2.stable.err
sql/test/leaks/Tests/check2.stable.out
sql/test/leaks/Tests/check2.stable.out.int128
sql/test/leaks/Tests/check3.reqtests
sql/test/leaks/Tests/check3.sql.src
sql/test/leaks/Tests/check3.stable.err
sql/test/leaks/Tests/check3.stable.out
sql/test/leaks/Tests/check3.stable.out.int128
sql/test/leaks/Tests/check4.reqtests
sql/test/leaks/Tests/check4.sql.src
sql/test/leaks/Tests/check4.stable.err
sql/test/leaks/Tests/check4.stable.out
sql/test/leaks/Tests/check4.stable.out.int128
sql/test/leaks/Tests/check5.reqtests
sql/test/leaks/Tests/check5.sql.src
sql/test/leaks/Tests/check5.stable.err
sql/test/leaks/Tests/check5.stable.out
sql/test/leaks/Tests/check5.stable.out.int128
sql/test/leaks/Tests/drop3.reqtests
sql/test/leaks/Tests/drop3.sql
sql/test/leaks/Tests/drop3.stable.err

MonetDB: Jul2017 - Remove support for default values in the Pyth...

[Mark Raasveldt]

Changeset: b3bc5cb7c32e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b3bc5cb7c32e
Modified Files:
sql/backends/monet5/Tests/pyloader01.sql
sql/backends/monet5/Tests/pyloader01.stable.out
sql/backends/monet5/UDF/pyapi/emit.c

sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.err

sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.out
Branch: Jul2017
Log Message:

Remove support for default values in the Python loader and throw an error 
message if they are encountered instead of adding incorrect values to the table.


diffs (242 lines):

diff --git a/sql/backends/monet5/Tests/pyloader01.sql 
b/sql/backends/monet5/Tests/pyloader01.sql
--- a/sql/backends/monet5/Tests/pyloader01.sql
+++ b/sql/backends/monet5/Tests/pyloader01.sql
@@ -1,7 +1,7 @@
 
 START TRANSACTION;
 
-CREATE TABLE mytable(a DOUBLE, d int, s STRING DEFAULT 'hello');
+CREATE TABLE mytable(a DOUBLE, d int, s STRING);
 
 CREATE LOADER myfunc() LANGUAGE PYTHON {
_emit.emit({'a':42,'d':1})
diff --git a/sql/backends/monet5/Tests/pyloader01.stable.out 
b/sql/backends/monet5/Tests/pyloader01.stable.out
--- a/sql/backends/monet5/Tests/pyloader01.stable.out
+++ b/sql/backends/monet5/Tests/pyloader01.stable.out
@@ -56,11 +56,11 @@ Ready.
 % sys.mytable, sys.mytable,sys.mytable # table_name
 % a,   d,  s # name
 % double,  int,clob # type
-% 24,  1,  7 # length
-[ 42,  1,  "'hello'"   ]
+% 24,  1,  5 # length
+[ 42,  1,  NULL]
 [ 46,  4,  "hello" ]
-[ 45,  3,  "'hello'"   ]
-[ 44,  2,  "'hello'"   ]
+[ 45,  3,  NULL]
+[ 44,  2,  NULL]
 #DROP TABLE mytable;
 #DROP ALL LOADER myfunc;
 #CREATE LOADER myfunc() LANGUAGE PYTHON {
diff --git a/sql/backends/monet5/UDF/pyapi/emit.c 
b/sql/backends/monet5/UDF/pyapi/emit.c
--- a/sql/backends/monet5/UDF/pyapi/emit.c
+++ b/sql/backends/monet5/UDF/pyapi/emit.c
@@ -338,21 +338,19 @@ PyObject *PyEmit_Emit(PyEmitObject *self
self->cols[i].b->tnonil = 1 - 
self->cols[i].b->tnil;
}
} else {
-   void* nill_value = ATOMnil(self->cols[i].b->ttype);
-   void* default_value = self->cols[i].def ?
-   
self->cols[i].def :
-   nill_value;
+   if (self->cols[i].def != NULL) {
+   msg = GDKstrdup("Inserting into columns with 
default values is not supported currently.");
+   goto wrapup;
+   }
for (ai = 0; ai < (size_t)el_count; ai++) {
if (BUNappend(self->cols[i].b,
- default_value,
+ 
ATOMnil(self->cols[i].b->ttype),
  0) != GDK_SUCCEED) {
goto wrapup;
}
}
-   if 
(BATatoms[self->cols[i].b->ttype].atomCmp(default_value, nill_value) == 0) {
-   self->cols[i].b->tnil = 1;
-   self->cols[i].b->tnonil = 0;
-   }
+   self->cols[i].b->tnil = 1;
+   self->cols[i].b->tnonil = 0;
}
BATsetcount(self->cols[i].b, self->nvals + el_count);
}
diff --git 
a/sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.err
 
b/sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.err
--- 
a/sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.err
+++ 
b/sql/test/BugTracker-2017/Tests/python_loader_clobbers_default_with_null.Bug-6464.stable.err
@@ -56,6 +56,102 @@ ERROR = !INSERT INTO: NOT NULL constrain
 MAPI  = (monetdb) /var/tmp/mtest-17575/.s.monetdb.32655
 QUERY = insert into t (a) values (-1);
 ERROR = !INSERT INTO: PRIMARY KEY constraint 't.t_a_pkey' violated
+MAPI  = (monetdb) /var/tmp/mtest-80053/.s.monetdb.38101
+QUERY = COPY LOADER INTO t FROM myloader(-30,'abcdeg');
+ERROR = !Python exception
+!  6. if j in y:
+!  7.   z[j] = x - i
+!> 8.   _emit.emit(z)
+!  9. 
+!Failed conversion: Inserting into columns with default values is not 
supported currently.
+MAPI  = (monetdb) /var/tmp/mtest-80053/.s.monetdb.38101
+QUERY = COPY LOADER INTO t FROM myloader(-40,'abcdfg');
+ERROR = !Python exception
+!  6. if j in y:
+!  7.   z[j] = x - i
+!> 8.   _emit.emit(z)
+!  9. 
+!Failed conversion: Inserting into columns with default 

MonetDB: default - Approve tests.

[Mark Raasveldt]

Changeset: fd8ababbe0b4 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fd8ababbe0b4
Modified Files:
sql/backends/monet5/Tests/pyloader07.stable.out
sql/backends/monet5/Tests/pyloader07.stable.out.Windows
sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_01.stable.out
sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_02.stable.out
sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_05.stable.out
Branch: default
Log Message:

Approve tests.


diffs (89 lines):

diff --git a/sql/backends/monet5/Tests/pyloader07.stable.out 
b/sql/backends/monet5/Tests/pyloader07.stable.out
--- a/sql/backends/monet5/Tests/pyloader07.stable.out
+++ b/sql/backends/monet5/Tests/pyloader07.stable.out
@@ -82,7 +82,6 @@ Ready.
 #_emit.emit({'d': ['2014-05-20'], 's': ['00:02:30'], 't': ['2014-05-20 
00:02:30']});
 #};
 #COPY LOADER INTO tstamp FROM pyloader07();
-[ 2]
 #SELECT * FROM tstamp;
 % sys.tstamp,  sys.tstamp, sys.tstamp # table_name
 % d,   s,  t # name
diff --git a/sql/backends/monet5/Tests/pyloader07.stable.out.Windows 
b/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
--- a/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
+++ b/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
@@ -82,7 +82,6 @@ Ready.
 #_emit.emit({'d': ['2014-05-20'], 's': ['00:02:30'], 't': ['2014-05-20 
00:02:30']});
 #};
 #COPY LOADER INTO tstamp FROM pyloader07();
-[ 2]
 #SELECT * FROM tstamp;
 % sys.tstamp,  sys.tstamp, sys.tstamp # table_name
 % d,   s,  t # name
diff --git a/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_01.stable.out 
b/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_01.stable.out
--- a/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_01.stable.out
+++ b/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_01.stable.out
@@ -44,23 +44,19 @@ Ready.
 % varchar, varchar,varchar,int,int,boolean,
boolean,boolean # type
 % 6,   32, 5,  1,  1,  5,  5,  5 # length
 [ "myfunc","{\n\t_emit.emit({'a':42,'d':1})\n};",  "pyapi",6,  
7,  true,   true,   false   ]
-#COPY INTO mytable FROM LOADER myfunc3(46, 'asdf', 3.2);
-[ 1]
-#COPY INTO mytable FROM LOADER myfunc2(45, 'asdf');
-[ 1]
-#COPY INTO mytable FROM LOADER myfunc1(44);
-[ 1]
-#COPY INTO mytable FROM LOADER myfunc();
-[ 1]
+#COPY LOADER INTO mytable FROM myfunc();
+#COPY LOADER INTO mytable FROM myfunc3(46, 'asdf', 3.2);
+#COPY LOADER INTO mytable FROM myfunc2(45, 'asdf');
+#COPY LOADER INTO mytable FROM myfunc1(44);
 #SELECT * FROM mytable;
 % sys.mytable, sys.mytable,sys.mytable # table_name
 % a,   d,  s # name
 % double,  int,clob # type
-% 24,  1,  5 # length
+% 24,  1,  7 # length
+[ 42,  1,  "'hello'"   ]
 [ 46,  4,  "hello" ]
-[ 45,  3,  NULL]
-[ 44,  2,  NULL]
-[ 42,  1,  NULL]
+[ 45,  3,  "'hello'"   ]
+[ 44,  2,  "'hello'"   ]
 #DROP TABLE mytable;
 #DROP ALL LOADER myfunc;
 #CREATE LOADER myfunc() LANGUAGE PYTHON {
diff --git a/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_02.stable.out 
b/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_02.stable.out
--- a/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_02.stable.out
+++ b/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_02.stable.out
@@ -35,12 +35,9 @@ Ready.
 #  res['a'+str(j+1)] = (i+1)*(j+1)
 #  _emit.emit(res)
 #};
-#COPY INTO mytable3 FROM LOADER myfunc(10, 3);
-[ 10   ]
-#COPY INTO mytable4 FROM LOADER myfunc(10, 3);
-[ 10   ]
-#COPY INTO mytable2 FROM LOADER myfunc(20, 2);
-[ 20   ]
+#COPY LOADER INTO mytable3 FROM myfunc(10, 3);
+#COPY LOADER INTO mytable4 FROM myfunc(10, 3);
+#COPY LOADER INTO mytable2 FROM myfunc(20, 2);
 #SELECT * FROM mytable4;
 % sys.mytable4,sys.mytable4,   sys.mytable4,   sys.mytable4 # 
table_name
 % a1,  a2, a3, a4 # name
diff --git a/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_05.stable.out 
b/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_05.stable.out
--- a/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_05.stable.out
+++ b/sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_05.stable.out
@@ -73,8 +73,7 @@ Ready.
 #_emit.emit({'s': numpy.array([u'\u00D6', 'hello', 33])});
 #_emit.emit({'s': numpy.arange(3).astype(numpy.float32)});
 #_emit.emit({'s': numpy.arange(3).astype(numpy.float64)});
-#COPY INTO pyloader05table FROM LOADER pyloader05();
-[ 35   ]
+#COPY LOADER INTO pyloader05table FROM pyloader05();
 #SELECT * FROM pyloader05table;
 % sys.pyloader05table # table_name
 % s # name
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Get correct number of columns.

[Mark Raasveldt]

Changeset: b5a46a7a7ac9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b5a46a7a7ac9
Modified Files:
sql/backends/monet5/UDF/pyapi/pyloader.c
Branch: default
Log Message:

Get correct number of columns.


diffs (21 lines):

diff --git a/sql/backends/monet5/UDF/pyapi/pyloader.c 
b/sql/backends/monet5/UDF/pyapi/pyloader.c
--- a/sql/backends/monet5/UDF/pyapi/pyloader.c
+++ b/sql/backends/monet5/UDF/pyapi/pyloader.c
@@ -162,7 +162,7 @@ PYFUNCNAME(PyAPIevalLoader)(Client cntxt
if (sqlmorefun->colnames) {
n = sqlmorefun->colnames->h;
n2 = sqlmorefun->coltypes->h;
-   ncols = pyapi_list_length(sqlmorefun->colnames);
+   ncols = pyapi_list_length(sqlmorefun->coltypes);
if (ncols == 0) {
msg = createException(MAL, "pyapi.eval_loader",
  "No columns 
supplied.");
@@ -174,7 +174,7 @@ PYFUNCNAME(PyAPIevalLoader)(Client cntxt
  
SQLSTATE(HY001) MAL_MALLOC_FAIL "column list");
goto wrapup;
}
-   assert(pyapi_list_length(sqlmorefun->colnames) == 
pyapi_list_length(sqlmorefun->coltypes));
+   assert(pyapi_list_length(sqlmorefun->colnames) == 
pyapi_list_length(sqlmorefun->coltypes) * 2);
i = 0;
while (n) {
sql_subtype* tpe = (sql_subtype*) n2->data;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jul2017 - Properly respect DEFAULT values in the Python...

[Mark Raasveldt]

Changeset: 4303f7489624 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4303f7489624
Modified Files:
sql/backends/monet5/Tests/pyloader01.sql
sql/backends/monet5/Tests/pyloader01.stable.out
sql/backends/monet5/UDF/pyapi/emit.c
sql/backends/monet5/UDF/pyapi/pyloader.c
sql/include/sql_catalog.h
sql/server/rel_updates.c
Branch: Jul2017
Log Message:

Properly respect DEFAULT values in the Python loader.


diffs (145 lines):

diff --git a/sql/backends/monet5/Tests/pyloader01.sql 
b/sql/backends/monet5/Tests/pyloader01.sql
--- a/sql/backends/monet5/Tests/pyloader01.sql
+++ b/sql/backends/monet5/Tests/pyloader01.sql
@@ -1,7 +1,7 @@
 
 START TRANSACTION;
 
-CREATE TABLE mytable(a DOUBLE, d int, s STRING);
+CREATE TABLE mytable(a DOUBLE, d int, s STRING DEFAULT 'hello');
 
 CREATE LOADER myfunc() LANGUAGE PYTHON {
_emit.emit({'a':42,'d':1})
@@ -23,10 +23,10 @@ SELECT name,func,mod,language,type,side_
 
 
 -- there is a reason for this, functions with 0, 1, 2 and 3+ arguments are 
handled differently.
+COPY LOADER INTO mytable FROM myfunc();
 COPY LOADER INTO mytable FROM myfunc3(46, 'asdf', 3.2);
 COPY LOADER INTO mytable FROM myfunc2(45, 'asdf');
 COPY LOADER INTO mytable FROM myfunc1(44);
-COPY LOADER INTO mytable FROM myfunc();
 
 SELECT * FROM mytable;
 
diff --git a/sql/backends/monet5/Tests/pyloader01.stable.out 
b/sql/backends/monet5/Tests/pyloader01.stable.out
--- a/sql/backends/monet5/Tests/pyloader01.stable.out
+++ b/sql/backends/monet5/Tests/pyloader01.stable.out
@@ -56,11 +56,11 @@ Ready.
 % sys.mytable, sys.mytable,sys.mytable # table_name
 % a,   d,  s # name
 % double,  int,clob # type
-% 24,  1,  5 # length
+% 24,  1,  7 # length
+[ 42,  1,  "'hello'"   ]
 [ 46,  4,  "hello" ]
-[ 45,  3,  NULL]
-[ 44,  2,  NULL]
-[ 42,  1,  NULL]
+[ 45,  3,  "'hello'"   ]
+[ 44,  2,  "'hello'"   ]
 #DROP TABLE mytable;
 #DROP ALL LOADER myfunc;
 #CREATE LOADER myfunc() LANGUAGE PYTHON {
diff --git a/sql/backends/monet5/UDF/pyapi/emit.c 
b/sql/backends/monet5/UDF/pyapi/emit.c
--- a/sql/backends/monet5/UDF/pyapi/emit.c
+++ b/sql/backends/monet5/UDF/pyapi/emit.c
@@ -199,6 +199,7 @@ PyObject *PyEmit_Emit(PyEmitObject *self
 
self->cols[self->ncols].b = COLnew(0, bat_type, 
0, TRANSIENT);
self->cols[self->ncols].name = GDKstrdup(val);
+   self->cols[self->ncols].def = NULL;
if (self->nvals > 0) {
// insert NULL values up until the 
current entry
for (ai = 0; ai < self->nvals; ai++) {
@@ -209,9 +210,9 @@ PyObject *PyEmit_Emit(PyEmitObject *self
goto wrapup;
}
}
-   self->cols[i].b->tnil = 1;
-   self->cols[i].b->tnonil = 0;
-   BATsetcount(self->cols[i].b, 
self->nvals);
+   self->cols[self->ncols].b->tnil = 1;
+   self->cols[self->ncols].b->tnonil = 0;
+   BATsetcount(self->cols[self->ncols].b, 
self->nvals);
}
self->ncols++;
}
@@ -337,14 +338,21 @@ PyObject *PyEmit_Emit(PyEmitObject *self
self->cols[i].b->tnonil = 1 - 
self->cols[i].b->tnil;
}
} else {
+   void* nill_value = ATOMnil(self->cols[i].b->ttype);
+   void* default_value = self->cols[i].def ?
+   
self->cols[i].def :
+   nill_value;
for (ai = 0; ai < (size_t)el_count; ai++) {
-   if (BUNappend(self->cols[i].b, 
ATOMnil(self->cols[i].b->ttype),
+   if (BUNappend(self->cols[i].b,
+ default_value,
  0) != GDK_SUCCEED) {
goto wrapup;
}
}
-   self->cols[i].b->tnil = 1;
-   self->cols[i].b->tnonil = 0;
+   if 
(BATatoms[self->cols[i].b->ttype].atomCmp(default_value, nill_value) == 0) {
+   self->cols[i].b->tnil = 1;
+   self->cols[i].b->tnonil = 0;
+   }
}
BATsetcount(self->cols[i].b, self->nvals + 

MonetDB: Jul2017 - Also properly convert Python objects to times...

[Mark Raasveldt]

Changeset: 39a8a3511b29 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=39a8a3511b29
Modified Files:
sql/backends/monet5/UDF/pyapi/convert_loops.h
Branch: Jul2017
Log Message:

Also properly convert Python objects to timestamps.


diffs (21 lines):

diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h 
b/sql/backends/monet5/UDF/pyapi/convert_loops.h
--- a/sql/backends/monet5/UDF/pyapi/convert_loops.h
+++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h
@@ -530,7 +530,7 @@ convert_and_append(BAT* b, const char* t
if (mask != NULL && 
   \
(mask[index_offset * ret->count + iu]) 
== TRUE) {  \
b->tnil = 1;
   \
-   if (BUNappend(b, str_nil, FALSE) != 
GDK_SUCCEED) { \
+   if (convert_and_append(b, str_nil, 
FALSE) != GDK_SUCCEED) { \
msg = createException(MAL, 
"pyapi.eval",   \

  "BUNappend failed.\n");  \
goto wrapup;
   \
@@ -541,7 +541,7 @@ convert_and_append(BAT* b, const char* t
((PyObject 
**)[(index_offset * ret->count + iu) * \

ret->memory_size]),\
utf8_size, _string);   
   \
-   if (BUNappend(b, utf8_string, FALSE) != 
GDK_SUCCEED) { \
+   if (convert_and_append(b, utf8_string, 
FALSE) != GDK_SUCCEED) { \
msg = createException(MAL, 
"pyapi.eval",   \

  "BUNappend failed.\n");  \
goto wrapup;
   \
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: 1e07a13940f9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1e07a13940f9
Added Files:

sql/test/BugTracker-2016/Tests/min-max-uuid-column-wrong-results.Bug-3953.sql
Removed Files:
sql/test/BugTracker-2016/Tests/data-uuid.csv.bz2

sql/test/BugTracker-2016/Tests/min-max-uuid-column-wrong-results.Bug-3953.sql.in
Modified Files:
NT/monetdb_config.h.in
clients/Tests/exports.stable.out
configure.ag
gdk/ChangeLog
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_align.c
gdk/gdk_atoms.c
gdk/gdk_atoms.h
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_calc.c
gdk/gdk_calc_compare.h
gdk/gdk_calc_private.h
gdk/gdk_firstn.c
gdk/gdk_group.c
gdk/gdk_hash.c
gdk/gdk_imprints.c
gdk/gdk_join.c
gdk/gdk_logger.c
gdk/gdk_logger.h
gdk/gdk_project.c
gdk/gdk_qsort.c
gdk/gdk_search.c
gdk/gdk_select.c
gdk/gdk_ssort.c
gdk/gdk_unique.c
gdk/gdk_utils.c
gdk/gdk_value.c
geom/monetdb5/geom.c
monetdb5/extras/rapi/converters.c.h
monetdb5/mal/mal_atom.c
monetdb5/mal/mal_authorize.c
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_parser.c
monetdb5/modules/atoms/color.c
monetdb5/modules/atoms/color.h
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/str.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/batcolor.c
monetdb5/modules/kernel/batmmath.c
monetdb5/modules/kernel/microbenchmark.c
monetdb5/modules/kernel/mmath.c
monetdb5/modules/kernel/mmath.h
monetdb5/modules/mal/batcalc.c
monetdb5/modules/mal/calc.c
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/iterator.c
monetdb5/modules/mal/language.c
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mkey.c
monetdb5/modules/mal/pcre.c
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/tablet.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_multiplex.c
monetdb5/optimizer/opt_remap.c
sql/ChangeLog
sql/backends/monet5/LSST/lsst.c
sql/backends/monet5/Tests/pyloader07.sql
sql/backends/monet5/Tests/pyloader07.stable.out
sql/backends/monet5/Tests/pyloader07.stable.out.Windows
sql/backends/monet5/UDF/pyapi/conversion.c
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/UDF/pyapi/emit.c
sql/backends/monet5/UDF/pyapi/type_conversion.c
sql/backends/monet5/UDF/udf/udf_impl.h
sql/backends/monet5/generator/generator.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_cast_impl_down_from_flt.h
sql/backends/monet5/sql_cast_impl_int.h
sql/backends/monet5/sql_cast_impl_up_to_flt.h
sql/backends/monet5/sql_cat.c
sql/backends/monet5/sql_fround.c
sql/backends/monet5/sql_fround_impl.h
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_round.c
sql/backends/monet5/sql_round_impl.h
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_user.c
sql/backends/monet5/vaults/bam/bam_lib.c
sql/backends/monet5/vaults/fits/fits.c
sql/backends/monet5/vaults/lidar/lidar.c
sql/backends/monet5/vaults/netcdf/netcdf.c
sql/backends/monet5/vaults/shp/shp.c
sql/server/sql_atom.c
sql/server/sql_mvc.c
sql/server/sql_parser.y
sql/server/sql_privileges.c
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_table.c
sql/storage/store.c
sql/storage/store_dependency.c
sql/test/BugTracker-2016/Tests/All
sql/test/BugTracker-2017/Tests/double-groupby-column.Bug-6207.sql
sql/test/BugTracker-2017/Tests/double-groupby-column.Bug-6207.stable.out
sql/test/Tests/fsum.sql
sql/test/Tests/fsum.stable.err
sql/test/Tests/fsum.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/mergetables/Tests/All
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out
testing/Mtest.py.in
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 8668 to 300 lines):

diff --git a/NT/monetdb_config.h.in b/NT/monetdb_config.h.in
--- a/NT/monetdb_config.h.in
+++ b/NT/monetdb_config.h.in
@@ -33,6 +33,12 @@
 #define _CRT_SECURE_NO_DEPRECATE 1
 #endif
 
+/* include Intel's mathimf.h early, before any other include files
+ * might try 

MonetDB: Jul2017 - Compilation fix.

[Mark Raasveldt]

Changeset: 8ef5d95741d7 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8ef5d95741d7
Modified Files:
sql/backends/monet5/UDF/pyapi/convert_loops.h
Branch: Jul2017
Log Message:

Compilation fix.


diffs (15 lines):

diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h 
b/sql/backends/monet5/UDF/pyapi/convert_loops.h
--- a/sql/backends/monet5/UDF/pyapi/convert_loops.h
+++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h
@@ -292,9 +292,10 @@ convert_and_append(BAT* b, const char* t
} else {
void* element = NULL;
int len = 0;
+   gdk_return ret;
 
BATatoms[b->ttype].atomFromStr(text, , );
-   gdk_return ret = BUNappend(b, element, force);
+   ret = BUNappend(b, element, force);
GDKfree(element);
return ret;
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jul2017 - Properly support dates, times and timestamps ...

[Mark Raasveldt]

Changeset: 839bd7161d60 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=839bd7161d60
Modified Files:
sql/backends/monet5/Tests/pyloader07.sql
sql/backends/monet5/Tests/pyloader07.stable.out
sql/backends/monet5/Tests/pyloader07.stable.out.Windows
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/UDF/pyapi/emit.c
Branch: Jul2017
Log Message:

Properly support dates, times and timestamps in Python loader functions.


diffs (247 lines):

diff --git a/sql/backends/monet5/Tests/pyloader07.sql 
b/sql/backends/monet5/Tests/pyloader07.sql
--- a/sql/backends/monet5/Tests/pyloader07.sql
+++ b/sql/backends/monet5/Tests/pyloader07.sql
@@ -9,3 +9,21 @@ SELECT * FROM pyloader07table;
 DROP TABLE pyloader07table;
 DROP LOADER pyloader07;
 ROLLBACK;
+
+START TRANSACTION;
+
+CREATE TABLE tstamp(d DATE, s TIME, t TIMESTAMP);
+
+
+CREATE LOADER pyloader07() LANGUAGE PYTHON {
+_emit.emit({'d': '2014-05-20', 's': '00:02:30', 't': '2014-05-20 
00:02:30'});
+_emit.emit({'d': ['2014-05-20'], 's': ['00:02:30'], 't': ['2014-05-20 
00:02:30']});
+};
+
+COPY LOADER INTO tstamp FROM pyloader07();
+
+SELECT * FROM tstamp;
+DROP TABLE tstamp;
+DROP LOADER pyloader07;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/pyloader07.stable.out 
b/sql/backends/monet5/Tests/pyloader07.stable.out
--- a/sql/backends/monet5/Tests/pyloader07.stable.out
+++ b/sql/backends/monet5/Tests/pyloader07.stable.out
@@ -75,6 +75,24 @@ Ready.
 #DROP TABLE pyloader07table;
 #DROP LOADER pyloader07;
 #ROLLBACK;
+#START TRANSACTION;
+#CREATE TABLE tstamp(d DATE, s TIME, t TIMESTAMP);
+#CREATE LOADER pyloader07() LANGUAGE PYTHON {
+#_emit.emit({'d': '2014-05-20', 's': '00:02:30', 't': '2014-05-20 
00:02:30'});
+#_emit.emit({'d': ['2014-05-20'], 's': ['00:02:30'], 't': ['2014-05-20 
00:02:30']});
+#};
+#COPY LOADER INTO tstamp FROM pyloader07();
+[ 2]
+#SELECT * FROM tstamp;
+% sys.tstamp,  sys.tstamp, sys.tstamp # table_name
+% d,   s,  t # name
+% date,time,   timestamp # type
+% 10,  8,  26 # length
+[ 2014-05-20,  00:02:30,   2014-05-20 00:02:30.00  ]
+[ 2014-05-20,  00:02:30,   2014-05-20 00:02:30.00  ]
+#DROP TABLE tstamp;
+#DROP LOADER pyloader07;
+#ROLLBACK;
 
 # 15:06:46 >  
 # 15:06:46 >  "Done."
diff --git a/sql/backends/monet5/Tests/pyloader07.stable.out.Windows 
b/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
--- a/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
+++ b/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
@@ -75,6 +75,24 @@ Ready.
 #DROP TABLE pyloader07table;
 #DROP LOADER pyloader07;
 #ROLLBACK;
+#START TRANSACTION;
+#CREATE TABLE tstamp(d DATE, s TIME, t TIMESTAMP);
+#CREATE LOADER pyloader07() LANGUAGE PYTHON {
+#_emit.emit({'d': '2014-05-20', 's': '00:02:30', 't': '2014-05-20 
00:02:30'});
+#_emit.emit({'d': ['2014-05-20'], 's': ['00:02:30'], 't': ['2014-05-20 
00:02:30']});
+#};
+#COPY LOADER INTO tstamp FROM pyloader07();
+[ 2]
+#SELECT * FROM tstamp;
+% sys.tstamp,  sys.tstamp, sys.tstamp # table_name
+% d,   s,  t # name
+% date,time,   timestamp # type
+% 10,  8,  26 # length
+[ 2014-05-20,  00:02:30,   2014-05-20 00:02:30.00  ]
+[ 2014-05-20,  00:02:30,   2014-05-20 00:02:30.00  ]
+#DROP TABLE tstamp;
+#DROP LOADER pyloader07;
+#ROLLBACK;
 
 # 15:06:46 >  
 # 15:06:46 >  "Done."
diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h 
b/sql/backends/monet5/UDF/pyapi/convert_loops.h
--- a/sql/backends/monet5/UDF/pyapi/convert_loops.h
+++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h
@@ -282,6 +282,24 @@
}   
   \
}
 
+
+static gdk_return
+convert_and_append(BAT* b, const char* text, bit force) {
+   if (b->ttype == TYPE_str) {
+   return BUNappend(b, text, force);   
+   } else if (text == str_nil) {
+   return BUNappend(b, BATatoms[b->ttype].atomNull, force);
+   } else {
+   void* element = NULL;
+   int len = 0;
+
+   BATatoms[b->ttype].atomFromStr(text, , );
+   gdk_return ret = BUNappend(b, element, force);
+   GDKfree(element);
+   return ret;
+   }
+}
+
 // This #define is for converting a numeric numpy array into a string BAT.
 // 'conv' is a function that turns a numeric value of type 'mtpe' to a char*
 // array.
@@ -291,7 +309,7 @@
snprintf(utf8_string, utf8string_minlength, fmt,
   \
 *((mtpe *)[(index_offset * 
ret->count + iu) *\
 
ret->memory_size]));  \
-   if (BUNappend(bat, utf8_string, FALSE) != GDK_SUCCEED) 
{   \
+   if (convert_and_append(bat, utf8_string, 

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: 53bb444c8ee1 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=53bb444c8ee1
Added Files:
sql/test/BugTracker-2009/Tests/AVG_of_SQRT.SF-2757642.sql

sql/test/BugTracker-2017/Tests/insert_into_multiple_subqueries.Bug-6448.sql

sql/test/BugTracker-2017/Tests/insert_into_multiple_subqueries.Bug-6448.stable.err

sql/test/BugTracker-2017/Tests/insert_into_multiple_subqueries.Bug-6448.stable.out

sql/test/BugTracker-2017/Tests/sqlitelogictest-cast-decimal.Bug-6445.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-cast-decimal.Bug-6445.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6449.sql
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6449.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6449.stable.out
sql/test/BugTracker-2017/Tests/table_returning_with.Bug-6444.sql
sql/test/BugTracker-2017/Tests/table_returning_with.Bug-6444.stable.err
sql/test/BugTracker-2017/Tests/table_returning_with.Bug-6444.stable.out
sql/test/Tests/fsum.sql
sql/test/Tests/fsum.stable.err
sql/test/Tests/fsum.stable.out
sql/test/Tests/fsum1.sql.src
sql/test/Tests/fsum1.stable.err.src
sql/test/Tests/fsum1.stable.out.src
sql/test/mergetables/Tests/sqlsmith.Bug-6451.sql
sql/test/mergetables/Tests/sqlsmith.Bug-6451.stable.err
sql/test/mergetables/Tests/sqlsmith.Bug-6451.stable.out
sql/test/mergetables/Tests/sqlsmith.Bug-6453.sql
sql/test/mergetables/Tests/sqlsmith.Bug-6453.stable.err
sql/test/mergetables/Tests/sqlsmith.Bug-6453.stable.out
sql/test/mergetables/Tests/sqlsmith.Bug-6455.sql
sql/test/mergetables/Tests/sqlsmith.Bug-6455.stable.err
sql/test/mergetables/Tests/sqlsmith.Bug-6455.stable.out
Removed Files:
sql/test/BugTracker-2009/Tests/AVG_of_SQRT.SF-2757642.SQL.py
sql/test/BugTracker-2011/Tests/and-power.Bug-3013.stable.out.int128
Modified Files:
MonetDB.spec
NT/monetdb_config.h.in
buildtools/selinux/monetdb.te
clients/Tests/exports.stable.out
configure.ag
gdk/ChangeLog.Jul2017
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_atoms.c
gdk/gdk_atoms.h
gdk/gdk_bat.c
gdk/gdk_calc.c
gdk/gdk_group.c
gdk/gdk_hash.c
gdk/gdk_join.c
gdk/gdk_orderidx.c
gdk/gdk_utils.c
geom/monetdb5/geom.c
geom/monetdb5/geom.h
monetdb5/mal/mal_parser.c
monetdb5/modules/kernel/mmath.h
sql/backends/monet5/sql_gencode.c
sql/server/rel_dump.c
sql/server/rel_optimizer.c
sql/server/rel_rel.c
sql/server/rel_select.c
sql/server/rel_sequence.c
sql/server/rel_updates.c
sql/server/sql_parser.h
sql/server/sql_parser.y
sql/server/sql_privileges.c
sql/server/sql_scan.c
sql/test/BugTracker-2009/Tests/AVG_of_SQRT.SF-2757642.stable.out
sql/test/BugTracker-2011/Tests/and-power.Bug-3013.stable.out
sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.err
sql/test/BugTracker-2017/Tests/All

sql/test/BugTracker-2017/Tests/extract_quarter_week_from_date.Bug-3831.sql
sql/test/BugTracker/Tests/drop_schema_crash.SF-1504794.stable.err
sql/test/BugTracker/Tests/set_a_new_user_password.SF-1844050.stable.err
sql/test/Dependencies/Tests/Dependencies.stable.err
sql/test/Tests/All
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128
sql/test/mergetables/Tests/All
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
testing/Mtest.py.in
tools/merovingian/daemon/merovingian.c
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 25152 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -885,7 +885,7 @@ developer, but if you do want to test, t
 
 %if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} >= 7
 %package selinux
-Summary: MonetDB - Monet Database Management System
+Summary: SELinux policy files for MonetDB
 Group: Applications/Databases
 %if "%{_selinux_policy_version}" != ""
 Requires:   selinux-policy >= %{_selinux_policy_version}
@@ -1002,6 

MonetDB: jitudf - Approve exports.

[Mark Raasveldt]

Changeset: a0223ae484e9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a0223ae484e9
Modified Files:
clients/Tests/exports.stable.out
Branch: jitudf
Log Message:

Approve exports.


diffs (69 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -237,6 +237,7 @@ char *GDKfilepath(int farmid, const char
 void GDKfree(void *blk);
 char *GDKgetenv(const char *name);
 int GDKgetenv_int(const char *name, int def);
+int GDKgetenv_istext(const char *name, const char *text);
 int GDKgetenv_istrue(const char *name);
 int GDKgetenv_isyes(const char *name);
 gdk_return GDKgetsem(int sem_id, int count, int *semid, str *msg);
@@ -1973,7 +1974,10 @@ str batmalRef;
 str batmmathRef;
 str batmtimeRef;
 str batpyapi3Ref;
+str batpyapi3Ref;
 str batpyapiRef;
+str batpyapiRef;
+str batrapiRef;
 str batrapiRef;
 str batsqlRef;
 str batstrRef;
@@ -1996,7 +2000,7 @@ int bstream_read_wrap(int *res, Bstream 
 str bstream_read_wrapwrap(int *res, Bstream *BS, int *size);
 str calcRef;
 str callMAL(Client cntxt, MalBlkPtr mb, MalStkPtr *glb, ValPtr argv[], char 
debug);
-int callString(Client c, str s, int listing);
+str callString(Client c, str s, int listing);
 str capiRef;
 str catalogRef;
 void chkDeclarations(MalBlkPtr mb);
@@ -2065,6 +2069,7 @@ str deleteRef;
 void deleteSymbol(Module scope, Symbol prg);
 str deltaRef;
 str dense_rankRef;
+str dense_rankRef;
 malType destinationType(MalBlkPtr mb, InstrPtr p);
 str diffRef;
 str differenceRef;
@@ -2436,8 +2441,12 @@ str putName(const char *nme);
 str putNameLen(const char *nme, size_t len);
 str putRef;
 str pyapi3Ref;
+str pyapi3Ref;
+str pyapi3mapRef;
 str pyapi3mapRef;
 str pyapiRef;
+str pyapiRef;
+str pyapimapRef;
 str pyapimapRef;
 int qtop;
 str queryRef;
@@ -2445,6 +2454,8 @@ str querylogRef;
 str raiseRef;
 str rangejoinRef;
 str rankRef;
+str rankRef;
+str rapiRef;
 str rapiRef;
 int readConsole(Client cntxt);
 MalStkPtr reallocGlobalStack(MalStkPtr s, int cnt);
@@ -2546,6 +2557,7 @@ str subcountRef;
 str subdeltaRef;
 str subdiffRef;
 str subeval_aggrRef;
+str subeval_aggrRef;
 str subgroupRef;
 str subgroupdoneRef;
 str subinterRef;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Fixes after merge.

[Mark Raasveldt]

Changeset: 7a364f69e1c7 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7a364f69e1c7
Modified Files:
gdk/gdk_utils.c
gdk/gdk_utils.h
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.err
sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.out
sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.err
sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.out
Branch: jitudf
Log Message:

Fixes after merge.


diffs (119 lines):

diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -101,25 +101,26 @@ GDKgetenv(const char *name)
 }
 
 int
-GDKgetenv_isyes(const char *name)
+GDKgetenv_istext(const char *name, const char* text)
 {
char *val = GDKgetenv(name);
 
-   if (val && strcasecmp(val, "yes") == 0) {
+   if (val && strcasecmp(val, text) == 0) {
return 1;
}
return 0;
 }
 
 int
+GDKgetenv_isyes(const char *name)
+{
+   return GDKgetenv_istext(name, "yes");
+}
+
+int
 GDKgetenv_istrue(const char *name)
 {
-   char *val = GDKgetenv(name);
-
-   if (val && strcasecmp(val, "true") == 0) {
-   return 1;
-   }
-   return 0;
+   return GDKgetenv_istext(name, "true");
 }
 
 int
diff --git a/gdk/gdk_utils.h b/gdk/gdk_utils.h
--- a/gdk/gdk_utils.h
+++ b/gdk/gdk_utils.h
@@ -17,8 +17,8 @@ gdk_export BAT *GDKval;
 
 gdk_export char *GDKgetenv(const char *name);
 
+gdk_export int GDKgetenv_istext(const char *name, const char* text);
 gdk_export int GDKgetenv_isyes(const char *name);
-
 gdk_export int GDKgetenv_istrue(const char *name);
 
 gdk_export int GDKgetenv_int(const char *name, int def);
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -62,6 +62,7 @@ typedef struct _cached_functions {
 
 static cached_functions *function_cache[FUNCTION_CACHE_SIZE];
 static MT_Lock cache_lock;
+static int cudf_initialized = 0;
 
 static str CUDFeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
bit grouped);
@@ -79,7 +80,10 @@ str CUDFevalAggr(Client cntxt, MalBlkPtr
 str CUDFprelude(void *ret)
 {
(void)ret;
-   MT_lock_init(_lock, "cache_lock");
+   if (!cudf_initialized) {
+   MT_lock_init(_lock, "cache_lock");
+   cudf_initialized = true;
+   }
return MAL_SUCCEED;
 }
 
@@ -1455,7 +1459,7 @@ static str CUDFeval(Client cntxt, MalBlk
// return the BAT from the function
if (isaBatType(getArgType(mb, pci, i))) {
*getArgReference_bat(stk, pci, i) = b->batCacheid;
-   // BBPkeepref(b->batCacheid);
+   BBPkeepref(b->batCacheid);
} else {
// single value return, only for non-grouped 
aggregations
BATiter li = bat_iterator(b);
diff --git a/sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.err 
b/sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.err
--- a/sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.err
+++ b/sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.err
@@ -1,4 +1,4 @@
-stderr of test 'udf-fuse` in directory 'sql/backends/monet5/UDF/cudf` itself:
+stderr of test 'udf-fuse` in directory 'sql/backends/monet5/UDF/udf` itself:
 
 
 # 18:49:07 >  
diff --git a/sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.out 
b/sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.out
--- a/sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.out
+++ b/sql/backends/monet5/UDF/udf/Tests/udf-fuse.stable.out
@@ -1,4 +1,4 @@
-stdout of test 'udf-fuse` in directory 'sql/backends/monet5/UDF/cudf` itself:
+stdout of test 'udf-fuse` in directory 'sql/backends/monet5/UDF/udf` itself:
 
 
 # 18:49:07 >  
diff --git a/sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.err 
b/sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.err
--- a/sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.err
+++ b/sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.err
@@ -1,4 +1,4 @@
-stderr of test 'udf-reverse` in directory 'sql/backends/monet5/UDF/cudf` 
itself:
+stderr of test 'udf-reverse` in directory 'sql/backends/monet5/UDF/udf` itself:
 
 
 # 18:44:21 >  
diff --git a/sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.out 
b/sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.out
--- a/sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.out
+++ b/sql/backends/monet5/UDF/udf/Tests/udf-reverse.stable.out
@@ -1,4 +1,4 @@
-stdout of test 'udf-reverse` in directory 'sql/backends/monet5/UDF/cudf` 
itself:
+stdout of test 'udf-reverse` in directory 'sql/backends/monet5/UDF/udf` itself:
 
 
 # 18:44:21 >  
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Merge w3ith default.

[Mark Raasveldt]

Changeset: 7a853f58866d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7a853f58866d
Added Files:
sql/jdbc/tests/Tests/Bug_PrepStmtSetString_6382.SQL.bat
sql/jdbc/tests/Tests/Bug_PrepStmtSetString_6382.SQL.sh
sql/jdbc/tests/Tests/Bug_PrepStmtSetString_6382.stable.err
sql/jdbc/tests/Tests/Bug_PrepStmtSetString_6382.stable.out
sql/test/BugTracker-2017/Tests/copy_into_crash.Bug-6404.sql
sql/test/BugTracker-2017/Tests/copy_into_crash.Bug-6404.stable.err
sql/test/BugTracker-2017/Tests/copy_into_crash.Bug-6404.stable.out
sql/test/BugTracker-2017/Tests/corr_coersion.Bug-6287.sql
sql/test/BugTracker-2017/Tests/corr_coersion.Bug-6287.stable.err
sql/test/BugTracker-2017/Tests/corr_coersion.Bug-6287.stable.out
sql/test/BugTracker-2017/Tests/double-groupby-column.Bug-6207.sql
sql/test/BugTracker-2017/Tests/double-groupby-column.Bug-6207.stable.err
sql/test/BugTracker-2017/Tests/double-groupby-column.Bug-6207.stable.out
sql/test/BugTracker-2017/Tests/null_matches_in_outer.Bug-6398.sql
sql/test/BugTracker-2017/Tests/null_matches_in_outer.Bug-6398.stable.err
sql/test/BugTracker-2017/Tests/null_matches_in_outer.Bug-6398.stable.out
sql/test/BugTracker-2017/Tests/side-effect.Bug-6397.sql
sql/test/BugTracker-2017/Tests/side-effect.Bug-6397.stable.err
sql/test/BugTracker-2017/Tests/side-effect.Bug-6397.stable.out

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-distinct-coalesce.Bug-6431.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-distinct-coalesce.Bug-6431.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-distinct-coalesce.Bug-6431.stable.out

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-distinct.Bug-6411.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-distinct.Bug-6411.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-distinct.Bug-6411.stable.out

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-having-avg.Bug-6428.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-having-avg.Bug-6428.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-having-avg.Bug-6428.stable.out

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-not-in.Bug-6416.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-not-in.Bug-6416.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-aggregation-not-in.Bug-6416.stable.out
sql/test/BugTracker-2017/Tests/sqlitelogictest-cast-decimal.Bug-6445.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-having-with-in.Bug-6410.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-having-with-in.Bug-6410.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-having-with-in.Bug-6410.stable.out

sql/test/BugTracker-2017/Tests/sqlitelogictest-having-with-not-in.Bug-6409.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-having-with-not-in.Bug-6409.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-having-with-not-in.Bug-6409.stable.out
sql/test/BugTracker-2017/Tests/sqlitelogictest-select-in.Bug-6433.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-select-in.Bug-6433.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-select-in.Bug-6433.stable.out

sql/test/BugTracker-2017/Tests/sqlitelogictest-select-not-in.Bug-6435.sql

sql/test/BugTracker-2017/Tests/sqlitelogictest-select-not-in.Bug-6435.stable.err

sql/test/BugTracker-2017/Tests/sqlitelogictest-select-not-in.Bug-6435.stable.out
sql/test/BugTracker-2017/Tests/sqllitelogistest_case.Bug-6335.stable.err
sql/test/BugTracker-2017/Tests/sqllitelogistest_case.Bug-6335.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith-semijoin-constant.bug-6417.sql

sql/test/BugTracker-2017/Tests/sqlsmith-semijoin-constant.bug-6417.stable.err

sql/test/BugTracker-2017/Tests/sqlsmith-semijoin-constant.bug-6417.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6418.sql
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6418.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6418.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6423.sql
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6423.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6423.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6424.sql
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6424.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6424.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6425.sql
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6425.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith.Bug-6425.stable.out
 

MonetDB: jitudf - Try to mute strange GCC warning.

[Mark Raasveldt]

Changeset: ec2552b9a090 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec2552b9a090
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Try to mute strange GCC warning.


diffs (14 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -22,6 +22,10 @@
 #include 
 #include 
 
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic ignored "-Wclobbered"
+#endif
+
 struct _allocated_region;
 typedef struct _allocated_region {
struct _allocated_region *next;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: e2e6994c1796 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e2e6994c1796
Added Files:

sql/test/BugTracker-2017/Tests/too-large-decimals.Bug-6192.stable.err.int128
Removed Files:
geom/sql/conformance/Tests/T3.stable.out.int128
Modified Files:
clients/Tests/MAL-signatures.stable.out
geom/sql/conformance/Tests/T3.stable.out

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.err.Windows

monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows

sql/jdbc/tests/Tests/Bug_Connect_as_voc_getMetaData_Failure_Bug_6388.stable.out
sql/test/BugTracker-2009/Tests/bit_and.SF-2850341.stable.err

sql/test/BugTracker-2012/Tests/large-number-operation-strange-results.Bug-2929.stable.err
sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.err
sql/test/BugTracker-2014/Tests/hexadecimal_literals.Bug-3621.stable.out
sql/test/BugTracker-2014/Tests/round.Bug-3542.stable.err
sql/test/BugTracker-2016/Tests/convert-function-test.Bug-3460.stable.err
sql/test/BugTracker-2016/Tests/storagemodel.stable.out
sql/test/BugTracker-2017/Tests/too-large-decimals.Bug-6192.stable.err
sql/test/BugTracker/Tests/explain.SF-1739353.stable.out
sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out
sql/test/BugTracker/Tests/limits_of_decimal.SF-1685357.stable.err
sql/test/pg_regress/Tests/float8.stable.err
sql/test/pg_regress/Tests/int8.stable.err
sql/test/pg_regress/Tests/int8.stable.out
sql/test/pg_regress/Tests/interval.stable.err
sql/test/pg_regress/Tests/interval.stable.out
sql/test/pg_regress/Tests/numeric_big.stable.err
sql/test/pg_regress/Tests/oid.stable.err
sql/test/pg_regress/Tests/strings.stable.err
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 10509 to 300 lines):

diff --git a/clients/Tests/MAL-signatures.stable.out 
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -7825,20 +7825,20 @@ Ready.
 [ "json",  "unfold",   "pattern json.unfold(val:json) (o:bat[:oid], 
k:bat[:str], v:bat[:json]) ",  "JSONunfold;",  "Expands the outermost JSON 
object into key-value pairs."   ]
 [ "json",  "valuearray",   "command json.valuearray(val:json):json ",  
"JSONvalueArray;",  "Expands the outermost JSON object values into a JSON 
value array." ]
 [ "json",  "values",   "command json.values(val:json):bat[:json] ",
"JSONvalueTable;",  "Expands the outermost JSON values."]
-[ "language",  "assert",   "unsafe command language.assert(v:bit, 
term:str):void ","MALassertBit;",""  ]
-[ "language",  "assert",   "unsafe command language.assert(v:int, 
term:str):void ","MALassertInt;",""  ]
-[ "language",  "assert",   "unsafe command language.assert(v:lng, 
term:str):void ","MALassertLng;",""  ]
-[ "language",  "assert",   "unsafe command language.assert(v:oid, 
term:str):void ","MALassertOid;",""  ]
-[ "language",  "assert",   "unsafe command language.assert(v:sht, 
term:str):void ","MALassertSht;",""  ]
-[ "language",  "assert",   "unsafe command language.assert(v:str, 
term:str):void ","MALassertStr;",""  ]
-[ "language",  "assert",   "unsafe pattern language.assert(v:any_1, 
pname:str, oper:str, val:any_2):void ","MALassertTriple;", 
"Assertion test."   ]
+[ "language",  "assert",   "command language.assert(v:bit, term:str):void 
",   "MALassertBit;",""  ]
+[ "language",  "assert",   "command language.assert(v:int, term:str):void 
",   "MALassertInt;",""  ]
+[ "language",  "assert",   "command language.assert(v:lng, term:str):void 
",   "MALassertLng;",""  ]
+[ "language",  "assert",   "command language.assert(v:oid, term:str):void 
",   "MALassertOid;",""  ]
+[ "language",  "assert",   "command language.assert(v:sht, term:str):void 
",   "MALassertSht;",""  ]
+[ "language",  "assert",   "command language.assert(v:str, term:str):void 
",   "MALassertStr;",""  ]
+[ "language",  "assert",   "pattern language.assert(v:any_1, pname:str, 
oper:str, val:any_2):void ",   "MALassertTriple;", "Assertion test."
   ]
 [ "language",  "block","pattern language.block(v:int, w:any...):int ", 
"deblockdataflow;", "Block on availability of all variables w, and then 
pass on v"  ]
 [ "language",  "call", "pattern language.call(s:bat[:str]):void ", 
"CMDcallBAT;",  "Evaluate a program stored in a BAT."   ]
 [ "language",  "call", "pattern language.call(m:str, f:str):void ",
"CMDcallFunction;", ""  

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: 844f9f1fb92e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=844f9f1fb92e
Added Files:
monetdb5/modules/mal/Tests/remote13.malC
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_00.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_01.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_02.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_03.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_04.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_05.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_06.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_07.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_09.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_10.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_11.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_12.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_13.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_15.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_16.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_17.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_18.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_19.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_20.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_21.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_23.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_24.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_25.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_26.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_27.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_28.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_29.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_30.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_31.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_32.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_01.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_02.sql.src
sql/backends/monet5/UDF/pyapi3/Tests/pyloader3_05.sql.src

sql/jdbc/tests/Tests/Bug_Connect_as_voc_getMetaData_Failure_Bug_6388.SQL.bat

sql/jdbc/tests/Tests/Bug_Connect_as_voc_getMetaData_Failure_Bug_6388.SQL.sh

sql/jdbc/tests/Tests/Bug_Connect_as_voc_getMetaData_Failure_Bug_6388.stable.err

sql/jdbc/tests/Tests/Bug_Connect_as_voc_getMetaData_Failure_Bug_6388.stable.out

sql/test/BugTracker-2017/Tests/aggr_udf_with_more_than_2params.Bug-6385.sql

sql/test/BugTracker-2017/Tests/aggr_udf_with_more_than_2params.Bug-6385.stable.err

sql/test/BugTracker-2017/Tests/aggr_udf_with_more_than_2params.Bug-6385.stable.out
sql/test/BugTracker-2017/Tests/caching_stats_bug.6374.sql
sql/test/BugTracker-2017/Tests/caching_stats_bug.6374.stable.err
sql/test/BugTracker-2017/Tests/caching_stats_bug.6374.stable.out
sql/test/BugTracker-2017/Tests/case.Bug-6386.sql
sql/test/BugTracker-2017/Tests/case.Bug-6386.stable.err
sql/test/BugTracker-2017/Tests/case.Bug-6386.stable.out
sql/test/BugTracker-2017/Tests/exists.Bug-6392.sql
sql/test/BugTracker-2017/Tests/exists.Bug-6392.stable.err
sql/test/BugTracker-2017/Tests/exists.Bug-6392.stable.out

sql/test/BugTracker-2017/Tests/extract_quarter_week_from_date.Bug-3831.sql

sql/test/BugTracker-2017/Tests/extract_quarter_week_from_date.Bug-3831.stable.err

sql/test/BugTracker-2017/Tests/extract_quarter_week_from_date.Bug-3831.stable.out
sql/test/BugTracker-2017/Tests/name-interference.Bug-6348.stable.err
sql/test/BugTracker-2017/Tests/name-interference.Bug-6348.stable.out
sql/test/BugTracker-2017/Tests/parsererror.Bug-6381.sql
sql/test/BugTracker-2017/Tests/spurious_error.Bug-6344.stable.err
sql/test/BugTracker-2017/Tests/spurious_error.Bug-6344.stable.out
sql/test/BugTracker-2017/Tests/sqllitelogistest_case.Bug-6335.sql
sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128.src
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.32bit.src
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128.src
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.src
sql/test/emptydb-upgrade-hge/Tests/check.stable.out.int128.src
sql/test/emptydb-upgrade/Tests/check.stable.out.32bit.src
sql/test/emptydb-upgrade/Tests/check.stable.out.int128.src
sql/test/emptydb-upgrade/Tests/check.stable.out.src
Removed Files:
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_00.SQL.sh
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_01.SQL.sh
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_02.SQL.sh
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_03.SQL.sh

MonetDB: jitudf - Merge.

[Mark Raasveldt]

Changeset: fcd5e4ac0da5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fcd5e4ac0da5
Modified Files:
MonetDB.spec
buildtools/selinux/monetdb.te
clients/ChangeLog
clients/mapiclient/stethoscope.c
monetdb5/mal/mal_profiler.c
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
tools/merovingian/utils/properties.c
tools/merovingian/utils/utils.c
Branch: jitudf
Log Message:

Merge.


diffs (truncated from 39730 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -998,6 +998,7 @@ fi
 
 make %{?_smp_mflags}
 
+%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} >= 7
 cd buildtools/selinux
 for selinuxvariant in %{selinux_variants}
 do
@@ -1006,6 +1007,7 @@ do
   make NAME=${selinuxvariant} -f /usr/share/selinux/devel/Makefile clean
 done
 cd -
+%endif
 
 %install
 %make_install
@@ -1022,6 +1024,7 @@ rm -f %{buildroot}%{_libdir}/monetdb5/*.
 # internal development stuff
 rm -f %{buildroot}%{_bindir}/Maddlog
 
+%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} >= 7
 for selinuxvariant in %{selinux_variants}
 do
   install -d %{buildroot}%{_datadir}/selinux/${selinuxvariant}
@@ -1029,6 +1032,7 @@ do
 %{buildroot}%{_datadir}/selinux/${selinuxvariant}/monetdb.pp
 done
 /usr/sbin/hardlink -cv %{buildroot}%{_datadir}/selinux
+%endif
 
 %post -p /sbin/ldconfig
 
diff --git a/buildtools/selinux/monetdb.te b/buildtools/selinux/monetdb.te
--- a/buildtools/selinux/monetdb.te
+++ b/buildtools/selinux/monetdb.te
@@ -14,7 +14,7 @@ init_daemon_domain(monetdbd_t, monetdbd_
 # mserver5_exec_t which must then transition to the mserver5_t domain
 type mserver5_t;   # mserver5 when run by monetdbd (monetdbd_t)
 type mserver5_exec_t;  # /usr/bin/mserver5
-application_domain(mserver5_t, mserver5_exec_t)
+init_daemon_domain(mserver5_t, mserver5_exec_t)
 # see 
https://selinuxproject.org/page/NB_Domain_and_Object_Transitions#Domain_Transition
 allow monetdbd_t mserver5_t:process transition;
 allow monetdbd_t mserver5_exec_t:file { execute open read getattr };
diff --git a/clients/ChangeLog b/clients/ChangeLog
--- a/clients/ChangeLog
+++ b/clients/ChangeLog
@@ -1,3 +1,10 @@
 # ChangeLog file for clients
 # This file is updated with Maddlog
 
+* Mon Jul 17 2017 Panagiotis Koutsourakis 
+- Add a new pretty printing option to stethoscope
+  Running stethoscope with the flag -j will produce not pretty printed
+  output (one json object per line). Running with the -y flag will produce
+  pretty printed output. Running with neither, wiil produce the legacy,
+  line oriented format
+
diff --git a/clients/mapiclient/stethoscope.c b/clients/mapiclient/stethoscope.c
--- a/clients/mapiclient/stethoscope.c
+++ b/clients/mapiclient/stethoscope.c
@@ -72,6 +72,7 @@ static char hostname[128];
 static char *filename = NULL;
 static int beat = 0;
 static int json = 0;
+static int stream_mode = 1;
 static Mapi dbh;
 static MapiHdl hdl = NULL;
 static FILE *trace = NULL;
@@ -201,11 +202,12 @@ usageStethoscope(void)
 fprintf(stderr, "  -h | --host=\n");
 fprintf(stderr, "  -c | --convert=\n");
 fprintf(stderr, "  -j | --json\n");
+fprintf(stderr, "  -y | --pretty (implies --json)\n");
 fprintf(stderr, "  -o | --output=\n");
-   fprintf(stderr, "  -b | --beat= in milliseconds (default 50)\n");
-   fprintf(stderr, "  -D | --debug\n");
+fprintf(stderr, "  -b | --beat= in milliseconds (default 50)\n");
+fprintf(stderr, "  -D | --debug\n");
 fprintf(stderr, "  -? | --help\n");
-   exit(-1);
+exit(-1);
 }
 
 /* Any signal should be captured and turned into a graceful
@@ -242,7 +244,7 @@ main(int argc, char **argv)
int done = 0;
EventRecord *ev = malloc(sizeof(EventRecord));
 
-   static struct option long_options[12] = {
+   static struct option long_options[13] = {
{ "dbname", 1, 0, 'd' },
{ "user", 1, 0, 'u' },
{ "port", 1, 0, 'p' },
@@ -251,6 +253,7 @@ main(int argc, char **argv)
{ "help", 0, 0, '?' },
{ "convert", 1, 0, 'c'},
{ "json", 0, 0, 'j'},
+   { "pretty", 0, 0, 'y'},
{ 

MonetDB: jitudf - Place temporary files in a new TEMPDIR instead...

[Mark Raasveldt]

Changeset: 06624855292e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=06624855292e
Modified Files:
gdk/gdk.h
gdk/gdk_bbp.c
sql/backends/monet5/UDF/capi/Tests/capi00.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Place temporary files in a new TEMPDIR instead of using the existing DELDIR, 
because the BBPsync assumes DELDIR is empty.


diffs (99 lines):

diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -391,18 +391,22 @@
 #define GDKisalnum(c)  isalnum((int) (unsigned char) (c))
 #define GDKisdigit(c)  (((unsigned char) (c)) >= '0' && ((unsigned char) (c)) 
<= '9')
 
+#define TEMPDIR_NAME "TEMP_DATA"
+
 #ifndef NATIVE_WIN32
 #define BATDIR "bat"
 #define DELDIR "bat/DELETE_ME"
 #define BAKDIR "bat/BACKUP"
 #define SUBDIR "bat/BACKUP/SUBCOMMIT"
 #define LEFTDIR"bat/LEFTOVERS"
+#define TEMPDIR "bat/"TEMPDIR_NAME
 #else
 #define BATDIR "bat"
 #define DELDIR "bat\\DELETE_ME"
 #define BAKDIR "bat\\BACKUP"
 #define SUBDIR "bat\\BACKUP\\SUBCOMMIT"
 #define LEFTDIR"bat\\LEFTOVERS"
+#define TEMPDIR "bat\\"TEMPDIR_NAME
 #endif
 
 #ifdef MAXPATHLEN
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1355,6 +1355,9 @@ BBPinit(void)
BBPaddfarm(".", 1 << TRANSIENT);
}
 
+   if (GDKremovedir(0, TEMPDIR) != GDK_SUCCEED)
+   GDKfatal("BBPinit: cannot remove directory %s\n", TEMPDIR);
+
if (GDKremovedir(0, DELDIR) != GDK_SUCCEED)
GDKfatal("BBPinit: cannot remove directory %s\n", DELDIR);
 
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi00.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi00.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi00.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi00.sql
@@ -1,4 +1,3 @@
-
 
 START TRANSACTION;
 
@@ -16,5 +15,6 @@ INSERT INTO integers VALUES (1), (2), (3
 SELECT i, capi00(i) FROM integers;
 
 DROP FUNCTION capi00;
+DROP TABLE integers;
 
 ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -603,13 +603,13 @@ static str CUDFeval(Client cntxt, MalBlk
// we have to compile it
 
// first generate the names of the files
-   // we place the temporary files in the LEFTOVERS directory
+   // we place the temporary files in the DELDIR directory
// because this will be removed again upon server startup
const int RANDOM_NAME_SIZE = 32;
char *path = NULL;
-   const char *prefix = "DELETE_ME" DIR_SEP_STR;
+   const char *prefix = TEMPDIR_NAME DIR_SEP_STR;
size_t prefix_size = strlen(prefix);
-   char *leftdirpath;
+   char *deldirpath;
 
memcpy(buf, prefix, sizeof(char) * strlen(prefix));
// generate a random 32-character name for the temporary files
@@ -640,18 +640,18 @@ static str CUDFeval(Client cntxt, MalBlk
strcpy(libname, path);
GDKfree(path);
 
-   // if LEFTOVERS directory does not exist, create it
-   leftdirpath = GDKfilepath(0, NULL, DELDIR, NULL);
-   if (!leftdirpath) {
+   // if DELDIR directory does not exist, create it
+   deldirpath = GDKfilepath(0, NULL, TEMPDIR, NULL);
+   if (!deldirpath) {
msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
goto wrapup;
}
-   if (mkdir(leftdirpath, 0755) < 0 && errno != EEXIST) {
+   if (mkdir(deldirpath, 0755) < 0 && errno != EEXIST) {
msg = createException(MAL, "cudf.eval",
- "cannot 
create directory %s\n", leftdirpath);
+ "cannot 
create directory %s\n", deldirpath);
goto wrapup;
}
-   GDKfree(leftdirpath);
+   GDKfree(deldirpath);
 
// now generate the source file
f = fopen(fname, "w+");
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Put temporary data in DELDIR instead of LEFTDIR.

[Mark Raasveldt]

Changeset: 3814162d083b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3814162d083b
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Put temporary data in DELDIR instead of LEFTDIR.


diffs (21 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -607,7 +607,7 @@ static str CUDFeval(Client cntxt, MalBlk
// because this will be removed again upon server startup
const int RANDOM_NAME_SIZE = 32;
char *path = NULL;
-   const char *prefix = "LEFTOVERS" DIR_SEP_STR;
+   const char *prefix = "DELETE_ME" DIR_SEP_STR;
size_t prefix_size = strlen(prefix);
char *leftdirpath;
 
@@ -641,7 +641,7 @@ static str CUDFeval(Client cntxt, MalBlk
GDKfree(path);
 
// if LEFTOVERS directory does not exist, create it
-   leftdirpath = GDKfilepath(0, NULL, LEFTDIR, NULL);
+   leftdirpath = GDKfilepath(0, NULL, DELDIR, NULL);
if (!leftdirpath) {
msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
goto wrapup;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

sql/test/BugTracker/Tests/explain.SF-1739353.stable.out
sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out
sql/test/Dependencies/Tests/Dependencies.stable.out
sql/test/Dependencies/Tests/Dependencies.stable.out.int128
sql/test/SQLite_regress/sqllogictest/Tests/select3.test.stable.out

sql/test/SQLite_regress/sqllogictest/Tests/select3.test.stable.out.int128
sql/test/Tests/systemfunctions.stable.out.int128
sql/test/emptydb/Tests/check.stable.out
sql/test/emptydb/Tests/check.stable.out.32bit
sql/test/emptydb/Tests/check.stable.out.int128
sql/test/leaks/Tests/check1.stable.out
sql/test/leaks/Tests/check1.stable.out.int128
sql/test/leaks/Tests/select1.stable.out
sql/test/leaks/Tests/select1.stable.out.int128
sql/test/leaks/Tests/select2.stable.out
sql/test/leaks/Tests/select2.stable.out.int128
sql/test/leaks/Tests/temp1.stable.out
sql/test/leaks/Tests/temp1.stable.out.int128
sql/test/mergetables/Tests/mergequery.stable.out
sql/test/pg_regress/Tests/alter_table.stable.err
sql/test/pg_regress/Tests/alter_table.stable.out
sql/test/pg_regress/Tests/insert.stable.err
sql/test/remote/Tests/ssbm.stable.out
testing/Mtest.py.in
tools/merovingian/ChangeLog-Archive
tools/merovingian/ChangeLog.Jul2017
tools/merovingian/daemon/Makefile.ag
tools/merovingian/daemon/client.c
tools/merovingian/daemon/connections.c
tools/merovingian/daemon/controlrunner.c
tools/merovingian/daemon/handlers.c
tools/merovingian/daemon/merovingian.c
tools/merovingian/daemon/monetdbd.service.in
tools/merovingian/daemon/multiplex-funnel.c
tools/merovingian/daemon/proxy.c
tools/merovingian/utils/control.c
tools/merovingian/utils/utils.c
vertoo.data
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 47710 to 300 lines):

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -64,6 +64,10 @@ lex.yy.c
 *.dll
 
 # other files we don't want
+GTAGS
+gtags.files
+GPATH
+GRTAGS
 TAGS
 tags
 *.pyo
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -673,3 +673,5 @@ c57454c6c6f6e77fbbdf70de32a7cc845b67d1dd
 05f4e62bc67911a25f19be7fe742009ab1ee41c7 Dec2016_SP4_release
 8b3d3f7c487cb6e34dab1167797a61862cba2124 Dec2016_23
 8b3d3f7c487cb6e34dab1167797a61862cba2124 Dec2016_SP5_release
+e6552cdd3180bcd35f959954807cab986a6042a5 Jul2017_1
+e6552cdd3180bcd35f959954807cab986a6042a5 Jul2017_release
diff --git a/ChangeLog-Archive b/ChangeLog-Archive
--- a/ChangeLog-Archive
+++ b/ChangeLog-Archive
@@ -1,6 +1,31 @@
 # DO NOT EDIT THIS FILE -- MAINTAINED AUTOMATICALLY
 # This file contains past ChangeLog entries
 
+* Thu Apr 13 2017 Mark Raasveldt <m.raasve...@cwi.nl> - 11.27.1-20170705
+- Added a new server-side protocol implementation. The new protocol
+  is backwards compatible with the old protocol. Clients can choose
+  whether they want to use the old or the new protocol during the initial
+  handshake with the server. The new protocol is a binary column-based
+  protocol that is significantly faster than the old protocol when
+  transferring large result sets. In addition, the new protocol supports
+  compression using Snappy or LZ4.
+
+* Mon Feb 20 2017 Sjoerd Mullender <sjo...@acm.org> - 11.27.1-20170705
+- Moved the sphinx extension module to its own repository.
+  See https://dev.monetdb.org/hg/MonetDB-sphinx/.
+
+* Thu Feb 16 2017 Sjoerd Mullender <sjo...@acm.org> - 11.27.1-20170705
+- Removed GSL module: it's now a separate (extension) package.
+  See https://dev.monetdb.org/hg/MonetDB-gsl/.
+- The PCRE library is now optional for systems that support POSIX regular
+  expressions.
+
+* Thu Jan 12 2017 Martin van Dinther <martin.van.dint...@monetdbsolutions.com> 
- 11.27.1-20170705
+- Added 5 new sys schema tables: function_languages, function_types,
+  key_types, index_types and privilege_codes.  They are pre-loaded with
+  static content and contain descriptive names for the various integer
+  type and code values.  See also sql/scripts/51_sys_schema_extension.sql
+
 * Thu Sep 22 2016 Sjoerd Mullender <sjo...@acm.org> - 11.25.1-20161214
 - The Perl, PHP, and Python clients, and the JDBC driver each now have
   their own repositories and release cycles.  The Python client is
diff --git a/ChangeLog.Jul2017 b/ChangeLog.Jul2017
--- a/ChangeLog.Jul2017
+++ b/ChangeLog.Jul2017
@@ -1,27 +1,3 @@
 # ChangeLog file for devel
 # This file is updated with Maddlog
 
-* Thu Apr 13 2017 Mark Raasveldt <m.raasve...@cwi.nl>
-- Added a new server-side protocol implementation. The new protocol
-  is backwards compatible with the old protocol. Clients can choose
-  whether they want to use the old or the new protocol during the initial
-  handshake with the server. The new protocol 

MonetDB: jitudf - For numeric columns, create result BATs immedi...

[Mark Raasveldt]

Changeset: e1d0cb52187a for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1d0cb52187a
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/capi/cheader.h
Branch: jitudf
Log Message:

For numeric columns, create result BATs immediately instead of manually 
allocating a region and then assigning that region to a BAT.

This is to allow the GDK to choose whether to malloc or mmap for a specific 
BAT, rather than always mallocing.


diffs (213 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -195,6 +195,27 @@ static void *wrapped_GDK_zalloc_nojump(s
return add_allocated_region(ptr);
 }
 
+#define GENERATE_NUMERIC_FUNCTIONS(type, tpename) \
+   static void tpename##_initialize(struct cudf_data_struct_##tpename 
*self,  \
+size_t 
count) \
+   {   
   \
+   BAT* b; 
   \
+   if (self->bat) {
   \
+   BBPunfix(((BAT*)self->bat)->batCacheid);
   \
+   self->bat = NULL;   
   \
+   }   
   \
+   b = COLnew(0, TYPE_##tpename, count, TRANSIENT);
   \
+   if (!b) {   
   \
+   longjmp(jump_buffer[THRgettid()], 2);   
   \
+   }   
   \
+   self->bat = (void*) b;  
   \
+   self->count = count;
   \
+   self->data = (type*) b->theap.base; 
   \
+   BATsetcount(b, count);  
   \
+   }   
   \
+   static int tpename##_is_null(type value) { return value == 
tpename##_nil; }
+
+
 #define GENERATE_BASE_HEADERS(type, tpename)   
\
static int tpename##_is_null(type value);   
   \
static void tpename##_initialize(struct cudf_data_struct_##tpename 
*self,  \
@@ -208,13 +229,14 @@ static void *wrapped_GDK_zalloc_nojump(s
GENERATE_BASE_HEADERS(tpe, tpename);
   \
static int tpename##_is_null(tpe value) { return value == 
tpename##_nil; }
 
-GENERATE_BASE_FUNCTIONS(bte, bte);
-GENERATE_BASE_FUNCTIONS(sht, sht);
-GENERATE_BASE_FUNCTIONS(int, int);
-GENERATE_BASE_FUNCTIONS(lng, lng);
-GENERATE_BASE_FUNCTIONS(flt, flt);
-GENERATE_BASE_FUNCTIONS(dbl, dbl);
-GENERATE_BASE_FUNCTIONS(oid, oid);
+GENERATE_NUMERIC_FUNCTIONS(bit, bit);
+GENERATE_NUMERIC_FUNCTIONS(bte, bte);
+GENERATE_NUMERIC_FUNCTIONS(sht, sht);
+GENERATE_NUMERIC_FUNCTIONS(int, int);
+GENERATE_NUMERIC_FUNCTIONS(lng, lng);
+GENERATE_NUMERIC_FUNCTIONS(flt, flt);
+GENERATE_NUMERIC_FUNCTIONS(dbl, dbl);
+GENERATE_NUMERIC_FUNCTIONS(oid, oid);
 
 GENERATE_BASE_HEADERS(char *, str);
 GENERATE_BASE_HEADERS(cudf_data_date, date);
@@ -226,7 +248,7 @@ static void blob_initialize(struct cudf_
 
 #define GENERATE_BAT_INPUT_BASE(tpe)   
\
struct cudf_data_struct_##tpe *bat_data =   
   \
-   GDKmalloc(sizeof(struct cudf_data_struct_##tpe));   
   \
+   GDKzalloc(sizeof(struct cudf_data_struct_##tpe));   
   \
if (!bat_data) {
   \
msg = createException(MAL, "cudf.eval", MAL_MALLOC_FAIL);   
   \
goto wrapup;
   \
@@ -235,6 +257,7 @@ static void blob_initialize(struct cudf_
bat_data->is_null = tpe##_is_null;  
   \
bat_data->scale =   
   \
argnode ? pow(10, ((sql_arg *)argnode->data)->type.scale) : 1;  
   \
+   bat_data->bat = NULL;   
   \
bat_data->initialize = (void (*)(void *, size_t))tpe##_initialize;
 
 #define GENERATE_BAT_INPUT(b, tpe) 
\
@@ -282,7 +305,7 @@ static void blob_initialize(struct cudf_
 
 #define GENERATE_BAT_OUTPUT_BASE(tpe)  

MonetDB: stratified_sampling - Merge with default.

[Mark Raasveldt]

/leaks/Tests/select1.stable.out.int128
sql/test/leaks/Tests/select2.stable.out
sql/test/leaks/Tests/select2.stable.out.int128
sql/test/leaks/Tests/temp1.stable.out
sql/test/leaks/Tests/temp1.stable.out.int128
sql/test/mergetables/Tests/mergequery.stable.out
sql/test/pg_regress/Tests/alter_table.stable.err
sql/test/pg_regress/Tests/alter_table.stable.out
sql/test/pg_regress/Tests/insert.stable.err
tools/merovingian/ChangeLog-Archive
tools/merovingian/ChangeLog.Jul2017
tools/merovingian/daemon/Makefile.ag
vertoo.data
Branch: stratified_sampling
Log Message:

Merge with default.


diffs (truncated from 46612 to 300 lines):

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -63,6 +63,10 @@ lex.yy.c
 *.dll
 
 # other files we don't want
+GTAGS
+gtags.files
+GPATH
+GRTAGS
 TAGS
 tags
 *.pyo
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -673,3 +673,5 @@ c57454c6c6f6e77fbbdf70de32a7cc845b67d1dd
 05f4e62bc67911a25f19be7fe742009ab1ee41c7 Dec2016_SP4_release
 8b3d3f7c487cb6e34dab1167797a61862cba2124 Dec2016_23
 8b3d3f7c487cb6e34dab1167797a61862cba2124 Dec2016_SP5_release
+e6552cdd3180bcd35f959954807cab986a6042a5 Jul2017_1
+e6552cdd3180bcd35f959954807cab986a6042a5 Jul2017_release
diff --git a/ChangeLog-Archive b/ChangeLog-Archive
--- a/ChangeLog-Archive
+++ b/ChangeLog-Archive
@@ -1,6 +1,31 @@
 # DO NOT EDIT THIS FILE -- MAINTAINED AUTOMATICALLY
 # This file contains past ChangeLog entries
 
+* Thu Apr 13 2017 Mark Raasveldt <m.raasve...@cwi.nl> - 11.27.1-20170705
+- Added a new server-side protocol implementation. The new protocol
+  is backwards compatible with the old protocol. Clients can choose
+  whether they want to use the old or the new protocol during the initial
+  handshake with the server. The new protocol is a binary column-based
+  protocol that is significantly faster than the old protocol when
+  transferring large result sets. In addition, the new protocol supports
+  compression using Snappy or LZ4.
+
+* Mon Feb 20 2017 Sjoerd Mullender <sjo...@acm.org> - 11.27.1-20170705
+- Moved the sphinx extension module to its own repository.
+  See https://dev.monetdb.org/hg/MonetDB-sphinx/.
+
+* Thu Feb 16 2017 Sjoerd Mullender <sjo...@acm.org> - 11.27.1-20170705
+- Removed GSL module: it's now a separate (extension) package.
+  See https://dev.monetdb.org/hg/MonetDB-gsl/.
+- The PCRE library is now optional for systems that support POSIX regular
+  expressions.
+
+* Thu Jan 12 2017 Martin van Dinther <martin.van.dint...@monetdbsolutions.com> 
- 11.27.1-20170705
+- Added 5 new sys schema tables: function_languages, function_types,
+  key_types, index_types and privilege_codes.  They are pre-loaded with
+  static content and contain descriptive names for the various integer
+  type and code values.  See also sql/scripts/51_sys_schema_extension.sql
+
 * Thu Sep 22 2016 Sjoerd Mullender <sjo...@acm.org> - 11.25.1-20161214
 - The Perl, PHP, and Python clients, and the JDBC driver each now have
   their own repositories and release cycles.  The Python client is
diff --git a/ChangeLog.Jul2017 b/ChangeLog.Jul2017
--- a/ChangeLog.Jul2017
+++ b/ChangeLog.Jul2017
@@ -1,27 +1,3 @@
 # ChangeLog file for devel
 # This file is updated with Maddlog
 
-* Thu Apr 13 2017 Mark Raasveldt <m.raasve...@cwi.nl>
-- Added a new server-side protocol implementation. The new protocol
-  is backwards compatible with the old protocol. Clients can choose
-  whether they want to use the old or the new protocol during the initial
-  handshake with the server. The new protocol is a binary column-based
-  protocol that is significantly faster than the old protocol when
-  transferring large result sets. In addition, the new protocol supports
-  compression using Snappy or LZ4.
-
-* Mon Feb 20 2017 Sjoerd Mullender <sjo...@acm.org>
-- Moved the sphinx extension module to its own repository.
-  See https://dev.monetdb.org/hg/MonetDB-sphinx/.
-
-* Thu Feb 16 2017 Sjoerd Mullender <sjo...@acm.org>
-- Removed GSL module: it's now a separate (extension) package.
-  See https://dev.monetdb.org/hg/MonetDB-gsl/.
-- The PCRE library is now optional for systems that support POSIX regular
-  expressions.
-
-* Thu Jan 12 2017 Martin van Dinther <martin.van.dint...@monetdbsolutions.com>
-- Added 5 new sys schema tables: function_languages, function_types,
-  key_types, index_types and privilege_codes.  They are pre-loaded with
-  static content and contain descriptive names for the various integer
-  type and code values.  See also sql/scripts/51_sys_schema_extension.sql
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -132,7 +132,7 @@ Vendor: MonetDB BV <i...@monetdb.org>
 Group: Applications/Databases
 License: MPLv2.0
 URL: https://www.monetdb.org/
-Source: 
https://www.monetdb.org/downloads/sources/Dec2016-SP5/%{name}-%{version}

MonetDB: jitudf - Compilation fixes.

[Mark Raasveldt]

Changeset: 04ab6a1a1900 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=04ab6a1a1900
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Compilation fixes.


diffs (11 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -95,6 +95,7 @@ static void handler(int sig, siginfo_t *
int tid = THRgettid();
 
(void)sig;
+   (void)si;
(void)unused;
 
longjmp(jump_buffer[tid], 1);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - When directly overwriting the heap storage of ...

[Mark Raasveldt]

Changeset: b7f43b0e47c8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b7f43b0e47c8
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

When directly overwriting the heap storage of a BAT do not allocate space for 
"count" variables.


diffs (256 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -1281,24 +1281,17 @@ static str CUDFeval(Client cntxt, MalBlk
  "Data has 
different cardinalities.");
goto wrapup;
}
-   b = COLnew(0, bat_type, count, TRANSIENT);
-   if (!b) {
-   msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
-   goto wrapup;
-   }
-   b->tnil = 0;
-   b->tnonil = 0;
-   b->tkey = 0;
-   b->tsorted = 0;
-   b->trevsorted = 0;
-
if (bat_type == TYPE_bit || bat_type == TYPE_bte ||
bat_type == TYPE_sht || bat_type == TYPE_int ||
bat_type == TYPE_oid || bat_type == TYPE_lng ||
bat_type == TYPE_flt || bat_type == TYPE_dbl) {
+   b = COLnew(0, bat_type, 0, TRANSIENT);
+   if (!b) {
+   msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
+   goto wrapup;
+   }
// we pass the data we have directly into the BAT for 
simple
-   // numeric
-   // types
+   // numeric types
// this way we do not need to copy any data 
unnecessarily
// free the current (initial) storage
GDKfree(b->theap.base);
@@ -1311,115 +1304,127 @@ static str CUDFeval(Client cntxt, MalBlk
b->batCount = (BUN)count;
b->batCapacity = (BUN)count;
b->batCopiedtodisk = false;
-   } else if (bat_type == TYPE_date) {
-   date *baseptr = (date *)Tloc(b, 0);
-   cudf_data_date *source_base = (cudf_data_date *)data;
-   for (j = 0; j < count; j++) {
-   baseptr[j] = date_from_data(source_base + j);
-   }
-   BATsetcount(b, count);
-   GDKfree(data);
-   } else if (bat_type == TYPE_daytime) {
-   daytime *baseptr = (daytime *)Tloc(b, 0);
-   cudf_data_time *source_base = (cudf_data_time *)data;
-   for (j = 0; j < count; j++) {
-   baseptr[j] = time_from_data(source_base + j);
-   }
-   BATsetcount(b, count);
-   GDKfree(data);
-   } else if (bat_type == TYPE_timestamp) {
-   timestamp *baseptr = (timestamp *)Tloc(b, 0);
-   cudf_data_timestamp *source_base = (cudf_data_timestamp 
*)data;
-   for (j = 0; j < count; j++) {
-   baseptr[j] = timestamp_from_data(source_base + 
j);
+   } else {
+   b = COLnew(0, bat_type, count, TRANSIENT);
+   if (!b) {
+   msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
+   goto wrapup;
}
-   BATsetcount(b, count);
-   GDKfree(data);
-   } else if (bat_type == TYPE_str) {
-   char **source_base = (char **)data;
-   for (j = 0; j < count; j++) {
-   const char *ptr = source_base[j];
-   if (!ptr) {
-   ptr = str_nil;
-   }
-   if (BUNappend(b, ptr, FALSE) != GDK_SUCCEED) {
-   msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
-   goto wrapup;
+   if (bat_type == TYPE_date) {
+   date *baseptr = (date *)Tloc(b, 0);
+   cudf_data_date *source_base = (cudf_data_date 
*)data;
+   for (j = 0; j < count; j++) {
+   baseptr[j] = date_from_data(source_base 
+ j);
}
-   }
-   GDKfree(data);
-   } else if (bat_type == TYPE_blob || bat_type == TYPE_sqlblob) {
-  

MonetDB: jitudf - Fix compilation issues.

[Mark Raasveldt]

Changeset: b122b265f4de for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b122b265f4de
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Fix compilation issues.


diffs (37 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -40,7 +40,6 @@ static char *mprotect_region(void *addr,
 static char *clear_mprotect(void *addr, size_t len);
 
 static allocated_region *allocated_regions[THREADS];
-static mprotected_region **actual_mprotected_regions[THREADS];
 static jmp_buf jump_buffer[THREADS];
 
 typedef char *(*jitted_function)(void **inputs, void **outputs,
@@ -111,8 +110,6 @@ static char *mprotect_region(void *addr,
 mprotected_region 
**regions)
 {
mprotected_region *region;
-   int pagesize;
-   void *page_begin;
if (len == 0)
return NULL;
 
@@ -410,7 +407,6 @@ static str CUDFeval(Client cntxt, MalBlk
 
(void)cntxt;
 
-   actual_mprotected_regions[tid] = 
allocated_regions[tid] = NULL;
 
// we need to be able to catch segfaults and bus errors
@@ -1244,8 +1240,6 @@ static str CUDFeval(Client cntxt, MalBlk
regions = next;
}
 
-   actual_mprotected_regions[tid] = NULL;
-
// clear the signal handlers
if (sigaction(SIGSEGV, , NULL) == -1 ||
sigaction(SIGBUS, , NULL) == -1) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Copy data that is not page aligned because we ...

[Mark Raasveldt]

Changeset: bfcd5e0e842f for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bfcd5e0e842f
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Copy data that is not page aligned because we can only protect entire pages.


diffs (truncated from 301 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
@@ -49,3 +49,4 @@ INSERT INTO strings VALUES ('ello'), ('o
 SELECT capi04(i) FROM strings;
 
 ROLLBACK;
+
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -32,9 +32,6 @@ typedef struct _mprotected_region {
void *addr;
size_t len;
 
-   void *actual_addr;
-   size_t actual_len;
-
struct _mprotected_region *next;
 } mprotected_region;
 
@@ -96,58 +93,18 @@ static bool WriteTextToFile(FILE *f, con
 
 static void handler(int sig, siginfo_t *si, void *unused)
 {
-   int actually_protected_area = false;
-   mprotected_region *found_region = NULL;
int tid = THRgettid();
 
(void)sig;
(void)unused;
-   // we caught a segfault or bus error
-   // this can be either because
-   // (1) the function accessed a protected piece of memory
-   // (2) the function caused a segfault by e.g. dereferencing a NULL 
pointer
-   // in the first case, this *might* be a valid memory access
-   // this is because we are required to align our mprotects on page 
boundaries
-   // thus sometimes we mprotect a page where only part of the page
-   // should actually be protected. Thus for this case we check if the 
access
-   // was actually an error
-   if (actual_mprotected_regions[tid]) {
-   mprotected_region *region = *actual_mprotected_regions[tid];
-   while (region) {
-   if (si->si_addr >= region->addr &&
-   (char *)si->si_addr <= (char *)region->addr + 
region->len) {
-   // the address belongs to this mprotected region
-   found_region = region;
-   if (si->si_addr >= region->actual_addr &&
-   (char *)si->si_addr <=
-   (char *)region->actual_addr + 
region->actual_len) {
-   // and the address is actually supposed 
to be protected
-   actually_protected_area = true;
-   break;
-   }
-   }
-   region = region->next;
-   }
-   }
 
-   if (found_region && !actually_protected_area) {
-   // this is NOT an actually protected area
-   // thus the segfault/bus error is invalid
-   // the nasty part here is that we have to unprotect the entire 
page now
-   // thus opening us up to future modifications of the data
+   longjmp(jump_buffer[tid], 1);
+}
 
-   mprotected_region *region = *actual_mprotected_regions[tid];
-   while (region) {
-   if (si->si_addr >= region->addr &&
-   (char *)si->si_addr <= (char *)region->addr + 
region->len) {
-   // the address belongs to this mprotected region
-   clear_mprotect(found_region->addr, 
found_region->len);
-   }
-   region = region->next;
-   }
-   } else {
-   longjmp(jump_buffer[tid], 1);
-   }
+static bool can_mprotect_region(void* addr) {
+   int pagesize = getpagesize();
+   void* page_begin = (void *)((size_t)addr - (size_t)addr % pagesize);
+   return page_begin == addr;
 }
 
 static char *mprotect_region(void *addr, size_t len,
@@ -156,23 +113,10 @@ static char *mprotect_region(void *addr,
mprotected_region *region;
int pagesize;
void *page_begin;
-   void *actual_addr = addr;
-   size_t actual_len = len;
if (len == 0)
return NULL;
 
-   // we must mprotect an entire page
-   // thus here we check if the region is page-aligned
-   // and if it is not, we page-align it
-   pagesize = getpagesize();
-   page_begin = (void *)((size_t)addr - (size_t)addr % pagesize);
-   if (page_begin != addr) {
-   // data is not page-aligned
-   len += ((size_t)addr - (size_t)page_begin);
-   addr = page_begin;
-   }
-   // page align len
-   len = len % pagesize == 0 ? len : len - len % 

MonetDB: Jul2017 - Release the GIL during loopback queries.

[Mark Raasveldt]

Changeset: a144c611 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a144c611
Modified Files:
sql/backends/monet5/UDF/pyapi/connection.c
Branch: Jul2017
Log Message:

Release the GIL during loopback queries.


diffs (25 lines):

diff --git a/sql/backends/monet5/UDF/pyapi/connection.c 
b/sql/backends/monet5/UDF/pyapi/connection.c
--- a/sql/backends/monet5/UDF/pyapi/connection.c
+++ b/sql/backends/monet5/UDF/pyapi/connection.c
@@ -33,12 +33,18 @@ static PyObject *_connection_execute(Py_
char *res = NULL;
char *query;
 #ifndef IS_PY3K
-   query = ((PyStringObject *)args)->ob_sval;
+   query = GDKstrdup(((PyStringObject *)args)->ob_sval);
 #else
-   query = PyUnicode_AsUTF8(args);
+   query = GDKstrdup(PyUnicode_AsUTF8(args));
 #endif
-
+   if (!query) {
+   PyErr_Format(PyExc_Exception, MAL_MALLOC_FAIL);
+   return NULL;
+   }
+Py_BEGIN_ALLOW_THREADS;
res = _connection_query(self->cntxt, query, );
+Py_END_ALLOW_THREADS;
+   GDKfree(query);
if (res != MAL_SUCCEED) {
PyErr_Format(PyExc_Exception, "SQL Query Failed: %s",
 (res ? res : ""));
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jul2017 - Support boolean types in loopback queries.

[Mark Raasveldt]

Changeset: 8f19f442c63c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8f19f442c63c
Added Files:
sql/backends/monet5/Tests/pyapi34.sql
sql/backends/monet5/Tests/pyapi34.stable.err
sql/backends/monet5/Tests/pyapi34.stable.out
Modified Files:
sql/backends/monet5/Tests/All
sql/backends/monet5/UDF/pyapi/conversion.c
Branch: Jul2017
Log Message:

Support boolean types in loopback queries.


diffs (178 lines):

diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All
--- a/sql/backends/monet5/Tests/All
+++ b/sql/backends/monet5/Tests/All
@@ -53,6 +53,7 @@ HAVE_LIBPY?pyapi29
 HAVE_LIBPY?pyapi30
 HAVE_LIBPY?pyapi32
 HAVE_LIBPY?pyapi33
+HAVE_LIBPY?pyapi34
 
 HAVE_LIBPY?pyloader01
 HAVE_LIBPY?pyloader02
diff --git a/sql/backends/monet5/Tests/pyapi34.sql 
b/sql/backends/monet5/Tests/pyapi34.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi34.sql
@@ -0,0 +1,18 @@
+
+START TRANSACTION;
+
+CREATE TABLE booleans(a BOOLEAN);
+INSERT INTO booleans VALUES (1), (0), (1);
+
+CREATE FUNCTION pyapi34a(inp BOOLEAN) RETURNS BOOLEAN LANGUAGE PYTHON {
+   results = _conn.execute('SELECT * FROM booleans;')
+   return {'result': numpy.logical_xor(inp, results['a']) };
+};
+
+CREATE FUNCTION pyapi34b(inp BOOLEAN) RETURNS BOOLEAN LANGUAGE PYTHON {
+   return {'result': inp};
+};
+
+SELECT a, pyapi34a(a), pyapi34b(a) FROM booleans;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/pyapi34.stable.err 
b/sql/backends/monet5/Tests/pyapi34.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi34.stable.err
@@ -0,0 +1,35 @@
+stderr of test 'pyapi34` in directory 'sql/backends/monet5` itself:
+
+
+# 11:20:10 >  
+# 11:20:10 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37278" "--set" 
"mapi_usock=/var/tmp/mtest-61818/.s.monetdb.37278" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5" 
"--set" "embedded_py=true"
+# 11:20:10 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37278
+# cmdline opt  mapi_usock = /var/tmp/mtest-61818/.s.monetdb.37278
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5
+# cmdline opt  embedded_py = true
+# cmdline opt  gdk_debug = 536870922
+
+# 11:20:10 >  
+# 11:20:10 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-61818" "--port=37278"
+# 11:20:10 >  
+
+
+# 11:20:10 >  
+# 11:20:10 >  "Done."
+# 11:20:10 >  
+
diff --git a/sql/backends/monet5/Tests/pyapi34.stable.out 
b/sql/backends/monet5/Tests/pyapi34.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi34.stable.out
@@ -0,0 +1,86 @@
+stdout of test 'pyapi34` in directory 'sql/backends/monet5` itself:
+
+
+# 11:20:10 >  
+# 11:20:10 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37278" "--set" 
"mapi_usock=/var/tmp/mtest-61818/.s.monetdb.37278" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5" 
"--set" "embedded_py=true"
+# 11:20:10 >  
+
+# MonetDB 5 server v11.27.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:37278/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-61818/.s.monetdb.37278
+# MonetDB/SQL module loaded
+# MonetDB/Python2 module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 

MonetDB: stratified_sampling - Fix dll linkage issue.

[Mark Raasveldt]

Changeset: 65ad22367605 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=65ad22367605
Modified Files:
common/utils/mtwist.h
Branch: stratified_sampling
Log Message:

Fix dll linkage issue.


diffs (12 lines):

diff --git a/common/utils/mtwist.h b/common/utils/mtwist.h
--- a/common/utils/mtwist.h
+++ b/common/utils/mtwist.h
@@ -32,7 +32,7 @@
 #define _SEEN_MTWIST_H 1
 
 #ifdef NATIVE_WIN32
-#if !defined(LIBMUTILS) && !defined(LIBGDK) && !defined(LIBMEROUTIL)
+#if !defined(LIBMTWIST) && !defined(LIBMUTILS) && !defined(LIBGDK) && 
!defined(LIBMEROUTIL)
 #define mtwist_export extern __declspec(dllimport)
 #else
 #define mtwist_export extern __declspec(dllexport)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - Approve exports.

[Mark Raasveldt]

Changeset: da686b4181c1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=da686b4181c1
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
Branch: stratified_sampling
Log Message:

Approve exports.


diffs (73 lines):

diff --git a/clients/Tests/MAL-signatures.stable.out 
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -8270,8 +8270,10 @@ Ready.
 [ "sabaoth",   "marchScenario","command 
sabaoth.marchScenario(lang:str):void ","SABmarchScenario;",
"Publishes the given language as available for this server" ]
 [ "sabaoth",   "prelude",  "command sabaoth.prelude():void ",  
"SABprelude;",  "Initialise the sabaoth module" ]
 [ "sabaoth",   "retreatScenario",  "command 
sabaoth.retreatScenario(lang:str):void ",  "SABretreatScenario;",  
"Unpublishes the given language as available for this server"   ]
-[ "sample","subuniform",   "command sample.subuniform(b:bat[:any], 
s:lng):bat[:oid] ", "SAMPLEuniform;",   "Returns the oids of a uniform 
sample of size s"]
-[ "sample","subuniform",   "command sample.subuniform(b:bat[:any], 
p:dbl):bat[:oid] ", "SAMPLEuniform_dbl;",   "Returns the oids of a uniform 
sample of size = (p x count(b)), where 0 <= p <= 1.0"]
+[ "sample","subuniform",   "command sample.subuniform(b:bat[:any], 
s:lng):bat[:oid] ", "SAMPLEuniform;",   "Returns the oids of a uniform 
sample (without replacement) of size s"  ]
+[ "sample","subuniform",   "command sample.subuniform(b:bat[:any], 
p:dbl):bat[:oid] ", "SAMPLEuniform_dbl;",   "Returns the oids of a uniform 
sample (without replacement) of size = (p x count(b)), where 0 <= p <= 1.0" 
 ]
+[ "sample","subweighted",  "command sample.subweighted(b:bat[:any], s:lng, 
w:bat[:any]):bat[:oid] ",   "SAMPLEweighted;",  "Returns the oids of a 
weighted sample (without replacement) of size s" ]
+[ "sample","subweighted",  "command sample.subweighted(b:bat[:any], p:dbl, 
w:bat[:any]):bat[:oid] ",   "SAMPLEweighted_dbl;",  "Returns the oids of a 
weighted sample (without replacement) of size = (p x count(b)), where 0 <= p <= 
1.0" ]
 [ "shp",   "attach",   "pattern shp.attach(filename:str):void ",   
"SHPattach;",   "Register an ESRI Shapefile in the vault catalog"   ]
 [ "shp",   "import",   "pattern shp.import(fileid:int):void ", 
"SHPimport;",   "Import an ESRI Shapefile with given id into the vault" ]
 [ "shp",   "import",   "pattern shp.import(fileid:int, po:wkb):void ", 
"SHPpartialimport;","Partially import an ESRI Shapefile with given id into 
the vault"   ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128 
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -10632,8 +10632,10 @@ Ready.
 [ "sabaoth",   "marchScenario","command 
sabaoth.marchScenario(lang:str):void ","SABmarchScenario;",
"Publishes the given language as available for this server" ]
 [ "sabaoth",   "prelude",  "command sabaoth.prelude():void ",  
"SABprelude;",  "Initialise the sabaoth module" ]
 [ "sabaoth",   "retreatScenario",  "command 
sabaoth.retreatScenario(lang:str):void ",  "SABretreatScenario;",  
"Unpublishes the given language as available for this server"   ]
-[ "sample","subuniform",   "command sample.subuniform(b:bat[:any], 
s:lng):bat[:oid] ", "SAMPLEuniform;",   "Returns the oids of a uniform 
sample of size s"]
-[ "sample","subuniform",   "command sample.subuniform(b:bat[:any], 
p:dbl):bat[:oid] ", "SAMPLEuniform_dbl;",   "Returns the oids of a uniform 
sample of size = (p x count(b)), where 0 <= p <= 1.0"]
+[ "sample","subuniform",   "command sample.subuniform(b:bat[:any], 
s:lng):bat[:oid] ", "SAMPLEuniform;",   "Returns the oids of a uniform 
sample (without replacement) of size s"  ]
+[ "sample","subuniform",   "command sample.subuniform(b:bat[:any], 
p:dbl):bat[:oid] ", "SAMPLEuniform_dbl;",   "Returns the oids of a uniform 
sample (without replacement) of size = (p x count(b)), where 0 <= p <= 1.0" 
 ]
+[ "sample","subweighted",  "command sample.subweighted(b:bat[:any], s:lng, 
w:bat[:any]):bat[:oid] ",   "SAMPLEweighted;",  "Returns the oids of a 
weighted sample (without replacement) of size s" ]
+[ "sample","subweighted",  "command sample.subweighted(b:bat[:any], p:dbl, 
w:bat[:any]):bat[:oid] ",   "SAMPLEweighted_dbl;",  "Returns the oids of a 
weighted sample (without replacement) of size = (p x count(b)), where 0 <= p <= 
1.0" ]
 [ "shp",   "attach",   "pattern shp.attach(filename:str):void ",   
"SHPattach;",   "Register an ESRI Shapefile 

MonetDB: stratified_sampling - Fix compilation issues.

[Mark Raasveldt]

Changeset: f2d7422ecbfe for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f2d7422ecbfe
Modified Files:
common/utils/mtwist.h
gdk/gdk_sample.c
Branch: stratified_sampling
Log Message:

Fix compilation issues.


diffs (65 lines):

diff --git a/common/utils/mtwist.h b/common/utils/mtwist.h
--- a/common/utils/mtwist.h
+++ b/common/utils/mtwist.h
@@ -31,6 +31,17 @@
 #ifndef _SEEN_MTWIST_H
 #define _SEEN_MTWIST_H 1
 
+#ifdef NATIVE_WIN32
+#if !defined(LIBMUTILS) && !defined(LIBGDK) && !defined(LIBMEROUTIL)
+#define mtwist_export extern __declspec(dllimport)
+#else
+#define mtwist_export extern __declspec(dllexport)
+#endif
+#else
+#define mtwist_export extern
+#endif
+
+
 #define MTWIST_N 624
 #define MTWIST_M 397
 
@@ -53,15 +64,15 @@ struct mtwist_s {
 typedef struct mtwist_s mtwist;
 
 /* constructor */
-mtwist* mtwist_new(void);
+mtwist_export mtwist* mtwist_new(void);
 
 /* destructor */
-void mtwist_free(mtwist* mt);
+mtwist_export void mtwist_free(mtwist* mt);
 
 /* methods */
-void mtwist_seed(mtwist* mt, unsigned int seed);
-unsigned int mtwist_u32rand(mtwist* mt);
-double mtwist_drand(mtwist* mt);
-int mtwist_uniform_int(mtwist* mt, int a, int b);
+mtwist_export void mtwist_seed(mtwist* mt, unsigned int seed);
+mtwist_export unsigned int mtwist_u32rand(mtwist* mt);
+mtwist_export double mtwist_drand(mtwist* mt);
+mtwist_export int mtwist_uniform_int(mtwist* mt, int a, int b);
 
 #endif
diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -23,16 +23,13 @@
  * values that should be omitted from the sample.
  */
 
-#include 
-
 #include "monetdb_config.h"
 #include "gdk.h"
 #include "gdk_private.h"
 
-#include "mtwist.h"
+#include 
 
-#undef BATsample
-
+#include "mtwist.h"
 
 //TODO share these with gkd_firstn.c
 #define siftup(OPER, START, SWAP)  \
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - Revert parser change.

[Mark Raasveldt]

Changeset: b2b9d5737695 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b2b9d5737695
Modified Files:
sql/server/sql_parser.y
Branch: stratified_sampling
Log Message:

Revert parser change.


diffs (59 lines):

diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -4669,25 +4669,45 @@ literal:
  sql_find_subtype(, "boolean", 0, 0 );
  $$ = _newAtomNode( atom_bool(SA, , TRUE)); }
  | sqlDBL
-   {
-   sql_subtype *t= sql_bind_localtype("dbl");
+   { char *s = strip_extra_zeros(sa_strdup(SA, $1));
+ char *dot = strchr(s, '.');
+ int digits = _strlen(s) - 1;
+ int scale = digits - (int) (dot-s);
+ sql_subtype t;
+
+ if (digits <= 0)
+   digits = 1;
+ if (digits <= MAX_DEC_DIGITS) {
+   double val = strtod($1,NULL);
+#ifdef HAVE_HGE
+   hge value = decimal_from_str(s, NULL);
+#else
+   lng value = decimal_from_str(s, NULL);
+#endif
+
+   if (*s == '+' || *s == '-')
+   digits --;
+   sql_find_subtype(, "decimal", digits, scale );
+   $$ = _newAtomNode( atom_dec(SA, , value, val));
+  } else {
+   char *p = $1;
double val;
-   char *p = $1;
-   
+
errno = 0;
val = strtod($1,);
-   if(errno) {
-   char *msg = sql_message("\b22003!double value 
could not be parsed (%s)", $1);
-   
+   if (p == $1 || val == dbl_nil || (errno == ERANGE && 
(val < -1 || val > 1))) {
+   char *msg = sql_message("\b22003!double value 
too large or not a number (%s)", $1);
+
yyerror(m, msg);
_DELETE(msg);
$$ = NULL;
YYABORT;
}
-   
-   $$ = _newAtomNode( atom_float(SA, t, val));
+   sql_find_subtype(, "double", 51, 0 );
+   $$ = _newAtomNode(atom_float(SA, , val));
+  }
}
- ;
+;
 
 interval_expression:
INTERVAL opt_sign string interval_qualifier { 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - Add testcase.

[Mark Raasveldt]

Changeset: 0307b09d883d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0307b09d883d
Added Files:
sql/backends/monet5/Tests/sample06.sql
sql/backends/monet5/Tests/sample06.stable.err
sql/backends/monet5/Tests/sample06.stable.out
Modified Files:
sql/backends/monet5/Tests/All
Branch: stratified_sampling
Log Message:

Add testcase.


diffs (146 lines):

diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All
--- a/sql/backends/monet5/Tests/All
+++ b/sql/backends/monet5/Tests/All
@@ -6,6 +6,7 @@ sample02
 sample03
 sample04
 HAVE_LIBPY?sample05
+sample06
 
 HAVE_LIBR_WIN32?rapi00
 HAVE_LIBR_WIN32?rapi01
diff --git a/sql/backends/monet5/Tests/sample06.sql 
b/sql/backends/monet5/Tests/sample06.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample06.sql
@@ -0,0 +1,10 @@
+# aggregate as weights
+
+START TRANSACTION;
+
+CREATE TABLE vals(w DOUBLE, value INTEGER);
+INSERT INTO vals VALUES (1, 100), (0, 50);
+
+SELECT * FROM vals SAMPLE 0.5 USING WEIGHTS SUM(w);
+
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/sample06.stable.err 
b/sql/backends/monet5/Tests/sample06.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample06.stable.err
@@ -0,0 +1,38 @@
+stderr of test 'sample06` in directory 'sql/backends/monet5` itself:
+
+
+# 13:34:43 >  
+# 13:34:43 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37944" "--set" 
"mapi_usock=/var/tmp/mtest-72078/.s.monetdb.37944" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5" 
"--set" "embedded_py=true"
+# 13:34:43 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37944
+# cmdline opt  mapi_usock = /var/tmp/mtest-72078/.s.monetdb.37944
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5
+# cmdline opt  embedded_py = true
+# cmdline opt  gdk_debug = 536870922
+
+# 13:34:43 >  
+# 13:34:43 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-72078" "--port=37944"
+# 13:34:43 >  
+
+MAPI  = (monetdb) /var/tmp/mtest-72078/.s.monetdb.37944
+QUERY = SELECT * FROM vals SAMPLE 0.5 USING WEIGHTS SUM(w);
+ERROR = !Weights could not be resolved correctly.
+
+# 13:34:43 >  
+# 13:34:43 >  "Done."
+# 13:34:43 >  
+
diff --git a/sql/backends/monet5/Tests/sample06.stable.out 
b/sql/backends/monet5/Tests/sample06.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample06.stable.out
@@ -0,0 +1,72 @@
+stdout of test 'sample06` in directory 'sql/backends/monet5` itself:
+
+
+# 13:34:43 >  
+# 13:34:43 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37944" "--set" 
"mapi_usock=/var/tmp/mtest-72078/.s.monetdb.37944" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5" 
"--set" "embedded_py=true"
+# 13:34:43 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:37944/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-72078/.s.monetdb.37944
+# MonetDB/SQL module loaded
+# MonetDB/Python2 module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 27_rejects.sql
+# loading sql script: 39_analytics.sql
+# loading sql script: 39_analytics_hge.sql
+# loading sql script: 40_json.sql
+# loading sql script: 40_json_hge.sql
+# loading sql script: 

MonetDB: stratified_sampling - Merge with default.

[Mark Raasveldt]

Changeset: 004835f67db3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=004835f67db3
Added Files:
sql/test/BugTracker-2017/Tests/default-insert-update.Bug-6249.sql
Removed Files:
sql/test/pg_regress/Tests/oid.stable.out.oid32
sql/test/pg_regress/Tests/without_oid.stable.out.oid32
Modified Files:
sql/server/rel_dump.c
sql/server/rel_optimizer.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/test/BugTracker-2017/Tests/All

sql/test/BugTracker-2017/Tests/crash-select_after_MAL_error.Bug-6332.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
Branch: stratified_sampling
Log Message:

Merge with default.


diffs (truncated from 19763 to 300 lines):

diff --git a/sql/server/rel_dump.c b/sql/server/rel_dump.c
--- a/sql/server/rel_dump.c
+++ b/sql/server/rel_dump.c
@@ -1182,7 +1182,9 @@ rel_read(mvc *sql, char *r, int *pos, li
 
gexps = read_exps(sql, nrel, NULL, NULL, r, pos, '[', 0);
skipWS(r, pos);
-   exps = read_exps(sql, nrel, NULL, NULL, r, pos, '[', 1);
+   exps = read_exps(sql, nrel, NULL, gexps, r, pos, '[', 1);
+   if (!exps)
+   return NULL;
 
rel = rel_groupby(sql, nrel, gexps);
rel->exps = exps;
diff --git a/sql/server/rel_optimizer.c b/sql/server/rel_optimizer.c
--- a/sql/server/rel_optimizer.c
+++ b/sql/server/rel_optimizer.c
@@ -2461,7 +2461,8 @@ static sql_exp *
 math_unsafe_fixup( mvc *sql, sql_exp *e, sql_exp *cond, int lr )
 {
list *args = e->l;
-   if (args->h->next)
+
+   if (args && args->h && args->h->next)
return math_unsafe_fixup_binop(sql, e, args->h->data, 
args->h->next->data, cond, lr);
else
return math_unsafe_fixup_unop(sql, e, args->h->data, cond, lr);
diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c
--- a/sql/server/rel_updates.c
+++ b/sql/server/rel_updates.c
@@ -23,6 +23,12 @@ insert_value(mvc *sql, sql_column *c, sq
 {
if (s->token == SQL_NULL) {
return exp_atom(sql->sa, atom_general(sql->sa, >type, NULL));
+   } else if (s->token == SQL_DEFAULT) {
+   if (c->def) {
+   return rel_parse_val(sql, sa_message(sql->sa, "select 
CAST(%s AS %s);", c->def, c->type.type->sqlname), sql->emode);
+   } else {
+   return sql_error(sql, 02, "INSERT INTO: column '%s' has 
no valid default value", c->base.name);
+   }
} else {
int is_last = 0;
exp_kind ek = {type_value, card_value, FALSE};
@@ -971,11 +977,19 @@ update_table(mvc *sql, dlist *qname, dli
int status = sql->session->status;
exp_kind ek = {type_value, 
(single)?card_column:card_relation, FALSE};
 
-   if (single) 
+   if(single && a->token == SQL_DEFAULT) {
+   char *colname = 
assignment->h->next->data.sval;
+   sql_column *col = mvc_bind_column(sql, 
t, colname);
+   if (col->def) {
+   v = rel_parse_val(sql, 
sa_message(sql->sa, "select CAST(%s AS %s);", col->def, 
col->type.type->sqlname), sql->emode);
+   } else {
+   return sql_error(sql, 02, 
"UPDATE: column '%s' has no valid default value", col->base.name);
+   }
+   } else if (single) {
v = rel_value_exp(sql, _val, a, 
sql_sel, ek);
-   else
+   } else {
rel_val = rel_subquery(sql, NULL, a, 
ek, APPLY_JOIN);
-
+   }
if (!v) {
sql->errstr[0] = 0;
sql->session->status = status;
diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -2893,6 +2893,7 @@ simple_atom:
 
 insert_atom:
 simple_atom
+ |  DEFAULT{ $$ = _symbol_create(SQL_DEFAULT, NULL ); }
  ;
 
 value:
@@ -2913,7 +2914,12 @@ assignment_commalist:
  ;
 
 assignment:
-   column '=' search_condition
+   column '=' DEFAULT
+   { dlist *l = L();
+ append_symbol(l, _symbol_create(SQL_DEFAULT, NULL ) );
+ append_string(l, $1);
+ $$ = _symbol_create_list( SQL_ASSIGN, l); }
+ |  column '=' search_condition
{ dlist *l = L();
  append_symbol(l, $3 );
  append_string(l, $1);
diff --git 

MonetDB: stratified_sampling - Approve tests.

[Mark Raasveldt]

Changeset: 9ff1a0391c9c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9ff1a0391c9c
Added Files:
sql/backends/monet5/Tests/sample00.stable.err
sql/backends/monet5/Tests/sample00.stable.out
sql/backends/monet5/Tests/sample01.stable.err
sql/backends/monet5/Tests/sample01.stable.out
sql/backends/monet5/Tests/sample02.stable.err
sql/backends/monet5/Tests/sample02.stable.out
sql/backends/monet5/Tests/sample03.stable.err
sql/backends/monet5/Tests/sample03.stable.out
sql/backends/monet5/Tests/sample05.stable.err
sql/backends/monet5/Tests/sample05.stable.out
Modified Files:
sql/backends/monet5/Tests/All
Branch: stratified_sampling
Log Message:

Approve tests.


diffs (truncated from 613 to 300 lines):

diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All
--- a/sql/backends/monet5/Tests/All
+++ b/sql/backends/monet5/Tests/All
@@ -1,5 +1,12 @@
 optimizers
 
+sample00
+sample01
+sample02
+sample03
+sample04
+HAVE_LIBPY?sample05
+
 HAVE_LIBR_WIN32?rapi00
 HAVE_LIBR_WIN32?rapi01
 HAVE_LIBR_WIN32?rapi02
diff --git a/sql/backends/monet5/Tests/sample00.stable.err 
b/sql/backends/monet5/Tests/sample00.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample00.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'sample00` in directory 'sql/backends/monet5` itself:
+
+
+# 17:16:50 >  
+# 17:16:50 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30118" "--set" 
"mapi_usock=/var/tmp/mtest-15360/.s.monetdb.30118" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5"
+# 17:16:50 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 30118
+# cmdline opt  mapi_usock = /var/tmp/mtest-15360/.s.monetdb.30118
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5
+# cmdline opt  gdk_debug = 536870922
+
+# 17:16:50 >  
+# 17:16:50 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-15360" "--port=30118"
+# 17:16:50 >  
+
+
+# 17:16:50 >  
+# 17:16:50 >  "Done."
+# 17:16:50 >  
+
diff --git a/sql/backends/monet5/Tests/sample00.stable.out 
b/sql/backends/monet5/Tests/sample00.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample00.stable.out
@@ -0,0 +1,64 @@
+stdout of test 'sample00` in directory 'sql/backends/monet5` itself:
+
+
+# 17:16:50 >  
+# 17:16:50 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30118" "--set" 
"mapi_usock=/var/tmp/mtest-15360/.s.monetdb.30118" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/Users/myth/opt/var/MonetDB/mTests_sql_backends_monet5"
+# 17:16:50 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:30118/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-15360/.s.monetdb.30118
+# MonetDB/SQL module loaded
+
+Ready.
+
+# 17:16:50 >  
+# 17:16:50 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-15360" "--port=30118"
+# 17:16:50 >  
+
+#START TRANSACTION;
+#CREATE TABLE vals(w DOUBLE, value INTEGER);
+#INSERT INTO vals VALUES (1, 100), (0, 50);
+[ 2]
+#SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
+% sys.vals,sys.vals # table_name
+% w,   value # name
+% double,  int # type
+% 24,  3 # length
+[ 1,   100 ]
+#SELECT * FROM vals SAMPLE 0.5 USING WEIGHTS w;
+% sys.vals,sys.vals # table_name
+% w,   value # name
+% double,  int # type
+% 24,  3 # length
+[ 1,   100 ]
+#SELECT * FROM vals SAMPLE 0.5 USING WEIGHTS (1 - w);
+% sys.vals,sys.vals # table_name
+% w,   value # name
+% double,  int # type
+% 24,  2 # length
+[ 0,   50  ]
+#ROLLBACK;
+#START TRANSACTION;
+#CREATE TABLE vals(w DECIMAL(10,3), value INTEGER);
+#INSERT INTO vals VALUES (10.77, 100), (0, 50);
+[ 2]
+#SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
+% sys.vals,sys.vals # table_name
+% w,   value # name
+% decimal, int # type
+% 12,  3 # length
+[ 10.770,  100 ]

MonetDB: stratified_sampling - Don't optimize away columns that ...

[Mark Raasveldt]

Changeset: 67bddbf2ee39 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=67bddbf2ee39
Modified Files:
gdk/gdk_sample.c
sql/backends/monet5/Tests/sample00.sql
sql/backends/monet5/Tests/sample01.sql
sql/backends/monet5/Tests/sample02.sql
sql/backends/monet5/Tests/sample03.sql
sql/server/rel_optimizer.c
sql/server/rel_rel.c
sql/server/rel_select.c
Branch: stratified_sampling
Log Message:

Don't optimize away columns that we need.


diffs (198 lines):

diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -255,6 +255,8 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
 
oid minoid = b->hseqbase;
 
+   ERRORcheck(n > BATcount(b), "BATsample: Sample size bigger than 
table!", NULL);
+
BATcheck(b, "BATsample", NULL);
BATcheck(w, "BATsample", NULL);
 
@@ -312,6 +314,10 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
}
oids[i] = (oid)(j + minoid);
keys[i] = pow(mtwist_drand(mt_rng), 1.0 / w_ptr[j]);//TODO cast 
1.0 to dbl?
+   if (keys[i] == 1) {
+   GDKerror("BATsample: weight overflow\n");
+   goto bailout;
+   }
i++;
}
 
diff --git a/sql/backends/monet5/Tests/sample00.sql 
b/sql/backends/monet5/Tests/sample00.sql
--- a/sql/backends/monet5/Tests/sample00.sql
+++ b/sql/backends/monet5/Tests/sample00.sql
@@ -2,7 +2,6 @@
 START TRANSACTION;
 
 CREATE TABLE vals(w DOUBLE, value INTEGER);
-
 INSERT INTO vals VALUES (1, 100), (0, 50);
 
 SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
@@ -15,7 +14,6 @@ ROLLBACK;
 START TRANSACTION;
 
 CREATE TABLE vals(w DECIMAL(10,3), value INTEGER);
-
 INSERT INTO vals VALUES (10.77, 100), (0, 50);
 
 SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
diff --git a/sql/backends/monet5/Tests/sample01.sql 
b/sql/backends/monet5/Tests/sample01.sql
--- a/sql/backends/monet5/Tests/sample01.sql
+++ b/sql/backends/monet5/Tests/sample01.sql
@@ -5,7 +5,8 @@ CREATE TABLE vals(id DOUBLE, type INTEGE
 
 INSERT INTO vals VALUES (1, 100), (0, 50);
 
-SELECT type, COUNT(*) FROM vals GROUP BY type SAMPLE 1 USING WEIGHTS cast(id 
AS double);
+plan SELECT type FROM vals GROUP BY type SAMPLE 1 USING WEIGHTS id;
+SELECT type FROM vals GROUP BY type SAMPLE 1 USING WEIGHTS id;
 
 ROLLBACK;
 
diff --git a/sql/backends/monet5/Tests/sample02.sql 
b/sql/backends/monet5/Tests/sample02.sql
--- a/sql/backends/monet5/Tests/sample02.sql
+++ b/sql/backends/monet5/Tests/sample02.sql
@@ -6,7 +6,7 @@ CREATE TABLE vals(id DOUBLE, type INTEGE
 
 INSERT INTO vals VALUES (1, 100), (0, 50);
 
-SELECT SUM(id) FROM vals SAMPLE 1 USING WEIGHTS cast(type AS double);
+SELECT SUM(id) FROM vals SAMPLE 1 USING WEIGHTS id;
 
 ROLLBACK;
 
diff --git a/sql/backends/monet5/Tests/sample03.sql 
b/sql/backends/monet5/Tests/sample03.sql
--- a/sql/backends/monet5/Tests/sample03.sql
+++ b/sql/backends/monet5/Tests/sample03.sql
@@ -8,6 +8,15 @@ INSERT INTO vals VALUES (1, 100), (-1, 5
 
 SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
 
+ROLLBACK;
+
+# fewer weights are non-zero than size of sample
+START TRANSACTION;
+
+CREATE TABLE vals(w DOUBLE, value INTEGER);
+INSERT INTO vals VALUES (1, 100), (0, 50);
+
+SELECT * FROM vals SAMPLE 2 USING WEIGHTS w;
 
 ROLLBACK;
 
@@ -20,3 +29,45 @@ INSERT INTO vals VALUES ('#1 sample', 10
 SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
 
 ROLLBACK;
+
+# sample size bigger than amount of elements
+START TRANSACTION;
+
+CREATE TABLE vals(w DOUBLE, value INTEGER);
+INSERT INTO vals VALUES (1, 100), (1, 50);
+
+SELECT * FROM vals SAMPLE 10 USING WEIGHTS w;
+
+ROLLBACK;
+
+# weight overflow
+START TRANSACTION;
+
+CREATE TABLE vals(w DOUBLE, value INTEGER);
+INSERT INTO vals VALUES (1.7976931348623157E+306, 100), (0, 50);
+
+SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
+
+ROLLBACK;
+
+# intigus maximus
+START TRANSACTION;
+
+CREATE TABLE vals(w DOUBLE, value INTEGER);
+INSERT INTO vals VALUES (1, 100), (0, 50);
+
+
+SELECT * FROM vals SAMPLE 18446744073709551617 USING WEIGHTS w;
+
+ROLLBACK;
+
+# biggus samplus
+START TRANSACTION;
+
+CREATE TABLE vals(w DOUBLE, value INTEGER);
+INSERT INTO vals VALUES (1, 100), (0, 50);
+
+
+SELECT * FROM vals SAMPLE 2147483648 USING WEIGHTS w;
+
+ROLLBACK;
diff --git a/sql/server/rel_optimizer.c b/sql/server/rel_optimizer.c
--- a/sql/server/rel_optimizer.c
+++ b/sql/server/rel_optimizer.c
@@ -5783,8 +5783,13 @@ rel_mark_used(mvc *sql, sql_rel *rel, in
}
break;
 
+   case op_sample:
+   if (rel->exps->cnt == 2) {
+   // weighted sample
+   exps_mark_used(sql->sa, rel, rel->l);
+   }
+   /* fall through */
case op_topn:
-   case op_sample:
if (proj) {
rel = rel ->l;
rel_mark_used(sql, rel, proj);
diff --git 

MonetDB: stratified_sampling - Correctly check for negative weig...

[Mark Raasveldt]

Changeset: 097357023ff2 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=097357023ff2
Modified Files:
gdk/gdk_sample.c
Branch: stratified_sampling
Log Message:

Correctly check for negative weights in weighted sample.


diffs (73 lines):

diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -242,12 +242,12 @@ BATsample(BAT *b, BUN n)
 BAT *
 BATweightedsample(BAT *b, BUN n, BAT *w)
 {//TODO test correctness extensively
-   BAT* weights;
-   bit weights_are_cast;
-   BAT* sample;
-   oid* oids;/* points to the oids in sample */
-   dbl* w_ptr;//TODO types of w
-   dbl* keys;/* keys as defined in Alg-A-exp */
+   BAT* weights = NULL;
+   bit weights_are_cast = 0;
+   BAT* sample = NULL;
+   oid* oids = NULL;  /* points to the oids in sample */
+   dbl* w_ptr = NULL; //TODO types of w
+   dbl* keys = NULL;  /* keys as defined in Alg-A-exp */
BUN cnt, i, j;
BUN pos, childpos;
oid item;
@@ -298,20 +298,20 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
}
 
BATsetcount(sample, n);
-   /* obtain sample */
+   /* obtain sample */
 
/* Initialize prioqueue */
-   i=0;/* i indices the initial sample (filled with elements with non-zero 
weight) */
-   /* j indices the oids and weights */
-   for(j=0; i < n && j < cnt; j++) {
+   i = 0; /* i indices the initial sample (filled with elements with 
non-zero weight) */
+ /* j indices the oids and weights */
+   for(j = 0; i < n && j < cnt; j++) {
if(w_ptr[j] == 0.0)
continue;
if(w_ptr[j] < 0.0) {
GDKerror("BATsample: w contains negative weights\n");
goto bailout;
}
-   oids[i] = (oid)(j+minoid);
-   keys[i] = pow(mtwist_drand(mt_rng),1.0/w_ptr[j]);//TODO cast 
1.0 to dbl?
+   oids[i] = (oid)(j + minoid);
+   keys[i] = pow(mtwist_drand(mt_rng), 1.0 / w_ptr[j]);//TODO cast 
1.0 to dbl?
i++;
}
 
@@ -325,8 +325,13 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
while(true) {
r = mtwist_drand(mt_rng);
xw = log(r)/log(keys[0]);
-   for(;j= w_ptr[j]; j++)
+   for(; j < cnt && xw >= w_ptr[j]; j++) {
+   if(w_ptr[j] < 0.0) {
+   GDKerror("BATsample: w contains negative 
weights\n");
+   goto bailout;
+   }
xw -= w_ptr[j];
+   }
if(j >= cnt) break;
 
/* At this point:
@@ -358,7 +363,7 @@ BATweightedsample(BAT *b, BUN n, BAT *w)
 
return sample;
 
-  bailout:
+bailout:
 if(weights_are_cast && weights)//if weights where converted, delete 
converted BAT
BBPunfix(weights->batCacheid);
 if(keys)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - Remove old code.

[Mark Raasveldt]

Changeset: 189631c86054 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=189631c86054
Removed Files:
sql/backends/monet5/sample/80_sample.sql
sql/backends/monet5/sample/Makefile.ag
sql/backends/monet5/sample/Tests/All
sql/backends/monet5/sample/Tests/weightedsample.sql
sql/backends/monet5/sample/sample.c
sql/backends/monet5/sample/sample.h
Modified Files:
sql/backends/monet5/Makefile.ag
Branch: stratified_sampling
Log Message:

Remove old code.


diffs (168 lines):

diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag
--- a/sql/backends/monet5/Makefile.ag
+++ b/sql/backends/monet5/Makefile.ag
@@ -4,7 +4,7 @@
 #
 # Copyright 1997 - July 2008 CWI, August 2008 - 2017 MonetDB B.V.
 
-SUBDIRS = NOT_WIN32?vaults UDF LSST generator sample
+SUBDIRS = NOT_WIN32?vaults UDF LSST generator
 
 INCLUDES = ../../include ../../common ../../storage ../../server \
   ../../../monetdb5/modules/atoms \
diff --git a/sql/backends/monet5/sample/80_sample.sql 
b/sql/backends/monet5/sample/80_sample.sql
deleted file mode 100644
--- a/sql/backends/monet5/sample/80_sample.sql
+++ /dev/null
@@ -1,13 +0,0 @@
--- This Source Code Form is subject to the terms of the Mozilla Public
--- License, v. 2.0.  If a copy of the MPL was not distributed with this
--- file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
--- Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
-
--- add function signatures to SQL catalog
-
-
--- Reverse a string
-create function weighted_sample(src double, cnt bigint)
-returns boolean external name libsample.weighted_sample;--TODO possibly 
nolibsample
-
diff --git a/sql/backends/monet5/sample/Makefile.ag 
b/sql/backends/monet5/sample/Makefile.ag
deleted file mode 100644
--- a/sql/backends/monet5/sample/Makefile.ag
+++ /dev/null
@@ -1,36 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
-
-INCLUDES = .. \
-../../../include \
-   ../../../common \
-   ../../../storage \
-   ../../../server \
-   ../../../../monetdb5/modules/atoms \
-   ../../../../monetdb5/modules/kernel \
-   ../../../../monetdb5/mal \
-   ../../../../monetdb5/modules/mal \
-   ../../../../monetdb5/optimizer \
-   ../../../../common/options \
-   ../../../../common/stream \
-   ../../../../gdk
-
-lib__sample = {
-   MODULE
-   DIR = libdir/monetdb5
-   SOURCES = sample.c sample.h sample_impl.h
-   LIBS = ../../../../monetdb5/tools/libmonetdb5 \
-  ../../../../gdk/libbat
-}
-
-headers_sql = {
-   HEADERS = sql
-   DIR = libdir/monetdb5/createdb
-   SOURCES = 80_sample.sql
-}
-
-EXTRA_DIST_DIR = Tests
-
diff --git a/sql/backends/monet5/sample/Tests/All 
b/sql/backends/monet5/sample/Tests/All
deleted file mode 100644
--- a/sql/backends/monet5/sample/Tests/All
+++ /dev/null
@@ -1,1 +0,0 @@
-weightedsample
diff --git a/sql/backends/monet5/sample/Tests/weightedsample.sql 
b/sql/backends/monet5/sample/Tests/weightedsample.sql
deleted file mode 100644
--- a/sql/backends/monet5/sample/Tests/weightedsample.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-set optimizer = 'sequential_pipe';
--- ADD FLAG TO DISALLOW PARALLELIZATION (MITOSIS) FOR weighted_sample
-CREATE TABLE wsample (i INTEGER, weights DOUBLE);
-INSERT INTO wsample VALUES (1, 1), (2, 1), (3, 1), (4, 1), (5, 1);
-
-
-explain SELECT i FROM wsample WHERE weighted_sample(weights, 2);
-SELECT i FROM wsample WHERE weighted_sample(weights, 2);
-
diff --git a/sql/backends/monet5/sample/sample.c 
b/sql/backends/monet5/sample/sample.c
deleted file mode 100644
--- a/sql/backends/monet5/sample/sample.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0.  If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
- */
-
-/* monetdb_config.h must be the first include in each .c file */
-
-#include "monetdb_config.h"
-#include "sample.h"
-
-#ifdef notdefined //!!!TODO
-
-/* MAL wrapper */
-char *
-UDFBATweightedsample(bat *ret, const bat *arg, const lng *cnt)
-{//bat = identifier, BAT is actual bat, BATdescriptor turns ID into BAT
-   BAT *res = NULL, *src = NULL;
-   char *msg = NULL;
-
-   /* assert calling sanity */
-   assert(ret != NULL && arg != NULL);
-
-   /* bat-id -> BAT-descriptor */
-   if ((src = BATdescriptor(*arg)) == NULL)
-   throw(MAL, "batudf.reverse", RUNTIME_OBJECT_MISSING);
-   printf("Count: %lld\n", *cnt);
-
-   //TODO Type checking
-   /* do the work */
-   //msg = UDFBATreverse_ ( , src 

MonetDB: stratified_sampling - Add some testcases.

[Mark Raasveldt]

Changeset: 4c81f2dca357 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4c81f2dca357
Added Files:
sql/backends/monet5/Tests/sample00.sql
sql/backends/monet5/Tests/sample01.sql
sql/backends/monet5/Tests/sample02.sql
sql/backends/monet5/Tests/sample03.sql
Branch: stratified_sampling
Log Message:

Add some testcases.


diffs (88 lines):

diff --git a/sql/backends/monet5/Tests/sample00.sql 
b/sql/backends/monet5/Tests/sample00.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample00.sql
@@ -0,0 +1,23 @@
+
+START TRANSACTION;
+
+CREATE TABLE vals(w DOUBLE, value INTEGER);
+
+INSERT INTO vals VALUES (1, 100), (0, 50);
+
+SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
+SELECT * FROM vals SAMPLE 0.5 USING WEIGHTS w;
+SELECT * FROM vals SAMPLE 0.5 USING WEIGHTS (1 - w);
+
+ROLLBACK;
+
+# different types of weights
+START TRANSACTION;
+
+CREATE TABLE vals(w DECIMAL(10,3), value INTEGER);
+
+INSERT INTO vals VALUES (10.77, 100), (0, 50);
+
+SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/sample01.sql 
b/sql/backends/monet5/Tests/sample01.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample01.sql
@@ -0,0 +1,11 @@
+
+START TRANSACTION;
+
+CREATE TABLE vals(id DOUBLE, type INTEGER);
+
+INSERT INTO vals VALUES (1, 100), (0, 50);
+
+SELECT type, COUNT(*) FROM vals GROUP BY type SAMPLE 1 USING WEIGHTS cast(id 
AS double);
+
+ROLLBACK;
+
diff --git a/sql/backends/monet5/Tests/sample02.sql 
b/sql/backends/monet5/Tests/sample02.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample02.sql
@@ -0,0 +1,12 @@
+
+
+START TRANSACTION;
+
+CREATE TABLE vals(id DOUBLE, type INTEGER);
+
+INSERT INTO vals VALUES (1, 100), (0, 50);
+
+SELECT SUM(id) FROM vals SAMPLE 1 USING WEIGHTS cast(type AS double);
+
+ROLLBACK;
+
diff --git a/sql/backends/monet5/Tests/sample03.sql 
b/sql/backends/monet5/Tests/sample03.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/sample03.sql
@@ -0,0 +1,22 @@
+# test various error conditions
+
+# negative weights
+START TRANSACTION;
+
+CREATE TABLE vals(w DOUBLE, value INTEGER);
+INSERT INTO vals VALUES (1, 100), (-1, 50);
+
+SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
+
+
+ROLLBACK;
+
+# string weights
+START TRANSACTION;
+
+CREATE TABLE vals(w STRING, value INTEGER);
+INSERT INTO vals VALUES ('#1 sample', 100), ('#2 sample', 50);
+
+SELECT * FROM vals SAMPLE 1 USING WEIGHTS w;
+
+ROLLBACK;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: stratified_sampling - Merge.

[Mark Raasveldt]

Changeset: e99bdbf8867b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e99bdbf8867b
Added Files:
sql/backends/monet5/sample/80_sample.sql
sql/backends/monet5/sample/Makefile.ag
sql/backends/monet5/sample/Tests/All
sql/backends/monet5/sample/Tests/weightedsample.sql
sql/backends/monet5/sample/sample.c
sql/backends/monet5/sample/sample.h
Modified Files:
sql/server/sql_parser.y
Branch: stratified_sampling
Log Message:

Merge.


diffs (183 lines):

diff --git a/sql/backends/monet5/sample/80_sample.sql 
b/sql/backends/monet5/sample/80_sample.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/sample/80_sample.sql
@@ -0,0 +1,13 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0.  If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+--
+-- Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
+
+-- add function signatures to SQL catalog
+
+
+-- Reverse a string
+create function weighted_sample(src double, cnt bigint)
+returns boolean external name libsample.weighted_sample;--TODO possibly 
nolibsample
+
diff --git a/sql/backends/monet5/sample/Makefile.ag 
b/sql/backends/monet5/sample/Makefile.ag
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/sample/Makefile.ag
@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
+
+INCLUDES = .. \
+../../../include \
+   ../../../common \
+   ../../../storage \
+   ../../../server \
+   ../../../../monetdb5/modules/atoms \
+   ../../../../monetdb5/modules/kernel \
+   ../../../../monetdb5/mal \
+   ../../../../monetdb5/modules/mal \
+   ../../../../monetdb5/optimizer \
+   ../../../../common/options \
+   ../../../../common/stream \
+   ../../../../gdk
+
+lib__sample = {
+   MODULE
+   DIR = libdir/monetdb5
+   SOURCES = sample.c sample.h sample_impl.h
+   LIBS = ../../../../monetdb5/tools/libmonetdb5 \
+  ../../../../gdk/libbat
+}
+
+headers_sql = {
+   HEADERS = sql
+   DIR = libdir/monetdb5/createdb
+   SOURCES = 80_sample.sql
+}
+
+EXTRA_DIST_DIR = Tests
+
diff --git a/sql/backends/monet5/sample/Tests/All 
b/sql/backends/monet5/sample/Tests/All
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/sample/Tests/All
@@ -0,0 +1,1 @@
+weightedsample
diff --git a/sql/backends/monet5/sample/Tests/weightedsample.sql 
b/sql/backends/monet5/sample/Tests/weightedsample.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/sample/Tests/weightedsample.sql
@@ -0,0 +1,9 @@
+set optimizer = 'sequential_pipe';
+-- ADD FLAG TO DISALLOW PARALLELIZATION (MITOSIS) FOR weighted_sample
+CREATE TABLE wsample (i INTEGER, weights DOUBLE);
+INSERT INTO wsample VALUES (1, 1), (2, 1), (3, 1), (4, 1), (5, 1);
+
+
+explain SELECT i FROM wsample WHERE weighted_sample(weights, 2);
+SELECT i FROM wsample WHERE weighted_sample(weights, 2);
+
diff --git a/sql/backends/monet5/sample/sample.c 
b/sql/backends/monet5/sample/sample.c
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/sample/sample.c
@@ -0,0 +1,47 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
+ */
+
+/* monetdb_config.h must be the first include in each .c file */
+
+#include "monetdb_config.h"
+#include "sample.h"
+
+#ifdef notdefined //!!!TODO
+
+/* MAL wrapper */
+char *
+UDFBATweightedsample(bat *ret, const bat *arg, const lng *cnt)
+{//bat = identifier, BAT is actual bat, BATdescriptor turns ID into BAT
+   BAT *res = NULL, *src = NULL;
+   char *msg = NULL;
+
+   /* assert calling sanity */
+   assert(ret != NULL && arg != NULL);
+
+   /* bat-id -> BAT-descriptor */
+   if ((src = BATdescriptor(*arg)) == NULL)
+   throw(MAL, "batudf.reverse", RUNTIME_OBJECT_MISSING);
+   printf("Count: %lld\n", *cnt);
+
+   //TODO Type checking
+   /* do the work */
+   //msg = UDFBATreverse_ ( , src );//TODO
+   throw(MAL, "batudf.reverse", "LOLFAIL");//TODO
+   res = _BATsample(arg, *cnt, BAT *cdf)
+   
+   /* release input BAT-descriptor */
+   //BBPunfix(src->batCacheid);
+
+   //if (msg == MAL_SUCCEED) {
+   /* register result BAT in buffer pool */
+   //  BBPkeepref((*ret = res->batCacheid));
+   //}
+   return msg;
+}
+
+#endif
diff --git a/sql/backends/monet5/sample/sample.h 
b/sql/backends/monet5/sample/sample.h
new file mode 100644
--- 

MonetDB: stratified_sampling - Merge with default.

[Mark Raasveldt]

com>
-- Added 5 new sys schema tables: function_languages, function_types,
-  key_types, index_types and privilege_codes.  They are pre-loaded with
-  static content and contain descriptive names for the various integer
-  type and code values.  See also sql/scripts/51_sys_schema_extension.sql
diff --git a/ChangeLog b/ChangeLog.Jul2017
copy from ChangeLog
copy to ChangeLog.Jul2017
--- a/ChangeLog
+++ b/ChangeLog.Jul2017
@@ -1,6 +1,25 @@
 # ChangeLog file for devel
 # This file is updated with Maddlog
 
+* Thu Apr 13 2017 Mark Raasveldt <m.raasve...@cwi.nl>
+- Added a new server-side protocol implementation. The new protocol
+  is backwards compatible with the old protocol. Clients can choose
+  whether they want to use the old or the new protocol during the initial
+  handshake with the server. The new protocol is a binary column-based
+  protocol that is significantly faster than the old protocol when
+  transferring large result sets. In addition, the new protocol supports
+  compression using Snappy or LZ4.
+
+* Mon Feb 20 2017 Sjoerd Mullender <sjo...@acm.org>
+- Moved the sphinx extension module to its own repository.
+  See https://dev.monetdb.org/hg/MonetDB-sphinx/.
+
+* Thu Feb 16 2017 Sjoerd Mullender <sjo...@acm.org>
+- Removed GSL module: it's now a separate (extension) package.
+  See https://dev.monetdb.org/hg/MonetDB-gsl/.
+- The PCRE library is now optional for systems that support POSIX regular
+  expressions.
+
 * Thu Jan 12 2017 Martin van Dinther <martin.van.dint...@monetdbsolutions.com>
 - Added 5 new sys schema tables: function_languages, function_types,
   key_types, index_types and privilege_codes.  They are pre-loaded with
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -1,5 +1,5 @@
 %define name MonetDB
-%define version 11.26.0
+%define version 11.28.0
 %{!?buildno: %global buildno %(date +%Y%m%d)}
 
 # groups of related archs
@@ -131,8 +131,8 @@ Vendor: MonetDB BV <i...@monetdb.org>
 
 Group: Applications/Databases
 License: MPLv2.0
-URL: http://www.monetdb.org/
-Source: 
http://dev.monetdb.org/downloads/sources/Dec2016-SP4/%{name}-%{version}.tar.bz2
+URL: https://www.monetdb.org/
+Source: 
https://www.monetdb.org/downloads/sources/Dec2016-SP5/%{name}-%{version}.tar.bz2
 
 # we need systemd for the _unitdir macro to exist
 %if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} >= 7
@@ -311,6 +311,11 @@ MonetDB, you will very likely need this 
 Summary: MonetDB - Monet Database Management System Client Programs
 Group: Applications/Databases
 Requires: %{name}-client%{?_isa} = %{version}-%{release}
+%if (0%{?fedora} >= 22)
+# tomograph executes these two:
+Recommends: /usr/bin/gs
+Recommends: /usr/bin/gnuplot
+%endif
 
 %description client-tools
 MonetDB is a database management system that is developed from a
@@ -953,6 +958,49 @@ rm -f %{buildroot}%{_bindir}/Maddlog
 %postun -p /sbin/ldconfig
 
 %changelog
+* Mon May 29 2017 Panagiotis Koutsourakis <kutsu...@monetdbsolutions.com> - 
11.25.23-20170529
+- Rebuilt.
+- BZ#6290: Crash (and assertion failure) with a correlated subquery with
+  NOT IN in the select-list
+- BZ#6291: crash if executes a function with sample operator
+- BZ#6294: Sqlitelogictest crash
+- BZ#6296: Another sqllitelogictest crash :(
+- BZ#6297: 7th sqllitelogictest crash :(
+- BZ#6300: Protect against missing BATs (sqlsmith)
+- BZ#6314: Lateral crash report (sqlsmith)
+- BZ#6315: Exist operator on type bigint missing (sqlsmith)
+- BZ#6316: Coalesc and limit error (sqlsmith)
+- BZ#6317: Two-column integer aggregation extremely slow
+- BZ#6318: Daemon crashes if .merovignian_properties for a database
+  contains more than 42 entries
+- BZ#6320: The daemon ignores all but the last entry in the
+  .merovignian_properties file when spawning mserver5
+
+* Tue Apr 25 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.21-20170425
+- Rebuilt.
+- BZ#6260: Sqlitelogictest crash
+- BZ#6288: Function cannot find column in merge table
+- BZ#6295: msqldump writes unescaped timestamp values when using inserts
+
+* Wed Apr 19 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.21-20170425
+- monetdb5: Fixed a bug causing a crash during cleanup when mserver5 is stopped
+  with monetdb stop database.
+
+* Tue Apr 18 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.19-20170418
+- Rebuilt.
+- BZ#6259: crash on select query from sqlitelogictests
+
+* Tue Apr 18 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.19-20170418
+- gdk: A potential deadlock was fixed in order index creation.
+- gdk: A bug that could happen during recovery of the write-ahead log (WAL)
+  was fixed.  See changeset 98ad79c555cc for details.
+
+* Tue Apr 18 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.19-20170418
+- monetdb5: Some memory leaks were plugged.
+
+* Tue Apr 18 2017 Sjoerd Mullender <sjo...@acm.org> - 11.25.19-20170418
+- sql: Some memory leaks were plugged.
+
 * Tue Apr 11 2017 Sjoerd Mullender <sjo...@acm.o

MonetDB: jitudf - Fix memory leak in non-grouped aggregate.

[Mark Raasveldt]

Changeset: 7412c6606722 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7412c6606722
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Fix memory leak in non-grouped aggregate.


diffs (35 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -246,6 +246,15 @@ static void *wrapped_GDK_malloc_nojump(s
return add_allocated_region(ptr);
 }
 
+static void *wrapped_GDK_zalloc_nojump(size_t size)
+{
+   void *ptr = GDKzalloc(size + sizeof(allocated_region));
+   if (!ptr) {
+   return NULL;
+   }
+   return add_allocated_region(ptr);
+}
+
 #define GENERATE_BASE_HEADERS(type, tpename)   
\
static int tpename##_is_null(type value);   
   \
static void tpename##_initialize(struct cudf_data_struct_##tpename 
*self,  \
@@ -1132,9 +1141,13 @@ static str CUDFeval(Client cntxt, MalBlk
if (non_grouped_aggregate) {
GENERATE_BAT_INPUT_BASE(oid);
bat_data->count = input_size;
+   bat_data->null_value = oid_nil;
bat_data->data =
-   GDKzalloc(bat_data->count * 
sizeof(bat_data->null_value));
-   bat_data->null_value = oid_nil;
+   wrapped_GDK_zalloc_nojump(bat_data->count * 
sizeof(bat_data->null_value));
+   if (!bat_data->data) {
+   msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
+   goto wrapup;
+   }
}
 
argnode = sqlfun ? sqlfun->res->h : NULL;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Approve exports.

[Mark Raasveldt]

Changeset: 55bdba4efc8f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=55bdba4efc8f
Modified Files:
clients/Tests/exports.stable.out
Branch: jitudf
Log Message:

Approve exports.


diffs (37 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1446,6 +1446,8 @@ str MTIMEepilogue(void *ret);
 str MTIMEepoch2int(int *res, const timestamp *ts);
 str MTIMEepoch2lng(lng *res, const timestamp *ts);
 str MTIMEepoch_bulk(bat *ret, bat *bid);
+void MTIMEfromdate(date n, int *d, int *m, int *y);
+void MTIMEfromtime(daytime n, int *hour, int *min, int *sec, int *msec);
 str MTIMElocal_timezone(lng *res);
 str MTIMEmonth_from_str(int *ret, const char *const *month);
 str MTIMEmonth_to_str(str *ret, const int *month);
@@ -1514,6 +1516,8 @@ str MTIMEtimestamp_to_str(str *s, const 
 str MTIMEtimestamp_year(int *ret, const timestamp *t);
 str MTIMEtimestamplng(timestamp *ret, const lng *sec);
 str MTIMEtimezone(tzone *z, const char *const *name);
+date MTIMEtodate(int day, int month, int year);
+daytime MTIMEtotime(int hour, int min, int sec, int msec);
 str MTIMEtzone_create(tzone *ret, const int *minutes);
 str MTIMEtzone_create_dst(tzone *ret, const int *minutes, const rule *start, 
const rule *end);
 str MTIMEtzone_create_lng(tzone *ret, const lng *minutes);
@@ -1954,6 +1958,7 @@ void batBeforeCall(ValPtr v, ValPtr bak)
 str batRef;
 str batalgebraRef;
 str batcalcRef;
+str batcapiRef;
 str batmalRef;
 str batmmathRef;
 str batmtimeRef;
@@ -1982,6 +1987,7 @@ str bstream_read_wrapwrap(int *res, Bstr
 str calcRef;
 str callMAL(Client cntxt, MalBlkPtr mb, MalStkPtr *glb, ValPtr argv[], char 
debug);
 int callString(Client c, str s, int listing);
+str capiRef;
 str catalogRef;
 str catchKernelException(Client cntxt, str ret);
 void chkDeclarations(stream *out, MalBlkPtr mb);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: b375ba69e94c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b375ba69e94c
Added Files:

sql/test/BugTracker-2015/Tests/crash_timestamp_convert.Bug-3816.stable.err.Windows

sql/test/BugTracker-2015/Tests/crash_timestamp_convert.Bug-3816.stable.out.Windows
sql/test/BugTracker-2017/Tests/crash-dce.Bug-6330.stable.err
sql/test/BugTracker-2017/Tests/crash-dce.Bug-6330.stable.out
sql/test/BugTracker-2017/Tests/crash-select_after_MAL_error.Bug-6332.sql

sql/test/BugTracker-2017/Tests/crash-select_after_MAL_error.Bug-6332.stable.err

sql/test/BugTracker-2017/Tests/crash-select_after_MAL_error.Bug-6332.stable.out

sql/test/BugTracker-2017/Tests/create_table_as-missing_not_null.Bug-6329.sql

sql/test/BugTracker-2017/Tests/create_table_as-missing_not_null.Bug-6329.stable.err

sql/test/BugTracker-2017/Tests/create_table_as-missing_not_null.Bug-6329.stable.out
sql/test/BugTracker-2017/Tests/statistics_nils_not_eq_zero.Bug-6331.sql

sql/test/BugTracker-2017/Tests/statistics_nils_not_eq_zero.Bug-6331.stable.err

sql/test/BugTracker-2017/Tests/statistics_nils_not_eq_zero.Bug-6331.stable.out
sql/test/malloc_fail/Tests/setmemorylimit-fail2.stable.err
sql/test/malloc_fail/Tests/setmemorylimit-fail2.stable.out
Modified Files:
NT/mkgeomwxs.py
NT/mkodbcwxs.py
NT/mksqlwxs.py
buildtools/conf/monetdb.ico
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
clients/mapiclient/dump.c
clients/mapiclient/mclient.c
clients/mapiclient/mhelp.c
clients/mapiclient/mhelp.h
clients/mapiclient/tomograph.c
clients/odbc/driver/SQLColumnPrivileges.c
clients/odbc/driver/SQLColumns.c
clients/odbc/driver/SQLExecute.c
clients/odbc/driver/SQLSpecialColumns.c
gdk/gdk_bbp.c
gdk/gdk_join.c
gdk/gdk_logger.c
gdk/gdk_private.h
gdk/gdk_search.c
gdk/gdk_select.c
geom/monetdb5/geom_upgrade.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_parser.c
monetdb5/modules/mal/00_language_hge.mal
monetdb5/modules/mal/manifold.c
monetdb5/modules/mal/manifold.h
monetdb5/optimizer/Tests/inline06.stable.out
monetdb5/optimizer/opt_costModel.c
monetdb5/optimizer/opt_dataflow.c
monetdb5/optimizer/opt_evaluate.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_multiplex.c
monetdb5/optimizer/opt_support.c
sql/backends/monet5/Tests/pyapi19.stable.err
sql/backends/monet5/Tests/pyapi19.stable.out
sql/backends/monet5/Tests/pyapi25.stable.err
sql/backends/monet5/Tests/pyapi25.stable.err.Windows
sql/backends/monet5/Tests/pyapi25.stable.out
sql/backends/monet5/Tests/pyapi29.stable.err
sql/backends/monet5/Tests/pyapi29.stable.out
sql/backends/monet5/UDF/pyapi/connection.c
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/backends/monet5/UDF/pyapi/pyapi.mal
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_19.stable.err
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_19.stable.out
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_25.stable.err
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_25.stable.out
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_29.stable.err
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_29.stable.out
sql/backends/monet5/UDF/pyapi3/pyapi3.mal
sql/backends/monet5/sql_assert.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_statistics.c
sql/backends/monet5/sql_upgrades.c
sql/common/sql_types.c
sql/storage/store.c
sql/test/BugTracker-2010/Tests/count-and-not.Bug-2663.stable.out
sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552.stable.out

sql/test/BugTracker-2010/Tests/not-in-union-except-union.Bug-2577.stable.out
sql/test/BugTracker-2017/Tests/All
sql/test/BugTracker-2017/Tests/crash-dce.Bug-6330.sql
sql/test/BugTracker-2017/Tests/nestedoperators.Bug-6292.stable.err
sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/check.stable.out
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.32bit
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/check.stable.out

MonetDB: jitudf - Make non-deterministic tests deterministic.

[Mark Raasveldt]

Changeset: f277a120089d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f277a120089d
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi03.sql
sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/Tests/capi04.stable.err
sql/backends/monet5/UDF/capi/Tests/capi10.sql
sql/backends/monet5/UDF/capi/Tests/capi10.stable.err
Branch: jitudf
Log Message:

Make non-deterministic tests deterministic.


diffs (152 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi03.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi03.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi03.sql
@@ -50,37 +50,3 @@ INSERT INTO integers VALUES (1), (2), (3
 SELECT capi03(i) FROM integers;
 
 ROLLBACK;
-
-START TRANSACTION;
-
-# Modify input data
-CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
-size_t i;
-inp.data[0] = 10;
-result->initialize(result, inp.count);
-for(i = 0; i < inp.count; i++) {
-result->data[i] = inp.data[i] * 2;
-}
-};
-
-CREATE TABLE integers(i INTEGER);
-INSERT INTO integers VALUES (1), (2), (3), (4), (5);
-
-SELECT capi03(i) FROM integers;
-
-ROLLBACK;
-
-START TRANSACTION;
-
-# Trigger a segfault
-CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
-int x = *((int*)NULL);
-};
-
-CREATE TABLE integers(i INTEGER);
-INSERT INTO integers VALUES (1), (2), (3), (4), (5);
-
-SELECT capi03(i) FROM integers;
-
-ROLLBACK;
-
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
--- a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
+++ b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
@@ -36,12 +36,6 @@ ERROR = !No data returned.
 MAPI  = (monetdb) /var/tmp/mtest-2836/.s.monetdb.35915
 QUERY = SELECT capi03(i) FROM integers;
 ERROR = !Something went wrong!
-MAPI  = (monetdb) /var/tmp/mtest-25067/.s.monetdb.39212
-QUERY = SELECT capi03(i) FROM integers;
-ERROR = !Attempting to write to the input or triggered a segfault/bus error
-MAPI  = (monetdb) /var/tmp/mtest-25067/.s.monetdb.39212
-QUERY = SELECT capi03(i) FROM integers;
-ERROR = !Attempting to write to the input or triggered a segfault/bus error
 
 # 17:24:55 >  
 # 17:24:55 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
--- a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
+++ b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
@@ -87,26 +87,6 @@ Ready.
 #INSERT INTO integers VALUES (1), (2), (3), (4), (5);
 [ 5]
 #ROLLBACK;
-#START TRANSACTION;
-#CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
-#  inp.data[0] = 10;
-#  result->initialize(result, inp.count);
-#  for(size_t i = 0; i < inp.count; i++) {
-#  result->data[i] = inp.data[i] * 2;
-#  }
-#};
-#CREATE TABLE integers(i INTEGER);
-#INSERT INTO integers VALUES (1), (2), (3), (4), (5);
-[ 5]
-#ROLLBACK;
-#START TRANSACTION;
-#CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
-#  int x = *((int*)NULL);
-#};
-#CREATE TABLE integers(i INTEGER);
-#INSERT INTO integers VALUES (1), (2), (3), (4), (5);
-[ 5]
-#ROLLBACK;
 
 # 17:24:55 >  
 # 17:24:55 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
@@ -25,16 +25,6 @@ SELECT capi04(i) FROM strings;
 
 DROP FUNCTION capi04;
 
-# try to modify one of the input strings
-CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
-size_t i;
-result->initialize(result, inp.count);
-for(i = 0; i < inp.count; i++) {
-result->data[i][0] = 'h';
-}
-};
-
-SELECT capi04(i) FROM strings;
 ROLLBACK;
 
 START TRANSACTION;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.err
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.stable.err
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.err
@@ -27,9 +27,6 @@ stderr of test 'capi04` in directory 'sq
 # 15:50:30 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-20003" "--port=32751"
 # 15:50:30 >  
 
-MAPI  = (monetdb) /var/tmp/mtest-20003/.s.monetdb.32751
-QUERY = SELECT capi04(i) FROM strings;
-ERROR = !Attempting to write to the input or triggered a segfault/bus error
 
 # 15:50:31 >  
 # 15:50:31 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi10.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi10.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi10.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi10.sql
@@ -31,15 +31,4 @@ SELECT capi10(i) FROM blobs;
 
 DROP FUNCTION capi10;
 

MonetDB: jitudf - Smaller decimals in tests for non-hge systems.

[Mark Raasveldt]

Changeset: 6ebfa942608f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6ebfa942608f
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi11.sql
Branch: jitudf
Log Message:

Smaller decimals in tests for non-hge systems.


diffs (12 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi11.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi11.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi11.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi11.sql
@@ -21,7 +21,7 @@ CREATE FUNCTION capi11(inp DECIMAL) RETU
 }
 };
 
-CREATE TABLE decimals(d DECIMAL(20,3));
+CREATE TABLE decimals(d DECIMAL(18,3));
 INSERT INTO decimals VALUES (10.3), (10.5), (NULL), (10.7);
 
 SELECT capi11(d) FROM decimals;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Compilation fixes and clean up extra bats crea...

[Mark Raasveldt]

Changeset: f8fdbc13df60 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f8fdbc13df60
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Compilation fixes and clean up extra bats created for groupless aggregations.


diffs (132 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -35,8 +35,6 @@ typedef struct _mprotected_region {
void *actual_addr;
size_t actual_len;
 
-   bool is_protected;
-
struct _mprotected_region *next;
 } mprotected_region;
 
@@ -113,7 +111,6 @@ static void handler(int sig, siginfo_t *
// thus sometimes we mprotect a page where only part of the page
// should actually be protected. Thus for this case we check if the 
access
// was actually an error
-
if (actual_mprotected_regions[tid]) {
mprotected_region *region = *actual_mprotected_regions[tid];
while (region) {
@@ -132,13 +129,22 @@ static void handler(int sig, siginfo_t *
region = region->next;
}
}
+
if (found_region && !actually_protected_area) {
// this is NOT an actually protected area
// thus the segfault/bus error is invalid
// the nasty part here is that we have to unprotect the entire 
page now
// thus opening us up to future modifications of the data
-   clear_mprotect(found_region->addr, found_region->len);
-   found_region->addr = NULL;
+
+   mprotected_region *region = *actual_mprotected_regions[tid];
+   while (region) {
+   if (si->si_addr >= region->addr &&
+   (char *)si->si_addr <= (char *)region->addr + 
region->len) {
+   // the address belongs to this mprotected region
+   clear_mprotect(found_region->addr, 
found_region->len);
+   }
+   region = region->next;
+   }
} else {
longjmp(jump_buffer[tid], 1);
}
@@ -177,7 +183,6 @@ static char *mprotect_region(void *addr,
region->next = *regions;
region->actual_addr = actual_addr;
region->actual_len = actual_len;
-   region->is_protected = false;
*regions = region;
return NULL;
 }
@@ -386,16 +391,16 @@ static str CUDFeval(Client cntxt, MalBlk
int seengrp = FALSE;
FILE *f = NULL;
void *handle = NULL;
-   jitted_function func = NULL;
+   jitted_function volatile func = NULL;
int ret;
 
FILE *compiler = NULL;
int compiler_return_code;
 
-   void **inputs = NULL;
-   size_t input_count = 0;
-   void **outputs = NULL;
-   size_t output_count = 0;
+   void ** volatile inputs = NULL;
+   size_t volatile input_count = 0;
+   void ** volatile outputs = NULL;
+   size_t volatile output_count = 0;
BAT **input_bats = NULL;
mprotected_region *regions = NULL, *region_iter = NULL;
 
@@ -425,7 +430,7 @@ static str CUDFeval(Client cntxt, MalBlk
 
BUN expression_hash = 0, funcname_hash = 0;
cached_functions *cached_function;
-   char *function_parameters = NULL;
+   char* volatile function_parameters = NULL;
int tid = THRgettid();
size_t input_size = 0;
bit non_grouped_aggregate = 0;
@@ -434,7 +439,7 @@ static str CUDFeval(Client cntxt, MalBlk
int bat_type = 0;
const char* tpe = NULL;
 
-   size_t extra_inputs = 0;
+   size_t volatile extra_inputs = 0;
 
(void)cntxt;
 
@@ -1222,7 +1227,6 @@ static str CUDFeval(Client cntxt, MalBlk
if (mprotect(region_iter->addr, region_iter->len, PROT_READ) < 
0) {
goto wrapup;
}
-   region_iter->is_protected = true;
region_iter = region_iter->next;
}
 
@@ -1232,9 +1236,7 @@ static str CUDFeval(Client cntxt, MalBlk
// clear any mprotected regions
while (regions) {
mprotected_region *next = regions->next;
-   if (regions->is_protected) {
-   clear_mprotect(regions->addr, regions->len);
-   }
+   clear_mprotect(regions->addr, regions->len);
GDKfree(regions);
regions = next;
}
@@ -1456,9 +1458,7 @@ wrapup:
// clear any mprotected regions
while (regions) {
mprotected_region *next = regions->next;
-   if (regions->is_protected) {
-   clear_mprotect(regions->addr, regions->len);
-   }
+   clear_mprotect(regions->addr, regions->len);
GDKfree(regions);
regions = next;
   

MonetDB: jitudf - Properly free input bats.

[Mark Raasveldt]

Changeset: ced7106e3dfb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ced7106e3dfb
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Properly free input bats.


diffs (36 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -434,6 +434,8 @@ static str CUDFeval(Client cntxt, MalBlk
int bat_type = 0;
const char* tpe = NULL;
 
+   size_t extra_inputs = 0;
+
(void)cntxt;
 
actual_mprotected_regions[tid] = 
@@ -901,7 +903,7 @@ static str CUDFeval(Client cntxt, MalBlk
}
if (input_count > 0) {
// add "aggr_group" for non-grouped aggregates
-   size_t extra_inputs = non_grouped_aggregate ? 1 : 0;
+   extra_inputs = non_grouped_aggregate ? 1 : 0;
input_bats = GDKzalloc(sizeof(BAT *) * (input_count + 
extra_inputs));
inputs = GDKzalloc(sizeof(void *) * (input_count + 
extra_inputs));
if (!inputs || !input_bats) {
@@ -1485,6 +1487,14 @@ wrapup:
}
GDKfree(output_names);
}
+   if (input_bats) {
+   for(i = 0; i < input_count + extra_inputs; i++) {
+   if (input_bats[i]) {
+   BBPunfix(input_bats[i]->batCacheid);
+   }
+   }
+   GDKfree(input_bats);
+   }
// input data
if (inputs) {
for (i = 0; i < (size_t)input_count; i++) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jul2017 - Disallow loopback queries in parallel Python ...

[Mark Raasveldt]

Changeset: fab1f7a16a9f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fab1f7a16a9f
Modified Files:
sql/backends/monet5/Tests/pyapi19.stable.err
sql/backends/monet5/Tests/pyapi19.stable.out
sql/backends/monet5/Tests/pyapi25.stable.err
sql/backends/monet5/Tests/pyapi25.stable.err.Windows
sql/backends/monet5/Tests/pyapi25.stable.out
sql/backends/monet5/Tests/pyapi29.stable.err
sql/backends/monet5/Tests/pyapi29.stable.out
sql/backends/monet5/UDF/pyapi/connection.c
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/backends/monet5/UDF/pyapi/pyapi.mal
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_19.stable.err
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_19.stable.out
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_25.stable.err
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_25.stable.out
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_29.stable.err
sql/backends/monet5/UDF/pyapi3/Tests/pyapi3_29.stable.out
sql/backends/monet5/UDF/pyapi3/pyapi3.mal
Branch: Jul2017
Log Message:

Disallow loopback queries in parallel Python UDFs and mark single-threaded UDFs 
as unsafe on the MAL layer.


diffs (truncated from 1704 to 300 lines):

diff --git a/sql/backends/monet5/Tests/pyapi19.stable.err 
b/sql/backends/monet5/Tests/pyapi19.stable.err
--- a/sql/backends/monet5/Tests/pyapi19.stable.err
+++ b/sql/backends/monet5/Tests/pyapi19.stable.err
@@ -31,6 +31,64 @@ stderr of test 'pyapi19` in directory 's
 # 11:59:04 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-5346" "--port=35428"
 # 11:59:04 >  
 
+MAPI  = (monetdb) /var/tmp/mtest-23452/.s.monetdb.39765
+QUERY = SELECT * FROM mapped_query();
+ERROR = !Python exception
+!
+!  1. def pyfun(_columns,_column_types,_conn):
+!> 2.   _conn.execute('CREATE TABLE pyapi19_integers(i INTEGER);')
+!  3.   _conn.execute('INSERT INTO pyapi19_integers VALUES (0), (1), 
(2);')
+!  4.   return(1)
+!Loopback queries are not supported in parallel.
+MAPI  = (monetdb) /var/tmp/mtest-23452/.s.monetdb.39765
+QUERY = SELECT * FROM pyapi19_integers;
+ERROR = !current transaction is aborted (please ROLLBACK)
+MAPI  = (monetdb) /var/tmp/mtest-23452/.s.monetdb.39765
+QUERY = CREATE FUNCTION mapped_result_query() returns table (i integer) 
LANGUAGE PYTHON_MAP
+{
+res = _conn.execute('SELECT * FROM pyapi19_integers;')
+return res['i']
+};
+ERROR = !current transaction is aborted (please ROLLBACK)
+MAPI  = (monetdb) /var/tmp/mtest-23452/.s.monetdb.39765
+QUERY = SELECT * FROM mapped_result_query();
+ERROR = !current transaction is aborted (please ROLLBACK)
+MAPI  = (monetdb) /var/tmp/mtest-23452/.s.monetdb.39765
+QUERY = SELECT COUNT(i) FROM pyapi19_load_table();
+ERROR = !Python exception
+!
+!  1. def pyfun(_columns,_column_types,_conn):
+!> 2.   res = _conn.execute('SELECT * FROM pyapi19_integers;')
+!  3.   return res['i']
+!  4. 
+!Loopback queries are not supported in parallel.
+MAPI  = (monetdb) /var/tmp/mtest-23452/.s.monetdb.39765
+QUERY = SELECT * FROM mapped_result_query();
+ERROR = !Python exception
+!
+!  1. def pyfun(_columns,_column_types,_conn):
+!> 2.   res = _conn.execute('SELECT * FROM pyapi19_strings;')
+!  3.   return res['s']
+!  4. 
+!Loopback queries are not supported in parallel.
+MAPI  = (monetdb) /var/tmp/mtest-23452/.s.monetdb.39765
+QUERY = SELECT * FROM mapped_result_query();
+ERROR = !Python exception
+!
+!  1. def pyfun(_columns,_column_types,_conn):
+!> 2.   res = _conn.execute('SELECT * FROM pyapi19_dates;')
+!  3.   return res
+!  4. 
+!Loopback queries are not supported in parallel.
+MAPI  = (monetdb) /var/tmp/mtest-23452/.s.monetdb.39765
+QUERY = SELECT * FROM pyapi19_load_table() LIMIT 100;
+ERROR = !Python exception
+!
+!  1. def pyfun(_columns,_column_types,_conn):
+!> 2.   res = _conn.execute('SELECT * FROM pyapi19_integers;')
+!  3.   return res
+!  4. 
+!Loopback queries are not supported in parallel.
 
 # 11:59:05 >  
 # 11:59:05 >  "Done."
diff --git a/sql/backends/monet5/Tests/pyapi19.stable.out 
b/sql/backends/monet5/Tests/pyapi19.stable.out
--- a/sql/backends/monet5/Tests/pyapi19.stable.out
+++ b/sql/backends/monet5/Tests/pyapi19.stable.out
@@ -36,33 +36,6 @@ Ready.
 #_conn.execute('INSERT INTO pyapi19_integers VALUES (0), (1), (2);')
 #return(1)
 #};
-#SELECT * FROM mapped_query();
-% . # table_name
-% i # name
-% int # type
-% 1 # length
-[ 1]
-#SELECT * FROM pyapi19_integers;
-% sys.pyapi19_integers # table_name
-% i # name
-% int # type
-% 1 # length
-[ 0]
-[ 1]
-[ 2]
-#CREATE FUNCTION mapped_result_query() returns table (i integer) LANGUAGE 
PYTHON_MAP
-#{
-#res = _conn.execute('SELECT * 

MonetDB: jitudf - Move indexes out of for loop declarations for ...

[Mark Raasveldt]

Changeset: 548ac6db1314 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=548ac6db1314
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi00.sql
sql/backends/monet5/UDF/capi/Tests/capi00.stable.out
sql/backends/monet5/UDF/capi/Tests/capi01.sql
sql/backends/monet5/UDF/capi/Tests/capi02.sql
sql/backends/monet5/UDF/capi/Tests/capi03.sql
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/Tests/capi05.sql
sql/backends/monet5/UDF/capi/Tests/capi06.sql
sql/backends/monet5/UDF/capi/Tests/capi07.sql
sql/backends/monet5/UDF/capi/Tests/capi08.sql
sql/backends/monet5/UDF/capi/Tests/capi09.sql
sql/backends/monet5/UDF/capi/Tests/capi10.sql
sql/backends/monet5/UDF/capi/Tests/capi11.sql
sql/backends/monet5/UDF/capi/Tests/capi12.sql
Branch: jitudf
Log Message:

Move indexes out of for loop declarations for GCC.


diffs (truncated from 592 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi00.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi00.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi00.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi00.sql
@@ -3,16 +3,17 @@
 START TRANSACTION;
 
 CREATE FUNCTION capi00(inp INTEGER) RETURNS INTEGER LANGUAGE C {
-   result->initialize(result, inp.count);
-   for(size_t i = 0; i < inp.count; i++) {
-   result->data[i] = inp.data[i] * 2;
-   }
+size_t i;
+result->initialize(result, inp.count);
+for(i = 0; i < inp.count; i++) {
+result->data[i] = inp.data[i] * 2;
+}
 };
 
 CREATE TABLE integers(i INTEGER);
 INSERT INTO integers VALUES (1), (2), (3), (4), (5);
 
-SELECT capi00(i) FROM integers;
+SELECT i, capi00(i) FROM integers;
 
 DROP FUNCTION capi00;
 
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi00.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi00.stable.out
--- a/sql/backends/monet5/UDF/capi/Tests/capi00.stable.out
+++ b/sql/backends/monet5/UDF/capi/Tests/capi00.stable.out
@@ -73,16 +73,17 @@ Ready.
 #CREATE TABLE integers(i INTEGER);
 #INSERT INTO integers VALUES (1), (2), (3), (4), (5);
 [ 5]
-#SELECT capi00(i) FROM integers;
-% sys.L2 # table_name
-% L2 # name
-% int # type
-% 2 # length
-[ 2]
-[ 4]
-[ 6]
-[ 8]
-[ 10   ]
+#SELECT i, capi00(i) FROM integers;
+% sys.integers,sys.L3 # table_name
+% i,   L3 # name
+% int, int # type
+% 1,   2 # length
+[ 1,   2   ]
+[ 2,   4   ]
+[ 3,   6   ]
+[ 4,   8   ]
+[ 5,   10  ]
+#DROP FUNCTION capi00;
 #ROLLBACK;
 
 # 11:56:00 >  
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi01.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi01.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi01.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi01.sql
@@ -5,13 +5,14 @@ CREATE FUNCTION capi01(inp INTEGER) RETU
 language C
 {
 #include 
-   size_t count = inp.data[0];
-   i->initialize(i, count);
-   d->initialize(d, count);
-   for(size_t j = 0; j < count; j++) {
-   i->data[j] = j;
-   d->data[j] = round(j > 0 ? 42.0 / j : 42.0);
-   }
+size_t j;
+size_t count = inp.data[0];
+i->initialize(i, count);
+d->initialize(d, count);
+for(j = 0; j < count; j++) {
+i->data[j] = j;
+d->data[j] = round(j > 0 ? 42.0 / j : 42.0);
+}
 };
 
 SELECT i,d FROM capi01(42) AS R;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi02.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi02.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi02.sql
@@ -4,24 +4,25 @@ START TRANSACTION;
 # dates
 # dates have the type
 #typedef struct {
-#  unsigned char day;
-#  unsigned char month;
-#  int year;
+#   unsigned char day;
+#   unsigned char month;
+#   int year;
 #} cudf_data_date;
 
 CREATE FUNCTION capi02_increment_year(d DATE) RETURNS DATE
 language C
 {
-   result->initialize(result, d.count);
-   for(size_t i = 0; i < result->count; i++) {
-   if (d.is_null(d.data[i])) {
-   result->data[i] = result->null_value;
-   } else {
-   result->data[i].year = d.data[i].year + 1;
-   result->data[i].month = d.data[i].month;
-   result->data[i].day = d.data[i].day;
-   }
-   }
+size_t i;
+result->initialize(result, d.count);
+for(i = 0; i < result->count; i++) {
+if (d.is_null(d.data[i])) {
+result->data[i] = result->null_value;
+} else {
+result->data[i].year = d.data[i].year + 1;
+result->data[i].month = d.data[i].month;
+result->data[i].day = d.data[i].day;
+}
+}
 };
 
 
@@ -36,26 +37,27 @@ DROP TABLE dates;
 #time
 #time has the type:
 #typedef struct {
-#  unsigned int ms;
-#  unsigned char seconds;
-#  unsigned char minutes;
-#  

MonetDB: jitudf - Compilation fixes.

[Mark Raasveldt]

Changeset: 8db98a7ce945 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8db98a7ce945
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Compilation fixes.


diffs (76 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -426,6 +426,7 @@ static str CUDFeval(Client cntxt, MalBlk
 
size_t index = 0;
int bat_type = 0;
+   const char* tpe = NULL;
 
(void)cntxt;
 
@@ -732,11 +733,11 @@ static str CUDFeval(Client cntxt, MalBlk
bat_type = !isaBatType(getArgType(mb, pci, i))
   ? getArgType(mb, 
pci, i)
   : 
getBatType(getArgType(mb, pci, i));
-   const char *tpe = GetTypeName(bat_type);
+   tpe = GetTypeName(bat_type);
assert(tpe);
if (tpe) {
snprintf(buf, sizeof(buf),
-"%s%s %s = 
*((%s%s*)__inputs[%zu]);\n", struct_prefix,
+"\t%s%s %s = 
*((%s%s*)__inputs[%zu]);\n", struct_prefix,
 tpe, args[i], struct_prefix, 
tpe,
 i - (pci->retc + ARG_OFFSET));
ATTEMPT_TO_WRITE_TO_FILE(f, buf);
@@ -745,11 +746,11 @@ static str CUDFeval(Client cntxt, MalBlk
if (non_grouped_aggregate) {
// manually add "aggr_group" for non-grouped aggregates
bat_type = TYPE_oid;
-   const char *tpe = GetTypeName(bat_type);
+   tpe = GetTypeName(bat_type);
assert(tpe);
if (tpe) {
snprintf(buf, sizeof(buf),
-"%s%s %s = 
*((%s%s*)__inputs[%zu]);\n", struct_prefix,
+"\t%s%s %s = 
*((%s%s*)__inputs[%zu]);\n", struct_prefix,
 tpe, "aggr_group", 
struct_prefix, tpe, input_count);
ATTEMPT_TO_WRITE_TO_FILE(f, buf);
}
@@ -757,11 +758,11 @@ static str CUDFeval(Client cntxt, MalBlk
// output types
for (i = 0; i < (size_t)pci->retc; i++) {
bat_type = getBatType(getArgType(mb, pci, i));
-   const char *tpe = GetTypeName(bat_type);
+   tpe = GetTypeName(bat_type);
assert(tpe);
if (tpe) {
snprintf(buf, sizeof(buf),
-"%s%s* %s = 
((%s%s*)__outputs[%zu]);\n", struct_prefix,
+"\t%s%s* %s = 
((%s%s*)__outputs[%zu]);\n", struct_prefix,
 tpe, output_names[i], 
struct_prefix, tpe, i);
ATTEMPT_TO_WRITE_TO_FILE(f, buf);
}
@@ -1087,8 +1088,8 @@ static str CUDFeval(Client cntxt, MalBlk
argnode = argnode ? argnode->next : NULL;
}
 
+   index = input_count;
if (non_grouped_aggregate) {
-   index = input_count;
GENERATE_BAT_INPUT_BASE(oid);
bat_data->count = input_size;
bat_data->data =
@@ -1224,10 +1225,11 @@ static str CUDFeval(Client cntxt, MalBlk
 
// create the output bats from the returned results
for (i = 0; i < (size_t)pci->retc; i++) {
-   bat_type = getBatType(getArgType(mb, pci, i));
size_t count;
void *data;
BAT *b;
+   bat_type = getBatType(getArgType(mb, pci, i));
+   
if (!outputs[i]) {
msg = createException(MAL, "cudf.eval", "No data 
returned.");
goto wrapup;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jul2017 - Initialize sys.argv, because tensorflow requi...

[Mark Raasveldt]

Changeset: 44e0bc7dab59 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=44e0bc7dab59
Modified Files:
sql/backends/monet5/UDF/pyapi/pyapi.c
Branch: Jul2017
Log Message:

Initialize sys.argv, because tensorflow requires this.


diffs (20 lines):

diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c 
b/sql/backends/monet5/UDF/pyapi/pyapi.c
--- a/sql/backends/monet5/UDF/pyapi/pyapi.c
+++ b/sql/backends/monet5/UDF/pyapi/pyapi.c
@@ -1425,9 +1425,16 @@ PYFUNCNAME(PyAPIprelude)(void *ret) {
MT_lock_init(, "query_lock");
MT_lock_set();
if (!pyapiInitialized) {
+#ifdef IS_PY3K
+   wchar_t* program = Py_DecodeLocale("mserver5", NULL);
+   wchar_t* argv[] = { program };
+#else
+   char* argv[] = {"mserver5"};
+#endif
str msg = MAL_SUCCEED;
PyObject *tmp;
Py_Initialize();
+   PySys_SetArgvEx(1, argv, 0);
_import_array();
msg = _connection_init();
if (msg != MAL_SUCCEED) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - More compilation fixes.

[Mark Raasveldt]

Changeset: c2c56a8d02c5 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c2c56a8d02c5
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

More compilation fixes.


diffs (118 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -424,6 +424,9 @@ static str CUDFeval(Client cntxt, MalBlk
size_t input_size = 0;
bit non_grouped_aggregate = 0;
 
+   size_t index = 0;
+   int bat_type = 0;
+
(void)cntxt;
 
actual_mprotected_regions[tid] = 
@@ -726,7 +729,7 @@ static str CUDFeval(Client cntxt, MalBlk
// of the function
// first convert the input
for (i = pci->retc + ARG_OFFSET; i < (size_t)pci->argc; i++) {
-   int bat_type = !isaBatType(getArgType(mb, pci, i))
+   bat_type = !isaBatType(getArgType(mb, pci, i))
   ? getArgType(mb, 
pci, i)
   : 
getBatType(getArgType(mb, pci, i));
const char *tpe = GetTypeName(bat_type);
@@ -741,7 +744,7 @@ static str CUDFeval(Client cntxt, MalBlk
}
if (non_grouped_aggregate) {
// manually add "aggr_group" for non-grouped aggregates
-   int bat_type = TYPE_oid;
+   bat_type = TYPE_oid;
const char *tpe = GetTypeName(bat_type);
assert(tpe);
if (tpe) {
@@ -753,7 +756,7 @@ static str CUDFeval(Client cntxt, MalBlk
}
// output types
for (i = 0; i < (size_t)pci->retc; i++) {
-   int bat_type = getBatType(getArgType(mb, pci, i));
+   bat_type = getBatType(getArgType(mb, pci, i));
const char *tpe = GetTypeName(bat_type);
assert(tpe);
if (tpe) {
@@ -882,8 +885,8 @@ static str CUDFeval(Client cntxt, MalBlk
// create the inputs
argnode = sqlfun ? sqlfun->ops->h : NULL;
for (i = pci->retc + ARG_OFFSET; i < (size_t)pci->argc; i++) {
-   size_t index = i - (pci->retc + ARG_OFFSET);
-   int bat_type = getArgType(mb, pci, i);
+   index = i - (pci->retc + ARG_OFFSET);
+   bat_type = getArgType(mb, pci, i);
if (!isaBatType(bat_type)) {
// scalar input
// create a temporary BAT
@@ -1085,7 +1088,7 @@ static str CUDFeval(Client cntxt, MalBlk
}
 
if (non_grouped_aggregate) {
-   size_t index = input_count;
+   index = input_count;
GENERATE_BAT_INPUT_BASE(oid);
bat_data->count = input_size;
bat_data->data =
@@ -1096,8 +1099,8 @@ static str CUDFeval(Client cntxt, MalBlk
argnode = sqlfun ? sqlfun->res->h : NULL;
// output types
for (i = 0; i < output_count; i++) {
-   size_t index = i;
-   int bat_type = getBatType(getArgType(mb, pci, i));
+   index = i;
+   bat_type = getBatType(getArgType(mb, pci, i));
if (bat_type == TYPE_bit || bat_type == TYPE_bte) {
GENERATE_BAT_OUTPUT(bte);
} else if (bat_type == TYPE_sht) {
@@ -1151,18 +1154,16 @@ static str CUDFeval(Client cntxt, MalBlk
msg = createException(MAL, "cudf.eval", "Attempting to 
write to "

"the input or triggered a "

"segfault/bus error");
-   goto wrapup;
} else if (ret == 2) {
msg = createException(MAL, "cudf.eval",
  "Malloc 
failure in internal function!");
-   goto wrapup;
} else {
// we jumped here
msg = createException(MAL, "cudf.eval", "We longjumped 
here "

"because of an error, but "

"we don't know which!");
-   goto wrapup;
}
+   goto wrapup;
}
 
// set up the signal handler for catching segfaults
@@ -1223,7 +1224,7 @@ static str CUDFeval(Client cntxt, MalBlk
 
// create the output bats from the returned results
   

MonetDB: jitudf - Compilation fixes.

[Mark Raasveldt]

Changeset: 659abbe2473d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=659abbe2473d
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Compilation fixes.


diffs (93 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -1,14 +1,14 @@
 
-capi00
-capi01
-capi02
-capi03
-capi04
-capi05
-capi06
-capi07
-capi08
-capi09
-capi10
-capi11
-capi12
+NOT_WIN32?capi00
+NOT_WIN32?capi01
+NOT_WIN32?capi02
+NOT_WIN32?capi03
+NOT_WIN32?capi04
+NOT_WIN32?capi05
+NOT_WIN32?capi06
+NOT_WIN32?capi07
+NOT_WIN32?capi08
+NOT_WIN32?capi09
+NOT_WIN32?capi10
+NOT_WIN32?capi11
+NOT_WIN32?capi12
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -16,7 +16,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 #include 
 #include 
@@ -348,7 +347,6 @@ const char *cpp_flag = "capi_cpp";
 
 static size_t GetTypeCount(int type, void *struct_ptr);
 static void *GetTypeData(int type, void *struct_ptr);
-static const char *GetTypeDefinition(int type);
 static const char *GetTypeName(int type);
 
 static void data_from_date(date d, cudf_data_date *ptr);
@@ -1521,42 +1519,6 @@ wrapup:
return msg;
 }
 
-static const char *GetTypeDefinition(int type)
-{
-   const char *tpe = NULL;
-   if (type == TYPE_bit) {
-   tpe = "signed char";
-   } else if (type == TYPE_bte) {
-   tpe = "signed char";
-   } else if (type == TYPE_sht) {
-   tpe = "short";
-   } else if (type == TYPE_int) {
-   tpe = "int";
-   } else if (type == TYPE_oid) {
-   tpe = "size_t";
-   } else if (type == TYPE_lng) {
-   tpe = "lng";
-   } else if (type == TYPE_flt) {
-   tpe = "float";
-   } else if (type == TYPE_dbl) {
-   tpe = "double";
-   } else if (type == TYPE_str) {
-   tpe = "char*";
-   } else if (type == TYPE_date) {
-   tpe = "cudf_data_date";
-   } else if (type == TYPE_daytime) {
-   tpe = "cudf_data_time";
-   } else if (type == TYPE_timestamp) {
-   tpe = "cudf_data_timestamp";
-   } else if (type == TYPE_blob || type == TYPE_sqlblob) {
-   tpe = "cudf_data_blob";
-   } else {
-   // unsupported type: string
-   tpe = "char*";
-   }
-   return tpe;
-}
-
 static const char *GetTypeName(int type)
 {
const char *tpe = NULL;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Treat scalar and bat input the same.

[Mark Raasveldt]

Changeset: ab770c3b2521 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ab770c3b2521
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi01.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Treat scalar and bat input the same.


diffs (truncated from 1010 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi01.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi01.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi01.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi01.sql
@@ -5,8 +5,7 @@ CREATE FUNCTION capi01(inp INTEGER) RETU
 language C
 {
 #include 
-
-   size_t count = inp;
+   size_t count = inp.data[0];
i->initialize(i, count);
d->initialize(d, count);
for(size_t j = 0; j < count; j++) {
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -33,7 +33,7 @@ typedef struct _mprotected_region {
void *addr;
size_t len;
 
-   void* actual_addr;
+   void *actual_addr;
size_t actual_len;
 
bool is_protected;
@@ -41,7 +41,8 @@ typedef struct _mprotected_region {
struct _mprotected_region *next;
 } mprotected_region;
 
-static char *mprotect_region(void *addr, size_t len, mprotected_region 
**regions);
+static char *mprotect_region(void *addr, size_t len,
+mprotected_region 
**regions);
 static char *clear_mprotect(void *addr, size_t len);
 
 static allocated_region *allocated_regions[THREADS];
@@ -55,8 +56,8 @@ struct _cached_functions;
 typedef struct _cached_functions {
jitted_function function;
BUN expression_hash;
-   char* parameters;
-   void* dll_handle;
+   char *parameters;
+   void *dll_handle;
struct _cached_functions *next;
 } cached_functions;
 
@@ -65,7 +66,6 @@ typedef struct _cached_functions {
 static cached_functions *function_cache[FUNCTION_CACHE_SIZE];
 static MT_Lock cache_lock;
 
-
 static str CUDFeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
bit grouped);
 
@@ -100,13 +100,13 @@ static bool WriteTextToFile(FILE *f, con
 static void handler(int sig, siginfo_t *si, void *unused)
 {
int actually_protected_area = false;
-   mprotected_region* found_region = NULL;
+   mprotected_region *found_region = NULL;
int tid = THRgettid();
 
(void)sig;
(void)unused;
// we caught a segfault or bus error
-   // this can be either because 
+   // this can be either because
// (1) the function accessed a protected piece of memory
// (2) the function caused a segfault by e.g. dereferencing a NULL 
pointer
// in the first case, this *might* be a valid memory access
@@ -116,13 +116,15 @@ static void handler(int sig, siginfo_t *
// was actually an error
 
if (actual_mprotected_regions[tid]) {
-   mprotected_region* region = *actual_mprotected_regions[tid];
-   while(region) {
-   if (si->si_addr >= region->addr && (char*) si->si_addr 
<= (char*) region->addr + region->len) {
+   mprotected_region *region = *actual_mprotected_regions[tid];
+   while (region) {
+   if (si->si_addr >= region->addr &&
+   (char *)si->si_addr <= (char *)region->addr + 
region->len) {
// the address belongs to this mprotected region
found_region = region;
-   if (si->si_addr >= region->actual_addr && 
-   (char*) si->si_addr <= (char*) 
region->actual_addr + region->actual_len) {
+   if (si->si_addr >= region->actual_addr &&
+   (char *)si->si_addr <=
+   (char *)region->actual_addr + 
region->actual_len) {
// and the address is actually supposed 
to be protected
actually_protected_area = true;
break;
@@ -143,12 +145,13 @@ static void handler(int sig, siginfo_t *
}
 }
 
-static char *mprotect_region(void *addr, size_t len, mprotected_region 
**regions)
+static char *mprotect_region(void *addr, size_t len,
+mprotected_region 
**regions)
 {
mprotected_region *region;
int pagesize;
void *page_begin;
-   void* actual_addr = addr;
+   void *actual_addr = addr;
size_t actual_len = len;
if (len == 0)
return NULL;
@@ -182,7 +185,8 @@ static char *mprotect_region(void *addr,
 
 static char *clear_mprotect(void *addr, size_t len)
 {
-   if 

MonetDB: jitudf - No rand() calls in tests.

[Mark Raasveldt]

Changeset: 8268c9af5f70 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8268c9af5f70
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi02.stable.out
Branch: jitudf
Log Message:

No rand() calls in tests.


diffs (27 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out
--- a/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out
+++ b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out
@@ -76,9 +76,9 @@ Ready.
 % L2 # name
 % time # type
 % 8 # length
-[ 22:52:18 ]
+[ 10:34:34 ]
 [ NULL ]
-[ 13:38:50 ]
+[ 12:36:36 ]
 #DROP FUNCTION capi02_randomize_time;
 #DROP TABLE times;
 #CREATE FUNCTION capi02_increment_timestamp(d TIMESTAMP) RETURNS TIMESTAMP
@@ -104,9 +104,9 @@ Ready.
 % L2 # name
 % timestamp # type
 % 26 # length
-[ 1993-09-20 22:52:18.417000   ]
+[ 1993-09-20 10:34:34.234000   ]
 [ NULL ]
-[ 2001-03-10 13:38:50.913000   ]
+[ 2001-03-10 12:36:36.236000   ]
 #DROP FUNCTION capi02_increment_timestamp;
 #DROP TABLE times;
 #ROLLBACK;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Only mprotect regions after the signal handler...

[Mark Raasveldt]

Changeset: 0a98aecb2ea2 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0a98aecb2ea2
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi02.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Only mprotect regions after the signal handlers have been set.


diffs (186 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi02.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi02.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi02.sql
@@ -45,16 +45,15 @@ DROP TABLE dates;
 CREATE FUNCTION capi02_randomize_time(d TIME) RETURNS TIME
 language C
 {
-   srand(1234);
result->initialize(result, d.count);
for(size_t i = 0; i < result->count; i++) {
if (d.is_null(d.data[i])) {
result->data[i] = result->null_value;
} else {
-   result->data[i].hours = rand() % 24;
-   result->data[i].minutes = rand() % 60;
-   result->data[i].seconds = rand() % 60;
-   result->data[i].ms = rand() % 1000;
+   result->data[i].hours = (i + 1234) % 24;
+   result->data[i].minutes = (i + 1234) % 60;
+   result->data[i].seconds = (i + 1234) % 60;
+   result->data[i].ms = (i + 1234) % 1000;
}
}
 };
@@ -77,7 +76,6 @@ DROP TABLE times;
 CREATE FUNCTION capi02_increment_timestamp(d TIMESTAMP) RETURNS TIMESTAMP
 language C
 {
-   srand(1234);
result->initialize(result, d.count);
for(size_t i = 0; i < result->count; i++) {
if (d.is_null(d.data[i])) {
@@ -87,10 +85,10 @@ language C
result->data[i].date.month = d.data[i].date.month;
result->data[i].date.day = d.data[i].date.day;
 
-   result->data[i].time.hours = rand() % 24;
-   result->data[i].time.minutes = rand() % 60;
-   result->data[i].time.seconds = rand() % 60;
-   result->data[i].time.ms = rand() % 1000;
+   result->data[i].time.hours = (i + 1234) % 24;
+   result->data[i].time.minutes = (i + 1234) % 60;
+   result->data[i].time.seconds = (i + 1234) % 60;
+   result->data[i].time.ms = (i + 1234) % 1000;
}
}
 };
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -36,11 +36,12 @@ typedef struct _mprotected_region {
void* actual_addr;
size_t actual_len;
 
+   bool is_protected;
+
struct _mprotected_region *next;
 } mprotected_region;
 
-static char *mprotect_region(void *addr, size_t len, int flags,
-mprotected_region 
**regions);
+static char *mprotect_region(void *addr, size_t len, mprotected_region 
**regions);
 static char *clear_mprotect(void *addr, size_t len);
 
 static allocated_region *allocated_regions[THREADS];
@@ -142,8 +143,7 @@ static void handler(int sig, siginfo_t *
}
 }
 
-static char *mprotect_region(void *addr, size_t len, int flags,
-mprotected_region 
**regions)
+static char *mprotect_region(void *addr, size_t len, mprotected_region 
**regions)
 {
mprotected_region *region;
int pagesize;
@@ -170,15 +170,12 @@ static char *mprotect_region(void *addr,
if (!region) {
return MAL_MALLOC_FAIL;
}
-   if (mprotect(addr, len, flags) < 0) {
-   GDKfree(region);
-   return strerror(errno);
-   }
region->addr = addr;
region->len = len;
region->next = *regions;
region->actual_addr = actual_addr;
region->actual_len = actual_len;
+   region->is_protected = false;
*regions = region;
return NULL;
 }
@@ -298,7 +295,7 @@ GENERATE_BASE_HEADERS(cudf_data_blob, bl
bat_data->data = (tpe *)Tloc(b, 0); 
   \
mprotect_retval = mprotect_region(  
   \
bat_data->data, bat_data->count * 
sizeof(bat_data->null_value),\
-   PROT_READ, );   
   \
+   );  
\
if (mprotect_retval) {  
   \
msg = createException(MAL, "cudf.eval", 
   \
  
"Failed to mprotect region: %s", \
@@ -392,7 

MonetDB: jitudf - Compilation fixes.

[Mark Raasveldt]

Changeset: edf6d672dd00 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=edf6d672dd00
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi00.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Compilation fixes.


diffs (44 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi00.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi00.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi00.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi00.sql
@@ -14,4 +14,6 @@ INSERT INTO integers VALUES (1), (2), (3
 
 SELECT capi00(i) FROM integers;
 
+DROP FUNCTION capi00;
+
 ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -228,14 +228,12 @@ static void* add_allocated_region(void* 
 
 static void *wrapped_GDK_malloc(size_t size)
 {
-   allocated_region *region;
void *ptr = jump_GDK_malloc(size + sizeof(allocated_region));
return add_allocated_region(ptr);
 }
 
 static void *wrapped_GDK_malloc_nojump(size_t size)
 {
-   allocated_region *region;
void *ptr = GDKmalloc(size + sizeof(allocated_region));
if (!ptr) {
return NULL;
@@ -1076,7 +1074,6 @@ static str CUDFeval(Client cntxt, MalBlk
// unsupported type: convert to string
BATiter li;
BUN p = 0, q = 0;
-   str mprotect_retval;
GENERATE_BAT_INPUT_BASE(str);
bat_data->count = BATcount(input_bats[index]);
bat_data->null_value = NULL;
@@ -1372,7 +1369,6 @@ static str CUDFeval(Client cntxt, MalBlk
if (!ptr || strcmp(ptr, str_nil) == 0) {
appended_element = (void*) 
BATatoms[bat_type].atomNull;
} else {
-   int len = 0;
if (BATatoms[bat_type].atomFromStr(ptr, 
, ) == 0) {
msg = createException(MAL, 
"cudf.eval", "Failed to convert output element from string: %s", ptr);
goto wrapup;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Always create aggr_group input and add test ca...

[Mark Raasveldt]

Changeset: d9e067c1bc99 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d9e067c1bc99
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi13.sql
sql/backends/monet5/UDF/capi/Tests/capi13.stable.err
sql/backends/monet5/UDF/capi/Tests/capi13.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Always create aggr_group input and add test case for grouped/ungrouped 
aggregates.


diffs (truncated from 326 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi13.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi13.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi13.sql
@@ -0,0 +1,28 @@
+
+START TRANSACTION;
+
+# grouped aggregate: sum
+
+CREATE AGGREGATE capi13(inp INTEGER) RETURNS BIGINT LANGUAGE C {
+#include 
+   // initialize one aggregate per group
+   result->initialize(result, aggr_group.count);
+   // zero initialize the sums
+   memset(result->data, 0, result->count * sizeof(result->null_value));
+   // gather the sums for each of the groups
+   for(size_t i = 0; i < inp.count; i++) {
+   result->data[aggr_group.data[i]] += inp.data[i];
+   }
+};
+
+CREATE TABLE vals(grp INTEGER, value INTEGER);
+INSERT INTO vals VALUES (1, 100), (2, 200), (1, 50), (2, 300);
+
+SELECT grp, capi13(value) FROM vals GROUP BY grp;
+
+SELECT capi13(value) FROM vals;
+
+DROP AGGREGATE capi13;
+DROP TABLE vals;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi13.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi13.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi13.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi13` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 14:43:45 >  
+# 14:43:45 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=35341" "--set" 
"mapi_usock=/var/tmp/mtest-91643/.s.monetdb.35341" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 14:43:45 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 35341
+# cmdline opt  mapi_usock = /var/tmp/mtest-91643/.s.monetdb.35341
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 14:43:46 >  
+# 14:43:46 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-91643" "--port=35341"
+# 14:43:46 >  
+
+
+# 14:43:47 >  
+# 14:43:47 >  "Done."
+# 14:43:47 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi13.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi13.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi13.stable.out
@@ -0,0 +1,97 @@
+stdout of test 'capi13` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 14:43:45 >  
+# 14:43:45 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=35341" "--set" 
"mapi_usock=/var/tmp/mtest-91643/.s.monetdb.35341" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 14:43:45 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:35341/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-91643/.s.monetdb.35341
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 

MonetDB: jitudf - Deal with dense bats correctly and support gro...

[Mark Raasveldt]

Changeset: 496a7405c39d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=496a7405c39d
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi07.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Deal with dense bats correctly and support grouped aggregates.


diffs (174 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi07.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi07.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi07.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi07.sql
@@ -2,7 +2,7 @@
 
 START TRANSACTION;
 
-CREATE AGGREGATE capi06(inp INTEGER) RETURNS BIGINT LANGUAGE C {
+CREATE AGGREGATE capi07(inp INTEGER) RETURNS BIGINT LANGUAGE C {
lng sum = 0;
for(size_t i = 0; i < inp.count; i++) {
sum += inp.data[i];
@@ -14,6 +14,6 @@ CREATE AGGREGATE capi06(inp INTEGER) RET
 CREATE TABLE integers(i INTEGER);
 INSERT INTO integers VALUES (3), (4), (1), (2), (5), (6);
 
-SELECT capi06(i) FROM integers;
+SELECT capi07(i) FROM integers;
 
 ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -216,16 +216,31 @@ static void *jump_GDK_malloc(size_t size
return ptr;
 }
 
+static void* add_allocated_region(void* ptr)
+{
+   allocated_region *region;
+   int tid = THRgettid();
+   region = (allocated_region *)ptr;
+   region->next = allocated_regions[tid];
+   allocated_regions[tid] = region;
+   return (char*)ptr + sizeof(allocated_region);
+}
+
 static void *wrapped_GDK_malloc(size_t size)
 {
allocated_region *region;
void *ptr = jump_GDK_malloc(size + sizeof(allocated_region));
-   int tid = THRgettid();
-   region = (allocated_region *)ptr;
-   region->next = allocated_regions[tid];
-   allocated_regions[tid] = region;
+   return add_allocated_region(ptr);
+}
 
-   return (char*)ptr + sizeof(allocated_region);
+static void *wrapped_GDK_malloc_nojump(size_t size)
+{
+   allocated_region *region;
+   void *ptr = GDKmalloc(size + sizeof(allocated_region));
+   if (!ptr) {
+   return NULL;
+   }
+   return add_allocated_region(ptr);
 }
 
 #define GENERATE_BASE_HEADERS(type, tpename)   
\
@@ -272,17 +287,27 @@ GENERATE_BASE_HEADERS(cudf_data_blob, bl
char *mprotect_retval;  
   \
GENERATE_BAT_INPUT_BASE(b, tpe);
   \
bat_data->count = BATcount(b);  
   \
-   bat_data->data = (tpe *)Tloc(b, 0); 
   \
bat_data->null_value = tpe##_nil;   
   \
-   mprotect_retval = mprotect_region(  
   \
-   bat_data->data, bat_data->count * 
sizeof(bat_data->null_value),\
-   PROT_READ, );   
   \
-   if (mprotect_retval) {  
   \
-   msg = createException(MAL, "cudf.eval", 
   \
- "Failed to 
mprotect region: %s", \
- 
mprotect_retval);\
-   goto wrapup;
   \
-   }   
   \
+   if (b->tdense && !b->tnodense) { \
+   size_t it = 0; \
+   tpe val = b->T.seq; \
+   /* bat is dense, materialize it */ \
+   bat_data->data = 
wrapped_GDK_malloc_nojump(bat_data->count * sizeof(bat_data->null_value)); \
+   for(it = 0; it < bat_data->count; it++) { \
+   bat_data->data[it] = val++; \
+   } \
+   } else { \
+   bat_data->data = (tpe *)Tloc(b, 0); 
   \
+   mprotect_retval = mprotect_region(  
   \
+   bat_data->data, bat_data->count * 
sizeof(bat_data->null_value),\
+   PROT_READ, );   
   \
+   if (mprotect_retval) {  
   \
+   msg = createException(MAL, "cudf.eval", 
   \
+ 
"Failed to mprotect region: %s", \
+   

MonetDB: jitudf - OIDs.

[Mark Raasveldt]

Changeset: b7f5722c5a13 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b7f5722c5a13
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi12.sql
sql/backends/monet5/UDF/capi/Tests/capi12.stable.err
sql/backends/monet5/UDF/capi/Tests/capi12.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/capi/cheader.h
Branch: jitudf
Log Message:

OIDs.


diffs (252 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -11,3 +11,4 @@ capi08
 capi09
 capi10
 capi11
+capi12
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi12.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi12.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi12.sql
@@ -0,0 +1,23 @@
+
+START TRANSACTION;
+
+# oids
+CREATE FUNCTION capi12(inp OID) RETURNS BOOLEAN LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+if (inp.data[i] == inp.null_value) {
+result->data[i] = 0;
+} else {
+result->data[i] = 1;
+}
+}
+};
+CREATE TABLE oids(i OID);
+INSERT INTO oids(i) VALUES (100), (NULL), (200);
+
+SELECT * FROM oids WHERE capi12(i);
+
+DROP FUNCTION capi12;
+DROP TABLE oids;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi12.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi12.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi12.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi12` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 13:43:07 >  
+# 13:43:07 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=34982" "--set" 
"mapi_usock=/var/tmp/mtest-84742/.s.monetdb.34982" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 13:43:07 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 34982
+# cmdline opt  mapi_usock = /var/tmp/mtest-84742/.s.monetdb.34982
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 13:43:07 >  
+# 13:43:07 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-84742" "--port=34982"
+# 13:43:07 >  
+
+
+# 13:43:08 >  
+# 13:43:08 >  "Done."
+# 13:43:08 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi12.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi12.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi12.stable.out
@@ -0,0 +1,90 @@
+stdout of test 'capi12` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 13:43:07 >  
+# 13:43:07 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=34982" "--set" 
"mapi_usock=/var/tmp/mtest-84742/.s.monetdb.34982" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 13:43:07 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:34982/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-84742/.s.monetdb.34982
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql 

MonetDB: jitudf - Use NULL instead of str_nil for string NULL va...

[Mark Raasveldt]

Changeset: 6822b8d5a1e2 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6822b8d5a1e2
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Use NULL instead of str_nil for string NULL values.


diffs (82 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -292,6 +292,8 @@ GENERATE_BASE_HEADERS(cudf_data_blob, bl
goto wrapup;
   \
}   
   \
outputs[index] = bat_data;  
   \
+   bat_data->count = 0;
   \
+   bat_data->data = NULL;  
   \
bat_data->is_null = tpe##_is_null;  
   \
bat_data->scale = argnode ? pow(10, ((sql_arg 
*)argnode->data)->type.scale) : 1;\
bat_data->initialize = (void (*)(void *, size_t))tpe##_initialize;
@@ -299,8 +301,6 @@ GENERATE_BASE_HEADERS(cudf_data_blob, bl
 #define GENERATE_BAT_OUTPUT(tpe)   
\
{   
   \
GENERATE_BAT_OUTPUT_BASE(tpe);  
   \
-   bat_data->count = 0;
   \
-   bat_data->data = NULL;  
   \
bat_data->null_value = (tpe)tpe##_nil;  
   \
}
 
@@ -910,6 +910,7 @@ static str CUDFeval(Client cntxt, MalBlk
GENERATE_BAT_INPUT_BASE(input_bats[index], str);
bat_data->count = BATcount(input_bats[index]);
bat_data->data = GDKmalloc(sizeof(char *) * 
bat_data->count);
+   bat_data->null_value = NULL;
if (!bat_data->data) {
msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
goto wrapup;
@@ -920,7 +921,11 @@ static str CUDFeval(Client cntxt, MalBlk
BATloop(input_bats[index], p, q)
{
char *t = (char *)BUNtail(li, p);
-   bat_data->data[j] = t;
+   if (strcmp(t, str_nil) == 0) {
+   bat_data->data[j] = NULL;
+   } else {
+   bat_data->data[j] = t;  
+   }
j++;
}
// for string columns, mprotect the varheap of 
the BAT
@@ -1078,7 +1083,8 @@ static str CUDFeval(Client cntxt, MalBlk
} else if (bat_type == TYPE_dbl) {
GENERATE_BAT_OUTPUT(dbl);
} else if (bat_type == TYPE_str) {
-   GENERATE_BAT_OUTPUT(str);
+   GENERATE_BAT_OUTPUT_BASE(str);
+   bat_data->null_value = NULL;
} else if (bat_type == TYPE_date) {
GENERATE_BAT_OUTPUT_BASE(date);
data_from_date(date_nil, _data->null_value);
@@ -1094,7 +1100,8 @@ static str CUDFeval(Client cntxt, MalBlk
bat_data->null_value.data = NULL;
} else {
// unsupported type, convert from string output
-   GENERATE_BAT_OUTPUT(str);
+   GENERATE_BAT_OUTPUT_BASE(str);
+   bat_data->null_value = NULL;
}
argnode = argnode ? argnode->next : NULL;
}
@@ -1254,7 +1261,7 @@ static str CUDFeval(Client cntxt, MalBlk
if (!ptr) {
ptr = str_nil;
}
-   if (BUNappend(b, source_base[j], FALSE) != 
GDK_SUCCEED) {
+   if (BUNappend(b, ptr, FALSE) != GDK_SUCCEED) {
msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
goto wrapup;
}
@@ -1664,7 +1671,9 @@ int timestamp_is_null(cudf_data_timestam
return ts_isnil(timestamp_from_data());
 }
 
-int str_is_null(char *value) { return strcmp(value, str_nil) == 0; }
+int str_is_null(char *value) {
+   return value == NULL;
+}
 
 int 

MonetDB: jitudf - Support all types by converting them to strings.

[Mark Raasveldt]

Changeset: fd2793b0d353 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fd2793b0d353
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi09.sql
sql/backends/monet5/UDF/capi/Tests/capi09.stable.err
sql/backends/monet5/UDF/capi/Tests/capi09.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/Tests/capi10.stable.err
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Support all types by converting them to strings.


diffs (truncated from 330 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -8,3 +8,6 @@ capi05
 capi06
 capi07
 capi08
+capi09
+capi10
+capi11
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi09.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi09.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi09.sql
@@ -0,0 +1,26 @@
+
+START TRANSACTION;
+
+# uuids
+# other (unsupported) types are simply converted to/from strings
+
+CREATE FUNCTION capi12(inp UUID) RETURNS UUID LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+if (inp.data[i] == inp.null_value) {
+result->data[i] = result->null_value;
+} else {
+result->data[i] = inp.data[i];
+}
+}
+};
+
+CREATE TABLE uuids(d UUID);
+INSERT INTO uuids VALUES ('ad887b3d-08f7-c308-7285-354a1857cbc8'), (NULL);
+
+SELECT capi12(d) FROM uuids;
+
+DROP FUNCTION capi12;
+DROP TABLE uuids;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi09.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi09.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi09.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi09` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 13:29:00 >  
+# 13:29:00 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=38148" "--set" 
"mapi_usock=/var/tmp/mtest-81291/.s.monetdb.38148" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 13:29:00 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 38148
+# cmdline opt  mapi_usock = /var/tmp/mtest-81291/.s.monetdb.38148
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 13:29:00 >  
+# 13:29:00 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-81291" "--port=38148"
+# 13:29:00 >  
+
+
+# 13:29:01 >  
+# 13:29:01 >  "Done."
+# 13:29:01 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi09.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi09.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi09.stable.out
@@ -0,0 +1,54 @@
+stdout of test 'capi09` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 13:29:00 >  
+# 13:29:00 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=38148" "--set" 
"mapi_usock=/var/tmp/mtest-81291/.s.monetdb.38148" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 13:29:00 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:38148/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-81291/.s.monetdb.38148
+# MonetDB/SQL module loaded
+
+Ready.
+
+# 13:29:00 >  
+# 13:29:00 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-81291" "--port=38148"
+# 13:29:00 >  
+
+#START TRANSACTION;
+#CREATE FUNCTION capi12(inp UUID) RETURNS UUID LANGUAGE C {
+#result->initialize(result, inp.count);
+#for(size_t i = 0; i < inp.count; i++) {
+#if (inp.data[i] == inp.null_value) {
+#result->data[i] = result->null_value;
+#} else {
+#

MonetDB: jitudf - Decimals.

[Mark Raasveldt]

Changeset: 8827543d9c36 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8827543d9c36
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi11.sql
sql/backends/monet5/UDF/capi/Tests/capi11.stable.err
sql/backends/monet5/UDF/capi/Tests/capi11.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi10.sql
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/capi/cheader.h
Branch: jitudf
Log Message:

Decimals.


diffs (truncated from 331 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi10.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi10.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi10.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi10.sql
@@ -1,5 +1,5 @@
 
-# test other SQL types
+# test blob type
 
 START TRANSACTION;
 
@@ -29,13 +29,15 @@ INSERT INTO blobs VALUES (BLOB '0000
 SELECT capi10(i) FROM blobs;
 
 DROP FUNCTION capi10;
-DROP TABLE blobs;
-
-
-# decimal
 
-# uuid
-
-# json
-
+# attempt to modify input of blobs
+CREATE FUNCTION capi10(inp BLOB) RETURNS BLOB LANGUAGE C {
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   if (inp.data[i].data && inp.data[i].size > 0) {
+   ((char*)inp.data[i].data)[0] = 'h';
+   }
+   }
+};
+SELECT capi10(i) FROM blobs;
 ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi11.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi11.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi11.sql
@@ -0,0 +1,62 @@
+
+START TRANSACTION;
+
+# decimal
+# decimals are integer types (bit, short, int or lng)
+# each decimal type has a "scale" (supplied as double)
+# this scale is the scaling factor of the value
+# for example, the value "10.3" may be stored as "103" with scaling factor "10"
+# to convert to double: 103 / 10.0 = 10.3
+# to convert a constant back into the decimal: 10 * 10 = 100 (which is equal 
to )
+
+CREATE FUNCTION capi11(inp DECIMAL) RETURNS DECIMAL(11,1) LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+if (inp.data[i] == inp.null_value) {
+result->data[i] = result->null_value;
+} else {
+result->data[i] = (inp.data[i] / inp.scale) * result->scale;
+}
+}
+};
+
+CREATE TABLE decimals(d DECIMAL(20,3));
+INSERT INTO decimals VALUES (10.3), (10.5), (NULL), (10.7);
+
+SELECT capi11(d) FROM decimals;
+
+DROP FUNCTION capi11;
+
+# to convert from a decimal to a double simply divide the value by the scale
+CREATE FUNCTION _dec2dbl(inp DECIMAL) RETURNS DOUBLE LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+if (inp.data[i] == inp.null_value) {
+result->data[i] = result->null_value;
+} else {
+result->data[i] = inp.data[i] / inp.scale;
+}
+}
+};
+
+SELECT _dec2dbl(d) FROM decimals;
+
+DROP TABLE decimals;
+DROP FUNCTION _dec2dbl;
+
+# to convert a double value to a decimal multiply by the scale
+CREATE FUNCTION _dbl2dec(inp DOUBLE) RETURNS DECIMAL LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+result->data[i] = inp.data[i] * result->scale;
+}
+};
+
+CREATE TABLE doubles(d DOUBLE);
+INSERT INTO doubles VALUES (10.3), (10.5), (10.7);
+
+
+SELECT _dbl2dec(d) FROM doubles;
+
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi11.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi11.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi11.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi11` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 11:45:23 >  
+# 11:45:23 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37236" "--set" 
"mapi_usock=/var/tmp/mtest-63830/.s.monetdb.37236" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 11:45:23 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37236
+# cmdline opt  mapi_usock = /var/tmp/mtest-63830/.s.monetdb.37236
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 11:45:23 >  
+# 11:45:23 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-63830" 

MonetDB: jitudf - Blobs.

[Mark Raasveldt]

Changeset: d2f6194479b6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d2f6194479b6
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi10.sql
sql/backends/monet5/UDF/capi/Tests/capi10.stable.err
sql/backends/monet5/UDF/capi/Tests/capi10.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi02.sql
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/capi/cheader.h
Branch: jitudf
Log Message:

Blobs.


diffs (truncated from 419 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi02.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi02.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi02.sql
@@ -1,6 +1,14 @@
 # Dates, times, timestamps
 START TRANSACTION;
 
+# dates
+# dates have the type
+#typedef struct {
+#  unsigned char day;
+#  unsigned char month;
+#  int year;
+#} cudf_data_date;
+
 CREATE FUNCTION capi02_increment_year(d DATE) RETURNS DATE
 language C
 {
@@ -25,6 +33,15 @@ DROP FUNCTION capi02_increment_year;
 DROP TABLE dates;
 
 
+#time
+#time has the type:
+#typedef struct {
+#  unsigned int ms;
+#  unsigned char seconds;
+#  unsigned char minutes;
+#  unsigned char hours;
+#} cudf_data_time;
+
 CREATE FUNCTION capi02_randomize_time(d TIME) RETURNS TIME
 language C
 {
@@ -50,6 +67,12 @@ SELECT capi02_randomize_time(i) FROM tim
 DROP FUNCTION capi02_randomize_time;
 DROP TABLE times;
 
+#timestamps
+#timestamps have the type:
+#typedef struct {
+#  cudf_data_date date;
+#  cudf_data_time time;
+#} cudf_data_timestamp;
 
 CREATE FUNCTION capi02_increment_timestamp(d TIMESTAMP) RETURNS TIMESTAMP
 language C
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi10.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi10.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi10.sql
@@ -0,0 +1,41 @@
+
+# test other SQL types
+
+START TRANSACTION;
+
+# blob
+# blobs have the type:
+#typedef struct {
+#  size_t size;
+#  void* data;
+#} cudf_data_blob;
+
+CREATE FUNCTION capi10(inp BLOB) RETURNS BLOB LANGUAGE C {
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   if (inp.is_null(inp.data[i])) {
+   result->data[i] = result->null_value;
+   } else {
+   result->data[i].data = malloc(inp.data[i].size);
+   memcpy(result->data[i].data, inp.data[i].data, 
inp.data[i].size);
+   result->data[i].size = inp.data[i].size;
+   }
+   }
+};
+
+CREATE TABLE blobs(i BLOB);
+INSERT INTO blobs VALUES (BLOB '0000'), (NULL), (BLOB '');
+
+SELECT capi10(i) FROM blobs;
+
+DROP FUNCTION capi10;
+DROP TABLE blobs;
+
+
+# decimal
+
+# uuid
+
+# json
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi10.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi10.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi10.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi10` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 11:25:01 >  
+# 11:25:01 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30547" "--set" 
"mapi_usock=/var/tmp/mtest-24716/.s.monetdb.30547" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 11:25:01 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 30547
+# cmdline opt  mapi_usock = /var/tmp/mtest-24716/.s.monetdb.30547
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 11:25:01 >  
+# 11:25:01 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-24716" "--port=30547"
+# 11:25:01 >  
+
+
+# 11:25:02 >  
+# 11:25:02 >  "Done."
+# 11:25:02 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi10.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi10.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi10.stable.out
@@ -0,0 +1,93 @@
+stdout of test 'capi10` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 11:25:01 >  
+# 11:25:01 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30547" "--set" 
"mapi_usock=/var/tmp/mtest-24716/.s.monetdb.30547" "--set" "monet_prompt=" 

MonetDB: jitudf - Properly handle changing parameter names with ...

[Mark Raasveldt]

Changeset: 32a3577c9a37 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=32a3577c9a37
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/Tests/capi08.sql
sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Properly handle changing parameter names with function caching as well.


diffs (152 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
@@ -45,8 +45,6 @@ CREATE FUNCTION capi04(inp STRING) RETUR
if (inp.is_null(inp.data[i])) {
result->data[i] = result->null_value;
} else {
-   result->data[i] = malloc(strlen(inp.data[i]) + 2);
-   strcpy(result->data[i] + 1, inp.data[i]);
result->data[i] = "hello";
}
}
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi08.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi08.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi08.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi08.sql
@@ -31,4 +31,28 @@ SELECT capi08(i) FROM integers;
 SELECT capi08(i) FROM integers;
 
 
+DROP FUNCTION capi08;
+CREATE FUNCTION capi08(inp1 INTEGER, inp2 INTEGER) RETURNS INTEGER LANGUAGE C {
+result->initialize(result, inp1.count);
+for(size_t i = 0; i < inp1.count; i++) {
+result->data[i] = inp1.data[i] / inp2.data[i];
+}
+};
+
+SELECT capi08(i * 2, i) FROM integers;
+
+DROP FUNCTION capi08;
+CREATE FUNCTION capi08(inp2 INTEGER, inp1 INTEGER) RETURNS INTEGER LANGUAGE C {
+result->initialize(result, inp1.count);
+for(size_t i = 0; i < inp1.count; i++) {
+result->data[i] = inp1.data[i] / inp2.data[i];
+}
+};
+
+# same function body and parameter types, but switched parameter names
+# should still give the same result
+SELECT capi08(i, i * 2) FROM integers;
+
+
+
 ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
--- a/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
+++ b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
@@ -116,6 +116,40 @@ Ready.
 [ 6]
 [ 8]
 [ 10   ]
+#DROP FUNCTION capi08;
+#CREATE FUNCTION capi08(inp1 INTEGER, inp2 INTEGER) RETURNS INTEGER LANGUAGE C 
{
+#result->initialize(result, inp1.count);
+#for(size_t i = 0; i < inp1.count; i++) {
+#result->data[i] = inp1.data[i] / inp2.data[i];
+#}
+#};
+#SELECT capi08(i * 2, i) FROM integers;
+% sys.L2 # table_name
+% L2 # name
+% int # type
+% 1 # length
+[ 2]
+[ 2]
+[ 2]
+[ 2]
+[ 2]
+#DROP FUNCTION capi08;
+#CREATE FUNCTION capi08(inp2 INTEGER, inp1 INTEGER) RETURNS INTEGER LANGUAGE C 
{
+#result->initialize(result, inp1.count);
+#for(size_t i = 0; i < inp1.count; i++) {
+#result->data[i] = inp1.data[i] / inp2.data[i];
+#}
+#};
+#SELECT capi08(i, i * 2) FROM integers;
+% sys.L2 # table_name
+% L2 # name
+% int # type
+% 1 # length
+[ 2]
+[ 2]
+[ 2]
+[ 2]
+[ 2]
 #ROLLBACK;
 
 # 16:28:18 >  
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -151,8 +151,10 @@ static char *mprotect_region(void *addr,
size_t actual_len = len;
if (len == 0)
return NULL;
-   // check if the region is page-aligned
 
+   // we must mprotect an entire page
+   // thus here we check if the region is page-aligned
+   // and if it is not, we page-align it
pagesize = getpagesize();
page_begin = (void *)((size_t)addr - (size_t)addr % pagesize);
if (page_begin != addr) {
@@ -420,8 +422,7 @@ static str CUDFeval(Client cntxt, MalBlk
funcname = sqlfun ? sqlfun->base.name : "yet_another_c_function";
 
args = (str *)GDKzalloc(sizeof(str) * pci->argc);
-   output_names =
-   pci->retc > 0 ? (str *)GDKzalloc(sizeof(str) * pci->retc) : 
NULL;
+   output_names = (str *)GDKzalloc(sizeof(str) * pci->argc);
if (!args || !output_names) {
throw(MAL, "cudf.eval", MAL_MALLOC_FAIL);
}
@@ -496,7 +497,17 @@ static str CUDFeval(Client cntxt, MalBlk
GDK_STRHASH(exprStr, expression_hash);
GDK_STRHASH(funcname, funcname_hash);
funcname_hash = funcname_hash % FUNCTION_CACHE_SIZE;
-   function_parameters = GDKzalloc((input_count + output_count + 1) * 
sizeof(char));
+   j = 0;
+   for (i = 0; i < (size_t)pci->argc; i++) {
+   if (args[i]) {
+   j += strlen(args[i]);
+   }
+   if (output_names[i]) {
+   j += strlen(output_names[i]);
+   

MonetDB: jitudf - When mprotect triggers, check if the address a...

[Mark Raasveldt]

Changeset: c5b67bdd8bcd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c5b67bdd8bcd
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

When mprotect triggers, check if the address actually falls in the area we 
protected.


diffs (192 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -27,8 +27,24 @@ typedef struct _allocated_region {
struct _allocated_region *next;
 } allocated_region;
 
-static __thread allocated_region *allocated_regions;
-static __thread jmp_buf jump_buffer;
+struct _mprotected_region;
+typedef struct _mprotected_region {
+   void *addr;
+   size_t len;
+
+   void* actual_addr;
+   size_t actual_len;
+
+   struct _mprotected_region *next;
+} mprotected_region;
+
+static char *mprotect_region(void *addr, size_t len, int flags,
+mprotected_region 
**regions);
+static char *clear_mprotect(void *addr, size_t len);
+
+static allocated_region *allocated_regions[THREADS];
+static mprotected_region **actual_mprotected_regions[THREADS];
+static jmp_buf jump_buffer[THREADS];
 
 typedef char *(*jitted_function)(void **inputs, void **outputs,
 
malloc_function_ptr malloc);
@@ -81,19 +97,49 @@ static bool WriteTextToFile(FILE *f, con
 
 static void handler(int sig, siginfo_t *si, void *unused)
 {
+   int actually_protected_area = false;
+   mprotected_region* found_region = NULL;
+   int tid = THRgettid();
+
(void)sig;
-   (void)si;
(void)unused;
// we caught a segfault or bus error
-   longjmp(jump_buffer, 1);
-}
+   // this can be either because 
+   // (1) the function accessed a protected piece of memory
+   // (2) the function caused a segfault by e.g. dereferencing a NULL 
pointer
+   // in the first case, this *might* be a valid memory access
+   // this is because we are required to align our mprotects on page 
boundaries
+   // thus sometimes we mprotect a page where only part of the page
+   // should actually be protected. Thus for this case we check if the 
access
+   // was actually an error
 
-struct _mprotected_region;
-typedef struct _mprotected_region {
-   void *addr;
-   size_t len;
-   struct _mprotected_region *next;
-} mprotected_region;
+   if (actual_mprotected_regions[tid]) {
+   mprotected_region* region = *actual_mprotected_regions[tid];
+   while(region) {
+   if (si->si_addr >= region->addr && (char*) si->si_addr 
<= (char*) region->addr + region->len) {
+   // the address belongs to this mprotected region
+   found_region = region;
+   if (si->si_addr >= region->actual_addr && 
+   (char*) si->si_addr <= (char*) 
region->actual_addr + region->actual_len) {
+   // and the address is actually supposed 
to be protected
+   actually_protected_area = true;
+   break;
+   }
+   }
+   region = region->next;
+   }
+   }
+   if (found_region && !actually_protected_area) {
+   // this is NOT an actually protected area
+   // thus the segfault/bus error is invalid
+   // the nasty part here is that we have to unprotect the entire 
page now
+   // thus opening us up to future modifications of the data
+   clear_mprotect(found_region->addr, found_region->len);
+   found_region->addr = NULL;
+   } else {
+   longjmp(jump_buffer[tid], 1);
+   }
+}
 
 static char *mprotect_region(void *addr, size_t len, int flags,
 mprotected_region 
**regions)
@@ -101,6 +147,8 @@ static char *mprotect_region(void *addr,
mprotected_region *region;
int pagesize;
void *page_begin;
+   void* actual_addr = addr;
+   size_t actual_len = len;
if (len == 0)
return NULL;
// check if the region is page-aligned
@@ -126,12 +174,16 @@ static char *mprotect_region(void *addr,
region->addr = addr;
region->len = len;
region->next = *regions;
+   region->actual_addr = actual_addr;
+   region->actual_len = actual_len;
*regions = region;
return NULL;
 }
 
 static char *clear_mprotect(void *addr, size_t len)
 {
+   if (!addr) return NULL;
+
if (mprotect(addr, len, PROT_READ | PROT_WRITE) < 0) {
return strerror(errno);
}
@@ -156,7 +208,7 @@ static 

MonetDB: jitudf - Merge with default.

[Mark Raasveldt]

Changeset: c4a6d8f49f54 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c4a6d8f49f54
Added Files:
sql/test/BugTracker-2017/Tests/crash-dce.Bug-6330.sql
sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.sql

sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.err

sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.out
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.sql
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.stable.err
sql/test/BugTracker-2017/Tests/push_subslice.Bug-6322.stable.out
sql/test/Tests/ifexists-views.sql
sql/test/Tests/ifexists-views.stable.err
sql/test/Tests/ifexists-views.stable.out
Modified Files:
MonetDB.spec
clients/ChangeLog.Dec2016
common/stream/stream.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_pushselect.c
sql/backends/monet5/sql_cat.c
sql/backends/monet5/sql_upgrades.c
sql/include/sql_catalog.h
sql/scripts/51_sys_schema_extension.sql
sql/server/rel_optimizer.c
sql/server/rel_schema.c
sql/server/rel_select.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/test/BugTracker-2017/Tests/All
sql/test/Tests/All
sql/test/Tests/copy-into-fwf.sql.in
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out
tools/merovingian/daemon/controlrunner.c
tools/merovingian/utils/control.c
Branch: jitudf
Log Message:

Merge with default.


diffs (truncated from 1128 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -311,6 +311,11 @@ MonetDB, you will very likely need this 
 Summary: MonetDB - Monet Database Management System Client Programs
 Group: Applications/Databases
 Requires: %{name}-client%{?_isa} = %{version}-%{release}
+%if (0%{?fedora} >= 22)
+# tomograph executes these two:
+Recommends: /usr/bin/gs
+Recommends: /usr/bin/gnuplot
+%endif
 
 %description client-tools
 MonetDB is a database management system that is developed from a
diff --git a/clients/ChangeLog.Dec2016 b/clients/ChangeLog.Dec2016
--- a/clients/ChangeLog.Dec2016
+++ b/clients/ChangeLog.Dec2016
@@ -1,3 +1,7 @@
 # ChangeLog file for clients
 # This file is updated with Maddlog
 
+* Mon Jun 12 2017 Sjoerd Mullender 
+- A bug was fixed in the implementation of the functions SQLTables in
+  the ODBC driver where not enough memory was allocated.
+
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -5444,11 +5444,11 @@ typedef struct {
char filler;
// state
size_t line_len;
-   char* in_buf;
-   char* out_buf;
+   char *in_buf;
+   char *out_buf;
size_t out_buf_start;
size_t out_buf_remaining;
-   char* nl_buf;
+   char nl_buf[1];
 } stream_fwf_data;
 
 
@@ -5458,10 +5458,11 @@ stream_fwf_read(stream *s, void *buf, si
stream_fwf_data *fsd;
size_t to_write = cnt;
size_t buf_written = 0;
-   if (strcmp(s->name, STREAM_FWF_NAME) != 0 || elmsize != 1) {
+
+   fsd = (stream_fwf_data*) s->stream_data.p;
+   if (fsd == NULL || elmsize != 1) {
return -1;
}
-   fsd = (stream_fwf_data*) s->stream_data.p;
 
while (to_write > 0) {
// input conversion
@@ -5499,12 +5500,12 @@ stream_fwf_read(stream *s, void *buf, si
 
// now we know something is in output_buf so deliver it
if (fsd->out_buf_remaining <= to_write) {
-   memcpy((char*)buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, fsd->out_buf_remaining);
to_write -= fsd->out_buf_remaining;
buf_written += fsd->out_buf_remaining;
fsd->out_buf_remaining = 0;
} else {
-   memcpy((char*) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
+   memcpy((char *) buf + buf_written, fsd->out_buf + 
fsd->out_buf_start, to_write);
fsd->out_buf_start += to_write;
fsd->out_buf_remaining -= to_write;
buf_written += to_write;
@@ -5518,25 +5519,35 @@ stream_fwf_read(stream *s, void *buf, si
 static void
 stream_fwf_close(stream *s)
 {
-   if (strcmp(s->name, STREAM_FWF_NAME) == 0) {
+   stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
+
+   if (fsd != NULL) {
stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p;
mnstr_close(fsd->s);
+   

MonetDB: jitudf - Linux compilation fixes.

[Mark Raasveldt]

Changeset: ec71c22bf9bc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec71c22bf9bc
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi06.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Linux compilation fixes.


diffs (49 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi06.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi06.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi06.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi06.sql
@@ -5,15 +5,16 @@ START TRANSACTION;
 
 CREATE FUNCTION capi06(inp INTEGER) RETURNS INTEGER LANGUAGE CPP {
 #include 
-   std::vector elements;
-   for(size_t i = 0; i < inp.count; i++) {
-   elements.push_back(inp.data[i]);
-   }
-   std::sort(elements.begin(), elements.end());
-   result->initialize(result, inp.count);
-   for(size_t i = 0; i < inp.count; i++) {
-   result->data[i] = elements[i];
-   }
+#include 
+std::vector elements;
+for(size_t i = 0; i < inp.count; i++) {
+elements.push_back(inp.data[i]);
+}
+std::sort(elements.begin(), elements.end());
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+result->data[i] = elements[i];
+}
 };
 
 CREATE TABLE integers(i INTEGER);
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -169,7 +169,7 @@ static void *wrapped_GDK_malloc(size_t s
region->next = allocated_regions;
allocated_regions = region;
 
-   return ptr + sizeof(allocated_region);
+   return (char*)ptr + sizeof(allocated_region);
 }
 
 #define GENERATE_BASE_HEADERS(type, tpename)   
\
@@ -571,6 +571,7 @@ static str CUDFeval(Client cntxt, MalBlk
ATTEMPT_TO_WRITE_DATA_TO_FILE(f,

  exprStr + preprocessor_start,

  i - preprocessor_start);
+   ATTEMPT_TO_WRITE_TO_FILE(f, 
"\n");
for (j = preprocessor_start; j 
< i; j++) {
// now overwrite the 
preprocessor directive in the
// expression string 
with spaces
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Add function cache to avoid recompiling functi...

[Mark Raasveldt]

Changeset: 63378d373390 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=63378d373390
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi08.sql
sql/backends/monet5/UDF/capi/Tests/capi08.stable.err
sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Add function cache to avoid recompiling functions constantly.


diffs (truncated from 780 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -7,3 +7,4 @@ capi04
 capi05
 capi06
 capi07
+capi08
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi08.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi08.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi08.sql
@@ -0,0 +1,34 @@
+# test caching behavior
+
+START TRANSACTION;
+
+CREATE FUNCTION capi08(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+result->data[i] = inp.data[i] * 2;
+}
+};
+
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+
+SELECT capi08(i) FROM integers;
+# this function should be loaded from the cache
+SELECT capi08(i) FROM integers;
+
+DROP FUNCTION capi08;
+
+CREATE FUNCTION capi08(inp DOUBLE) RETURNS DOUBLE LANGUAGE C {
+result->initialize(result, inp.count);
+for(size_t i = 0; i < inp.count; i++) {
+result->data[i] = inp.data[i] * 2;
+}
+};
+
+# same code and same function name, but different parameters
+# this needs to be recompiled now
+SELECT capi08(i) FROM integers;
+SELECT capi08(i) FROM integers;
+
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi08.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi08` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 16:28:17 >  
+# 16:28:17 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37555" "--set" 
"mapi_usock=/var/tmp/mtest-59671/.s.monetdb.37555" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 16:28:17 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37555
+# cmdline opt  mapi_usock = /var/tmp/mtest-59671/.s.monetdb.37555
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 16:28:17 >  
+# 16:28:17 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-59671" "--port=37555"
+# 16:28:17 >  
+
+
+# 16:28:18 >  
+# 16:28:18 >  "Done."
+# 16:28:18 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi08.stable.out
@@ -0,0 +1,124 @@
+stdout of test 'capi08` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 16:28:17 >  
+# 16:28:17 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37555" "--set" 
"mapi_usock=/var/tmp/mtest-59671/.s.monetdb.37555" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 16:28:17 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:37555/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-59671/.s.monetdb.37555
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# 

MonetDB: jitudf - Correctly support aggregates and add testcases.

[Mark Raasveldt]

Changeset: 579afc3ecc82 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=579afc3ecc82
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi06.sql
sql/backends/monet5/UDF/capi/Tests/capi06.stable.err
sql/backends/monet5/UDF/capi/Tests/capi06.stable.out
sql/backends/monet5/UDF/capi/Tests/capi07.sql
sql/backends/monet5/UDF/capi/Tests/capi07.stable.err
sql/backends/monet5/UDF/capi/Tests/capi07.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Correctly support aggregates and add testcases.


diffs (truncated from 331 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -5,3 +5,5 @@ capi02
 capi03
 capi04
 capi05
+capi06
+capi07
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi06.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi06.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi06.sql
@@ -0,0 +1,24 @@
+
+# test CPP UDFs
+
+START TRANSACTION;
+
+CREATE FUNCTION capi06(inp INTEGER) RETURNS INTEGER LANGUAGE CPP {
+#include 
+   std::vector elements;
+   for(size_t i = 0; i < inp.count; i++) {
+   elements.push_back(inp.data[i]);
+   }
+   std::sort(elements.begin(), elements.end());
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   result->data[i] = elements[i];
+   }
+};
+
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (3), (4), (1), (2), (5);
+
+SELECT capi06(i) FROM integers;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi06.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi06.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi06.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi06` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 14:53:38 >  
+# 14:53:38 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=38582" "--set" 
"mapi_usock=/var/tmp/mtest-20999/.s.monetdb.38582" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 14:53:38 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 38582
+# cmdline opt  mapi_usock = /var/tmp/mtest-20999/.s.monetdb.38582
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 14:53:38 >  
+# 14:53:38 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-20999" "--port=38582"
+# 14:53:38 >  
+
+
+# 14:53:39 >  
+# 14:53:39 >  "Done."
+# 14:53:39 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi06.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi06.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi06.stable.out
@@ -0,0 +1,93 @@
+stdout of test 'capi06` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 14:53:38 >  
+# 14:53:38 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=38582" "--set" 
"mapi_usock=/var/tmp/mtest-20999/.s.monetdb.38582" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 14:53:38 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:38582/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-20999/.s.monetdb.38582
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 

MonetDB: jitudf - Formatting.

[Mark Raasveldt]

Changeset: 86e62d4bfe87 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=86e62d4bfe87
Modified Files:
sql/backends/monet5/UDF/capi/capi.c
sql/backends/monet5/UDF/capi/capi.h
sql/backends/monet5/UDF/capi/capi.mal
sql/backends/monet5/UDF/capi/cheader.h
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_statement.c
sql/include/sql_catalog.h
sql/server/rel_psm.c
sql/server/sql_parser.y
Branch: jitudf
Log Message:

Formatting.


diffs (truncated from 1538 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -24,67 +24,73 @@
 
 struct _allocated_region;
 typedef struct _allocated_region {
-   struct _allocated_region* next;
+   struct _allocated_region *next;
 } allocated_region;
 
-
-static __thread allocated_region* allocated_regions;
+static __thread allocated_region *allocated_regions;
 static __thread jmp_buf jump_buffer;
 
-static str
-CUDFeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped);
+static str CUDFeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
+   bit grouped);
 
-str CUDFevalStd(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) {
+str CUDFevalStd(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
return CUDFeval(cntxt, mb, stk, pci, 0);
 }
 
-str CUDFevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) {
+str CUDFevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
return CUDFeval(cntxt, mb, stk, pci, 1);
 }
 
-str CUDFprelude(void *ret) {
-   (void) ret;
+str CUDFprelude(void *ret)
+{
+   (void)ret;
return MAL_SUCCEED;
 }
 
-typedef char* (*jitted_function)(void** inputs, void** outputs, 
malloc_function_ptr malloc);
+typedef char *(*jitted_function)(void **inputs, void **outputs,
+
malloc_function_ptr malloc);
 
-static bool
-WriteDataToFile(FILE* f, const void* data, size_t data_size) {
+static bool WriteDataToFile(FILE *f, const void *data, size_t data_size)
+{
fwrite(data, data_size, 1, f);
return (!ferror(f));
 }
 
-static bool
-WriteTextToFile(FILE* f, const char* data) {
+static bool WriteTextToFile(FILE *f, const char *data)
+{
return WriteDataToFile(f, data, strlen(data));
 }
 
-static void handler(int sig, siginfo_t *si, void *unused) {
-   (void) sig;
-   (void) si;
-   (void) unused;
+static void handler(int sig, siginfo_t *si, void *unused)
+{
+   (void)sig;
+   (void)si;
+   (void)unused;
// we caught a segfault or bus error
longjmp(jump_buffer, 1);
 }
 
 struct _mprotected_region;
 typedef struct _mprotected_region {
-   void* addr;
+   void *addr;
size_t len;
-   struct _mprotected_region* next;
+   struct _mprotected_region *next;
 } mprotected_region;
 
-static char*
-mprotect_region(void* addr, size_t len, int flags, mprotected_region** 
regions) {
-   mprotected_region* region;
+static char *mprotect_region(void *addr, size_t len, int flags,
+mprotected_region 
**regions)
+{
+   mprotected_region *region;
int pagesize;
-   void* page_begin;
-   if (len == 0) return NULL;
+   void *page_begin;
+   if (len == 0)
+   return NULL;
// check if the region is page-aligned
-   
+
pagesize = getpagesize();
-   page_begin = (void*)((size_t)addr - (size_t)addr % pagesize);
+   page_begin = (void *)((size_t)addr - (size_t)addr % pagesize);
if (page_begin != addr) {
// data is not page-aligned
len += ((size_t)addr - (size_t)page_begin);
@@ -108,60 +114,60 @@ mprotect_region(void* addr, size_t len, 
return NULL;
 }
 
-static char*
-clear_mprotect(void* addr, size_t len) {
+static char *clear_mprotect(void *addr, size_t len)
+{
if (mprotect(addr, len, PROT_READ | PROT_WRITE) < 0) {
return strerror(errno);
}
return NULL;
 }
 
-#define ATTEMPT_TO_WRITE_TO_FILE(f, data) \
-   if (!WriteTextToFile(f, data)) { \
-   errno = 0; \
-   msg = createException(MAL, "cudf.eval", "Write error."); \
-   goto wrapup; \
+#define ATTEMPT_TO_WRITE_TO_FILE(f, data)  
\
+   if (!WriteTextToFile(f, data)) {
   \
+   errno = 0;  
   \
+   msg = createException(MAL, "cudf.eval", "Write error.");
   \
+   goto wrapup;
   \
}
 
-
-#define ATTEMPT_TO_WRITE_DATA_TO_FILE(f, data, 

MonetDB: jitudf - Add test for automatically freeing stray mallocs.

[Mark Raasveldt]

Changeset: 718daabfbf43 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=718daabfbf43
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi05.sql
sql/backends/monet5/UDF/capi/Tests/capi05.stable.err
sql/backends/monet5/UDF/capi/Tests/capi05.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
Branch: jitudf
Log Message:

Add test for automatically freeing stray mallocs.


diffs (165 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -4,3 +4,4 @@ capi01
 capi02
 capi03
 capi04
+capi05
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi05.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi05.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi05.sql
@@ -0,0 +1,20 @@
+# memory leak test
+# randomly allocate some buffers
+
+
+START TRANSACTION;
+
+CREATE FUNCTION capi05(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+   char* mydata = malloc(1000);
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   result->data[i] = inp.data[i] * 2;
+   }
+};
+
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+
+SELECT capi05(i) FROM integers;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi05.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi05.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi05.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi05` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 12:58:52 >  
+# 12:58:52 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=39823" "--set" 
"mapi_usock=/var/tmp/mtest-37374/.s.monetdb.39823" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 12:58:52 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 39823
+# cmdline opt  mapi_usock = /var/tmp/mtest-37374/.s.monetdb.39823
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 12:58:52 >  
+# 12:58:52 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-37374" "--port=39823"
+# 12:58:52 >  
+
+
+# 12:58:53 >  
+# 12:58:53 >  "Done."
+# 12:58:53 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi05.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi05.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi05.stable.out
@@ -0,0 +1,88 @@
+stdout of test 'capi05` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 12:58:52 >  
+# 12:58:52 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=39823" "--set" 
"mapi_usock=/var/tmp/mtest-37374/.s.monetdb.39823" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 12:58:52 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on 
mapi:monetdb://dhcp-21.eduroam.cwi.nl:39823/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-37374/.s.monetdb.39823
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 27_rejects.sql
+# loading sql script: 39_analytics.sql
+# loading sql 

MonetDB: jitudf - Automatically clean up any mallocs performed i...

[Mark Raasveldt]

Changeset: dce7fa42e62a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dce7fa42e62a
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Automatically clean up any mallocs performed in the jitted function.


diffs (190 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
@@ -10,8 +10,6 @@ CREATE FUNCTION capi04(inp STRING) RETUR
if (inp.is_null(inp.data[i])) {
result->data[i] = result->null_value;
} else {
-   // the contract says we must use "malloc" to allocate 
for the result strings
-   // "malloc" is a function pointer that actually points 
to GDKmalloc
result->data[i] = malloc(strlen(inp.data[i]) + 2);
strcpy(result->data[i] + 1, inp.data[i]);
result->data[i][0] = 'H';
@@ -35,8 +33,28 @@ CREATE FUNCTION capi04(inp STRING) RETUR
 };
 
 SELECT capi04(i) FROM strings;
+ROLLBACK;
+
+START TRANSACTION;
+# return constant strings, instead of allocated strings
+CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
+#include 
+
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   if (inp.is_null(inp.data[i])) {
+   result->data[i] = result->null_value;
+   } else {
+   result->data[i] = malloc(strlen(inp.data[i]) + 2);
+   strcpy(result->data[i] + 1, inp.data[i]);
+   result->data[i] = "hello";
+   }
+   }
+};
+
+CREATE TABLE strings(i STRING);
+INSERT INTO strings VALUES ('ello'), ('ow'), (NULL), ('onestly?'), ('annes');
+
+SELECT capi04(i) FROM strings;
 
 ROLLBACK;
-
-
-
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
@@ -54,14 +54,10 @@ Ready.
 # loading sql script: 90_generator.sql
 # loading sql script: 90_generator_hge.sql
 # loading sql script: 99_system.sql
-Hello
-How
-Honestly?
-Hannes
 
-# 15:50:30 >  
-# 15:50:30 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-20003" "--port=32751"
-# 15:50:30 >  
+# 12:56:26 >  
+# 12:56:26 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-37150" "--port=31836"
+# 12:56:26 >  
 
 #START TRANSACTION;
 #CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
@@ -98,6 +94,35 @@ Hannes
 #  }
 #};
 #ROLLBACK;
+#START TRANSACTION;
+#CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
+##include 
+#
+#  result->initialize(result, inp.count);
+#  for(size_t i = 0; i < inp.count; i++) {
+#  if (inp.is_null(inp.data[i])) {
+#  result->data[i] = result->null_value;
+#  } else {
+#  result->data[i] = malloc(strlen(inp.data[i]) + 2);
+#  strcpy(result->data[i] + 1, inp.data[i]);
+#  result->data[i] = "hello";
+#  }
+#  }
+#};
+#CREATE TABLE strings(i STRING);
+#INSERT INTO strings VALUES ('ello'), ('ow'), (NULL), ('onestly?'), ('annes');
+[ 5]
+#SELECT capi04(i) FROM strings;
+% sys.L2 # table_name
+% L2 # name
+% clob # type
+% 5 # length
+[ "hello"  ]
+[ "hello"  ]
+[ NULL ]
+[ "hello"  ]
+[ "hello"  ]
+#ROLLBACK;
 
 # 15:50:31 >  
 # 15:50:31 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -22,6 +22,13 @@
 #include 
 #include 
 
+struct _allocated_region;
+typedef struct _allocated_region {
+   struct _allocated_region* next;
+} allocated_region;
+
+
+static __thread allocated_region* allocated_regions;
 static __thread jmp_buf jump_buffer;
 
 static str
@@ -125,7 +132,7 @@ clear_mprotect(void* addr, size_t len) {
}
 
 
-static void* wrapped_GDK_malloc(size_t size) {
+static void* jump_GDK_malloc(size_t size) {
void* ptr = GDKmalloc(size);
if (!ptr) {
longjmp(jump_buffer, 2);
@@ -133,11 +140,21 @@ static void* wrapped_GDK_malloc(size_t s
return ptr;
 }
 
+static void* wrapped_GDK_malloc(size_t size) {
+   allocated_region* region;
+   void* ptr = jump_GDK_malloc(size + sizeof(allocated_region));
+   region = (allocated_region*)ptr;
+   region->next = allocated_regions;
+   allocated_regions = region;
+   
+   return ptr + sizeof(allocated_region);
+}
+
 #define 

MonetDB: jitudf - Create temporary files in the LEFTOVERS direct...

[Mark Raasveldt]

Changeset: 1b9f39d82528 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1b9f39d82528
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Create temporary files in the LEFTOVERS directory of the BBP farm.


diffs (151 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
--- a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
+++ b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err
@@ -36,6 +36,12 @@ ERROR = !No data returned.
 MAPI  = (monetdb) /var/tmp/mtest-2836/.s.monetdb.35915
 QUERY = SELECT capi03(i) FROM integers;
 ERROR = !Something went wrong!
+MAPI  = (monetdb) /var/tmp/mtest-25067/.s.monetdb.39212
+QUERY = SELECT capi03(i) FROM integers;
+ERROR = !Attempting to write to the input or triggered a segfault/bus error
+MAPI  = (monetdb) /var/tmp/mtest-25067/.s.monetdb.39212
+QUERY = SELECT capi03(i) FROM integers;
+ERROR = !Attempting to write to the input or triggered a segfault/bus error
 
 # 17:24:55 >  
 # 17:24:55 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
--- a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
+++ b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out
@@ -87,6 +87,26 @@ Ready.
 #INSERT INTO integers VALUES (1), (2), (3), (4), (5);
 [ 5]
 #ROLLBACK;
+#START TRANSACTION;
+#CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+#  inp.data[0] = 10;
+#  result->initialize(result, inp.count);
+#  for(size_t i = 0; i < inp.count; i++) {
+#  result->data[i] = inp.data[i] * 2;
+#  }
+#};
+#CREATE TABLE integers(i INTEGER);
+#INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+[ 5]
+#ROLLBACK;
+#START TRANSACTION;
+#CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+#  int x = *((int*)NULL);
+#};
+#CREATE TABLE integers(i INTEGER);
+#INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+[ 5]
+#ROLLBACK;
 
 # 17:24:55 >  
 # 17:24:55 >  "Done."
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi04.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
@@ -15,7 +15,6 @@ CREATE FUNCTION capi04(inp STRING) RETUR
result->data[i] = malloc(strlen(inp.data[i]) + 2);
strcpy(result->data[i] + 1, inp.data[i]);
result->data[i][0] = 'H';
-   printf("%s\n", result->data[i]);
}
}
 };
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -223,6 +223,8 @@ static daytime time_from_data(cudf_data_
 static void data_from_timestamp(timestamp d, cudf_data_timestamp* ptr);
 static timestamp timestamp_from_data(cudf_data_timestamp* ptr);
 
+static char valid_path_characters[] = "abcdefghijklmnopqrstuvwxyz";
+
 static str
 CUDFeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped) 
{
sql_func * sqlfun = NULL;
@@ -232,6 +234,7 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal
char argbuf[64];
char buf[BUFSIZ];
char fname[BUFSIZ];
+   char oname[BUFSIZ];
char libname[BUFSIZ];
char error_buf[BUFSIZ];
char total_error_buf[8192];
@@ -358,9 +361,52 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal
}
}
 
-   // name of the source file and lib file that will be generated
-   snprintf(fname, BUFSIZ, "%s.c", funcname);
-   snprintf(libname, BUFSIZ, "lib%s%s", funcname, SO_EXT);
+   {
+   const int RANDOM_NAME_SIZE = 32;
+   char* path = NULL;
+   const char* prefix = "LEFTOVERS"DIR_SEP_STR;
+   size_t prefix_size = strlen(prefix);
+
+   memcpy(buf, prefix, sizeof(char) * strlen(prefix));
+   // generate a random 32-character name for the temporary files
+   for(i = prefix_size; i < prefix_size + RANDOM_NAME_SIZE; i++) {
+   buf[i] = valid_path_characters[rand() % 
(sizeof(valid_path_characters) - 1)];
+   }
+   buf[i] = '\0';
+   path = GDKfilepath(0, BATDIR, buf, "c");
+   if (!path) {
+   msg = createException(MAL, "cudf.eval", 
MAL_MALLOC_FAIL);
+   goto wrapup;
+   }
+   strcpy(fname, path);
+   strcpy(oname, fname);
+   oname[strlen(oname) - 1] = 'o';
+   GDKfree(path);
+
+   memmove(buf + strlen(SO_PREFIX) + prefix_size, buf + 
prefix_size, i 

MonetDB: jitudf - Support strings in C UDFs.

[Mark Raasveldt]

Changeset: 6c8b7012d916 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6c8b7012d916
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi04.sql
sql/backends/monet5/UDF/capi/Tests/capi04.stable.err
sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Support strings in C UDFs.


diffs (truncated from 335 to 300 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -3,3 +3,4 @@ capi00
 capi01
 capi02
 capi03
+capi04
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.sql
@@ -0,0 +1,43 @@
+# test strings
+
+START TRANSACTION;
+
+CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
+#include 
+
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   if (inp.is_null(inp.data[i])) {
+   result->data[i] = result->null_value;
+   } else {
+   // the contract says we must use "malloc" to allocate 
for the result strings
+   // "malloc" is a function pointer that actually points 
to GDKmalloc
+   result->data[i] = malloc(strlen(inp.data[i]) + 2);
+   strcpy(result->data[i] + 1, inp.data[i]);
+   result->data[i][0] = 'H';
+   printf("%s\n", result->data[i]);
+   }
+   }
+};
+
+CREATE TABLE strings(i STRING);
+INSERT INTO strings VALUES ('ello'), ('ow'), (NULL), ('onestly?'), ('annes');
+
+SELECT capi04(i) FROM strings;
+
+DROP FUNCTION capi04;
+
+# try to modify one of the input strings
+CREATE FUNCTION capi04(inp STRING) RETURNS STRING LANGUAGE C {
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   result->data[i][0] = 'h';
+   }
+};
+
+SELECT capi04(i) FROM strings;
+
+ROLLBACK;
+
+
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.err
@@ -0,0 +1,37 @@
+stderr of test 'capi04` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 15:50:30 >  
+# 15:50:30 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=32751" "--set" 
"mapi_usock=/var/tmp/mtest-20003/.s.monetdb.32751" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 15:50:30 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 32751
+# cmdline opt  mapi_usock = /var/tmp/mtest-20003/.s.monetdb.32751
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 15:50:30 >  
+# 15:50:30 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-20003" "--port=32751"
+# 15:50:30 >  
+
+MAPI  = (monetdb) /var/tmp/mtest-20003/.s.monetdb.32751
+QUERY = SELECT capi04(i) FROM strings;
+ERROR = !Attempting to write to the input or triggered a segfault/bus error
+
+# 15:50:31 >  
+# 15:50:31 >  "Done."
+# 15:50:31 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi04.stable.out
@@ -0,0 +1,105 @@
+stdout of test 'capi04` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 15:50:30 >  
+# 15:50:30 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=32751" "--set" 
"mapi_usock=/var/tmp/mtest-20003/.s.monetdb.32751" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 15:50:30 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights 

MonetDB: jitudf - Testcases.

[Mark Raasveldt]

Changeset: 1e254184146c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1e254184146c
Added Files:
sql/backends/monet5/UDF/capi/Tests/capi02.stable.err
sql/backends/monet5/UDF/capi/Tests/capi02.stable.out
Branch: jitudf
Log Message:

Testcases.


diffs (161 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.stable.err 
b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'capi02` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 22:49:10 >  
+# 22:49:10 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37477" "--set" 
"mapi_usock=/var/tmp/mtest-45783/.s.monetdb.37477" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 22:49:10 >  
+
+# builtin opt  gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37477
+# cmdline opt  mapi_usock = /var/tmp/mtest-45783/.s.monetdb.37477
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi
+# cmdline opt  gdk_debug = 536870922
+
+# 22:49:11 >  
+# 22:49:11 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-45783" "--port=37477"
+# 22:49:11 >  
+
+
+# 22:49:11 >  
+# 22:49:11 >  "Done."
+# 22:49:11 >  
+
diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out 
b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out
@@ -0,0 +1,117 @@
+stdout of test 'capi02` in directory 'sql/backends/monet5/UDF/capi` itself:
+
+
+# 22:49:10 >  
+# 22:49:10 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37477" "--set" 
"mapi_usock=/var/tmp/mtest-45783/.s.monetdb.37477" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi"
+# 22:49:10 >  
+
+# MonetDB 5 server v11.28.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads
+# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers
+# Found 8.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit https://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://marks-mbp.home:37477/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-45783/.s.monetdb.37477
+# MonetDB/SQL module loaded
+
+Ready.
+
+# 22:49:11 >  
+# 22:49:11 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-45783" "--port=37477"
+# 22:49:11 >  
+
+#START TRANSACTION;
+#CREATE FUNCTION capi02_increment_year(d DATE) RETURNS DATE
+#language C
+#{
+#  result->initialize(result, d.count);
+#  for(size_t i = 0; i < result->count; i++) {
+#  if (d.is_null(d.data[i])) {
+#  result->data[i] = result->null_value;
+#  } else {
+#  result->data[i].year = d.data[i].year + 1;
+#  result->data[i].month = d.data[i].month;
+#  result->data[i].day = d.data[i].day;
+#  }
+#  }
+#};
+#CREATE TABLE dates(i DATE);
+#INSERT INTO dates VALUES ('1992-09-20'), ('2000-03-10'), ('1000-05-03'), 
(NULL);
+[ 4]
+#SELECT capi02_increment_year(i) FROM dates;
+% sys.L2 # table_name
+% L2 # name
+% date # type
+% 10 # length
+[ 1993-09-20   ]
+[ 2001-03-10   ]
+[ 1001-05-03   ]
+[ NULL ]
+#DROP FUNCTION capi02_increment_year;
+#DROP TABLE dates;
+#CREATE FUNCTION capi02_randomize_time(d TIME) RETURNS TIME
+#language C
+#{
+#  srand(1234);
+#  result->initialize(result, d.count);
+#  for(size_t i = 0; i < result->count; i++) {
+#  if (d.is_null(d.data[i])) {
+#  result->data[i] = result->null_value;
+#  } else {
+#  result->data[i].hours = rand() % 24;
+#  result->data[i].minutes = rand() % 60;
+#  result->data[i].seconds = rand() % 60;
+#  result->data[i].ms = rand() % 1000;
+#  }
+#  }
+#};
+#CREATE TABLE times(i TIME);
+#INSERT INTO times VALUES ('03:03:02.0101'), (NULL), ('03:03:02.0101');
+[ 3]
+#SELECT capi02_randomize_time(i) FROM times;
+% sys.L2 # table_name

MonetDB: jitudf - Add testcase for catching segfaults and fix ti...

[Mark Raasveldt]

Changeset: 1a325a0081e3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1a325a0081e3
Modified Files:
sql/backends/monet5/UDF/capi/Tests/All
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Add testcase for catching segfaults and fix timestamp null comparison.


diffs (19 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/All 
b/sql/backends/monet5/UDF/capi/Tests/All
--- a/sql/backends/monet5/UDF/capi/Tests/All
+++ b/sql/backends/monet5/UDF/capi/Tests/All
@@ -2,3 +2,4 @@
 capi00
 capi01
 capi02
+capi03
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -1089,6 +1089,6 @@ int time_is_null(cudf_data_time value) {
 }
 
 int timestamp_is_null(cudf_data_timestamp value) {
-   return date_is_null(value.date) && time_is_null(value.time);
+   return ts_isnil(timestamp_from_data());
 }
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: jitudf - Use pthread_sigmask to stop blocking SIGSEGV/S...

[Mark Raasveldt]

Changeset: 60ac438a9b58 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=60ac438a9b58
Modified Files:
sql/backends/monet5/UDF/capi/Tests/capi03.sql
sql/backends/monet5/UDF/capi/capi.c
Branch: jitudf
Log Message:

Use pthread_sigmask to stop blocking SIGSEGV/SIGBUS in C UDFs.

We want to catch these signals so we can detect and prevent writing to input 
BATs using mprotect.


diffs (208 lines):

diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.sql 
b/sql/backends/monet5/UDF/capi/Tests/capi03.sql
--- a/sql/backends/monet5/UDF/capi/Tests/capi03.sql
+++ b/sql/backends/monet5/UDF/capi/Tests/capi03.sql
@@ -50,20 +50,35 @@ SELECT capi03(i) FROM integers;
 
 ROLLBACK;
 
--- START TRANSACTION;
+START TRANSACTION;
 
--- # Modify input data
--- CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
--- inp.data[0] = 10;
--- result->initialize(result, inp.count);
--- for(size_t i = 0; i < inp.count; i++) {
--- result->data[i] = inp.data[i] * 2;
--- }
--- };
+# Modify input data
+CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+   inp.data[0] = 10;
+   result->initialize(result, inp.count);
+   for(size_t i = 0; i < inp.count; i++) {
+   result->data[i] = inp.data[i] * 2;
+   }
+};
 
--- CREATE TABLE integers(i INTEGER);
--- INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (1), (2), (3), (4), (5);
 
--- SELECT capi03(i) FROM integers;
+SELECT capi03(i) FROM integers;
 
--- ROLLBACK;
+ROLLBACK;
+
+START TRANSACTION;
+
+# Trigger a segfault
+CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C {
+   int x = *((int*)NULL);
+};
+
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (1), (2), (3), (4), (5);
+
+SELECT capi03(i) FROM integers;
+
+ROLLBACK;
+
diff --git a/sql/backends/monet5/UDF/capi/capi.c 
b/sql/backends/monet5/UDF/capi/capi.c
--- a/sql/backends/monet5/UDF/capi/capi.c
+++ b/sql/backends/monet5/UDF/capi/capi.c
@@ -12,7 +12,15 @@
 
 #include "mtime.h"
 
-#include "setjmp.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
 
 static __thread jmp_buf jump_buffer;
 
@@ -49,6 +57,7 @@ static void handler(int sig, siginfo_t *
(void) sig;
(void) si;
(void) unused;
+   // we caught a segfault or bus error
longjmp(jump_buffer, 1);
 }
 
@@ -61,24 +70,22 @@ typedef struct _mprotected_region {
 
 static char*
 mprotect_region(void* addr, size_t len, int flags, mprotected_region** 
regions) {
-#ifdef __APPLE__
-   // we don't mprotect on OSX for now
-   return NULL;
-#else
mprotected_region* region;
+   int pagesize;
+   void* page_begin;
if (len == 0) return NULL;
// check if the region is page-aligned
-   /*
-   int pagesize = getpagesize();
-   void* page_begin = (void*)((size_t)addr - (size_t)addr % pagesize);
+   
+   pagesize = getpagesize();
+   page_begin = (void*)((size_t)addr - (size_t)addr % pagesize);
if (page_begin != addr) {
// data is not page-aligned
-   len += (addr - page_begin);
+   len += ((size_t)addr - (size_t)page_begin);
addr = page_begin;
}
// page align len
len = len % pagesize == 0 ? len : len - len % pagesize + pagesize;
-*/
+
region = GDKmalloc(sizeof(mprotected_region));
if (!region) {
return MAL_MALLOC_FAIL;
@@ -92,7 +99,6 @@ mprotect_region(void* addr, size_t len, 
region->next = *regions;
*regions = region;
return NULL;
-#endif
 }
 
 static char*
@@ -202,7 +208,7 @@ GENERATE_BASE_HEADERS(cudf_data_timestam
 const char *debug_flag = "capi_use_debug";
 const char *cc_flag = "capi_cc";
 
-#define JIT_COMPILER_NAME "clang"
+#define JIT_COMPILER_NAME "cc"
 
 static size_t GetTypeCount(int type, void* struct_ptr);
 static void* GetTypeData(int type, void* struct_ptr);
@@ -252,7 +258,8 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal
 
lng initial_output_count = -1;
 
-   struct sigaction sa;
+   struct sigaction sa, oldsa, oldsb;
+   sigset_t signal_set;
 
 #ifdef NDEBUG
int debug_build = GDKgetenv_istrue(debug_flag) || 
GDKgetenv_isyes(debug_flag);
@@ -267,6 +274,16 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal
 
(void) cntxt;
 
+   // we need to be able to catch segfaults and bus errors
+   // so we can work with mprotect to prevent UDFs from changing
+   // the input data
+
+   // we remove them from the pthread_sigmask
+   (void) sigemptyset(_set);
+   (void) sigaddset(_set, SIGSEGV);
+   (void) sigaddset(_set, SIGBUS);
+   (void) pthread_sigmask(SIG_UNBLOCK, _set, NULL);
+
memset(, 0, sizeof(sa));
 
if (!grouped) {
@@ -687,7 +704,7 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal
goto wrapup;