MonetDB: jitudf - Testcases.
Changeset: 1e254184146c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1e254184146c Added Files: sql/backends/monet5/UDF/capi/Tests/capi02.stable.err sql/backends/monet5/UDF/capi/Tests/capi02.stable.out Branch: jitudf Log Message: Testcases. diffs (161 lines): diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.stable.err b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.err new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.err @@ -0,0 +1,34 @@ +stderr of test 'capi02` in directory 'sql/backends/monet5/UDF/capi` itself: + + +# 22:49:10 > +# 22:49:10 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=37477" "--set" "mapi_usock=/var/tmp/mtest-45783/.s.monetdb.37477" "--set" "monet_prompt=" "--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" +# 22:49:10 > + +# builtin opt gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 37477 +# cmdline opt mapi_usock = /var/tmp/mtest-45783/.s.monetdb.37477 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi +# cmdline opt gdk_debug = 536870922 + +# 22:49:11 > +# 22:49:11 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-45783" "--port=37477" +# 22:49:11 > + + +# 22:49:11 > +# 22:49:11 > "Done." +# 22:49:11 > + diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/UDF/capi/Tests/capi02.stable.out @@ -0,0 +1,117 @@ +stdout of test 'capi02` in directory 'sql/backends/monet5/UDF/capi` itself: + + +# 22:49:10 > +# 22:49:10 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=37477" "--set" "mapi_usock=/var/tmp/mtest-45783/.s.monetdb.37477" "--set" "monet_prompt=" "--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" +# 22:49:10 > + +# MonetDB 5 server v11.28.0 +# This is an unreleased version +# Serving database 'mTests_sql_backends_monet5_UDF_capi', using 4 threads +# Compiled for x86_64-apple-darwin15.6.0/64bit with 128bit integers +# Found 8.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved +# Visit https://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://marks-mbp.home:37477/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-45783/.s.monetdb.37477 +# MonetDB/SQL module loaded + +Ready. + +# 22:49:11 > +# 22:49:11 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-45783" "--port=37477" +# 22:49:11 > + +#START TRANSACTION; +#CREATE FUNCTION capi02_increment_year(d DATE) RETURNS DATE +#language C +#{ +# result->initialize(result, d.count); +# for(size_t i = 0; i < result->count; i++) { +# if (d.is_null(d.data[i])) { +# result->data[i] = result->null_value; +# } else { +# result->data[i].year = d.data[i].year + 1; +# result->data[i].month = d.data[i].month; +# result->data[i].day = d.data[i].day; +# } +# } +#}; +#CREATE TABLE dates(i DATE); +#INSERT INTO dates VALUES ('1992-09-20'), ('2000-03-10'), ('1000-05-03'), (NULL); +[ 4] +#SELECT capi02_increment_year(i) FROM dates; +% sys.L2 # table_name +% L2 # name +% date # type +% 10 # length +[ 1993-09-20 ] +[ 2001-03-10 ] +[ 1001-05-03 ] +[ NULL ] +#DROP FUNCTION capi02_increment_year; +#DROP TABLE dates; +#CREATE FUNCTION capi02_randomize_time(d TIME) RETURNS TIME +#language C +#{ +# srand(1234); +# result->initialize(result, d.count); +# for(size_t i = 0; i < result->count; i++) { +# if (d.is_null(d.data[i])) { +# result->data[i] = result->null_value; +# } else { +# result->data[i].hours = rand() % 24; +# result->data[i].minutes = rand() % 60; +# result->data[i].seconds = rand() % 60; +# result->data[i].ms = rand() % 1000; +# } +# } +#}; +#CREATE TABLE times(i TIME); +#INSERT INTO times VALUES ('03:03:02.0101'), (NULL), ('03:03:02.0101'); +[ 3] +#SELECT capi02_randomize_time(i) FROM times; +% sys.L2 # table_name
MonetDB: jitudf - Add testcase for catching segfaults and fix ti...
Changeset: 1a325a0081e3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1a325a0081e3 Modified Files: sql/backends/monet5/UDF/capi/Tests/All sql/backends/monet5/UDF/capi/capi.c Branch: jitudf Log Message: Add testcase for catching segfaults and fix timestamp null comparison. diffs (19 lines): diff --git a/sql/backends/monet5/UDF/capi/Tests/All b/sql/backends/monet5/UDF/capi/Tests/All --- a/sql/backends/monet5/UDF/capi/Tests/All +++ b/sql/backends/monet5/UDF/capi/Tests/All @@ -2,3 +2,4 @@ capi00 capi01 capi02 +capi03 diff --git a/sql/backends/monet5/UDF/capi/capi.c b/sql/backends/monet5/UDF/capi/capi.c --- a/sql/backends/monet5/UDF/capi/capi.c +++ b/sql/backends/monet5/UDF/capi/capi.c @@ -1089,6 +1089,6 @@ int time_is_null(cudf_data_time value) { } int timestamp_is_null(cudf_data_timestamp value) { - return date_is_null(value.date) && time_is_null(value.time); + return ts_isnil(timestamp_from_data(&value)); } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: jitudf - Use pthread_sigmask to stop blocking SIGSEGV/S...
Changeset: 60ac438a9b58 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=60ac438a9b58 Modified Files: sql/backends/monet5/UDF/capi/Tests/capi03.sql sql/backends/monet5/UDF/capi/capi.c Branch: jitudf Log Message: Use pthread_sigmask to stop blocking SIGSEGV/SIGBUS in C UDFs. We want to catch these signals so we can detect and prevent writing to input BATs using mprotect. diffs (208 lines): diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.sql b/sql/backends/monet5/UDF/capi/Tests/capi03.sql --- a/sql/backends/monet5/UDF/capi/Tests/capi03.sql +++ b/sql/backends/monet5/UDF/capi/Tests/capi03.sql @@ -50,20 +50,35 @@ SELECT capi03(i) FROM integers; ROLLBACK; --- START TRANSACTION; +START TRANSACTION; --- # Modify input data --- CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C { --- inp.data[0] = 10; --- result->initialize(result, inp.count); --- for(size_t i = 0; i < inp.count; i++) { --- result->data[i] = inp.data[i] * 2; --- } --- }; +# Modify input data +CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C { + inp.data[0] = 10; + result->initialize(result, inp.count); + for(size_t i = 0; i < inp.count; i++) { + result->data[i] = inp.data[i] * 2; + } +}; --- CREATE TABLE integers(i INTEGER); --- INSERT INTO integers VALUES (1), (2), (3), (4), (5); +CREATE TABLE integers(i INTEGER); +INSERT INTO integers VALUES (1), (2), (3), (4), (5); --- SELECT capi03(i) FROM integers; +SELECT capi03(i) FROM integers; --- ROLLBACK; +ROLLBACK; + +START TRANSACTION; + +# Trigger a segfault +CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C { + int x = *((int*)NULL); +}; + +CREATE TABLE integers(i INTEGER); +INSERT INTO integers VALUES (1), (2), (3), (4), (5); + +SELECT capi03(i) FROM integers; + +ROLLBACK; + diff --git a/sql/backends/monet5/UDF/capi/capi.c b/sql/backends/monet5/UDF/capi/capi.c --- a/sql/backends/monet5/UDF/capi/capi.c +++ b/sql/backends/monet5/UDF/capi/capi.c @@ -12,7 +12,15 @@ #include "mtime.h" -#include "setjmp.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include static __thread jmp_buf jump_buffer; @@ -49,6 +57,7 @@ static void handler(int sig, siginfo_t * (void) sig; (void) si; (void) unused; + // we caught a segfault or bus error longjmp(jump_buffer, 1); } @@ -61,24 +70,22 @@ typedef struct _mprotected_region { static char* mprotect_region(void* addr, size_t len, int flags, mprotected_region** regions) { -#ifdef __APPLE__ - // we don't mprotect on OSX for now - return NULL; -#else mprotected_region* region; + int pagesize; + void* page_begin; if (len == 0) return NULL; // check if the region is page-aligned - /* - int pagesize = getpagesize(); - void* page_begin = (void*)((size_t)addr - (size_t)addr % pagesize); + + pagesize = getpagesize(); + page_begin = (void*)((size_t)addr - (size_t)addr % pagesize); if (page_begin != addr) { // data is not page-aligned - len += (addr - page_begin); + len += ((size_t)addr - (size_t)page_begin); addr = page_begin; } // page align len len = len % pagesize == 0 ? len : len - len % pagesize + pagesize; -*/ + region = GDKmalloc(sizeof(mprotected_region)); if (!region) { return MAL_MALLOC_FAIL; @@ -92,7 +99,6 @@ mprotect_region(void* addr, size_t len, region->next = *regions; *regions = region; return NULL; -#endif } static char* @@ -202,7 +208,7 @@ GENERATE_BASE_HEADERS(cudf_data_timestam const char *debug_flag = "capi_use_debug"; const char *cc_flag = "capi_cc"; -#define JIT_COMPILER_NAME "clang" +#define JIT_COMPILER_NAME "cc" static size_t GetTypeCount(int type, void* struct_ptr); static void* GetTypeData(int type, void* struct_ptr); @@ -252,7 +258,8 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal lng initial_output_count = -1; - struct sigaction sa; + struct sigaction sa, oldsa, oldsb; + sigset_t signal_set; #ifdef NDEBUG int debug_build = GDKgetenv_istrue(debug_flag) || GDKgetenv_isyes(debug_flag); @@ -267,6 +274,16 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal (void) cntxt; + // we need to be able to catch segfaults and bus errors + // so we can work with mprotect to prevent UDFs from changing + // the input data + + // we remove them from the pthread_sigmask + (void) sigemptyset(&signal_set); + (void) sigaddset(&signal_set, SIGSEGV); + (void) sigaddset(&signal_set, SIGBUS); + (void) pthread_sigmask(SIG_UNBLOCK, &signal_set, NULL); + memset(&sa, 0, sizeof(sa)); if (!grouped) { @@ -687,7 +704,7 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal
MonetDB: malerrors - Put all SQLSTATE frameworks in place.
Changeset: 1f1101316e76 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1f1101316e76 Modified Files: geom/monetdb5/geom.c geom/monetdb5/geomBulk.c sql/backends/monet5/UDF/pyapi/connection.c sql/backends/monet5/UDF/pyapi/conversion.c sql/backends/monet5/UDF/pyapi/convert_loops.h sql/backends/monet5/UDF/pyapi/emit.c sql/backends/monet5/UDF/pyapi/formatinput.c sql/backends/monet5/UDF/pyapi/pyapi.c sql/backends/monet5/UDF/pyapi/pyheader.h sql/backends/monet5/UDF/pyapi/pyloader.c sql/backends/monet5/UDF/pyapi/type_conversion.c sql/backends/monet5/sql.c sql/backends/monet5/sql_assert.c sql/backends/monet5/sql_cast.c sql/backends/monet5/sql_cast_impl_int.h sql/backends/monet5/sql_cat.c sql/backends/monet5/sql_scenario.c sql/backends/monet5/vaults/bam/bam_export.c sql/backends/monet5/vaults/bam/bam_lib.c sql/backends/monet5/vaults/bam/bam_loader.c sql/backends/monet5/vaults/bam/bam_wrapper.c sql/backends/monet5/vaults/fits/fits.c sql/backends/monet5/vaults/lidar/lidar.c sql/backends/monet5/vaults/netcdf/netcdf.c sql/backends/monet5/vaults/shp/shp.c sql/backends/monet5/vaults/vault.c sql/backends/monet5/wlr.c Branch: malerrors Log Message: Put all SQLSTATE frameworks in place. The SQLSTATE - messages should be converted to a proper id using the primary conventio and vendor specif ones. diffs (truncated from 7834 to 300 lines): diff --git a/geom/monetdb5/geom.c b/geom/monetdb5/geom.c --- a/geom/monetdb5/geom.c +++ b/geom/monetdb5/geom.c @@ -71,7 +71,7 @@ transformCoordSeq(int idx, int coordinat if (!GEOSCoordSeq_getX(gcs_old, idx, &x) || !GEOSCoordSeq_getY(gcs_old, idx, &y) || (coordinatesNum > 2 && !GEOSCoordSeq_getZ(gcs_old, idx, &z))) - throw(MAL, "geom.Transform", "Couldn't get coordinates"); + throw(MAL, "geom.Transform", "SQLSTATE - !""Couldn't get coordinates"); /* check if the passed reference system is geographic (proj=latlong) * and change the degrees to radians because pj_transform works with radians*/ @@ -83,9 +83,9 @@ transformCoordSeq(int idx, int coordinat errorNum = pj_get_errno_ref(); if (*errorNum != 0) { if (coordinatesNum > 2) - throw(MAL, "geom.Transform", "Couldn't transform point (%f %f %f): %s\n", x, y, z, pj_strerrno(*errorNum)); + throw(MAL, "geom.Transform", "SQLSTATE - !""Couldn't transform point (%f %f %f): %s\n", x, y, z, pj_strerrno(*errorNum)); else - throw(MAL, "geom.Transform", "Couldn't transform point (%f %f): %s\n", x, y, pj_strerrno(*errorNum)); + throw(MAL, "geom.Transform", "SQLSTATE - !""Couldn't transform point (%f %f): %s\n", x, y, pj_strerrno(*errorNum)); } /* check if the destination reference system is geographic and change @@ -96,7 +96,7 @@ transformCoordSeq(int idx, int coordinat if (!GEOSCoordSeq_setX(gcs_new, idx, x) || !GEOSCoordSeq_setY(gcs_new, idx, y) || (coordinatesNum > 2 && !GEOSCoordSeq_setZ(gcs_new, idx, z))) - throw(MAL, "geom.Transform", "Couldn't set coordinates"); + throw(MAL, "geom.Transform", "SQLSTATE - !""Couldn't set coordinates"); return MAL_SUCCEED; } @@ -117,12 +117,12 @@ transformPoint(GEOSGeometry **transforme gcs_old = GEOSGeom_getCoordSeq(geosGeometry); if (gcs_old == NULL) - throw(MAL, "geom.Transform", "GEOSGeom_getCoordSeq failed"); + throw(MAL, "geom.Transform", "SQLSTATE - !""GEOSGeom_getCoordSeq failed"); /* create the coordinates sequence for the transformed geometry */ gcs_new = GEOSCoordSeq_create(1, coordinatesNum); if (gcs_new == NULL) - throw(MAL, "geom.Transform", "GEOSGeom_getCoordSeq failed"); + throw(MAL, "geom.Transform", "SQLSTATE - !""GEOSGeom_getCoordSeq failed"); /* create the transformed coordinates */ ret = transformCoordSeq(0, coordinatesNum, proj4_src, proj4_dst, gcs_old, gcs_new); @@ -135,7 +135,7 @@ transformPoint(GEOSGeometry **transforme *transformedGeometry = GEOSGeom_createPoint(gcs_new); if (*transformedGeometry == NULL) { GEOSCoordSeq_destroy(gcs_new); - throw(MAL, "geom.Transform", "GEOSGeom_getCoordSeq failed"); + throw(MAL, "geom.Transform", "SQLSTATE - !""GEOSGeom_getCoordSeq failed"); } return MAL_SUCCEED; @@ -155,16 +155,16 @@ transformLine(GEOSCoordSeq *gcs_new, con gcs_old = GEOSGeom_getCoordSeq(geosGeometry); if (gcs_old == NULL) - throw(MAL, "geom.Transform", "GEOSGeom_getCoordSeq failed"); +
MonetDB: Dec2016 - Adding 3 missing alternate key checks: the co...
Changeset: 02b10f3a1328 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=02b10f3a1328 Modified Files: sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out Branch: Dec2016 Log Message: Adding 3 missing alternate key checks: the combination of table_id and column number must be unique also. diffs (38 lines): diff --git a/sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql b/sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql --- a/sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql +++ b/sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql @@ -8,6 +8,9 @@ SELECT COUNT(*) AS duplicates, schema_id SELECT COUNT(*) AS duplicates, table_id, name FROM sys._columns GROUP BY table_id, name HAVING COUNT(*) > 1; SELECT COUNT(*) AS duplicates, table_id, name FROM tmp._columns GROUP BY table_id, name HAVING COUNT(*) > 1; SELECT COUNT(*) AS duplicates, table_id, name FROM sys.columns GROUP BY table_id, name HAVING COUNT(*) > 1; +SELECT COUNT(*) AS duplicates, table_id, number FROM sys._columns GROUP BY table_id, number HAVING COUNT(*) > 1; +SELECT COUNT(*) AS duplicates, table_id, number FROM tmp._columns GROUP BY table_id, number HAVING COUNT(*) > 1; +SELECT COUNT(*) AS duplicates, table_id, number FROM sys.columns GROUP BY table_id, number HAVING COUNT(*) > 1; -- The id values from sys.schemas, sys._tables, sys._columns and sys.functions combined must be exclusive (see FK from sys.privileges.obj_id) SELECT COUNT(*) AS duplicates, T.id FROM (SELECT id FROM sys.schemas UNION ALL SELECT id FROM sys._tables UNION ALL SELECT id FROM sys._columns UNION ALL SELECT id FROM sys.functions) T GROUP BY T.id HAVING COUNT(*) > 1; diff --git a/sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out b/sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out --- a/sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out +++ b/sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out @@ -64,6 +64,21 @@ Ready. % duplicates, table_id, name # name % bigint, int,varchar # type % 1, 1, 0 # length +#SELECT COUNT(*) AS duplicates, table_id, number FROM sys._columns GROUP BY table_id, number HAVING COUNT(*) > 1; +% sys.L3, sys._columns, sys._columns # table_name +% duplicates, table_id, number # name +% bigint, int,int # type +% 1, 1, 1 # length +#SELECT COUNT(*) AS duplicates, table_id, number FROM tmp._columns GROUP BY table_id, number HAVING COUNT(*) > 1; +% tmp.L3, tmp._columns, tmp._columns # table_name +% duplicates, table_id, number # name +% bigint, int,int # type +% 1, 1, 1 # length +#SELECT COUNT(*) AS duplicates, table_id, number FROM sys.columns GROUP BY table_id, number HAVING COUNT(*) > 1; +% .L15,.columns, .columns # table_name +% duplicates, table_id, number # name +% bigint, int,int # type +% 1, 1, 1 # length #SELECT COUNT(*) AS duplicates, T.id FROM (SELECT id FROM sys.schemas UNION ALL SELECT id FROM sys._tables UNION ALL SELECT id FROM sys._columns UNION ALL SELECT id FROM sys.functions) T GROUP BY T.id HAVING COUNT(*) > 1; % .L26,.t # table_name % duplicates, id # name ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2017 - Approve error output.
Changeset: 7e9cafa986b5 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7e9cafa986b5 Modified Files: sql/test/Tests/ifexists-views.sql sql/test/Tests/ifexists-views.stable.err Branch: Jul2017 Log Message: Approve error output. diffs (33 lines): diff --git a/sql/test/Tests/ifexists-views.sql b/sql/test/Tests/ifexists-views.sql --- a/sql/test/Tests/ifexists-views.sql +++ b/sql/test/Tests/ifexists-views.sql @@ -1,4 +1,4 @@ -# Test IF (NOT) EXISTS functionality for views +# Test DROP VIEW IF EXISTS functionality CREATE TABLE tab0 (col0 INTEGER, col2 TEXT); INSERT INTO tab0 VALUES (1,'lekker'), (2, 'heerlijk'), (3, 'smullen'); diff --git a/sql/test/Tests/ifexists-views.stable.err b/sql/test/Tests/ifexists-views.stable.err --- a/sql/test/Tests/ifexists-views.stable.err +++ b/sql/test/Tests/ifexists-views.stable.err @@ -28,14 +28,14 @@ stderr of test 'ifexists-views` in direc # 10:36:23 > MAPI = (monetdb) /var/tmp/mtest-29186/.s.monetdb.32036 -QUERY = SELECT * FROM view1; - +QUERY = SELECT * FROM view1; -- should fail +ERROR = !SELECT: no such table 'view1' MAPI = (monetdb) /var/tmp/mtest-29186/.s.monetdb.32036 -QUERY = DROP VIEW view2; - +QUERY = DROP VIEW view2; -- should fail +ERROR = !DROP VIEW: unknown view 'view2' MAPI = (monetdb) /var/tmp/mtest-29186/.s.monetdb.32036 -QUERY = SELECT * FROM view2; - +QUERY = SELECT * FROM view2; -- should fail +ERROR = !SELECT: no such table 'view2' # 10:36:23 > # 10:36:23 > "Done." ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2017 - Add test script for feature request / bug 3743
Changeset: 52df84bcf8ba for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=52df84bcf8ba Added Files: sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.sql sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.err sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.out Modified Files: sql/test/BugTracker-2017/Tests/All Branch: Jul2017 Log Message: Add test script for feature request / bug 3743 diffs (179 lines): diff --git a/sql/test/BugTracker-2017/Tests/All b/sql/test/BugTracker-2017/Tests/All --- a/sql/test/BugTracker-2017/Tests/All +++ b/sql/test/BugTracker-2017/Tests/All @@ -53,3 +53,4 @@ distinct_minus_count_distinct.Bug-6296 all_minus_count_star.Bug-6297 simplify_math.Bug-6324 push_subslice.Bug-6322 +delete_from_merge_table.Bug-3743 diff --git a/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.sql b/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.sql new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.sql @@ -0,0 +1,23 @@ +create table t1(t int); +insert into t1 values(1); + +create table t2(t int); +insert into t2 values(2); + +create merge table tt(t int); +alter table tt add table t1; +alter table tt add table t2; + +select * from tt; + +-- update tt set t = 3 where t = 1; +-- update tt set t = 4; + +delete from tt where t = 1; +delete from tt where t = 2; +delete from tt; + +drop table tt cascade; +drop table t1 cascade; +drop table t2 cascade; + diff --git a/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.err b/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.err new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.err @@ -0,0 +1,36 @@ +stderr of test 'delete_from_merge_table.Bug-3743` in directory 'sql/test/BugTracker-2017` itself: + + +# 17:32:33 > +# 17:32:33 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=31497" "--set" "mapi_usock=/var/tmp/mtest-19147/.s.monetdb.31497" "--set" "monet_prompt=" "--forcemito" "--dbpath=/export/scratch2/dinther/INSTALL/var/MonetDB/mTests_sql_test_BugTracker-2017" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 17:32:33 > + +# builtin opt gdk_dbpath = /export/scratch2/dinther/INSTALL/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 31497 +# cmdline opt mapi_usock = /var/tmp/mtest-19147/.s.monetdb.31497 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /export/scratch2/dinther/INSTALL/var/MonetDB/mTests_sql_test_BugTracker-2017 +# cmdline opt embedded_r = yes +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 17:32:34 > +# 17:32:34 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-19147" "--port=31497" +# 17:32:34 > + + +# 17:32:35 > +# 17:32:35 > "Done." +# 17:32:35 > + diff --git a/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.out b/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.out new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2017/Tests/delete_from_merge_table.Bug-3743.stable.out @@ -0,0 +1,97 @@ +stdout of test 'delete_from_merge_table.Bug-3743` in directory 'sql/test/BugTracker-2017` itself: + + +# 17:32:33 > +# 17:32:33 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=31497" "--set" "mapi_usock=/var/tmp/mtest-19147/.s.monetdb.31497" "--set" "monet_prompt=" "--forcemito" "--dbpath=/export/scratch2/dinther/INSTALL/var/MonetDB/mTests_sql_test_BugTracker-2017" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 17:32:33 > + +# MonetDB 5 server v11.27.0 +# This is an unreleased version +# Serving database 'mTests_sql_test_BugTracker-2017', using 8 threads +# Compiled for x86_64-unknown-linux-gnu/64bit with 128bit integers +# Found 15.588 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved +# Visit https://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://toulouse.da.cwi.nl:31497/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-19147/.s.monetdb.31497 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded +# MonetDB/Python2 module loaded +# MonetDB/R module loaded + +Ready. +# SQL catalog created, loading sql scripts once +# loading sql script: 09_like.sql +# l
MonetDB: jitudf - Linux compilation fixes.
Changeset: f95b44bcabc7 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f95b44bcabc7 Modified Files: sql/backends/monet5/UDF/capi/capi.c Branch: jitudf Log Message: Linux compilation fixes. diffs (187 lines): diff --git a/sql/backends/monet5/UDF/capi/capi.c b/sql/backends/monet5/UDF/capi/capi.c --- a/sql/backends/monet5/UDF/capi/capi.c +++ b/sql/backends/monet5/UDF/capi/capi.c @@ -46,6 +46,9 @@ WriteTextToFile(FILE* f, const char* dat } static void handler(int sig, siginfo_t *si, void *unused) { + (void) sig; + (void) si; + (void) unused; longjmp(jump_buffer, 1); } @@ -62,9 +65,10 @@ mprotect_region(void* addr, size_t len, // we don't mprotect on OSX for now return NULL; #else + mprotected_region* region; if (len == 0) return NULL; // check if the region is page-aligned - + /* int pagesize = getpagesize(); void* page_begin = (void*)((size_t)addr - (size_t)addr % pagesize); if (page_begin != addr) { @@ -74,12 +78,12 @@ mprotect_region(void* addr, size_t len, } // page align len len = len % pagesize == 0 ? len : len - len % pagesize + pagesize; - - mprotected_region* region = GDKmalloc(sizeof(mprotected_region)); +*/ + region = GDKmalloc(sizeof(mprotected_region)); if (!region) { return MAL_MALLOC_FAIL; } - if (mprotect(addr, len, PROT_READ) < 0) { + if (mprotect(addr, len, flags) < 0) { GDKfree(region); return strerror(errno); } @@ -115,7 +119,7 @@ clear_mprotect(void* addr, size_t len) { } -void* wrapped_GDK_malloc(size_t size) { +static void* wrapped_GDK_malloc(size_t size) { void* ptr = GDKmalloc(size); if (!ptr) { longjmp(jump_buffer, 2); @@ -259,6 +263,10 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal const char* compilation_flags = debug_build ? "-g -O0" : "-O2"; const char* c_compiler = GDKgetenv(cc_flag) ? GDKgetenv(cc_flag) : JIT_COMPILER_NAME; + const char* struct_prefix = "struct cudf_data_struct_"; + + (void) cntxt; + memset(&sa, 0, sizeof(sa)); if (!grouped) { @@ -294,13 +302,13 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal } // retrieve the output names argnode = sqlfun->res->h; - for (i = 0; i < sqlfun->res->cnt; i++) { + for (i = 0; i < (size_t) sqlfun->res->cnt; i++) { output_names[i] = GDKstrdup(((sql_arg *)argnode->data)->name); argnode = argnode->next; } } // name unnamed outputs - for(i = 0; i < pci->retc; i++) { + for(i = 0; i < (size_t) pci->retc; i++) { if (!output_names[i]) { if (pci->retc > 1) { snprintf(argbuf, sizeof(argbuf), "output%zu", i); @@ -312,7 +320,7 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal } } // the first unknown argument is the group, we don't really care for the rest. - for (i = pci->retc + 2; i < pci->argc; i++) { + for (i = pci->retc + 2; i < (size_t) pci->argc; i++) { if (args[i] == NULL) { if (!seengrp && grouped) { args[i] = GDKstrdup("aggr_group"); @@ -395,11 +403,10 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal ATTEMPT_TO_WRITE_TO_FILE(f, funcname); ATTEMPT_TO_WRITE_TO_FILE(f, "(void** __inputs, void** __outputs, malloc_function_ptr malloc) {\n"); - const char* struct_prefix = "struct cudf_data_struct_"; // now we convert the input arguments from void** to the proper input/output of the function // first convert the input // FIXME: deal with SQL types - for (i = pci->retc + 2; i < pci->argc; i++) { + for (i = pci->retc + 2; i < (size_t) pci->argc; i++) { if (!isaBatType(getArgType(mb, pci, i))) { // scalar input int scalar_type = getArgType(mb, pci, i); @@ -421,7 +428,7 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal } // output types // FIXME: deal with SQL types - for (i = 0; i < pci->retc; i++) { + for (i = 0; i < (size_t) pci->retc; i++) { int bat_type = getBatType(getArgType(mb, pci, i)); const char* tpe = GetTypeName(bat_type); assert(tpe); @@ -518,7 +525,7 @@ CUDFeval(Client cntxt, MalBlkPtr mb, Mal } } // create the inputs - for (i = pci->retc + 2; i < pci->argc; i++) { + for (i = pci->retc + 2; i < (size_t) pci->argc; i++) { size_t index = i - (pci->retc + 2); if (!isaBatType(getArgType(mb, pci, i))) { // deal with scalar input @@ -725,25 +732,28 @@ C
MonetDB: jitudf - Missing file.
Changeset: d96a77739818 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d96a77739818 Added Files: sql/backends/monet5/UDF/capi/cheader.h Branch: jitudf Log Message: Missing file. diffs (46 lines): diff --git a/sql/backends/monet5/UDF/capi/cheader.h b/sql/backends/monet5/UDF/capi/cheader.h new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/UDF/capi/cheader.h @@ -0,0 +1,41 @@ + +typedef void* (*malloc_function_ptr)(size_t); + +typedef struct { + unsigned char day; + unsigned char month; + int year; +} cudf_data_date; + +typedef struct { + unsigned int ms; + unsigned char seconds; + unsigned char minutes; + unsigned char hours; +} cudf_data_time; + +typedef struct { + cudf_data_date date; + cudf_data_time time; +} cudf_data_timestamp; + +#define DEFAULT_STRUCT_DEFINITION(type, typename) \ +struct cudf_data_struct_##typename { \ + type* data; \ + size_t count; \ + type null_value; \ + int (*is_null)(type value); \ + void (*initialize)(void* self, size_t count); \ +} + +DEFAULT_STRUCT_DEFINITION(signed char, bte); +DEFAULT_STRUCT_DEFINITION(short, sht); +DEFAULT_STRUCT_DEFINITION(int, int); +DEFAULT_STRUCT_DEFINITION(long long, lng); +DEFAULT_STRUCT_DEFINITION(float, flt); +DEFAULT_STRUCT_DEFINITION(double, dbl); +DEFAULT_STRUCT_DEFINITION(char*, str); +DEFAULT_STRUCT_DEFINITION(cudf_data_date, date); +DEFAULT_STRUCT_DEFINITION(cudf_data_time, time); +DEFAULT_STRUCT_DEFINITION(cudf_data_timestamp, timestamp); + ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: jitudf - Error handling in C UDFs.
Changeset: 485a1ab4415b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=485a1ab4415b Added Files: sql/backends/monet5/UDF/capi/Tests/capi03.sql sql/backends/monet5/UDF/capi/Tests/capi03.stable.err sql/backends/monet5/UDF/capi/Tests/capi03.stable.out Modified Files: sql/backends/monet5/UDF/capi/Tests/capi02.sql sql/backends/monet5/UDF/capi/capi.c Branch: jitudf Log Message: Error handling in C UDFs. diffs (truncated from 533 to 300 lines): diff --git a/sql/backends/monet5/UDF/capi/Tests/capi02.sql b/sql/backends/monet5/UDF/capi/Tests/capi02.sql --- a/sql/backends/monet5/UDF/capi/Tests/capi02.sql +++ b/sql/backends/monet5/UDF/capi/Tests/capi02.sql @@ -58,7 +58,6 @@ language C result->initialize(result, d.count); for(size_t i = 0; i < result->count; i++) { if (d.is_null(d.data[i])) { - printf("Null value!\n"); result->data[i] = result->null_value; } else { result->data[i].date.year = d.data[i].date.year + 1; diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.sql b/sql/backends/monet5/UDF/capi/Tests/capi03.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/UDF/capi/Tests/capi03.sql @@ -0,0 +1,69 @@ +# Failure cases + + +START TRANSACTION; + +# Return different amount of rows in table-producing functions + +CREATE FUNCTION capi03() RETURNS TABLE(i INTEGER, j INTEGER) LANGUAGE C { + i->initialize(i, 10); + j->initialize(j, 20); + for(size_t index = 0; index < i->count; index++) { + i->data[index] = 0; + } + for(size_t index = 0; index < j->count; index++) { + j->data[index] = 1; + } +}; + +SELECT * FROM capi03(); + +ROLLBACK; + +START TRANSACTION; + +# No return value +CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C { + +}; + +CREATE TABLE integers(i INTEGER); +INSERT INTO integers VALUES (1), (2), (3), (4), (5); + +SELECT capi03(i) FROM integers; + +ROLLBACK; + + +START TRANSACTION; + +# Manually return an error from the function + +CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C { + return "Something went wrong!"; +}; + +CREATE TABLE integers(i INTEGER); +INSERT INTO integers VALUES (1), (2), (3), (4), (5); + +SELECT capi03(i) FROM integers; + +ROLLBACK; + +-- START TRANSACTION; + +-- # Modify input data +-- CREATE FUNCTION capi03(inp INTEGER) RETURNS INTEGER LANGUAGE C { +-- inp.data[0] = 10; +-- result->initialize(result, inp.count); +-- for(size_t i = 0; i < inp.count; i++) { +-- result->data[i] = inp.data[i] * 2; +-- } +-- }; + +-- CREATE TABLE integers(i INTEGER); +-- INSERT INTO integers VALUES (1), (2), (3), (4), (5); + +-- SELECT capi03(i) FROM integers; + +-- ROLLBACK; diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.err @@ -0,0 +1,43 @@ +stderr of test 'capi03` in directory 'sql/backends/monet5/UDF/capi` itself: + + +# 17:24:54 > +# 17:24:54 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=35915" "--set" "mapi_usock=/var/tmp/mtest-2836/.s.monetdb.35915" "--set" "monet_prompt=" "--forcemito" "--dbpath=NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi" +# 17:24:54 > + +# builtin opt gdk_dbpath = /Users/myth/opt/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 35915 +# cmdline opt mapi_usock = /var/tmp/mtest-2836/.s.monetdb.35915 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /Users/myth/opt/mTests/sql/backends/monet5/UDF/capi/NONE/var/MonetDB/mTests_sql_backends_monet5_UDF_capi +# cmdline opt gdk_debug = 536870922 + +# 17:24:54 > +# 17:24:54 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-2836" "--port=35915" +# 17:24:54 > + +MAPI = (monetdb) /var/tmp/mtest-2836/.s.monetdb.35915 +QUERY = SELECT * FROM capi03(); +ERROR = !Data has different cardinalities. +MAPI = (monetdb) /var/tmp/mtest-2836/.s.monetdb.35915 +QUERY = SELECT capi03(i) FROM integers; +ERROR = !No data returned. +MAPI = (monetdb) /var/tmp/mtest-2836/.s.monetdb.35915 +QUERY = SELECT capi03(i) FROM integers; +ERROR = !Something went wrong! + +# 17:24:55 > +# 17:24:55 > "Done." +# 17:24:55 > + diff --git a/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out b/sql/backends/monet5/UDF/capi/Tests/capi03.stable.out new file mode 100644 --- /dev/null +++ b/sql/bac
MonetDB: timetrails - First steps into adding continuous queries...
Changeset: 76dc5cf78ca2 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=76dc5cf78ca2 Added Files: sql/backends/monet5/Tests/cqcreate.sql Modified Files: sql/backends/monet5/sql_cat.c sql/backends/monet5/sql_cquery.c sql/backends/monet5/sql_cquery.h sql/backends/monet5/sql_scenario.c sql/include/sql_catalog.h sql/scripts/15_querylog.sql sql/scripts/25_debug.sql sql/scripts/26_sysmon.sql sql/server/rel_psm.c sql/server/rel_schema.c sql/server/sql_parser.y sql/server/sql_scan.c sql/server/sql_scan.h Branch: timetrails Log Message: First steps into adding continuous queries into the SQL catalog, but I am still getting errors while registering :( Can a SQL layer veteran help me? diffs (truncated from 496 to 300 lines): diff --git a/sql/backends/monet5/Tests/cqcreate.sql b/sql/backends/monet5/Tests/cqcreate.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/cqcreate.sql @@ -0,0 +1,5 @@ +CREATE stream TABLE testing (a int); + +CREATE TABLE results (b int); + +CREATE CONTINUOUS QUERY stressing() BEGIN INSERT INTO results SELECT a FROM testing; END; diff --git a/sql/backends/monet5/sql_cat.c b/sql/backends/monet5/sql_cat.c --- a/sql/backends/monet5/sql_cat.c +++ b/sql/backends/monet5/sql_cat.c @@ -18,6 +18,7 @@ #include "sql_scenario.h" #include "sql_mvc.h" #include "sql_qc.h" +#include "sql_cquery.h" #include "sql_optimizer.h" #include "mal_namespace.h" #include "opt_prelude.h" @@ -446,13 +447,28 @@ static str drop_func(mvc *sql, char *sname, char *name, int fid, int type, int action) { sql_schema *s = NULL; - char is_aggr = (type == F_AGGR); - char is_func = (type != F_PROC); - char *F = is_aggr ? "AGGREGATE" : (is_func ? "FUNCTION" : "PROCEDURE"); - char *f = is_aggr ? "aggregate" : (is_func ? "function" : "procedure"); + char *F, *f; char *KF = type == F_FILT ? "FILTER " : type == F_UNION ? "UNION " : ""; char *kf = type == F_FILT ? "filter " : type == F_UNION ? "union " : ""; + switch (type) { + case F_AGGR: + F = "AGGREGATE"; + f = "aggregate"; + break; + case F_PROC: + F = "PROCEDURE"; + f = "procedure"; + break; + case F_CONTINUOUS_QUERY: + F = "CONTINUOUS QUERY"; + f = "continuous query"; + break; + default: + F = "FUNCTION"; + f = "function"; + } + if (sname && !(s = mvc_bind_schema(sql, sname))) return sql_message("3F000!DROP %s%s: no such schema '%s'", KF, F, sname); if (!s) @@ -497,12 +513,24 @@ create_func(mvc *sql, char *sname, char { sql_func *nf; sql_schema *s = NULL; - char is_aggr = (f->type == F_AGGR); - char is_func = (f->type != F_PROC); - char *F = is_aggr ? "AGGREGATE" : (is_func ? "FUNCTION" : "PROCEDURE"); - char *KF = f->type == F_FILT ? "FILTER " : f->type == F_UNION ? "UNION " : ""; + char *F, *KF = f->type == F_FILT ? "FILTER " : f->type == F_UNION ? "UNION " : ""; - (void)fname; + (void) fname; + + switch (f->type) { + case F_AGGR: + F = "AGGREGATE"; + break; + case F_PROC: + F = "PROCEDURE"; + break; + case F_CONTINUOUS_QUERY: + F = "CONTINUOUS QUERY"; + break; + default: + F = "FUNCTION"; + } + if (sname && !(s = mvc_bind_schema(sql, sname))) return sql_message("3F000!CREATE %s%s: no such schema '%s'", KF, F, sname); if (!s) @@ -546,6 +574,13 @@ create_func(mvc *sql, char *sname, char if (!backend_resolve_function(sql, nf)) return sql_message("3F000!CREATE %s%s: external name %s.%s not bound", KF, F, nf->mod, nf->base.name); } + if(f->type == F_CONTINUOUS_QUERY) { + Client cntxt = MCgetClient(sql->clientid); + char *err = CQregisterInternal(cntxt, (str) sname, f->base.name); + if (err != NULL) { + return sql_message("3F000!CREATE %s%s: continuous query register error: %s", KF, F, err); + } + } return MAL_SUCCEED; } diff --git a/sql/backends/monet5/sql_cquery.c b/sql/backends/monet5/sql_cquery.c --- a/sql/backends/monet5/sql_cquery.c +++ b/sql/backends/monet5/sql_cquery.c @@ -416,7 +416,7 @@ IOTprocedureStmt(Client cntxt, MalBlkPtr throw(SQL, "cquery.register", "SQL procedure missing"); } -str +/*str CQregister(Client cntxt, MalBlkPtr mb, MalStkPtr stk, I