MonetDB: strheapvacuum - merge default
Changeset: 50aee56dda65 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/50aee56dda65 Modified Files: gdk/gdk_bat.c sql/backends/monet5/sql.c sql/storage/bat/bat_storage.c Branch: strheapvacuum Log Message: merge default diffs (truncated from 5971 to 300 lines): diff --git a/clients/Tests/MAL-signatures.stable.out b/clients/Tests/MAL-signatures.stable.out --- a/clients/Tests/MAL-signatures.stable.out +++ b/clients/Tests/MAL-signatures.stable.out @@ -5752,7 +5752,7 @@ stdout of test 'MAL-signatures` in direc [ "batcalc", "timestamp","pattern batcalc.timestamp(X_0:bat[:timestamp], X_1:bat[:oid], X_2:int):bat[:timestamp] ", "timestamp_2time_timestamp;", "" ] [ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:str], X_1:bat[:oid]):bat[:uuid] ", "UUIDstr2uuid_bulk;", "" ] [ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:uuid], X_1:bat[:oid]):bat[:uuid] ", "UUIDuuid2uuid_bulk;", "" ] -[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:wkb], X_1:int, X_2:int):bat[:wkb] ", "geom_2_geom_bat;", "" ] +[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:wkb], X_1:bat[:oid], X_2:int, X_3:int):bat[:wkb] ", "geom_2_geom_bat;", "" ] [ "batcalc", "xml", "command batcalc.xml(X_0:bat[:str]):bat[:xml] ", "BATXMLstr2xml;", "" ] [ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit]):bat[:bit] ", "CMDbatXOR;", "" ] [ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit], X_2:bat[:oid], X_3:bat[:oid]):bat[:bit] ", "CMDbatXOR;", "" ] diff --git a/clients/Tests/MAL-signatures.stable.out.int128 b/clients/Tests/MAL-signatures.stable.out.int128 --- a/clients/Tests/MAL-signatures.stable.out.int128 +++ b/clients/Tests/MAL-signatures.stable.out.int128 @@ -8276,7 +8276,7 @@ stdout of test 'MAL-signatures` in direc [ "batcalc", "timestamp","pattern batcalc.timestamp(X_0:bat[:timestamp], X_1:bat[:oid], X_2:int):bat[:timestamp] ", "timestamp_2time_timestamp;", "" ] [ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:str], X_1:bat[:oid]):bat[:uuid] ", "UUIDstr2uuid_bulk;", "" ] [ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:uuid], X_1:bat[:oid]):bat[:uuid] ", "UUIDuuid2uuid_bulk;", "" ] -[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:wkb], X_1:int, X_2:int):bat[:wkb] ", "geom_2_geom_bat;", "" ] +[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:wkb], X_1:bat[:oid], X_2:int, X_3:int):bat[:wkb] ", "geom_2_geom_bat;", "" ] [ "batcalc", "xml", "command batcalc.xml(X_0:bat[:str]):bat[:xml] ", "BATXMLstr2xml;", "" ] [ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit]):bat[:bit] ", "CMDbatXOR;", "" ] [ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit], X_2:bat[:oid], X_3:bat[:oid]):bat[:bit] ", "CMDbatXOR;", "" ] diff --git a/clients/mapiclient/dump.c b/clients/mapiclient/dump.c --- a/clients/mapiclient/dump.c +++ b/clients/mapiclient/dump.c @@ -394,15 +394,15 @@ dump_foreign_keys(Mapi mid, const char * goto bailout; } snprintf(query, maxquerylen, -"SELECT ps.name, " /* 0 */ +"SELECT ps.name, " /* 0 */ "pkt.name, "/* 1 */ - "pkkc.name, " /* 2 */ - "fkkc.name, " /* 3 */ - "fkkc.nr, " /* 4 */ - "fkk.name, "/* 5 */ - "fkk.\"action\", " /* 6 */ - "fs.name, " /* 7 */ - "fkt.name " /* 8 */ + "pkkc.name, " /* 2 */ + "fkkc.name, " /* 3 */ + "fkkc.nr, " /* 4 */ + "fkk.name, "/* 5 */ + "fkk.\"action\", " /* 6 */ + "fs.name, " /* 7 */ + "fkt.name " /* 8 */ "FROM sys._tables fkt, " "sys.objects fkkc, " "sys.keys fkk, " @@ -430,15 +430,15 @@ dump_foreign_keys(Mapi mid, const char * if (query == NULL) goto bailout; snprintf(query, maxquerylen, -"SELECT ps.name, " /* 0 */ +"SELECT ps.name, " /* 0 */ "pkt.name, "/
MonetDB: Jun2020 - Initialize `end'.
Changeset: dd189b4e215e for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/dd189b4e215e Modified Files: monetdb5/modules/mal/tablet.c Branch: Jun2020 Log Message: Initialize `end'. diffs (12 lines): diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c --- a/monetdb5/modules/mal/tablet.c +++ b/monetdb5/modules/mal/tablet.c @@ -1225,7 +1225,7 @@ SQLproducer(void *p) bool blocked[MAXBUFFERS] = { false }; bool ateof[MAXBUFFERS] = { false }; BUN cnt = 0, bufcnt[MAXBUFFERS] = { 0 }; - char *end, *e = NULL, *s = NULL, *base; + char *end = NULL, *e = NULL, *s = NULL, *base; const char *rsep = task->rsep; size_t rseplen = strlen(rsep), partial = 0; char quote = task->quote; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Oct2020 - Merge with Jun2020 branch.
Changeset: bc6eb7167efe for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/bc6eb7167efe Modified Files: gdk/gdk_logger.c monetdb5/modules/mal/tablet.c Branch: Oct2020 Log Message: Merge with Jun2020 branch. diffs (54 lines): diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c --- a/monetdb5/modules/mal/tablet.c +++ b/monetdb5/modules/mal/tablet.c @@ -1235,12 +1235,13 @@ SQLproducer(void *p) bool blocked[MAXBUFFERS] = { false }; bool ateof[MAXBUFFERS] = { false }; BUN cnt = 0, bufcnt[MAXBUFFERS] = { 0 }; - char *end, *e, *s = NULL, *base; + char *end = NULL, *e = NULL, *s = NULL, *base; const char *rsep = task->rsep; size_t rseplen = strlen(rsep), partial = 0; char quote = task->quote; dfa_t rdfa; lng rowno = 0; + int more = 0; MT_sema_down(&task->producer); if (task->id < 0) { @@ -1447,6 +1448,7 @@ SQLproducer(void *p) task->cur = cur; task->ateof = ateof[cur]; task->cnt = bufcnt[cur]; + more = !ateof[cur] || (e && e < end && task->top[cur] == task->limit); /* TRC_DEBUG(MAL_SERVER, "SQL producer got buffer '%d' filled with '%d' records\n", cur, task->top[cur]);*/ MT_sema_up(&task->consumer); @@ -1464,7 +1466,7 @@ SQLproducer(void *p) /* TRC_DEBUG(MAL_SERVER, "Continue producer buffer: %d\n", cur);*/ /* we ran out of input? */ - if (task->ateof) { + if (task->ateof && !more) { /* TRC_DEBUG(MAL_SERVER, "Producer encountered eof\n");*/ GDKfree(rdfa); return; @@ -1528,7 +1530,7 @@ SQLload_file(Client cntxt, Tablet *as, b BUN i, attr; READERtask task; READERtask ptask[MAXWORKERS]; - int threads = (maxrow< 0 || maxrow > (1 << 16)) ? (GDKnr_threads < MAXWORKERS && GDKnr_threads > 1 ? GDKnr_threads - 1 : MAXWORKERS - 1) : 1; + int threads = (maxrow< 0 || maxrow > (1 << 16)) && GDKnr_threads > 1 ? (GDKnr_threads < MAXWORKERS ? GDKnr_threads - 1 : MAXWORKERS - 1) : 1; lng lio = 0, tio, t1 = 0, total = 0, iototal = 0; char name[MT_NAME_LEN]; @@ -1836,6 +1838,9 @@ SQLload_file(Client cntxt, Tablet *as, b task.maxrow = cnt; task.state = ENDOFCOPY; } + if (task.ateof && task.top[task.cur] < task.limit && cnt != task.maxrow) + break; + task.top[task.cur] = 0; MT_sema_up(&task.producer); } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2021 - Merge with Oct2020 branch.
Changeset: 010632ebf99f for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/010632ebf99f Modified Files: gdk/gdk_logger.c monetdb5/modules/mal/tablet.c Branch: Jul2021 Log Message: Merge with Oct2020 branch. diffs (45 lines): diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c --- a/monetdb5/modules/mal/tablet.c +++ b/monetdb5/modules/mal/tablet.c @@ -1232,12 +1232,13 @@ SQLproducer(void *p) bool blocked[MAXBUFFERS] = { false }; bool ateof[MAXBUFFERS] = { false }; BUN cnt = 0, bufcnt[MAXBUFFERS] = { 0 }; - char *end, *e, *s = NULL, *base; + char *end = NULL, *e = NULL, *s = NULL, *base; const char *rsep = task->rsep; size_t rseplen = strlen(rsep), partial = 0; char quote = task->quote; dfa_t rdfa; lng rowno = 0; + int more = 0; MT_sema_down(&task->producer); if (task->id < 0) { @@ -1444,6 +1445,7 @@ SQLproducer(void *p) task->cur = cur; task->ateof = ateof[cur]; task->cnt = bufcnt[cur]; + more = !ateof[cur] || (e && e < end && task->top[cur] == task->limit); /* TRC_DEBUG(MAL_SERVER, "SQL producer got buffer '%d' filled with '%d' records\n", cur, task->top[cur]);*/ MT_sema_up(&task->consumer); @@ -1461,7 +1463,7 @@ SQLproducer(void *p) /* TRC_DEBUG(MAL_SERVER, "Continue producer buffer: %d\n", cur);*/ /* we ran out of input? */ - if (task->ateof) { + if (task->ateof && !more) { /* TRC_DEBUG(MAL_SERVER, "Producer encountered eof\n");*/ GDKfree(rdfa); return; @@ -1833,6 +1835,9 @@ SQLload_file(Client cntxt, Tablet *as, b task.maxrow = cnt; task.state = ENDOFCOPY; } + if (task.ateof && task.top[task.cur] < task.limit && cnt != task.maxrow) + break; + task.top[task.cur] = 0; MT_sema_up(&task.producer); } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2021 - Use more bat iterators to access data.
Changeset: 72e66a52b74d for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/72e66a52b74d Modified Files: sql/backends/monet5/UDF/pyapi3/conversion3.c sql/backends/monet5/UDF/pyapi3/convert_loops.h sql/backends/monet5/UDF/pyapi3/pyapi3.c sql/backends/monet5/UDF/udf/udf.c sql/backends/monet5/UDF/udf/udf_impl.h sql/backends/monet5/sql.c sql/backends/monet5/sql_cat.c sql/backends/monet5/sql_datetrunc.c sql/backends/monet5/sql_fround_impl.h sql/backends/monet5/sql_rank.c sql/backends/monet5/sql_result.c sql/backends/monet5/sql_round_impl.h sql/backends/monet5/sql_subquery.c sql/backends/monet5/sql_time.c sql/storage/bat/bat_storage.c Branch: Jul2021 Log Message: Use more bat iterators to access data. diffs (truncated from 2071 to 300 lines): diff --git a/sql/backends/monet5/UDF/pyapi3/conversion3.c b/sql/backends/monet5/UDF/pyapi3/conversion3.c --- a/sql/backends/monet5/UDF/pyapi3/conversion3.c +++ b/sql/backends/monet5/UDF/pyapi3/conversion3.c @@ -388,10 +388,12 @@ PyObject *PyArrayObject_FromBAT(PyInput j = 0; npy_float64 *data = (npy_float64 *)PyArray_DATA((PyArrayObject *)vararray); - const hge *vals = (const hge *) Tloc(b, 0); + BATiter bi = bat_iterator(b); + const hge *vals = (const hge *) bi.base; BATloop(b, p, q) { data[j++] = (npy_float64)vals[p]; } + bat_iterator_end(&bi); break; } #endif diff --git a/sql/backends/monet5/UDF/pyapi3/convert_loops.h b/sql/backends/monet5/UDF/pyapi3/convert_loops.h --- a/sql/backends/monet5/UDF/pyapi3/convert_loops.h +++ b/sql/backends/monet5/UDF/pyapi3/convert_loops.h @@ -19,17 +19,19 @@ #define BAT_TO_NP(bat, mtpe, nptpe) \ do { \ + BATiter bi = bat_iterator(bat); \ if (copy) { \ vararray = PyArray_EMPTY(1, elements, nptpe, 0); \ - memcpy(PyArray_DATA((PyArrayObject *)vararray), Tloc(bat, 0), \ + memcpy(PyArray_DATA((PyArrayObject *)vararray), bi.base, \ sizeof(mtpe) * (t_end - t_start)); \ } else { \ vararray = \ PyArray_New(&PyArray_Type, 1, elements, nptpe, NULL, \ - &((mtpe *)Tloc(bat, 0))[t_start], 0, \ + &((mtpe *)bi.base)[t_start], 0,\ NPY_ARRAY_CARRAY || !NPY_ARRAY_WRITEABLE, NULL); \ } \ - } while(0) \ + bat_iterator_end(&bi); \ + } while(0) // This #define creates a new BAT with the internal data and mask from a Numpy // array, without copying the data diff --git a/sql/backends/monet5/UDF/pyapi3/pyapi3.c b/sql/backends/monet5/UDF/pyapi3/pyapi3.c --- a/sql/backends/monet5/UDF/pyapi3/pyapi3.c +++ b/sql/backends/monet5/UDF/pyapi3/pyapi3.c @@ -768,9 +768,10 @@ static str PyAPIeval(Client cntxt, MalBl // now split the columns one by one for (i = 0; i < named_columns; i++) { PyInput input = pyinput_values[i]; - void *basevals = Tloc(input.bat, 0); + if (!input.scalar) { + BATiter bi = bat_iterator(input.bat); + void *basevals = bi.base; - if (!input.scalar) { switch (input.bat_type) { case TYPE_void: NP_SPLIT_BAT(oid); @@ -845,9 +846,10 @@ static str PyAPIeval(Client cntxt, MalBl msg = createException( MAL, "pyapi3.eval", SQLSTATE(PY000) "Unrecognized BA
MonetDB: Jul2021 - Make LOCK_STATS viable again.
Changeset: a770c1ff8307 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/a770c1ff8307 Modified Files: gdk/gdk.h gdk/gdk_system.h gdk/gdk_tracer.h Branch: Jul2021 Log Message: Make LOCK_STATS viable again. diffs (51 lines): diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -331,6 +331,8 @@ #include /* for *_MIN and *_MAX */ #include /* for FLT_MAX and DBL_MAX */ +typedef enum { GDK_FAIL, GDK_SUCCEED } gdk_return; + #include "gdk_system.h" #include "gdk_posix.h" #include "stream.h" @@ -524,8 +526,6 @@ typedef size_t BUN; /* * @- Checking and Error definitions: */ -typedef enum { GDK_FAIL, GDK_SUCCEED } gdk_return; - #define ATOMextern(t) (ATOMstorage(t) >= TYPE_str) typedef enum { @@ -1592,6 +1592,8 @@ gdk_export BAT *BBPquickdesc(bat b, bool #include "gdk_tracer.h" +gdk_export gdk_return GDKtracer_fill_comp_info(BAT *id, BAT *component, BAT *log_level); + #define GDKerror(format, ...) \ GDKtracer_log(__FILE__, __func__, __LINE__, M_ERROR,\ GDK, NULL, format, ##__VA_ARGS__) diff --git a/gdk/gdk_system.h b/gdk/gdk_system.h --- a/gdk/gdk_system.h +++ b/gdk/gdk_system.h @@ -179,6 +179,7 @@ gdk_export int MT_join_thread(MT_Id t); #define USE_NATIVE_LOCKS 1 #ifdef LOCK_STATS +#include "gdk_tracer.h" #define _DBG_LOCK_COUNT_0(l) \ do {\ diff --git a/gdk/gdk_tracer.h b/gdk/gdk_tracer.h --- a/gdk/gdk_tracer.h +++ b/gdk/gdk_tracer.h @@ -259,6 +259,4 @@ gdk_export void GDKtracer_log(const char gdk_export gdk_return GDKtracer_flush_buffer(void); -gdk_export gdk_return GDKtracer_fill_comp_info(BAT *id, BAT *component, BAT *log_level); - #endif /* _GDK_TRACER_H_ */ ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2021 - fix optimized compilation with lock stats
Changeset: 8f08d7b1970c for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/8f08d7b1970c Modified Files: monetdb5/modules/atoms/batxml.c sql/backends/monet5/sql.c Branch: Jul2021 Log Message: fix optimized compilation with lock stats diffs (33 lines): diff --git a/monetdb5/modules/atoms/batxml.c b/monetdb5/modules/atoms/batxml.c --- a/monetdb5/modules/atoms/batxml.c +++ b/monetdb5/modules/atoms/batxml.c @@ -477,7 +477,7 @@ BATXMLoptions(bat *ret, const char * con str buf = GDKmalloc(BUFSIZ); str val = GDKmalloc(BUFSIZ); size_t size = BUFSIZ, len = strlen(*name); - BATiter bi; + BATiter bi = (BATiter) { .b = NULL }; const char *err = OPERATION_FAILED " During bulk options analysis"; if (val == NULL || buf == NULL) { @@ -1230,7 +1230,7 @@ static const char * BATxmlaggr(BAT **bnp, BAT *b, BAT *g, BAT *e, BAT *s, int skip_nils) { BAT *bn = NULL, *t1, *t2 = NULL; - BATiter bi; + BATiter bi = (BATiter) { .b = NULL }; oid min, max; BUN ngrp; BUN nils = 0; diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -930,7 +930,7 @@ mvc_bat_next_get_value(Client cntxt, Mal sql_schema *s = NULL; sql_sequence *seq = NULL; seqbulk *sb = NULL; - BATiter bi, ci; + BATiter bi = (BATiter) { .b = NULL }, ci = (BATiter) { .b = NULL }; bat *res = getArgReference_bat(stk, pci, 0); bat schid = 0, seqid = 0; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: iso - Always finish the remote plan transaction, so the...
Changeset: d1b95282d0f0 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/d1b95282d0f0 Modified Files: clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/exports.stable.out monetdb5/mal/mal_exception.c monetdb5/mal/mal_exception.h monetdb5/modules/mal/remote.c monetdb5/optimizer/opt_oltp.c monetdb5/optimizer/opt_prelude.c monetdb5/optimizer/opt_prelude.h sql/backends/monet5/sql.c sql/backends/monet5/sql_execute.c sql/backends/monet5/sql_execute.h sql/backends/monet5/sql_gencode.c sql/test/BugTracker-2015/Tests/local-remotetable-crash.Bug-3754.SQL.py sql/test/BugTracker-2015/Tests/schemadiff.Bug-3778.SQL.py sql/test/BugTracker-2019/Tests/remote-table-non-existent-column.Bug-6750.py sql/test/BugTracker-2020/Tests/table-udf-on-remote.Bug-6971.py Branch: iso Log Message: Always finish the remote plan transaction, so the pending dependencies can get a chance to be cleaned At the moment I changed the remote plan to throw an error to check the remote server log, because I can't figure out how to rethrow a MAL error. Please a MAL expert review this an make it throw the original exception. diffs (truncated from 482 to 300 lines): diff --git a/clients/Tests/MAL-signatures.stable.out b/clients/Tests/MAL-signatures.stable.out --- a/clients/Tests/MAL-signatures.stable.out +++ b/clients/Tests/MAL-signatures.stable.out @@ -9169,6 +9169,7 @@ stdout of test 'MAL-signatures` in direc [ "rapi", "eval_aggr","pattern rapi.eval_aggr(X_0:ptr, X_1:str, X_2:any...):any... ", "RAPIevalAggr;","" ] [ "rapi", "prelude", "command rapi.prelude():void ", "RAPIprelude;", "" ] [ "rapi", "subeval_aggr", "pattern rapi.subeval_aggr(X_0:ptr, X_1:str, X_2:any...):any... ", "RAPIevalAggr;","" ] +[ "remote","assert", "pattern remote.assert(X_0:bit, X_1:str):void ","RMTassert;", "" ] [ "remote","batbincopy", "pattern remote.batbincopy():bat[:any] ", "RMTbincopyfrom;", "" ] [ "remote","batbincopy", "pattern remote.batbincopy(X_0:bat[:any]):void ", "RMTbincopyto;","" ] [ "remote","batload", "pattern remote.batload(X_0:any_1, X_1:int):bat[:any_1] ", "RMTbatload;", "" ] @@ -9270,6 +9271,7 @@ stdout of test 'MAL-signatures` in direc [ "sql", "deltas", "pattern sql.deltas(X_0:str, X_1:str) (X_2:bat[:int], X_3:bat[:lng], X_4:bat[:lng], X_5:bat[:lng], X_6:bat[:lng], X_7:bat[:lng], X_8:bat[:int]) ", "mvc_delta_values;","" ] [ "sql", "deltas", "pattern sql.deltas(X_0:str, X_1:str, X_2:str) (X_3:bat[:int], X_4:bat[:lng], X_5:bat[:lng], X_6:bat[:lng], X_7:bat[:lng], X_8:bat[:lng], X_9:bat[:int]) ", "mvc_delta_values;","" ] [ "sql", "dense_rank", "pattern sql.dense_rank(X_0:any_1, X_1:bit, X_2:bit):int ", "SQLdense_rank;", "" ] +[ "sql", "deregister", "pattern sql.deregister():int ", "RAstatementEnd;", "" ] [ "sql", "diff", "pattern sql.diff(X_0:any_1):bit ", "SQLdiff;", "" ] [ "sql", "diff", "pattern sql.diff(X_0:bit, X_1:any_1):bit ", "SQLdiff;", "" ] [ "sql", "drop_hash","unsafe pattern sql.drop_hash(X_0:str, X_1:str):void ", "SQLdrop_hash;","" ] diff --git a/clients/Tests/MAL-signatures.stable.out.int128 b/clients/Tests/MAL-signatures.stable.out.int128 --- a/clients/Tests/MAL-signatures.stable.out.int128 +++ b/clients/Tests/MAL-signatures.stable.out.int128 @@ -12470,6 +12470,7 @@ stdout of test 'MAL-signatures` in direc [ "rapi", "eval_aggr","pattern rapi.eval_aggr(X_0:ptr, X_1:str, X_2:any...):any... ", "RAPIevalAggr;","" ] [ "rapi", "prelude", "command rapi.prelude():void ", "RAPIprelude;", "" ] [ "rapi", "subeval_aggr", "pattern rapi.subeval_aggr(X_0:ptr, X_1:str, X_2:any...):any... ", "RAPIevalAggr;","" ] +[ "remote","assert", "pattern remote.assert(X_0:bit, X_1:str):void ","RMTassert;", "" ] [ "remote","batbincopy", "pattern remote.batbincopy():bat[:any] ", "RMTbincopyfrom;", "" ] [ "remote","batbincopy", "pattern remote.batbincopy(X_0:bat[:any]):void ", "RMTbincopyto;","" ] [ "remote","batload", "pattern remote.batload(X_0:any_1, X_1:int):bat[:any_1] ", "RMTbatload;", "" ] @@ -12576,6 +12577,7 @@ stdout of test 'MAL-signatures` in direc [ "sql", "deltas", "pattern sql.deltas(X_0:str, X_1:str) (X_2:bat[:int], X_3:bat[:lng], X_4:bat[:lng], X_5:bat[:lng], X_6:bat[:lng], X_7:bat[:lng], X_8:bat[:int]) ", "mvc_delta_values;","" ] [ "sql", "deltas", "pattern sql.deltas(X_0:str, X_1:str, X_2:str) (X_3:bat[:int], X_4:bat[:l
MonetDB: iso - Merged with Jul2021
Changeset: 40a042bd8329 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/40a042bd8329 Modified Files: clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 sql/backends/monet5/sql.c sql/backends/monet5/sql_cat.c sql/backends/monet5/sql_user.c sql/storage/bat/bat_storage.c Branch: iso Log Message: Merged with Jul2021 diffs (truncated from 2485 to 300 lines): diff --git a/clients/Tests/MAL-signatures.stable.out b/clients/Tests/MAL-signatures.stable.out --- a/clients/Tests/MAL-signatures.stable.out +++ b/clients/Tests/MAL-signatures.stable.out @@ -5753,6 +5753,7 @@ stdout of test 'MAL-signatures` in direc [ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:str], X_1:bat[:oid]):bat[:uuid] ", "UUIDstr2uuid_bulk;", "" ] [ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:uuid], X_1:bat[:oid]):bat[:uuid] ", "UUIDuuid2uuid_bulk;", "" ] [ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:wkb], X_1:bat[:oid], X_2:int, X_3:int):bat[:wkb] ", "geom_2_geom_bat;", "" ] +[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:str], X_1:bat[:oid], X_2:int, X_3:int):bat[:wkb] ", "wkbFromText_bat_cand;","" ] [ "batcalc", "xml", "command batcalc.xml(X_0:bat[:str]):bat[:xml] ", "BATXMLstr2xml;", "" ] [ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit]):bat[:bit] ", "CMDbatXOR;", "" ] [ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit], X_2:bat[:oid], X_3:bat[:oid]):bat[:bit] ", "CMDbatXOR;", "" ] diff --git a/clients/Tests/MAL-signatures.stable.out.int128 b/clients/Tests/MAL-signatures.stable.out.int128 --- a/clients/Tests/MAL-signatures.stable.out.int128 +++ b/clients/Tests/MAL-signatures.stable.out.int128 @@ -8277,6 +8277,7 @@ stdout of test 'MAL-signatures` in direc [ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:str], X_1:bat[:oid]):bat[:uuid] ", "UUIDstr2uuid_bulk;", "" ] [ "batcalc", "uuid", "command batcalc.uuid(X_0:bat[:uuid], X_1:bat[:oid]):bat[:uuid] ", "UUIDuuid2uuid_bulk;", "" ] [ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:wkb], X_1:bat[:oid], X_2:int, X_3:int):bat[:wkb] ", "geom_2_geom_bat;", "" ] +[ "batcalc", "wkb", "command batcalc.wkb(X_0:bat[:str], X_1:bat[:oid], X_2:int, X_3:int):bat[:wkb] ", "wkbFromText_bat_cand;","" ] [ "batcalc", "xml", "command batcalc.xml(X_0:bat[:str]):bat[:xml] ", "BATXMLstr2xml;", "" ] [ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit]):bat[:bit] ", "CMDbatXOR;", "" ] [ "batcalc", "xor", "pattern batcalc.xor(X_0:bat[:bit], X_1:bat[:bit], X_2:bat[:oid], X_3:bat[:oid]):bat[:bit] ", "CMDbatXOR;", "" ] diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -331,6 +331,8 @@ #include /* for *_MIN and *_MAX */ #include /* for FLT_MAX and DBL_MAX */ +typedef enum { GDK_FAIL, GDK_SUCCEED } gdk_return; + #include "gdk_system.h" #include "gdk_posix.h" #include "stream.h" @@ -524,8 +526,6 @@ typedef size_t BUN; /* * @- Checking and Error definitions: */ -typedef enum { GDK_FAIL, GDK_SUCCEED } gdk_return; - #define ATOMextern(t) (ATOMstorage(t) >= TYPE_str) typedef enum { @@ -1592,6 +1592,8 @@ gdk_export BAT *BBPquickdesc(bat b, bool #include "gdk_tracer.h" +gdk_export gdk_return GDKtracer_fill_comp_info(BAT *id, BAT *component, BAT *log_level); + #define GDKerror(format, ...) \ GDKtracer_log(__FILE__, __func__, __LINE__, M_ERROR,\ GDK, NULL, format, ##__VA_ARGS__) diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c --- a/gdk/gdk_logger.c +++ b/gdk/gdk_logger.c @@ -2335,7 +2335,7 @@ log_constant(logger *lg, int type, ptr v static gdk_return string_writer(logger *lg, BAT *b, lng offset, lng nr) { - int sz = 0; + size_t sz = 0; BUN end = (BUN)(offset + nr); BATiter bi = bat_iterator(b); diff --git a/gdk/gdk_system.h b/gdk/gdk_system.h --- a/gdk/gdk_system.h +++ b/gdk/gdk_system.h @@ -179,6 +179,7 @@ gdk_export int MT_join_thread(MT_Id t); #define USE_NATIVE_LOCKS 1 #ifdef LOCK_STATS +#include "gdk_tracer.h" #define _DBG_LOCK_COUNT_0(l) \ do {\ diff --git a/gdk/gdk_tracer.h b/gdk/gdk_tracer.h --- a/gdk/gdk_tracer.h +++ b/gdk/gdk_tracer.h @@ -259,6 +259,4 @@ gdk_export void GDKtracer_log(const char gdk_export gdk_return GDKtracer_flush_buffer(void); -gdk_export gdk_return GDKtracer_fill_comp_info(BAT *id, BAT *component, BAT *log_level); - #endif /* _GDK_TRACER_H_ */ diff --git a/geom/monetdb5/geom.c b/geom/monetdb5/geom.c --- a/geom/monetdb5/geom.c +++ b/geom/monetdb5/geom.c @@ -6575,6 +6575,7 @@ static mel_func geom_init_funcs[] = { command(
MonetDB: iso - Fix lock and always destroy changes list on exit
Changeset: 4c19f5e8b2b9 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/4c19f5e8b2b9 Modified Files: sql/storage/store.c Branch: iso Log Message: Fix lock and always destroy changes list on exit diffs (41 lines): diff --git a/sql/storage/store.c b/sql/storage/store.c --- a/sql/storage/store.c +++ b/sql/storage/store.c @@ -2082,8 +2082,8 @@ store_exit(sqlstore *store) } else _DELETE(c); } - list_destroy(store->changes); } + list_destroy(store->changes); MT_lock_unset(&store->commit); os_destroy(store->cat->objects, store); os_destroy(store->cat->schemas, store); @@ -3410,14 +3410,16 @@ sql_trans_rollback(sql_trans *tr, int lo if (!locked) store_unlock(store); } else if (ATOMIC_GET(&store->nr_active) == 1) { /* just me cleanup */ - store_lock(store); + if (!locked) + store_lock(store); ulng oldest = store_timestamp(store); store_pending_changes(store, oldest); if (ATOMIC_GET(&store->nr_active) == 1 && !tr->parent) { id_hash_clear(store->dependencies); id_hash_clear(store->removals); } - store_unlock(store); + if (!locked) + store_unlock(store); } if (tr->localtmps.dset) { list_destroy2(tr->localtmps.dset, tr->store); @@ -3754,7 +3756,7 @@ sql_trans_commit(sql_trans *tr) if (locked) { store_unlock(store); MT_lock_unset(&store->commit); - } + } /* drop local temp tables with commit action CA_DROP, after cleanup */ if (cs_size(&tr->localtmps)) { for(node *n=tr->localtmps.set->h; n; ) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2021 - Always destroy store changes list and use tra...
Changeset: 2a64a51ae777 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/2a64a51ae777 Modified Files: sql/storage/store.c Branch: Jul2021 Log Message: Always destroy store changes list and use tracer instead of print calls diffs (21 lines): diff --git a/sql/storage/store.c b/sql/storage/store.c --- a/sql/storage/store.c +++ b/sql/storage/store.c @@ -1987,15 +1987,15 @@ store_exit(sqlstore *store) if (c->cleanup && !c->cleanup(store, c, oldest)) { /* try again with newer oldest, should cleanup any pending issues */ if (!c->cleanup(store, c, oldest+1)) - printf("not deleted\n"); + TRC_DEBUG(SQL_STORE, "not deleted\n"); else _DELETE(c); } else _DELETE(c); } - list_destroy(store->changes); } MT_lock_unset(&store->commit); + list_destroy(store->changes); os_destroy(store->cat->objects, store); os_destroy(store->cat->schemas, store); _DELETE(store->cat); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: iso - Merged with Jul2021
Changeset: 1190c4ecad0e for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/1190c4ecad0e Modified Files: sql/storage/store.c Branch: iso Log Message: Merged with Jul2021 diffs (21 lines): diff --git a/sql/storage/store.c b/sql/storage/store.c --- a/sql/storage/store.c +++ b/sql/storage/store.c @@ -2076,15 +2076,15 @@ store_exit(sqlstore *store) if (c->cleanup && !c->cleanup(store, c, oldest)) { /* try again with newer oldest, should cleanup any pending issues */ if (!c->cleanup(store, c, oldest+1)) - printf("not deleted\n"); + TRC_DEBUG(SQL_STORE, "not deleted\n"); else _DELETE(c); } else _DELETE(c); } } + MT_lock_unset(&store->commit); list_destroy(store->changes); - MT_lock_unset(&store->commit); os_destroy(store->cat->objects, store); os_destroy(store->cat->schemas, store); _DELETE(store->cat); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2021 - A little cleanup.
Changeset: 24c897fc5700 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/24c897fc5700 Modified Files: gdk/gdk.h gdk/gdk_aggr.c gdk/gdk_bat.c gdk/gdk_batop.c monetdb5/modules/atoms/batxml.c monetdb5/modules/mal/manifold.c sql/backends/monet5/sql.c Branch: Jul2021 Log Message: A little cleanup. diffs (217 lines): diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -1079,7 +1079,6 @@ gdk_export BUN BUNfnd(BAT *b, const void (BUN) (*(const oid*)(v) - (b)->tseqbase)) #define BATttype(b)(BATtdense(b) ? TYPE_oid : (b)->ttype) -#define Tbase(b) ((b)->tvheap->base) #define Tsize(b) ((b)->twidth) @@ -1410,8 +1409,8 @@ BATsettrivprop(BAT *b) int c; if (b->tvarsized) c = ATOMcmp(b->ttype, - Tbase(b) + VarHeapVal(Tloc(b, 0), 0, b->twidth), - Tbase(b) + VarHeapVal(Tloc(b, 0), 1, b->twidth)); + b->tvheap->base + VarHeapVal(Tloc(b, 0), 0, b->twidth), + b->tvheap->base + VarHeapVal(Tloc(b, 0), 1, b->twidth)); else c = ATOMcmp(b->ttype, Tloc(b, 0), Tloc(b, 1)); b->tsorted = c <= 0; diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c --- a/gdk/gdk_aggr.c +++ b/gdk/gdk_aggr.c @@ -910,7 +910,7 @@ BATgroupsum(BAT *b, BAT *g, BAT *e, BAT } if ((e == NULL || -(BATcount(e) == BATcount(b) && e->hseqbase == b->hseqbase)) && +(BATcount(e) == ncand && e->hseqbase == ci.hseq)) && (BATtdense(g) || (g->tkey && g->tnonil))) { /* trivial: singleton groups, so all results are equal * to the inputs (but possibly a different type) */ @@ -1586,7 +1586,7 @@ BATgroupprod(BAT *b, BAT *g, BAT *e, BAT } if ((e == NULL || -(BATcount(e) == BATcount(b) && e->hseqbase == b->hseqbase)) && +(BATcount(e) == ncand && e->hseqbase == ci.hseq)) && (BATtdense(g) || (g->tkey && g->tnonil))) { /* trivial: singleton groups, so all results are equal * to the inputs (but possibly a different type) */ @@ -4583,7 +4583,7 @@ dogroupstdev(BAT **avgb, BAT *b, BAT *g, } if ((e == NULL || -(BATcount(e) == BATcount(b) && e->hseqbase == b->hseqbase)) && +(BATcount(e) == ncand && e->hseqbase == ci.hseq)) && (BATtdense(g) || (g->tkey && g->tnonil)) && (issample || b->tnonil)) { /* trivial: singleton groups, so all results are equal diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c --- a/gdk/gdk_bat.c +++ b/gdk/gdk_bat.c @@ -174,7 +174,7 @@ void BATsetdims(BAT *b) { b->twidth = b->ttype == TYPE_str ? 1 : ATOMsize(b->ttype); - b->tshift = ATOMelmshift(Tsize(b)); + b->tshift = ATOMelmshift(b->twidth); assert_shift_width(b->tshift, b->twidth); b->tvarsized = b->ttype == TYPE_void || BATatoms[b->ttype].atomPut != NULL; } @@ -602,6 +602,7 @@ BATclear(BAT *b, bool force) PROPdestroy(b); /* we must dispose of all inserted atoms */ + MT_lock_set(&b->theaplock); if (force && BATatoms[b->ttype].atomDel == NULL) { assert(b->tvheap == NULL || b->tvheap->parentid == b->batCacheid); /* no stable elements: we do a quick heap clean */ @@ -621,12 +622,10 @@ BATclear(BAT *b, bool force) if (ATOMheap(b->ttype, th, 0) != GDK_SUCCEED) return GDK_FAIL; ATOMIC_INIT(&th->refs, 1); - MT_lock_set(&b->theaplock); th->parentid = b->tvheap->parentid; th->dirty = true; HEAPdecref(b->tvheap, false); b->tvheap = th; - MT_lock_unset(&b->theaplock); } } else { /* do heap-delete of all inserted atoms */ @@ -635,15 +634,14 @@ BATclear(BAT *b, bool force) /* TYPE_str has no del method, so we shouldn't get here */ assert(tatmdel == NULL || b->twidth == sizeof(var_t)); if (tatmdel) { - MT_lock_set(&b->theaplock); BATiter bi = bat_iterator_nolock(b); for (p = b->batInserted, q = BUNlast(b); p < q; p++) (*tatmdel)(b->tvheap, (var_t*) BUNtloc(bi,p)); b->tvheap->dirty = true; - MT_lock_unset(&b->theaplock); } } + MT_lock_unset(&b->theaplock); if (force) b->batInserted = 0; diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -409,7 +409,7 @@ insert_string_bat(BAT *b, BAT
MonetDB: Jul2021 - Create BBP.dir file during save stage with ba...
Changeset: 8adfdd8687f8 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/8adfdd8687f8 Modified Files: gdk/gdk_bbp.c gdk/gdk_private.h gdk/gdk_storage.c Branch: Jul2021 Log Message: Create BBP.dir file during save stage with bat locks in place. This should prevent the race condition where a bat was first saved, and then, in a separate action, written to the BBP.dir file, giving other threads the opportunity to change the bat between those actions. diffs (truncated from 429 to 300 lines): diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -114,7 +114,7 @@ static void BBPuncacheit(bat bid, bool u static gdk_return BBPprepare(bool subcommit); static BAT *getBBPdescriptor(bat i, bool lock); static gdk_return BBPbackup(BAT *b, bool subcommit); -static gdk_return BBPdir(int cnt, bat *restrict subcommit, BUN *restrict sizes, lng logno, lng transid); +static gdk_return BBPdir_init(void); static lng BBPlogno; /* two lngs of extra info in BBP.dir */ static lng BBPtransid; @@ -1109,7 +1109,7 @@ BBPinit(void) /* no BBP.bak (nor BBP.dir or BACKUP/BBP.dir): * create a new one */ TRC_DEBUG(IO_, "initializing BBP.\n"); - if (BBPdir(0, NULL, NULL, LL_CONSTANT(0), LL_CONSTANT(0)) != GDK_SUCCEED) { + if (BBPdir_init() != GDK_SUCCEED) { GDKfree(bbpdirstr); GDKfree(backupbbpdirstr); MT_lock_unset(&GDKtmLock); @@ -1401,99 +1401,146 @@ BBPdir_header(FILE *f, int n, lng logno, } static gdk_return -BBPdir_subcommit(int cnt, bat *restrict subcommit, BUN *restrict sizes, lng logno, lng transid) +BBPdir_first(bool subcommit, lng logno, lng transid, +FILE **obbpfp, FILE **nbbpfp) { - FILE *obbpf, *nbbpf; - bat j = 1; - char buf[3000]; - int n; + FILE *obbpf = NULL, *nbbpf = NULL; + int n = 0; lng ologno, otransid; #ifndef NDEBUG assert(islocked(&GDKtmLock)); - assert(subcommit != NULL); - for (n = 2; n < cnt; n++) - assert(subcommit[n - 1] < subcommit[n]); #endif - if ((nbbpf = GDKfilelocate(0, "BBP", "w", "dir")) == NULL) + if (obbpfp) + *obbpfp = NULL; + *nbbpfp = NULL; + + if ((nbbpf = GDKfilelocate(0, "BBP", "w", "dir")) == NULL) { return GDK_FAIL; - - /* we need to copy the backup BBP.dir to the new, but -* replacing the entries for the subcommitted bats */ - if ((obbpf = GDKfileopen(0, SUBDIR, "BBP", "dir", "r")) == NULL && - (obbpf = GDKfileopen(0, BAKDIR, "BBP", "dir", "r")) == NULL) { - GDKsyserror("subcommit attempted without backup BBP.dir."); - goto bailout; } - /* read first three lines */ - if (fgets(buf, sizeof(buf), obbpf) == NULL || /* BBP.dir, GDKversion %d */ - fgets(buf, sizeof(buf), obbpf) == NULL || /* SIZEOF_SIZE_T SIZEOF_OID SIZEOF_MAX_INT */ - fgets(buf, sizeof(buf), obbpf) == NULL) { /* BBPsize=%d */ - GDKerror("subcommit attempted with invalid backup BBP.dir."); - goto bailout; + + if (subcommit) { + char buf[512]; + + assert(obbpfp != NULL); + /* we need to copy the backup BBP.dir to the new, but +* replacing the entries for the subcommitted bats */ + if ((obbpf = GDKfileopen(0, SUBDIR, "BBP", "dir", "r")) == NULL && + (obbpf = GDKfileopen(0, BAKDIR, "BBP", "dir", "r")) == NULL) { + GDKsyserror("subcommit attempted without backup BBP.dir."); + goto bailout; + } + /* read first three lines */ + if (fgets(buf, sizeof(buf), obbpf) == NULL || /* BBP.dir, GDKversion %d */ + fgets(buf, sizeof(buf), obbpf) == NULL || /* SIZEOF_SIZE_T SIZEOF_OID SIZEOF_MAX_INT */ + fgets(buf, sizeof(buf), obbpf) == NULL) { /* BBPsize=%d */ + GDKerror("subcommit attempted with invalid backup BBP.dir."); + goto bailout; + } + /* third line contains BBPsize */ + if (sscanf(buf, "BBPsize=%d", &n) != 1) { + GDKerror("cannot read BBPsize in backup BBP.dir."); + goto bailout; + } + /* fourth line contains BBPinfo */ + if (fgets(buf, sizeof(buf), obbpf) == NULL || + sscanf(buf, "BBPinfo=" LLSCN " " LLSCN, &ologno, &otransid) != 2) { + GDKerror("cannot read BBPinfo in backup BBP.dir."); + goto bailout; + } } - /* third line co