Changeset: b25e1b621d3a for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/b25e1b621d3a Modified Files: clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/exports.stable.out gdk/gdk.h gdk/gdk_aggr.c gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_bbp.c gdk/gdk_group.c gdk/gdk_heap.c gdk/gdk_join.c gdk/gdk_select.c gdk/gdk_string.c sql/backends/monet5/sql.c sql/backends/monet5/sql.h sql/storage/bat/bat_logger.c sql/storage/bat/bat_storage.c sql/storage/sql_storage.h tools/monetdbe/monetdbe.c Branch: strheapvacuum Log Message:
merge jul2021 diffs (truncated from 2253 to 300 lines): diff --git a/clients/Tests/MAL-signatures.stable.out b/clients/Tests/MAL-signatures.stable.out --- a/clients/Tests/MAL-signatures.stable.out +++ b/clients/Tests/MAL-signatures.stable.out @@ -9209,7 +9209,7 @@ stdout of test 'MAL-signatures` in direc [ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng, X_2:str, X_3:str):void ", "sql_analyze;", "" ] [ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng, X_2:str, X_3:str, X_4:str):void ", "sql_analyze;", "" ] [ "sql", "any", "pattern sql.any(X_0:bit, X_1:bit, X_2:bit):bit ", "SQLany_cmp;", "" ] -[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str, X_3:str, X_4:bat[:oid], X_5:any):int ", "mvc_append_wrap;", "" ] +[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str, X_3:str, X_4:oid, X_5:bat[:oid], X_6:any):int ", "mvc_append_wrap;", "" ] [ "sql", "argRecord", "pattern sql.argRecord():str ", "SQLargRecord;", "" ] [ "sql", "argRecord", "pattern sql.argRecord(X_0:any...):str ", "SQLargRecord;", "" ] [ "sql", "assert", "pattern sql.assert(X_0:bit, X_1:str):void ", "SQLassert;", "" ] @@ -9233,7 +9233,7 @@ stdout of test 'MAL-signatures` in direc [ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str, X_2:str, X_3:str, X_4:int):bat[:any_1] ", "mvc_bind_idxbat_wrap;", "" ] [ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str, X_2:str, X_3:str, X_4:int, X_5:int, X_6:int) (X_7:bat[:oid], X_8:bat[:any_1]) ", "mvc_bind_idxbat_wrap;", "" ] [ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str, X_2:str, X_3:str, X_4:int, X_5:int, X_6:int):bat[:any_1] ", "mvc_bind_idxbat_wrap;", "" ] -[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str, X_2:str, X_3:lng):bat[:oid] ", "mvc_claim_wrap;", "" ] +[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str, X_2:str, X_3:lng) (X_4:oid, X_5:bat[:oid]) ", "mvc_claim_wrap;", "" ] [ "sql", "clear_table", "unsafe pattern sql.clear_table(X_0:str, X_1:str):lng ", "mvc_clear_table_wrap;", "" ] [ "sql", "commit", "unsafe pattern sql.commit():void ", "SQLcommit;", "" ] [ "sql", "copy_from", "unsafe pattern sql.copy_from(X_0:ptr, X_1:str, X_2:str, X_3:str, X_4:str, X_5:str, X_6:lng, X_7:lng, X_8:int, X_9:str, X_10:int, X_11:int):bat[:any]... ", "mvc_import_table_wrap;", "" ] @@ -9686,7 +9686,7 @@ stdout of test 'MAL-signatures` in direc [ "wlr", "alter_set_table", "pattern wlr.alter_set_table(X_0:str, X_1:str, X_2:int):void ", "WLRgeneric;", "" ] [ "wlr", "alter_table", "pattern wlr.alter_table(X_0:str, X_1:str, X_2:int):void ", "WLRgeneric;", "" ] [ "wlr", "alter_user", "pattern wlr.alter_user(X_0:str, X_1:str, X_2:int, X_3:str, X_4:str):void ", "WLRgeneric;", "" ] -[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str, X_3:any...):int ", "WLRappend;", "" ] +[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str, X_3:oid, X_4:bat[:oid], X_5:any...):int ", "WLRappend;", "" ] [ "wlr", "catalog", "pattern wlr.catalog(X_0:str):void ", "WLRcatalog;", "" ] [ "wlr", "clear_table", "pattern wlr.clear_table(X_0:str, X_1:str):int ", "WLRclear_table;", "" ] [ "wlr", "comment_on", "pattern wlr.comment_on(X_0:int, X_1:str):void ", "WLRgeneric;", "" ] diff --git a/clients/Tests/MAL-signatures.stable.out.int128 b/clients/Tests/MAL-signatures.stable.out.int128 --- a/clients/Tests/MAL-signatures.stable.out.int128 +++ b/clients/Tests/MAL-signatures.stable.out.int128 @@ -12510,7 +12510,7 @@ stdout of test 'MAL-signatures` in direc [ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng, X_2:str, X_3:str):void ", "sql_analyze;", "" ] [ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng, X_2:str, X_3:str, X_4:str):void ", "sql_analyze;", "" ] [ "sql", "any", "pattern sql.any(X_0:bit, X_1:bit, X_2:bit):bit ", "SQLany_cmp;", "" ] -[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str, X_3:str, X_4:bat[:oid], X_5:any):int ", "mvc_append_wrap;", "" ] +[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str, X_3:str, X_4:oid, X_5:bat[:oid], X_6:any):int ", "mvc_append_wrap;", "" ] [ "sql", "argRecord", "pattern sql.argRecord():str ", "SQLargRecord;", "" ] [ "sql", "argRecord", "pattern sql.argRecord(X_0:any...):str ", "SQLargRecord;", "" ] [ "sql", "assert", "pattern sql.assert(X_0:bit, X_1:str):void ", "SQLassert;", "" ] @@ -12536,7 +12536,7 @@ stdout of test 'MAL-signatures` in direc [ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str, X_2:str, X_3:str, X_4:int):bat[:any_1] ", "mvc_bind_idxbat_wrap;", "" ] [ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str, X_2:str, X_3:str, X_4:int, X_5:int, X_6:int) (X_7:bat[:oid], X_8:bat[:any_1]) ", "mvc_bind_idxbat_wrap;", "" ] [ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str, X_2:str, X_3:str, X_4:int, X_5:int, X_6:int):bat[:any_1] ", "mvc_bind_idxbat_wrap;", "" ] -[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str, X_2:str, X_3:lng):bat[:oid] ", "mvc_claim_wrap;", "" ] +[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str, X_2:str, X_3:lng) (X_4:oid, X_5:bat[:oid]) ", "mvc_claim_wrap;", "" ] [ "sql", "clear_table", "unsafe pattern sql.clear_table(X_0:str, X_1:str):lng ", "mvc_clear_table_wrap;", "" ] [ "sql", "commit", "unsafe pattern sql.commit():void ", "SQLcommit;", "" ] [ "sql", "copy_from", "unsafe pattern sql.copy_from(X_0:ptr, X_1:str, X_2:str, X_3:str, X_4:str, X_5:str, X_6:lng, X_7:lng, X_8:int, X_9:str, X_10:int, X_11:int):bat[:any]... ", "mvc_import_table_wrap;", "" ] @@ -13009,7 +13009,7 @@ stdout of test 'MAL-signatures` in direc [ "wlr", "alter_set_table", "pattern wlr.alter_set_table(X_0:str, X_1:str, X_2:int):void ", "WLRgeneric;", "" ] [ "wlr", "alter_table", "pattern wlr.alter_table(X_0:str, X_1:str, X_2:int):void ", "WLRgeneric;", "" ] [ "wlr", "alter_user", "pattern wlr.alter_user(X_0:str, X_1:str, X_2:int, X_3:str, X_4:str):void ", "WLRgeneric;", "" ] -[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str, X_3:any...):int ", "WLRappend;", "" ] +[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str, X_3:oid, X_4:bat[:oid], X_5:any...):int ", "WLRappend;", "" ] [ "wlr", "catalog", "pattern wlr.catalog(X_0:str):void ", "WLRcatalog;", "" ] [ "wlr", "clear_table", "pattern wlr.clear_table(X_0:str, X_1:str):int ", "WLRclear_table;", "" ] [ "wlr", "comment_on", "pattern wlr.comment_on(X_0:int, X_1:str):void ", "WLRgeneric;", "" ] diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -181,6 +181,7 @@ BAT *BATproject2(BAT *restrict l, BAT *r BAT *BATprojectchain(BAT **bats); gdk_return BATrangejoin(BAT **r1p, BAT **r2p, BAT *l, BAT *rl, BAT *rh, BAT *sl, BAT *sr, bool li, bool hi, bool anti, bool symmetric, BUN estimate) __attribute__((__warn_unused_result__)); gdk_return BATreplace(BAT *b, BAT *p, BAT *n, bool force) __attribute__((__warn_unused_result__)); +gdk_return BATreplacepos(BAT *b, const oid *positions, BAT *n, bool autoincr, bool force) __attribute__((__warn_unused_result__)); gdk_return BATroles(BAT *b, const char *tnme); BAT *BATsample(BAT *b, BUN n); BAT *BATsample_with_seed(BAT *b, BUN n, uint64_t seed); @@ -204,6 +205,7 @@ void BATundo(BAT *b); BAT *BATunique(BAT *b, BAT *s); BAT *BATunmask(BAT *b); gdk_return BATupdate(BAT *b, BAT *p, BAT *n, bool force) __attribute__((__warn_unused_result__)); +gdk_return BATupdatepos(BAT *b, const oid *positions, BAT *n, bool autoincr, bool force) __attribute__((__warn_unused_result__)); BBPrec *BBP[N_BBPINIT]; gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror); void BBPclear(bat bid); @@ -543,7 +545,8 @@ gdk_return log_bat_transient(logger *lg, gdk_return log_constant(logger *lg, int type, ptr val, log_id id, lng offset, lng cnt); gdk_return log_delta(logger *lg, BAT *uid, BAT *uval, log_id id); gdk_return log_sequence(logger *lg, int seq, lng id); -gdk_return log_tend(logger *lg, ulng commit_ts); +gdk_return log_tdone(logger *lg, ulng commit_ts); +gdk_return log_tend(logger *lg); gdk_return log_tstart(logger *lg, bool flush); gdk_return logger_activate(logger *lg); lng logger_changes(logger *lg); diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -1091,6 +1091,10 @@ gdk_export gdk_return BATreplace(BAT *b, __attribute__((__warn_unused_result__)); gdk_export gdk_return BATupdate(BAT *b, BAT *p, BAT *n, bool force) __attribute__((__warn_unused_result__)); +gdk_export gdk_return BATreplacepos(BAT *b, const oid *positions, BAT *n, bool autoincr, bool force) + __attribute__((__warn_unused_result__)); +gdk_export gdk_return BATupdatepos(BAT *b, const oid *positions, BAT *n, bool autoincr, bool force) + __attribute__((__warn_unused_result__)); /* Functions to perform a binary search on a sorted BAT. * See gdk_search.c for details. */ diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c --- a/gdk/gdk_aggr.c +++ b/gdk/gdk_aggr.c @@ -3739,7 +3739,7 @@ BATmin_skipnil(BAT *b, void *aggr, bit s if (BATcheckorderidx(b) || (/* DISABLES CODE */ (0) && VIEWtparent(b) && - (pb = BBPdescriptor(VIEWtparent(b))) != NULL && + (pb = BBP_cache(VIEWtparent(b))) != NULL && pb->tbaseoff == b->tbaseoff && BATcount(pb) == BATcount(b) && pb->hseqbase == b->hseqbase && @@ -3770,9 +3770,9 @@ BATmin_skipnil(BAT *b, void *aggr, bit s } } else if ((VIEWtparent(b) == 0 || (/* DISABLES CODE */ (0) && - BATcount(b) == BATcount(BBPdescriptor(VIEWtparent(b))))) && + BATcount(b) == BATcount(BBP_cache(VIEWtparent(b))))) && BATcheckimprints(b)) { - Imprints *imprints = VIEWtparent(b) ? BBPdescriptor(VIEWtparent(b))->timprints : b->timprints; + Imprints *imprints = VIEWtparent(b) ? BBP_cache(VIEWtparent(b))->timprints : b->timprints; int i; MT_thread_setalgorithm(VIEWtparent(b) ? "using parent imprints" : "using imprints"); @@ -3853,7 +3853,7 @@ BATmax_skipnil(BAT *b, void *aggr, bit s if (BATcheckorderidx(b) || (/* DISABLES CODE */ (0) && VIEWtparent(b) && - (pb = BBPdescriptor(VIEWtparent(b))) != NULL && + (pb = BBP_cache(VIEWtparent(b))) != NULL && pb->tbaseoff == b->tbaseoff && BATcount(pb) == BATcount(b) && pb->hseqbase == b->hseqbase && @@ -3875,9 +3875,9 @@ BATmax_skipnil(BAT *b, void *aggr, bit s } } else if ((VIEWtparent(b) == 0 || (/* DISABLES CODE */ (0) && - BATcount(b) == BATcount(BBPdescriptor(VIEWtparent(b))))) && + BATcount(b) == BATcount(BBP_cache(VIEWtparent(b))))) && BATcheckimprints(b)) { - Imprints *imprints = VIEWtparent(b) ? BBPdescriptor(VIEWtparent(b))->timprints : b->timprints; + Imprints *imprints = VIEWtparent(b) ? BBP_cache(VIEWtparent(b))->timprints : b->timprints; int i; MT_thread_setalgorithm(VIEWtparent(b) ? "using parent imprints" : "using imprints"); @@ -4170,7 +4170,7 @@ doBATgroupquantile(BAT *b, BAT *g, BAT * if (BATcheckorderidx(b) || (/* DISABLES CODE */ (0) && VIEWtparent(b) && - (pb = BBPdescriptor(VIEWtparent(b))) != NULL && + (pb = BBP_cache(VIEWtparent(b))) != NULL && pb->tbaseoff == b->tbaseoff && BATcount(pb) == BATcount(b) && pb->hseqbase == b->hseqbase && diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c --- a/gdk/gdk_bat.c +++ b/gdk/gdk_bat.c @@ -556,7 +556,7 @@ BATextend(BAT *b, BUN newcap) b->theap->filename, b->theap->size, theap_size); MT_lock_set(&b->theaplock); if (ATOMIC_GET(&b->theap->refs) == 1) { - rc = HEAPextend(b->theap, theap_size, true); + rc = HEAPextend(b->theap, theap_size, b->batRestricted == BAT_READ); } else { MT_lock_unset(&b->theaplock); Heap *h = HEAPgrow(b->theap, theap_size); @@ -1158,7 +1158,8 @@ setcolprops(BAT *b, const void *x) /* Append an array of values of length count to the bat. For * fixed-sized values, `values' is an array of values, for - * variable-sized values, `values' is an array of pointers to values. */ + * variable-sized values, `values' is an array of pointers to values. + * If values equals NULL, count times nil will be appended. */ gdk_return BUNappendmulti(BAT *b, const void *values, BUN count, bool force) { @@ -1182,18 +1183,20 @@ BUNappendmulti(BAT *b, const void *value if (b->ttype == TYPE_void && BATtdense(b)) { const oid *ovals = values; - bool dense = b->batCount == 0 || b->tseqbase + 1 == ovals[0]; - for (BUN i = 1; dense && i < count; i++) { - dense = ovals[i - 1] + 1 == ovals[i]; + bool dense = b->batCount == 0 || (ovals != NULL && b->tseqbase + 1 == ovals[0]); + if (ovals) { + for (BUN i = 1; dense && i < count; i++) { + dense = ovals[i - 1] + 1 == ovals[i]; + } } if (dense) { if (b->batCount == 0) - b->tseqbase = ovals[0]; + b->tseqbase = ovals ? ovals[0] : oid_nil; BATsetcount(b, BATcount(b) + count); return GDK_SUCCEED; } else { /* we need to materialize b; allocate enough capacity */ - b->batCapacity = BATcount(b) + 1; + b->batCapacity = BATcount(b) + count; if (BATmaterialize(b) != GDK_SUCCEED) return GDK_FAIL; } @@ -1219,9 +1222,12 @@ BUNappendmulti(BAT *b, const void *value BATrmprop(b, GDK_UNIQUE_ESTIMATE); b->theap->dirty |= count > 0; MT_rwlock_wrlock(&b->thashlock); + const void *t = b->ttype == TYPE_msk ? &(msk){false} : ATOMnilptr(b->ttype); for (BUN i = 0; i < count; i++) { - void *t = b->ttype && b->tvarsized ? ((void **) values)[i] : - (void *) ((char *) values + i * Tsize(b)); + if (values) { + t = b->ttype && b->tvarsized ? ((void **) values)[i] : + (void *) ((char *) values + i * Tsize(b)); + } setcolprops(b, t); gdk_return rc = bunfastapp_nocheck(b, p, t, Tsize(b)); if (rc != GDK_SUCCEED) { diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -56,7 +56,7 @@ unshare_varsized_heap(BAT *b) * of inserting individual strings. See the comments in the code for * more information. */ static gdk_return -insert_string_bat(BAT *b, BAT *n, struct canditer *ci, bool mayshare) +insert_string_bat(BAT *b, BAT *n, struct canditer *ci, bool force, bool mayshare) { BATiter ni; /* iterator */ size_t toff = ~(size_t) 0; /* tail offset */ @@ -146,14 +146,24 @@ insert_string_bat(BAT *b, BAT *n, struct (var_t) 1 << 17; MT_thread_setalgorithm("copy vheap, copy heap"); if (b->tvheap->size < ni.vh->free) { - Heap *h = HEAPgrow(b->tvheap, ni.vh->free); - if (h == NULL) { - bat_iterator_end(&ni); - return GDK_FAIL; + MT_lock_set(&b->theaplock); + if (ATOMIC_GET(&b->tvheap->refs) == 1) { + if (HEAPextend(b->tvheap, ni.vh->free, force) != GDK_SUCCEED) { + MT_lock_unset(&b->theaplock); + bat_iterator_end(&ni); + return GDK_FAIL; + } + } else { + MT_lock_unset(&b->theaplock); + Heap *h = HEAPgrow(b->tvheap, ni.vh->free); + if (h == NULL) { + bat_iterator_end(&ni); + return GDK_FAIL; + } + MT_lock_set(&b->theaplock); + HEAPdecref(b->tvheap, false); + b->tvheap = h; } - MT_lock_set(&b->theaplock); - HEAPdecref(b->tvheap, false); - b->tvheap = h; MT_lock_unset(&b->theaplock); } memcpy(b->tvheap->base, ni.vh->base, ni.vh->free); @@ -197,14 +207,24 @@ insert_string_bat(BAT *b, BAT *n, struct toff = (toff + GDK_VARALIGN - 1) & ~(GDK_VARALIGN - 1); /* if in "force" mode, the heap may be * shared when memory mapped */ - Heap *h = HEAPgrow(b->tvheap, toff + ni.vh->size); - if (h == NULL) { - bat_iterator_end(&ni); - return GDK_FAIL; + MT_lock_set(&b->theaplock); + if (ATOMIC_GET(&b->tvheap->refs) == 1) { + if (HEAPextend(b->tvheap, toff + ni.vh->size, force) != GDK_SUCCEED) { + MT_lock_unset(&b->theaplock); + bat_iterator_end(&ni); + return GDK_FAIL; + } + } else { + MT_lock_unset(&b->theaplock); + Heap *h = HEAPgrow(b->tvheap, toff + ni.vh->size); + if (h == NULL) { + bat_iterator_end(&ni); + return GDK_FAIL; + } + MT_lock_set(&b->theaplock); + HEAPdecref(b->tvheap, false); + b->tvheap = h; } - MT_lock_set(&b->theaplock); - HEAPdecref(b->tvheap, false); - b->tvheap = h; MT_lock_unset(&b->theaplock); MT_thread_setalgorithm("append vheap"); memcpy(b->tvheap->base + toff, ni.vh->base, ni.vh->free); @@ -891,7 +911,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool b->tnil |= n->tnil && cnt == ni.count; } if (b->ttype == TYPE_str) { _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list