Changeset: b25e1b621d3a for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b25e1b621d3a
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_group.c
gdk/gdk_heap.c
gdk/gdk_join.c
gdk/gdk_select.c
gdk/gdk_string.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
sql/storage/bat/bat_logger.c
sql/storage/bat/bat_storage.c
sql/storage/sql_storage.h
tools/monetdbe/monetdbe.c
Branch: strheapvacuum
Log Message:
merge jul2021
diffs (truncated from 2253 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -9209,7 +9209,7 @@ stdout of test 'MAL-signatures` in direc
[ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng,
X_2:str, X_3:str):void ", "sql_analyze;", "" ]
[ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng,
X_2:str, X_3:str, X_4:str):void ", "sql_analyze;", "" ]
[ "sql", "any", "pattern sql.any(X_0:bit, X_1:bit, X_2:bit):bit ",
"SQLany_cmp;", "" ]
-[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str,
X_3:str, X_4:bat[:oid], X_5:any):int ", "mvc_append_wrap;", "" ]
+[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str,
X_3:str, X_4:oid, X_5:bat[:oid], X_6:any):int ", "mvc_append_wrap;", ""
]
[ "sql", "argRecord", "pattern sql.argRecord():str ",
"SQLargRecord;", "" ]
[ "sql", "argRecord", "pattern sql.argRecord(X_0:any...):str ",
"SQLargRecord;", "" ]
[ "sql", "assert", "pattern sql.assert(X_0:bit, X_1:str):void ",
"SQLassert;", "" ]
@@ -9233,7 +9233,7 @@ stdout of test 'MAL-signatures` in direc
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int):bat[:any_1] ", "mvc_bind_idxbat_wrap;", ""
]
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int, X_5:int, X_6:int) (X_7:bat[:oid], X_8:bat[:any_1])
", "mvc_bind_idxbat_wrap;", "" ]
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int, X_5:int, X_6:int):bat[:any_1] ",
"mvc_bind_idxbat_wrap;", "" ]
-[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str,
X_2:str, X_3:lng):bat[:oid] ", "mvc_claim_wrap;", "" ]
+[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str,
X_2:str, X_3:lng) (X_4:oid, X_5:bat[:oid]) ", "mvc_claim_wrap;", ""
]
[ "sql", "clear_table", "unsafe pattern sql.clear_table(X_0:str,
X_1:str):lng ", "mvc_clear_table_wrap;", "" ]
[ "sql", "commit", "unsafe pattern sql.commit():void ",
"SQLcommit;", "" ]
[ "sql", "copy_from", "unsafe pattern sql.copy_from(X_0:ptr, X_1:str,
X_2:str, X_3:str, X_4:str, X_5:str, X_6:lng, X_7:lng, X_8:int, X_9:str,
X_10:int, X_11:int):bat[:any]... ", "mvc_import_table_wrap;", ""
]
@@ -9686,7 +9686,7 @@ stdout of test 'MAL-signatures` in direc
[ "wlr", "alter_set_table", "pattern wlr.alter_set_table(X_0:str,
X_1:str, X_2:int):void ", "WLRgeneric;", "" ]
[ "wlr", "alter_table", "pattern wlr.alter_table(X_0:str, X_1:str,
X_2:int):void ", "WLRgeneric;", "" ]
[ "wlr", "alter_user", "pattern wlr.alter_user(X_0:str, X_1:str,
X_2:int, X_3:str, X_4:str):void ", "WLRgeneric;", "" ]
-[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str,
X_3:any...):int ", "WLRappend;", "" ]
+[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str,
X_3:oid, X_4:bat[:oid], X_5:any...):int ", "WLRappend;", "" ]
[ "wlr", "catalog", "pattern wlr.catalog(X_0:str):void ",
"WLRcatalog;", "" ]
[ "wlr", "clear_table", "pattern wlr.clear_table(X_0:str, X_1:str):int
", "WLRclear_table;", "" ]
[ "wlr", "comment_on", "pattern wlr.comment_on(X_0:int, X_1:str):void
", "WLRgeneric;", "" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -12510,7 +12510,7 @@ stdout of test 'MAL-signatures` in direc
[ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng,
X_2:str, X_3:str):void ", "sql_analyze;", "" ]
[ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng,
X_2:str, X_3:str, X_4:str):void ", "sql_analyze;", "" ]
[ "sql", "any", "pattern sql.any(X_0:bit, X_1:bit, X_2:bit):bit ",
"SQLany_cmp;", "" ]
-[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str,
X_3:str, X_4:bat[:oid], X_5:any):int ", "mvc_append_wrap;", "" ]
+[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str,
X_3:str, X_4:oid, X_5:bat[:oid], X_6:any):int ", "mvc_append_wrap;", ""
]
[ "sql", "argRecord", "pattern sql.argRecord():str ",
"SQLargRecord;", "" ]
[ "sql", "argRecord", "pattern sql.argRecord(X_0:any...):str ",
"SQLargRecord;", "" ]
[ "sql", "assert", "pattern sql.assert(X_0:bit, X_1:str):void ",
"SQLassert;", "" ]
@@ -12536,7 +12536,7 @@ stdout of test 'MAL-signatures` in direc
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int):bat[:any_1] ", "mvc_bind_idxbat_wrap;", ""
]
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int, X_5:int, X_6:int) (X_7:bat[:oid], X_8:bat[:any_1])
", "mvc_bind_idxbat_wrap;", "" ]
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int, X_5:int, X_6:int):bat[:any_1] ",
"mvc_bind_idxbat_wrap;", "" ]
-[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str,
X_2:str, X_3:lng):bat[:oid] ", "mvc_claim_wrap;", "" ]
+[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str,
X_2:str, X_3:lng) (X_4:oid, X_5:bat[:oid]) ", "mvc_claim_wrap;", ""
]
[ "sql", "clear_table", "unsafe pattern sql.clear_table(X_0:str,
X_1:str):lng ", "mvc_clear_table_wrap;", "" ]
[ "sql", "commit", "unsafe pattern sql.commit():void ",
"SQLcommit;", "" ]
[ "sql", "copy_from", "unsafe pattern sql.copy_from(X_0:ptr, X_1:str,
X_2:str, X_3:str, X_4:str, X_5:str, X_6:lng, X_7:lng, X_8:int, X_9:str,
X_10:int, X_11:int):bat[:any]... ", "mvc_import_table_wrap;", ""
]
@@ -13009,7 +13009,7 @@ stdout of test 'MAL-signatures` in direc
[ "wlr", "alter_set_table", "pattern wlr.alter_set_table(X_0:str,
X_1:str, X_2:int):void ", "WLRgeneric;", "" ]
[ "wlr", "alter_table", "pattern wlr.alter_table(X_0:str, X_1:str,
X_2:int):void ", "WLRgeneric;", "" ]
[ "wlr", "alter_user", "pattern wlr.alter_user(X_0:str, X_1:str,
X_2:int, X_3:str, X_4:str):void ", "WLRgeneric;", "" ]
-[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str,
X_3:any...):int ", "WLRappend;", "" ]
+[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str,
X_3:oid, X_4:bat[:oid], X_5:any...):int ", "WLRappend;", "" ]
[ "wlr", "catalog", "pattern wlr.catalog(X_0:str):void ",
"WLRcatalog;", "" ]
[ "wlr", "clear_table", "pattern wlr.clear_table(X_0:str, X_1:str):int
", "WLRclear_table;", "" ]
[ "wlr", "comment_on", "pattern wlr.comment_on(X_0:int, X_1:str):void
", "WLRgeneric;", "" ]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -181,6 +181,7 @@ BAT *BATproject2(BAT *restrict l, BAT *r
BAT *BATprojectchain(BAT **bats);
gdk_return BATrangejoin(BAT **r1p, BAT **r2p, BAT *l, BAT *rl, BAT *rh, BAT
*sl, BAT *sr, bool li, bool hi, bool anti, bool symmetric, BUN estimate)
__attribute__((__warn_unused_result__));
gdk_return BATreplace(BAT *b, BAT *p, BAT *n, bool force)
__attribute__((__warn_unused_result__));
+gdk_return BATreplacepos(BAT *b, const oid *positions, BAT *n, bool autoincr,
bool force) __attribute__((__warn_unused_result__));
gdk_return BATroles(BAT *b, const char *tnme);
BAT *BATsample(BAT *b, BUN n);
BAT *BATsample_with_seed(BAT *b, BUN n, uint64_t seed);
@@ -204,6 +205,7 @@ void BATundo(BAT *b);
BAT *BATunique(BAT *b, BAT *s);
BAT *BATunmask(BAT *b);
gdk_return BATupdate(BAT *b, BAT *p, BAT *n, bool force)
__attribute__((__warn_unused_result__));
+gdk_return BATupdatepos(BAT *b, const oid *positions, BAT *n, bool autoincr,
bool force) __attribute__((__warn_unused_result__));
BBPrec *BBP[N_BBPINIT];
gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror);
void BBPclear(bat bid);
@@ -543,7 +545,8 @@ gdk_return log_bat_transient(logger *lg,
gdk_return log_constant(logger *lg, int type, ptr val, log_id id, lng offset,
lng cnt);
gdk_return log_delta(logger *lg, BAT *uid, BAT *uval, log_id id);
gdk_return log_sequence(logger *lg, int seq, lng id);
-gdk_return log_tend(logger *lg, ulng commit_ts);
+gdk_return log_tdone(logger *lg, ulng commit_ts);
+gdk_return log_tend(logger *lg);
gdk_return log_tstart(logger *lg, bool flush);
gdk_return logger_activate(logger *lg);
lng logger_changes(logger *lg);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1091,6 +1091,10 @@ gdk_export gdk_return BATreplace(BAT *b,
__attribute__((__warn_unused_result__));
gdk_export gdk_return BATupdate(BAT *b, BAT *p, BAT *n, bool force)
__attribute__((__warn_unused_result__));
+gdk_export gdk_return BATreplacepos(BAT *b, const oid *positions, BAT *n, bool
autoincr, bool force)
+ __attribute__((__warn_unused_result__));
+gdk_export gdk_return BATupdatepos(BAT *b, const oid *positions, BAT *n, bool
autoincr, bool force)
+ __attribute__((__warn_unused_result__));
/* Functions to perform a binary search on a sorted BAT.
* See gdk_search.c for details. */
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -3739,7 +3739,7 @@ BATmin_skipnil(BAT *b, void *aggr, bit s
if (BATcheckorderidx(b) ||
(/* DISABLES CODE */ (0) &&
VIEWtparent(b) &&
- (pb = BBPdescriptor(VIEWtparent(b))) != NULL &&
+ (pb = BBP_cache(VIEWtparent(b))) != NULL &&
pb->tbaseoff == b->tbaseoff &&
BATcount(pb) == BATcount(b) &&
pb->hseqbase == b->hseqbase &&
@@ -3770,9 +3770,9 @@ BATmin_skipnil(BAT *b, void *aggr, bit s
}
} else if ((VIEWtparent(b) == 0 ||
(/* DISABLES CODE */ (0) &&
- BATcount(b) ==
BATcount(BBPdescriptor(VIEWtparent(b))))) &&
+ BATcount(b) ==
BATcount(BBP_cache(VIEWtparent(b))))) &&
BATcheckimprints(b)) {
- Imprints *imprints = VIEWtparent(b) ?
BBPdescriptor(VIEWtparent(b))->timprints : b->timprints;
+ Imprints *imprints = VIEWtparent(b) ?
BBP_cache(VIEWtparent(b))->timprints : b->timprints;
int i;
MT_thread_setalgorithm(VIEWtparent(b) ? "using parent
imprints" : "using imprints");
@@ -3853,7 +3853,7 @@ BATmax_skipnil(BAT *b, void *aggr, bit s
if (BATcheckorderidx(b) ||
(/* DISABLES CODE */ (0) &&
VIEWtparent(b) &&
- (pb = BBPdescriptor(VIEWtparent(b))) != NULL &&
+ (pb = BBP_cache(VIEWtparent(b))) != NULL &&
pb->tbaseoff == b->tbaseoff &&
BATcount(pb) == BATcount(b) &&
pb->hseqbase == b->hseqbase &&
@@ -3875,9 +3875,9 @@ BATmax_skipnil(BAT *b, void *aggr, bit s
}
} else if ((VIEWtparent(b) == 0 ||
(/* DISABLES CODE */ (0) &&
- BATcount(b) ==
BATcount(BBPdescriptor(VIEWtparent(b))))) &&
+ BATcount(b) ==
BATcount(BBP_cache(VIEWtparent(b))))) &&
BATcheckimprints(b)) {
- Imprints *imprints = VIEWtparent(b) ?
BBPdescriptor(VIEWtparent(b))->timprints : b->timprints;
+ Imprints *imprints = VIEWtparent(b) ?
BBP_cache(VIEWtparent(b))->timprints : b->timprints;
int i;
MT_thread_setalgorithm(VIEWtparent(b) ? "using parent
imprints" : "using imprints");
@@ -4170,7 +4170,7 @@ doBATgroupquantile(BAT *b, BAT *g, BAT *
if (BATcheckorderidx(b) ||
(/* DISABLES CODE */ (0) &&
VIEWtparent(b) &&
- (pb = BBPdescriptor(VIEWtparent(b))) != NULL &&
+ (pb = BBP_cache(VIEWtparent(b))) != NULL &&
pb->tbaseoff == b->tbaseoff &&
BATcount(pb) == BATcount(b) &&
pb->hseqbase == b->hseqbase &&
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -556,7 +556,7 @@ BATextend(BAT *b, BUN newcap)
b->theap->filename, b->theap->size, theap_size);
MT_lock_set(&b->theaplock);
if (ATOMIC_GET(&b->theap->refs) == 1) {
- rc = HEAPextend(b->theap, theap_size, true);
+ rc = HEAPextend(b->theap, theap_size, b->batRestricted
== BAT_READ);
} else {
MT_lock_unset(&b->theaplock);
Heap *h = HEAPgrow(b->theap, theap_size);
@@ -1158,7 +1158,8 @@ setcolprops(BAT *b, const void *x)
/* Append an array of values of length count to the bat. For
* fixed-sized values, `values' is an array of values, for
- * variable-sized values, `values' is an array of pointers to values. */
+ * variable-sized values, `values' is an array of pointers to values.
+ * If values equals NULL, count times nil will be appended. */
gdk_return
BUNappendmulti(BAT *b, const void *values, BUN count, bool force)
{
@@ -1182,18 +1183,20 @@ BUNappendmulti(BAT *b, const void *value
if (b->ttype == TYPE_void && BATtdense(b)) {
const oid *ovals = values;
- bool dense = b->batCount == 0 || b->tseqbase + 1 == ovals[0];
- for (BUN i = 1; dense && i < count; i++) {
- dense = ovals[i - 1] + 1 == ovals[i];
+ bool dense = b->batCount == 0 || (ovals != NULL && b->tseqbase
+ 1 == ovals[0]);
+ if (ovals) {
+ for (BUN i = 1; dense && i < count; i++) {
+ dense = ovals[i - 1] + 1 == ovals[i];
+ }
}
if (dense) {
if (b->batCount == 0)
- b->tseqbase = ovals[0];
+ b->tseqbase = ovals ? ovals[0] : oid_nil;
BATsetcount(b, BATcount(b) + count);
return GDK_SUCCEED;
} else {
/* we need to materialize b; allocate enough capacity */
- b->batCapacity = BATcount(b) + 1;
+ b->batCapacity = BATcount(b) + count;
if (BATmaterialize(b) != GDK_SUCCEED)
return GDK_FAIL;
}
@@ -1219,9 +1222,12 @@ BUNappendmulti(BAT *b, const void *value
BATrmprop(b, GDK_UNIQUE_ESTIMATE);
b->theap->dirty |= count > 0;
MT_rwlock_wrlock(&b->thashlock);
+ const void *t = b->ttype == TYPE_msk ? &(msk){false} :
ATOMnilptr(b->ttype);
for (BUN i = 0; i < count; i++) {
- void *t = b->ttype && b->tvarsized ? ((void **) values)[i] :
- (void *) ((char *) values + i * Tsize(b));
+ if (values) {
+ t = b->ttype && b->tvarsized ? ((void **) values)[i] :
+ (void *) ((char *) values + i * Tsize(b));
+ }
setcolprops(b, t);
gdk_return rc = bunfastapp_nocheck(b, p, t, Tsize(b));
if (rc != GDK_SUCCEED) {
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -56,7 +56,7 @@ unshare_varsized_heap(BAT *b)
* of inserting individual strings. See the comments in the code for
* more information. */
static gdk_return
-insert_string_bat(BAT *b, BAT *n, struct canditer *ci, bool mayshare)
+insert_string_bat(BAT *b, BAT *n, struct canditer *ci, bool force, bool
mayshare)
{
BATiter ni; /* iterator */
size_t toff = ~(size_t) 0; /* tail offset */
@@ -146,14 +146,24 @@ insert_string_bat(BAT *b, BAT *n, struct
(var_t) 1 << 17;
MT_thread_setalgorithm("copy vheap, copy heap");
if (b->tvheap->size < ni.vh->free) {
- Heap *h = HEAPgrow(b->tvheap, ni.vh->free);
- if (h == NULL) {
- bat_iterator_end(&ni);
- return GDK_FAIL;
+ MT_lock_set(&b->theaplock);
+ if (ATOMIC_GET(&b->tvheap->refs) == 1) {
+ if (HEAPextend(b->tvheap, ni.vh->free,
force) != GDK_SUCCEED) {
+ MT_lock_unset(&b->theaplock);
+ bat_iterator_end(&ni);
+ return GDK_FAIL;
+ }
+ } else {
+ MT_lock_unset(&b->theaplock);
+ Heap *h = HEAPgrow(b->tvheap,
ni.vh->free);
+ if (h == NULL) {
+ bat_iterator_end(&ni);
+ return GDK_FAIL;
+ }
+ MT_lock_set(&b->theaplock);
+ HEAPdecref(b->tvheap, false);
+ b->tvheap = h;
}
- MT_lock_set(&b->theaplock);
- HEAPdecref(b->tvheap, false);
- b->tvheap = h;
MT_lock_unset(&b->theaplock);
}
memcpy(b->tvheap->base, ni.vh->base, ni.vh->free);
@@ -197,14 +207,24 @@ insert_string_bat(BAT *b, BAT *n, struct
toff = (toff + GDK_VARALIGN - 1) &
~(GDK_VARALIGN - 1);
/* if in "force" mode, the heap may be
* shared when memory mapped */
- Heap *h = HEAPgrow(b->tvheap, toff +
ni.vh->size);
- if (h == NULL) {
- bat_iterator_end(&ni);
- return GDK_FAIL;
+ MT_lock_set(&b->theaplock);
+ if (ATOMIC_GET(&b->tvheap->refs) == 1) {
+ if (HEAPextend(b->tvheap, toff +
ni.vh->size, force) != GDK_SUCCEED) {
+ MT_lock_unset(&b->theaplock);
+ bat_iterator_end(&ni);
+ return GDK_FAIL;
+ }
+ } else {
+ MT_lock_unset(&b->theaplock);
+ Heap *h = HEAPgrow(b->tvheap, toff +
ni.vh->size);
+ if (h == NULL) {
+ bat_iterator_end(&ni);
+ return GDK_FAIL;
+ }
+ MT_lock_set(&b->theaplock);
+ HEAPdecref(b->tvheap, false);
+ b->tvheap = h;
}
- MT_lock_set(&b->theaplock);
- HEAPdecref(b->tvheap, false);
- b->tvheap = h;
MT_lock_unset(&b->theaplock);
MT_thread_setalgorithm("append vheap");
memcpy(b->tvheap->base + toff, ni.vh->base,
ni.vh->free);
@@ -891,7 +911,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool
b->tnil |= n->tnil && cnt == ni.count;
}
if (b->ttype == TYPE_str) {
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
