Changeset: bf3a6cbfe1fe for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/bf3a6cbfe1fe Modified Files: gdk/gdk.h gdk/gdk_align.c gdk/gdk_batop.c gdk/gdk_bbp.c gdk/gdk_project.c gdk/gdk_utils.c Branch: default Log Message:
Merge with Jan2022 branch. diffs (truncated from 675 to 300 lines): diff --git a/gdk/ChangeLog.Jan2022 b/gdk/ChangeLog.Jan2022 --- a/gdk/ChangeLog.Jan2022 +++ b/gdk/ChangeLog.Jan2022 @@ -1,3 +1,7 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Tue May 10 2022 Sjoerd Mullender <sjo...@acm.org> +- When exiting, long running instructions are aborted using the same + mechanism that is used for query timeouts. + diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -2321,24 +2321,23 @@ gdk_export BAT *BATsample_with_seed(BAT #define GDK_CHECK_TIMEOUT_BODY(timeoffset, callback) \ do { \ - if (timeoffset && GDKusec() > timeoffset) { \ + if (GDKexiting() || \ + (timeoffset && GDKusec() > timeoffset)) { \ callback; \ } \ } while (0) #define GDK_CHECK_TIMEOUT(timeoffset, counter, callback) \ do { \ - if (timeoffset) { \ - if (counter > CHECK_QRY_TIMEOUT_STEP) { \ - GDK_CHECK_TIMEOUT_BODY(timeoffset, callback); \ - counter = 0; \ - } else { \ - counter++; \ - } \ + if (counter > CHECK_QRY_TIMEOUT_STEP) { \ + GDK_CHECK_TIMEOUT_BODY(timeoffset, callback); \ + counter = 0; \ + } else { \ + counter++; \ } \ } while (0) -/* here are some useful construct to iterate a number of times (the +/* here are some useful constructs to iterate a number of times (the * REPEATS argument--only evaluated once) and checking for a timeout * every once in a while; the TIMEOFFSET value is a variable of type lng * which is either 0 or the GDKusec() compatible time after which the @@ -2351,19 +2350,19 @@ gdk_export BAT *BATsample_with_seed(BAT * on each iteration */ #define TIMEOUT_LOOP_IDX(IDX, REPEATS, TIMEOFFSET) \ for (BUN REPS = (IDX = 0, (REPEATS)); REPS > 0; REPS = 0) /* "loops" at most once */ \ - for (BUN CTR1 = 0, END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET ? -1 : TIMEOFFSET) \ + for (BUN CTR1 = 0, END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET) ? -1 : TIMEOFFSET) \ for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++, IDX++) /* declare and use IDX as a loop variable, initializing it to 0 and * incrementing it on each iteration */ #define TIMEOUT_LOOP_IDX_DECL(IDX, REPEATS, TIMEOFFSET) \ for (BUN IDX = 0, REPS = (REPEATS); REPS > 0; REPS = 0) /* "loops" at most once */ \ - for (BUN CTR1 = 0, END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET ? -1 : TIMEOFFSET) \ + for (BUN CTR1 = 0, END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET) ? -1 : TIMEOFFSET) \ for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++, IDX++) /* there is no user-visible loop variable */ #define TIMEOUT_LOOP(REPEATS, TIMEOFFSET) \ - for (BUN CTR1 = 0, REPS = (REPEATS), END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET ? -1 : TIMEOFFSET) \ + for (BUN CTR1 = 0, REPS = (REPEATS), END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET) ? -1 : TIMEOFFSET) \ for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++) /* break out of the loop (cannot use do/while trick here) */ diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c --- a/gdk/gdk_align.c +++ b/gdk/gdk_align.c @@ -392,12 +392,18 @@ VIEWdestroy(BAT *b) VIEWunlink(b); MT_lock_set(&b->theaplock); + /* heaps that are left after VIEWunlink are ours, so need to be + * destroyed (and files deleted) */ if (b->theap) { - HEAPdecref(b->theap, false); + HEAPdecref(b->theap, true); b->theap = NULL; } if (b->tvheap) { - HEAPdecref(b->tvheap, false); + /* should never happen: if this heap exists, then it was + * our own (not a view), and then it doesn't make sense + * that the offset heap was a view (at least one of them + * had to be) */ + HEAPdecref(b->tvheap, true); b->tvheap = NULL; } MT_lock_unset(&b->theaplock); diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -58,7 +58,7 @@ unshare_varsized_heap(BAT *b) * of inserting individual strings. See the comments in the code for * more information. */ static gdk_return -insert_string_bat(BAT *b, BATiter *ni, struct canditer *ci, bool force, bool mayshare) +insert_string_bat(BAT *b, BATiter *ni, struct canditer *ci, bool force, bool mayshare, lng timeoffset) { size_t toff = ~(size_t) 0; /* tail offset */ BUN p, r; /* loop variables */ @@ -186,8 +186,7 @@ insert_string_bat(BAT *b, BATiter *ni, s MT_thread_setalgorithm("copy offset values"); r = b->batCount; - while (cnt > 0) { - cnt--; + TIMEOUT_LOOP(cnt, timeoffset) { p = canditer_next(ci) - ni->b->hseqbase; switch (ni->width) { case 1: @@ -244,8 +243,7 @@ insert_string_bat(BAT *b, BATiter *ni, s r = b->batCount; oid hseq = ni->b->hseqbase; MT_thread_setalgorithm("insert string values"); - while (cnt > 0) { - cnt--; + TIMEOUT_LOOP(cnt, timeoffset) { p = canditer_next(ci) - hseq; tp = BUNtvar(*ni, p); if (tfastins_nocheckVAR(b, r, tp) != GDK_SUCCEED) { @@ -262,8 +260,7 @@ insert_string_bat(BAT *b, BATiter *ni, s * offset, otherwise we insert normally. */ r = b->batCount; MT_thread_setalgorithm("insert string values with check"); - while (cnt > 0) { - cnt--; + TIMEOUT_LOOP(cnt, timeoffset) { p = canditer_next(ci) - ni->b->hseqbase; off = BUNtvaroff(*ni, p); /* the offset */ tp = ni->vh->base + off; /* the string */ @@ -306,6 +303,7 @@ insert_string_bat(BAT *b, BATiter *ni, s r++; } } + TIMEOUT_CHECK(timeoffset, TIMEOUT_HANDLER(GDK_FAIL)); MT_lock_set(&b->theaplock); BATsetcount(b, oldcnt + ci->ncand); assert(b->batCapacity >= b->batCount); @@ -670,6 +668,12 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool ATOMname(BATttype(n)), ATOMname(BATttype(b))); } + lng timeoffset = 0; + QryCtx *qry_ctx = MT_thread_get_qry_ctx(); + if (qry_ctx != NULL) { + timeoffset = (qry_ctx->starttime && qry_ctx->querytimeout) ? (qry_ctx->starttime + qry_ctx->querytimeout) : 0; + } + BATiter ni = bat_iterator(n); canditer_init(&ci, n, s); @@ -820,7 +824,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool } MT_lock_unset(&b->theaplock); if (b->ttype == TYPE_str) { - if (insert_string_bat(b, &ni, &ci, force, mayshare) != GDK_SUCCEED) { + if (insert_string_bat(b, &ni, &ci, force, mayshare, timeoffset) != GDK_SUCCEED) { bat_iterator_end(&ni); return GDK_FAIL; } @@ -862,7 +866,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool r++; } } else { - CAND_LOOP(&ci) { + TIMEOUT_LOOP(ci.ncand, timeoffset) { BUN p = canditer_next(&ci) - hseq; const void *t = BUNtail(ni, p); if (tfastins_nocheck(b, r, t) != GDK_SUCCEED) { @@ -874,6 +878,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool HASHappend_locked(b, r, t); r++; } + TIMEOUT_CHECK(timeoffset, GOTO_LABEL_TIMEOUT_HANDLER(bailout)); } MT_rwlock_wrunlock(&b->thashlock); MT_lock_set(&b->theaplock); @@ -889,6 +894,10 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool GDKusec() - t0); return GDK_SUCCEED; + bailout: + MT_rwlock_wrunlock(&b->thashlock); + bat_iterator_end(&ni); + return GDK_FAIL; } gdk_return diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -1397,6 +1397,7 @@ BBPtrim(bool aggressive) (b = BBP_cache(bid)) != NULL) { MT_lock_set(&b->theaplock); if (b->batSharecnt == 0 && + !isVIEW(b) && (!BATdirty(b) || (aggressive && b->theap->storage == STORE_MMAP && (b->tvheap == NULL || b->tvheap->storage == STORE_MMAP))) && !(BBP_status(bid) & flag) /*&& (BBP_status(bid) & BBPPERSISTENT || @@ -1777,28 +1778,24 @@ BBPexit(void) skipped = true; continue; } - if (isVIEW(b)) { - /* "manually" - * decrement parent - * references, since - * VIEWdestroy doesn't - * (and can't here due - * to locks) do it */ - bat tp = VIEWtparent(b); - bat vtp = VIEWvtparent(b); - if (tp) { - BBP_desc(tp)->batSharecnt--; - --BBP_lrefs(tp); - } - if (vtp) { - BBP_desc(vtp)->batSharecnt--; - --BBP_lrefs(vtp); - } - VIEWdestroy(b); - } else { - PROPdestroy(b); - BATfree(b); + MT_lock_set(&b->theaplock); + bat tp = VIEWtparent(b); + if (tp != 0) { + BBP_desc(tp)->batSharecnt--; + --BBP_lrefs(tp); + HEAPdecref(b->theap, false); + b->theap = NULL; } + tp = VIEWvtparent(b); + if (tp != 0) { + BBP_desc(tp)->batSharecnt--; + --BBP_lrefs(tp); + HEAPdecref(b->tvheap, false); + b->tvheap = NULL; + } + MT_lock_unset(&b->theaplock); + PROPdestroy(b); + BATfree(b); } BBP_pid(i) = 0; BBPuncacheit(i, true); @@ -3150,7 +3147,7 @@ BBPreclaim(BAT *b) assert(BBP_refs(i) == 1); - return decref(i, false, false, true, lock, __func__) <0; + return decref(i, false, false, true, lock, __func__) < 0; } /* @@ -3316,20 +3313,16 @@ BBPfree(BAT *b) assert(bid > 0); assert(BBPswappable(b)); + assert(!isVIEW(b)); BBP_unload_inc(); - /* write dirty BATs before being unloaded */ + /* write dirty BATs before unloading */ ret = BBPsave(b); if (ret == GDK_SUCCEED) { - if (isVIEW(b)) { /* physical view */ - VIEWdestroy(b); - } else { - if (BBP_cache(bid)) - BATfree(b); /* free memory */ - } + if (BBP_cache(bid)) + BATfree(b); /* free memory */ BBPuncacheit(bid, false); } - /* clearing bits can be done without the lock */ TRC_DEBUG(BAT_, "turn off unloading %d\n", bid); BBP_status_off(bid, BBPUNLOADING); BBP_unload_dec(); diff --git a/gdk/gdk_project.c b/gdk/gdk_project.c --- a/gdk/gdk_project.c +++ b/gdk/gdk_project.c @@ -22,9 +22,10 @@ #define project1_loop(TYPE) \ static gdk_return \ -project1_##TYPE(BAT *restrict bn, BATiter *restrict li, BATiter *restrict r1i) \ +project1_##TYPE(BAT *restrict bn, BATiter *restrict li, \ + BATiter *restrict r1i, lng timeoffset) \ { \ - BUN lo, hi; \ + BUN lo; \ const TYPE *restrict r1t; \ TYPE *restrict bt; \ _______________________________________________ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org