Changeset: 48ff5d61c03d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=48ff5d61c03d
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk_logger.c
gdk/gdk_logger.h
gdk/gdk_logger_internals.h
sql/common/sql_list.c
sql/server/rel_optimizer.c
sql/storage/bat/bat_logger.c
sql/storage/bat/bat_storage.c
sql/storage/sql_storage.h
sql/storage/store.c
Branch: Oct2020
Log Message:
merged with jun2020 (small changes to keep ABI change smaller)
diffs (truncated from 1041 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -514,6 +514,7 @@ gdk_return log_bat_clear(logger *lg, con
gdk_return log_bat_persists(logger *lg, BAT *b, const char *n, char tpe, oid
id);
gdk_return log_bat_transient(logger *lg, const char *n, char tpe, oid id);
gdk_return log_delta(logger *lg, BAT *uid, BAT *uval, const char *n, char tpe,
oid id);
+lng log_save_id(logger *lg);
gdk_return log_sequence(logger *lg, int seq, lng id);
gdk_return log_tend(logger *lg);
gdk_return log_tstart(logger *lg);
@@ -523,9 +524,8 @@ gdk_return logger_cleanup(logger *lg);
logger *logger_create(int debug, const char *fn, const char *logdir, int
version, preversionfix_fptr prefuncp, postversionfix_fptr postfuncp);
gdk_return logger_del_bat(logger *lg, log_bid bid)
__attribute__((__warn_unused_result__));
void logger_destroy(logger *lg);
-gdk_return logger_exit(logger *lg);
log_bid logger_find_bat(logger *lg, const char *name, char tpe, oid id);
-gdk_return logger_restart(logger *lg);
+gdk_return logger_flush(logger *lg, lng save_id);
int logger_sequence(logger *lg, int seq, lng *id);
gdk_return logger_upgrade_bat(logger *lg, const char *name, char tpe, oid id)
__attribute__((__warn_unused_result__));
void logger_with_ids(logger *lg);
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -160,7 +160,7 @@ typedef enum {LOG_OK, LOG_EOF, LOG_ERR}
* indicate that to the function wkbREAD during reading of the log. */
static bool geomisoldversion;
-static gdk_return bm_commit(logger *lg);
+static gdk_return bm_commit(logger *lg, lng save_id);
static gdk_return tr_grow(trans *tr);
static BUN
@@ -1401,7 +1401,7 @@ logger_readlogs(logger *lg, FILE *fp, ch
}
static gdk_return
-logger_commit(logger *lg)
+logger_commit(logger *lg, lng save_id)
{
if (lg->debug & 1)
fprintf(stderr, "#logger_commit\n");
@@ -1416,7 +1416,7 @@ logger_commit(logger *lg)
BATcommit(lg->snapshots_tid);
BATcommit(lg->dsnapshots);
}
- return bm_commit(lg);
+ return bm_commit(lg, save_id);
}
static gdk_return
@@ -1721,6 +1721,7 @@ logger_load(int debug, const char *fn, c
lg->snapshots_tid = NULL;
lg->dsnapshots = NULL;
lg->freed = NULL;
+ lg->freed_lid = NULL;
lg->seqs_id = NULL;
lg->seqs_val = NULL;
lg->dseqs = NULL;
@@ -2006,6 +2007,15 @@ logger_load(int debug, const char *fn, c
if (BBPrename(lg->freed->batCacheid, bak) < 0) {
goto error;
}
+ lg->freed_lid = logbat_new(TYPE_lng, 1, TRANSIENT);
+ if (lg->freed_lid == NULL) {
+ GDKerror("Logger_new: failed to create freed_lid bat");
+ goto error;
+ }
+ strconcat_len(bak, sizeof(bak), fn, "_freed_lid", NULL);
+ if (BBPrename(lg->freed_lid->batCacheid, bak) < 0) {
+ goto error;
+ }
snapshots_bid = logger_find_bat(lg, "snapshots_bid", 0, 0);
if (snapshots_bid == 0) {
lg->snapshots_bid = logbat_new(TYPE_int, 1, PERSISTENT);
@@ -2129,7 +2139,7 @@ logger_load(int debug, const char *fn, c
needcommit = true;
}
GDKdebug &= ~CHECKMASK;
- if (needcommit && bm_commit(lg) != GDK_SUCCEED) {
+ if (needcommit && bm_commit(lg, lg->tid+1) != GDK_SUCCEED) {
GDKerror("Logger_new: commit failed");
goto error;
}
@@ -2303,6 +2313,7 @@ logger_load(int debug, const char *fn, c
logbat_destroy(lg->snapshots_tid);
logbat_destroy(lg->dsnapshots);
logbat_destroy(lg->freed);
+ logbat_destroy(lg->freed_lid);
logbat_destroy(lg->seqs_id);
logbat_destroy(lg->seqs_val);
logbat_destroy(lg->dseqs);
@@ -2392,6 +2403,12 @@ logger_new(int debug, const char *fn, co
return NULL;
}
+static gdk_return
+logger_restart(logger *lg)
+{
+ return logger_flush(lg, lg->tid);
+}
+
/* Create a new logger */
logger *
logger_create(int debug, const char *fn, const char *logdir, int version,
preversionfix_fptr prefuncp, postversionfix_fptr postfuncp)
@@ -2456,6 +2473,7 @@ logger_destroy(logger *lg)
logbat_destroy(lg->catalog_oid);
logbat_destroy(lg->dcatalog);
logbat_destroy(lg->freed);
+ logbat_destroy(lg->freed_lid);
}
GDKfree(lg->fn);
GDKfree(lg->dir);
@@ -2463,8 +2481,8 @@ logger_destroy(logger *lg)
GDKfree(lg);
}
-gdk_return
-logger_exit(logger *lg)
+static gdk_return
+logger_exit(logger *lg, lng save_id)
{
FILE *fp;
char filename[FILENAME_MAX];
@@ -2472,7 +2490,7 @@ logger_exit(logger *lg)
if (LOG_DISABLED(lg)) {
logger_close(lg);
- if (logger_commit(lg) != GDK_SUCCEED) {
+ if (logger_commit(lg, save_id) != GDK_SUCCEED) {
TRC_CRITICAL(GDK, "logger_commit failed\n");
return GDK_FAIL;
}
@@ -2503,7 +2521,7 @@ logger_exit(logger *lg)
}
lg->id ++;
- if (logger_commit(lg) != GDK_SUCCEED) {
+ if (logger_commit(lg, save_id) != GDK_SUCCEED) {
(void) fclose(fp);
TRC_CRITICAL(GDK, "logger_commit failed\n");
return GDK_FAIL;
@@ -2557,9 +2575,9 @@ logger_exit(logger *lg)
}
gdk_return
-logger_restart(logger *lg)
+logger_flush(logger *lg, lng save_id)
{
- if (logger_exit(lg) == GDK_SUCCEED &&
+ if (logger_exit(lg, save_id) == GDK_SUCCEED &&
logger_open(lg) == GDK_SUCCEED)
return GDK_SUCCEED;
return GDK_FAIL;
@@ -3114,24 +3132,39 @@ log_sequence(logger *lg, int seq, lng va
}
static gdk_return
-bm_commit(logger *lg)
+bm_commit(logger *lg, lng save_id)
{
BUN p, q;
- BAT *b = lg->catalog_bid;
+ BAT *b = lg->catalog_bid, *nfreed = NULL, *nfreed_lid = NULL;
BAT *n = logbat_new(TYPE_str, BATcount(lg->freed), TRANSIENT);
gdk_return res;
const log_bid *bids;
+ const lng *lids;
+ BUN leftover = 0;
if (n == NULL)
return GDK_FAIL;
/* subcommit the freed bats */
bids = (const log_bid *) Tloc(lg->freed, 0);
+ lids = (const lng *) Tloc(lg->freed_lid, 0);
BATloop(lg->freed, p, q) {
bat bid = bids[p];
+ lng lid = lids[p];
BAT *lb = BATdescriptor(bid);
str name = BBPname(bid);
+ if (lid > save_id) {
+ leftover++;
+ continue;
+ }
+
+ if (lg->debug & 1) {
+ if (BBP_lrefs(bid) != 2) {
+ fprintf(stderr, "release %d %d\n", bid,
BBP_lrefs(bid));
+ assert(0);
+ }
+ }
if (lb == NULL ||
BATmode(lb, true) != GDK_SUCCEED) {
logbat_destroy(lb);
@@ -3150,6 +3183,29 @@ bm_commit(logger *lg)
}
BBPrelease(bid);
}
+ if (leftover) {
+ nfreed = logbat_new(TYPE_int, leftover, TRANSIENT);
+ nfreed_lid = logbat_new(TYPE_lng, leftover, TRANSIENT);
+
+ if (!nfreed || !nfreed_lid) {
+ logbat_destroy(n);
+ logbat_destroy(nfreed);
+ logbat_destroy(nfreed_lid);
+ return GDK_FAIL;
+ }
+ BATloop(lg->freed, p, q) {
+ lng lid = lids[p];
+
+ if (lid < save_id && (
+ BUNappend(nfreed, bids+p, false) != GDK_SUCCEED
||
+ BUNappend(nfreed_lid, &lid, false) !=
GDK_SUCCEED)) {
+ logbat_destroy(n);
+ logbat_destroy(nfreed);
+ logbat_destroy(nfreed_lid);
+ return GDK_FAIL;
+ }
+ }
+ }
bids = (log_bid *) Tloc(b, 0);
for (p = b->batInserted; p < BUNlast(b); p++) {
@@ -3167,6 +3223,8 @@ bm_commit(logger *lg)
BATmode(lb, false) != GDK_SUCCEED) {
logbat_destroy(lb);
logbat_destroy(n);
+ logbat_destroy(nfreed);
+ logbat_destroy(nfreed_lid);
return GDK_FAIL;
}
@@ -3180,8 +3238,24 @@ bm_commit(logger *lg)
res = bm_subcommit(lg, lg->catalog_bid, lg->catalog_nme,
lg->catalog_bid, lg->catalog_nme, lg->catalog_tpe, lg->catalog_oid,
lg->dcatalog, n, lg->debug);
BBPreclaim(n);
if (res == GDK_SUCCEED) {
- BATclear(lg->freed, false);
- BATcommit(lg->freed);
+ /* switch */
+ if (nfreed && nfreed_lid) {
+ if (logger_switch_bat(lg->freed, nfreed, lg->fn,
"freed") != GDK_SUCCEED ||
+ logger_switch_bat(lg->freed_lid, nfreed_lid, lg->fn,
"freed_lid") != GDK_SUCCEED) {
+ logbat_destroy(nfreed);
+ logbat_destroy(nfreed_lid);
+ return GDK_FAIL;
+ }
+ logbat_destroy(lg->freed);
+ logbat_destroy(lg->freed_lid);
+ lg->freed = nfreed;
+ lg->freed_lid = nfreed_lid;
+ } else {
+ BATclear(lg->freed, true);
+ BATclear(lg->freed_lid, true);
+ BATcommit(lg->freed);
+ BATcommit(lg->freed_lid);
+ }
return GDK_SUCCEED;
}
return GDK_FAIL;
@@ -3224,6 +3298,7 @@ logger_add_bat(logger *lg, BAT *b, const
BUNappend(lg->catalog_tpe, &tpe, false) != GDK_SUCCEED ||
BUNappend(lg->catalog_oid, &lid, false) != GDK_SUCCEED)
return GDK_FAIL;
+
BBPretain(bid);
return GDK_SUCCEED;
}
@@ -3253,6 +3328,7 @@ logger_del_bat(logger *lg, log_bid bid)
BAT *b = BATdescriptor(bid);
BUN p = log_find(lg->catalog_bid, lg->dcatalog, bid), q;
oid pos;
+ lng lid = lg->tid;
assert(p != BUN_NONE);
if (p == BUN_NONE) {
@@ -3274,14 +3350,16 @@ logger_del_bat(logger *lg, log_bid bid)
fprintf(stderr,
"#logger_del_bat release snapshot %d (%d)\n",
bid, BBP_lrefs(bid));
- if (BUNappend(lg->freed, &bid, false) != GDK_SUCCEED) {
+ if (BUNappend(lg->freed, &bid, false) != GDK_SUCCEED ||
+ BUNappend(lg->freed_lid, &lid, false) != GDK_SUCCEED) {
logbat_destroy(b);
return GDK_FAIL;
}
- } else if (p >= lg->catalog_bid->batInserted) {
+ } else if (p >= lg->catalog_bid->batInserted) { /* never became
persistent */
BBPrelease(bid);
} else {
- if (BUNappend(lg->freed, &bid, false) != GDK_SUCCEED) {
+ if (BUNappend(lg->freed, &bid, false) != GDK_SUCCEED ||
+ BUNappend(lg->freed_lid, &lid, false) != GDK_SUCCEED) {
logbat_destroy(b);
return GDK_FAIL;
}
@@ -3368,3 +3446,9 @@ geomversion_get(void)
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
