MonetDB: default - Merges second head

2023-11-07 Thread stefanos mavros via checkin-list 
Changeset: d8e8844f39e3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/d8e8844f39e3
Branch: default
Log Message:

Merges second head


diffs (truncated from 372 to 300 lines):

diff --git a/gdk/ChangeLog.Jun2023 b/gdk/ChangeLog.Jun2023
--- a/gdk/ChangeLog.Jun2023
+++ b/gdk/ChangeLog.Jun2023
@@ -1,3 +1,10 @@
 # ChangeLog file for GDK
 # This file is updated with Maddlog
 
+* Tue Nov  7 2023 Sjoerd Mullender 
+- When saving the SQL catalog during a low-level commit, we should
+  only save the part of the catalog that corresponds to the part of the
+  write-ahead log that has been processed.  What we did was save more,
+  which resulted in the catalog containing references to tables and
+  columns whose disk presence is otherwise only in the write-ahead log.
+
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -94,7 +94,7 @@ typedef struct logformat_t {
 
 typedef enum { LOG_OK, LOG_EOF, LOG_ERR } log_return;
 
-static gdk_return bm_commit(logger *lg, uint32_t *updated, BUN maxupdated);
+static gdk_return bm_commit(logger *lg, logged_range *pending, uint32_t 
*updated, BUN maxupdated);
 static gdk_return tr_grow(trans *tr);
 
 #define log_lock(lg)   MT_lock_set(&(lg)->lock)
@@ -1089,6 +1089,8 @@ log_open_output(logger *lg)
new_range->next = NULL;
logged_range *current = lg->current;
assert(current && current->next == NULL);
+   new_range->cnt = current->cnt;
+   new_range->deleted = current->deleted;
current->next = new_range;
ATOMIC_INC(&lg->nr_open_files);
return GDK_SUCCEED;
@@ -1381,11 +1383,11 @@ log_readlogs(logger *lg, const char *fil
 }
 
 static gdk_return
-log_commit(logger *lg, uint32_t *updated, BUN maxupdated)
+log_commit(logger *lg, logged_range *pending, uint32_t *updated, BUN 
maxupdated)
 {
TRC_DEBUG(WAL, "commit");
 
-   return bm_commit(lg, updated, maxupdated);
+   return bm_commit(lg, pending, updated, maxupdated);
 }
 
 static gdk_return
@@ -1492,7 +1494,7 @@ log_switch_bat(BAT *old, BAT *new, const
 static gdk_return
 bm_get_counts(logger *lg)
 {
-   BUN p, q, deleted = 0;
+   BUN p, q;
const log_bid *bids = (const log_bid *) Tloc(lg->catalog_bid, 0);
 
BATloop(lg->catalog_bid, p, q) {
@@ -1505,7 +1507,6 @@ bm_get_counts(logger *lg)
assert(b);
cnt = BATcount(b);
} else {
-   deleted++;
lid = BBP_desc(bids[p]) ? 1 : -1;
}
if (BUNappend(lg->catalog_cnt, &cnt, false) != GDK_SUCCEED)
@@ -1513,8 +1514,6 @@ bm_get_counts(logger *lg)
if (BUNappend(lg->catalog_lid, &lid, false) != GDK_SUCCEED)
return GDK_FAIL;
}
-   lg->deleted = deleted;
-   lg->cnt = BATcount(lg->catalog_bid);
return GDK_SUCCEED;
 }
 
@@ -1632,7 +1631,7 @@ cleanup_and_swap(logger *lg, int *r, con
r[rcnt++] = lg->catalog_id->batCacheid;
r[rcnt++] = lg->dcatalog->batCacheid;
 
-   assert(lg->deleted - cleanup == BATcount(ndels));
+   assert(BATcount(lg->dcatalog) - cleanup == BATcount(ndels));
 
logbat_destroy(lg->catalog_bid);
logbat_destroy(lg->catalog_id);
@@ -1659,21 +1658,25 @@ cleanup_and_swap(logger *lg, int *r, con
strconcat_len(bak, sizeof(bak), lg->fn, "_catalog_lid", NULL);
if (BBPrename(lg->catalog_lid, bak) < 0)
GDKclrerr();
-   lg->cnt = BATcount(lg->catalog_bid);
-   lg->deleted -= cleanup;
+   for (logged_range *p = lg->pending; p; p = p->next) {
+   p->cnt -= cleanup;
+   p->deleted -= cleanup;
+   }
return rcnt;
 }
 
 /* this function is called with log_lock() held; it releases the lock
  * before returning */
 static gdk_return
-bm_subcommit(logger *lg, uint32_t *updated, BUN maxupdated)
+bm_subcommit(logger *lg, logged_range *pending, uint32_t *updated, BUN 
maxupdated)
 {
+   BUN cnt = pending ? pending->cnt : BATcount(lg->catalog_bid);
+   BUN dcnt = pending ? pending->deleted : BATcount(lg->dcatalog);
BUN p, q;
BAT *catalog_bid = lg->catalog_bid;
BAT *catalog_id = lg->catalog_id;
BAT *dcatalog = lg->dcatalog;
-   BUN nn = 13 + BATcount(catalog_bid);
+   BUN nn = 13 + cnt;
bat *n = GDKmalloc(sizeof(bat) * nn);
bat *r = GDKmalloc(sizeof(bat) * nn);
BUN *sizes = GDKmalloc(sizeof(BUN) * nn);
@@ -1716,27 +1719,31 @@ bm_subcommit(logger *lg, uint32_t *updat
n[i++] = col;
}
/* now commit catalog, so it's also up to date on disk */
-   sizes[i] = lg->cnt;
+   sizes[i] = cnt;
n[i++] = catalog_bid->batCacheid;
-   sizes[i] = lg->cnt;
+   sizes[i] = cnt;
n[i++] = catalog_id->batCacheid;
-   sizes[i] = BATcount(dcatalog);
+   sizes[i] = dcnt;
n[i++] = dcatalog->batCacheid;
 
-   if (cleanup &&
-

MonetDB: default - Fixes bug for case of REMOTE tree with only r...

2023-11-07 Thread stefanos mavros via checkin-list 
Changeset: 42bde329ac0f for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/42bde329ac0f
Modified Files:
sql/server/rel_distribute.c
Branch: default
Log Message:

Fixes bug for case of REMOTE tree with only replicas


diffs (25 lines):

diff --git a/sql/server/rel_distribute.c b/sql/server/rel_distribute.c
--- a/sql/server/rel_distribute.c
+++ b/sql/server/rel_distribute.c
@@ -140,8 +140,21 @@ replica_rewrite(visitor *v, sql_table *t
 
for (node *m = uris->h; m && !res; m = m->next) {
if (strcmp(((tid_uri*)m->data)->uri, pt->query) 
== 0) {
+   /* we found a matching uri do the 
actual rewrite */
res = do_replica_rewrite(v->sql, exps, 
t, pt,
 
rpstate->no_rmt_branch_rpl_leaf ? true: false);
+   /* set to the REMOTE a list with a 
single uri (the matching one)
+* this is for the case that our REMOTE 
subtree has only replicas
+* with multiple remotes*/
+   if (list_length(rp->value.pval) > 1) {
+   list *uri = sa_list(v->sql->sa);
+   tid_uri *tu = 
SA_NEW(v->sql->sa, tid_uri);
+   tu->id = 0;
+   tu->uri = pt->query;
+   append(uri, tu);
+   rp->value.pval = uri;
+   break;
+   }
}
}
}
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: default - Adds tests with only remote replicas

2023-11-07 Thread stefanos mavros via checkin-list 
Changeset: eda81ddf2c22 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/eda81ddf2c22
Added Files:
sql/test/rel-optimizers/Tests/remote-replica.reqtests
sql/test/rel-optimizers/Tests/remote-replica.test
Branch: default
Log Message:

Adds tests with only remote replicas


diffs (102 lines):

diff --git a/sql/test/rel-optimizers/Tests/remote-replica.reqtests 
b/sql/test/rel-optimizers/Tests/remote-replica.reqtests
new file mode 100644
--- /dev/null
+++ b/sql/test/rel-optimizers/Tests/remote-replica.reqtests
@@ -0,0 +1,1 @@
+replicas-base
diff --git a/sql/test/rel-optimizers/Tests/remote-replica.test 
b/sql/test/rel-optimizers/Tests/remote-replica.test
new file mode 100644
--- /dev/null
+++ b/sql/test/rel-optimizers/Tests/remote-replica.test
@@ -0,0 +1,91 @@
+statement ok
+create replica table foo_rpl_rmt (n int, m text)
+
+statement ok
+alter table foo_rpl_rmt add table foo_r2
+
+statement ok
+alter table foo_rpl_rmt add table foo_r3
+
+statement ok
+create replica table foo_rpl_rmt_node2 (n int, m text)
+
+statement ok
+alter table foo_rpl_rmt_node2 add table foo_r2
+
+statement ok
+create replica table bar_rpl_rmt (n int, m text)
+
+statement ok
+alter table bar_rpl_rmt add table bar_r2
+
+statement ok
+alter table bar_rpl_rmt add table bar_r3
+
+statement ok
+create replica table bar_rpl_rmt_node3 (n int, m text)
+
+statement ok
+alter table bar_rpl_rmt_node3 add table bar_r3
+
+query T nosort
+plan select * from foo_rpl_rmt
+
+table (
+| project (
+| | REMOTE("sys"."foo_r2") [ "foo_r2"."n" as "foo_rpl_rmt"."n", "foo_r2"."m" 
as "foo_rpl_rmt"."m", "foo_r2"."%TID%" NOT NULL UNIQUE as "foo_rpl_rmt"."%TID%" 
]
+| ) [ "foo_rpl_rmt"."n", "foo_rpl_rmt"."m" ] REMOTE 
mapi:monetdb://localhost:50002/node2
+) [ "foo_rpl_rmt"."n", "foo_rpl_rmt"."m" ]
+
+query T nosort
+plan select * from foo_rpl_rmt, bar_rpl_rmt
+
+table (
+| project (
+| | crossproduct (
+| | | REMOTE("sys"."foo_r2") [ "foo_r2"."n" as "foo_rpl_rmt"."n", "foo_r2"."m" 
as "foo_rpl_rmt"."m", "foo_r2"."%TID%" NOT NULL UNIQUE as "foo_rpl_rmt"."%TID%" 
],
+| | | REMOTE("sys"."bar_r2") [ "bar_r2"."n" as "bar_rpl_rmt"."n", "bar_r2"."m" 
as "bar_rpl_rmt"."m", "bar_r2"."%TID%" NOT NULL UNIQUE as "bar_rpl_rmt"."%TID%" 
]
+| | ) [  ]
+| ) [ "foo_rpl_rmt"."n", "foo_rpl_rmt"."m", "bar_rpl_rmt"."n", 
"bar_rpl_rmt"."m" ] REMOTE mapi:monetdb://localhost:50002/node2
+) [ "foo_rpl_rmt"."n", "foo_rpl_rmt"."m", "bar_rpl_rmt"."n", "bar_rpl_rmt"."m" 
]
+
+query T nosort
+plan select * from foo_rpl_rmt_node2, bar_rpl_rmt_node3
+
+project (
+| crossproduct (
+| | table (
+| | | REMOTE("sys"."foo_r2") [ "foo_r2"."n" as "foo_rpl_rmt_node2"."n", 
"foo_r2"."m" as "foo_rpl_rmt_node2"."m", "foo_r2"."%TID%" NOT NULL UNIQUE as 
"foo_rpl_rmt_node2"."%TID%" ] REMOTE mapi:monetdb://localhost:50002/node2
+| | ) [ "foo_rpl_rmt_node2"."n", "foo_rpl_rmt_node2"."m", 
"foo_rpl_rmt_node2"."%TID%" NOT NULL UNIQUE ],
+| | table (
+| | | REMOTE("sys"."bar_r3") [ "bar_r3"."n" as "bar_rpl_rmt_node3"."n", 
"bar_r3"."m" as "bar_rpl_rmt_node3"."m", "bar_r3"."%TID%" NOT NULL UNIQUE as 
"bar_rpl_rmt_node3"."%TID%" ] REMOTE mapi:monetdb://localhost:50003/node3
+| | ) [ "bar_rpl_rmt_node3"."n", "bar_rpl_rmt_node3"."m", 
"bar_rpl_rmt_node3"."%TID%" NOT NULL UNIQUE ]
+| ) [  ]
+) [ "foo_rpl_rmt_node2"."n", "foo_rpl_rmt_node2"."m", "bar_rpl_rmt_node3"."n", 
"bar_rpl_rmt_node3"."m" ]
+
+query T nosort
+plan select * from foo_rpl_rmt, buz_rmt
+
+table (
+| project (
+| | crossproduct (
+| | | REMOTE("sys"."foo_r2") [ "foo_r2"."n" as "foo_rpl_rmt"."n", "foo_r2"."m" 
as "foo_rpl_rmt"."m", "foo_r2"."%TID%" NOT NULL UNIQUE as "foo_rpl_rmt"."%TID%" 
],
+| | | REMOTE("sys"."buz_rmt") [ "buz_rmt"."l" ]
+| | ) [  ]
+| ) [ "foo_rpl_rmt"."n", "foo_rpl_rmt"."m", "buz_rmt"."l" ] REMOTE 
mapi:monetdb://localhost:50002/node2
+) [ "foo_rpl_rmt"."n", "foo_rpl_rmt"."m", "buz_rmt"."l" ]
+
+query T nosort
+plan select * from bar_rpl_rmt_node3, buz_rmt
+
+project (
+| crossproduct (
+| | table (
+| | | REMOTE("sys"."bar_r3") [ "bar_r3"."n" as "bar_rpl_rmt_node3"."n", 
"bar_r3"."m" as "bar_rpl_rmt_node3"."m", "bar_r3"."%TID%" NOT NULL UNIQUE as 
"bar_rpl_rmt_node3"."%TID%" ] REMOTE mapi:monetdb://localhost:50003/node3
+| | ) [ "bar_rpl_rmt_node3"."n", "bar_rpl_rmt_node3"."m", 
"bar_rpl_rmt_node3"."%TID%" NOT NULL UNIQUE ],
+| | table (
+| | | REMOTE("sys"."buz_rmt") [ "buz_rmt"."l" ] REMOTE 
mapi:monetdb://localhost:50002/node2
+| | ) [ "buz_rmt"."l" ]
+| ) [  ]
+) [ "bar_rpl_rmt_node3"."n", "bar_rpl_rmt_node3"."m", "buz_rmt"."l" ]
+
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: default - Remove unused struct members.

2023-11-07 Thread Sjoerd Mullender via checkin-list 
Changeset: 3911bab6c4f3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/3911bab6c4f3
Modified Files:
gdk/gdk_logger.c
gdk/gdk_logger_internals.h
sql/storage/bat/bat_logger.c
Branch: default
Log Message:

Remove unused struct members.


diffs (94 lines):

diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -1494,7 +1494,7 @@ log_switch_bat(BAT *old, BAT *new, const
 static gdk_return
 bm_get_counts(logger *lg)
 {
-   BUN p, q, deleted = 0;
+   BUN p, q;
const log_bid *bids = (const log_bid *) Tloc(lg->catalog_bid, 0);
 
BATloop(lg->catalog_bid, p, q) {
@@ -1507,7 +1507,6 @@ bm_get_counts(logger *lg)
assert(b);
cnt = BATcount(b);
} else {
-   deleted++;
lid = BBP_desc(bids[p]) ? 1 : -1;
}
if (BUNappend(lg->catalog_cnt, &cnt, false) != GDK_SUCCEED)
@@ -1515,8 +1514,6 @@ bm_get_counts(logger *lg)
if (BUNappend(lg->catalog_lid, &lid, false) != GDK_SUCCEED)
return GDK_FAIL;
}
-   lg->deleted = deleted;
-   lg->cnt = BATcount(lg->catalog_bid);
return GDK_SUCCEED;
 }
 
@@ -1634,7 +1631,7 @@ cleanup_and_swap(logger *lg, int *r, con
r[rcnt++] = lg->catalog_id->batCacheid;
r[rcnt++] = lg->dcatalog->batCacheid;
 
-   assert(lg->deleted - cleanup == BATcount(ndels));
+   assert(BATcount(lg->dcatalog) - cleanup == BATcount(ndels));
 
logbat_destroy(lg->catalog_bid);
logbat_destroy(lg->catalog_id);
@@ -1661,8 +1658,6 @@ cleanup_and_swap(logger *lg, int *r, con
strconcat_len(bak, sizeof(bak), lg->fn, "_catalog_lid", NULL);
if (BBPrename(lg->catalog_lid, bak) < 0)
GDKclrerr();
-   lg->cnt = BATcount(lg->catalog_bid);
-   lg->deleted -= cleanup;
for (logged_range *p = lg->pending; p; p = p->next) {
p->cnt -= cleanup;
p->deleted -= cleanup;
@@ -3279,7 +3274,6 @@ log_add_bat(logger *lg, BAT *b, log_id i
|| BUNappend(lg->catalog_cnt, &cnt, false) != GDK_SUCCEED ||
BUNappend(lg->catalog_lid, &lid, false) != GDK_SUCCEED)
return GDK_FAIL;
-   lg->cnt++;
if (lg->current)
lg->current->cnt++;
BBPretain(bid);
@@ -3304,7 +3298,6 @@ log_del_bat(logger *lg, log_bid bid)
if (BUNreplace(lg->catalog_lid, p, &lid, false) != GDK_SUCCEED)
return GDK_FAIL;
if (BUNappend(lg->dcatalog, &pos, true) == GDK_SUCCEED) {
-   lg->deleted++;
if (lg->current)
lg->current->deleted++;
return GDK_SUCCEED;
diff --git a/gdk/gdk_logger_internals.h b/gdk/gdk_logger_internals.h
--- a/gdk/gdk_logger_internals.h
+++ b/gdk/gdk_logger_internals.h
@@ -76,8 +76,6 @@ struct logger {
BAT *catalog_cnt;   /* count of ondisk buns (transient) */
BAT *catalog_lid;   /* last tid, after which it gets 
released/destroyed */
BAT *dcatalog;  /* deleted from catalog table */
-   BUN cnt;/* number of persistent bats, incremented on 
log flushing */
-   BUN deleted;/* number of destroyed persistent bats, needed 
for catalog vacuum */
BAT *seqs_id;   /* int id column */
BAT *seqs_val;  /* lng value column */
BAT *dseqs; /* deleted from seqs table */
diff --git a/sql/storage/bat/bat_logger.c b/sql/storage/bat/bat_logger.c
--- a/sql/storage/bat/bat_logger.c
+++ b/sql/storage/bat/bat_logger.c
@@ -112,7 +112,6 @@ replace_bat(old_logger *old_lg, logger *

MT_rwlock_rdunlock(&cii.b->thashlock);
return GDK_FAIL;
}
-   lg->deleted++;
break;
}
}
@@ -123,7 +122,6 @@ replace_bat(old_logger *old_lg, logger *
(rc = BUNappend(lg->catalog_cnt, 
&(lng){BATcount(newcol)}, false)) == GDK_SUCCEED) {
BBPretain(newcol->batCacheid);
}
-   lg->cnt++;
}
}
return rc;
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: default - Merge with Jun2023 branch.

2023-11-07 Thread Sjoerd Mullender via checkin-list 
Changeset: 3f7be6f05660 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/3f7be6f05660
Modified Files:
gdk/gdk_logger.c
Branch: default
Log Message:

Merge with Jun2023 branch.


diffs (truncated from 305 to 300 lines):

diff --git a/gdk/ChangeLog.Jun2023 b/gdk/ChangeLog.Jun2023
--- a/gdk/ChangeLog.Jun2023
+++ b/gdk/ChangeLog.Jun2023
@@ -1,3 +1,10 @@
 # ChangeLog file for GDK
 # This file is updated with Maddlog
 
+* Tue Nov  7 2023 Sjoerd Mullender 
+- When saving the SQL catalog during a low-level commit, we should
+  only save the part of the catalog that corresponds to the part of the
+  write-ahead log that has been processed.  What we did was save more,
+  which resulted in the catalog containing references to tables and
+  columns whose disk presence is otherwise only in the write-ahead log.
+
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -94,7 +94,7 @@ typedef struct logformat_t {
 
 typedef enum { LOG_OK, LOG_EOF, LOG_ERR } log_return;
 
-static gdk_return bm_commit(logger *lg, uint32_t *updated, BUN maxupdated);
+static gdk_return bm_commit(logger *lg, logged_range *pending, uint32_t 
*updated, BUN maxupdated);
 static gdk_return tr_grow(trans *tr);
 
 #define log_lock(lg)   MT_lock_set(&(lg)->lock)
@@ -1089,6 +1089,8 @@ log_open_output(logger *lg)
new_range->next = NULL;
logged_range *current = lg->current;
assert(current && current->next == NULL);
+   new_range->cnt = current->cnt;
+   new_range->deleted = current->deleted;
current->next = new_range;
ATOMIC_INC(&lg->nr_open_files);
return GDK_SUCCEED;
@@ -1381,11 +1383,11 @@ log_readlogs(logger *lg, const char *fil
 }
 
 static gdk_return
-log_commit(logger *lg, uint32_t *updated, BUN maxupdated)
+log_commit(logger *lg, logged_range *pending, uint32_t *updated, BUN 
maxupdated)
 {
TRC_DEBUG(WAL, "commit");
 
-   return bm_commit(lg, updated, maxupdated);
+   return bm_commit(lg, pending, updated, maxupdated);
 }
 
 static gdk_return
@@ -1661,19 +1663,25 @@ cleanup_and_swap(logger *lg, int *r, con
GDKclrerr();
lg->cnt = BATcount(lg->catalog_bid);
lg->deleted -= cleanup;
+   for (logged_range *p = lg->pending; p; p = p->next) {
+   p->cnt -= cleanup;
+   p->deleted -= cleanup;
+   }
return rcnt;
 }
 
 /* this function is called with log_lock() held; it releases the lock
  * before returning */
 static gdk_return
-bm_subcommit(logger *lg, uint32_t *updated, BUN maxupdated)
+bm_subcommit(logger *lg, logged_range *pending, uint32_t *updated, BUN 
maxupdated)
 {
+   BUN cnt = pending ? pending->cnt : BATcount(lg->catalog_bid);
+   BUN dcnt = pending ? pending->deleted : BATcount(lg->dcatalog);
BUN p, q;
BAT *catalog_bid = lg->catalog_bid;
BAT *catalog_id = lg->catalog_id;
BAT *dcatalog = lg->dcatalog;
-   BUN nn = 13 + BATcount(catalog_bid);
+   BUN nn = 13 + cnt;
bat *n = GDKmalloc(sizeof(bat) * nn);
bat *r = GDKmalloc(sizeof(bat) * nn);
BUN *sizes = GDKmalloc(sizeof(BUN) * nn);
@@ -1716,27 +1724,31 @@ bm_subcommit(logger *lg, uint32_t *updat
n[i++] = col;
}
/* now commit catalog, so it's also up to date on disk */
-   sizes[i] = lg->cnt;
+   sizes[i] = cnt;
n[i++] = catalog_bid->batCacheid;
-   sizes[i] = lg->cnt;
+   sizes[i] = cnt;
n[i++] = catalog_id->batCacheid;
-   sizes[i] = BATcount(dcatalog);
+   sizes[i] = dcnt;
n[i++] = dcatalog->batCacheid;
 
-   if (cleanup &&
-   (rcnt = cleanup_and_swap(lg, r, bids, lids, cnts, catalog_bid,
-catalog_id, dcatalog, cleanup, updated,
-maxupdated)) < 0) {
-   GDKfree(n);
-   GDKfree(r);
-   GDKfree(sizes);
-   log_unlock(lg);
-   return GDK_FAIL;
+   if (cleanup) {
+   if ((rcnt = cleanup_and_swap(lg, r, bids, lids, cnts,
+catalog_bid, catalog_id, dcatalog,
+cleanup, updated,
+maxupdated)) < 0) {
+   GDKfree(n);
+   GDKfree(r);
+   GDKfree(sizes);
+   log_unlock(lg);
+   return GDK_FAIL;
+   }
+   cnt -= cleanup;
+   dcnt -= cleanup;
}
if (dcatalog != lg->dcatalog) {
-   i = subcommit_list_add(i, n, sizes, 
lg->catalog_bid->batCacheid, BATcount(lg->catalog_bid));
-   i = subcommit_list_add(i, n, sizes, lg->catalog_id->batCacheid, 
BATcount(lg->catalog_bid));
-   i = subcommit_list_add(i, n, sizes, lg->dcatalog->batCacheid, 
BATcount(lg->dcatalog));
+   i = subcommit_l

MonetDB: Jun2023 - Be more careful in which parts of sql_catalog...

2023-11-07 Thread Sjoerd Mullender via checkin-list 
Changeset: 81d4cd0063e7 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/81d4cd0063e7
Modified Files:
gdk/ChangeLog.Jun2023
gdk/gdk_logger.c
gdk/gdk_logger_internals.h
Branch: Jun2023
Log Message:

Be more careful in which parts of sql_catalog_bid/sql_catalog_id/sql_dcatalog 
is saved.


diffs (truncated from 305 to 300 lines):

diff --git a/gdk/ChangeLog.Jun2023 b/gdk/ChangeLog.Jun2023
--- a/gdk/ChangeLog.Jun2023
+++ b/gdk/ChangeLog.Jun2023
@@ -1,3 +1,10 @@
 # ChangeLog file for GDK
 # This file is updated with Maddlog
 
+* Tue Nov  7 2023 Sjoerd Mullender 
+- When saving the SQL catalog during a low-level commit, we should
+  only save the part of the catalog that corresponds to the part of the
+  write-ahead log that has been processed.  What we did was save more,
+  which resulted in the catalog containing references to tables and
+  columns whose disk presence is otherwise only in the write-ahead log.
+
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -94,7 +94,7 @@ typedef struct logformat_t {
 
 typedef enum { LOG_OK, LOG_EOF, LOG_ERR } log_return;
 
-static gdk_return bm_commit(logger *lg, uint32_t *updated, BUN maxupdated);
+static gdk_return bm_commit(logger *lg, logged_range *pending, uint32_t 
*updated, BUN maxupdated);
 static gdk_return tr_grow(trans *tr);
 
 #define log_lock(lg)   MT_lock_set(&(lg)->lock)
@@ -1089,6 +1089,8 @@ log_open_output(logger *lg)
new_range->next = NULL;
logged_range *current = lg->current;
assert(current && current->next == NULL);
+   new_range->cnt = current->cnt;
+   new_range->deleted = current->deleted;
current->next = new_range;
ATOMIC_INC(&lg->nr_open_files);
return GDK_SUCCEED;
@@ -1381,11 +1383,11 @@ log_readlogs(logger *lg, const char *fil
 }
 
 static gdk_return
-log_commit(logger *lg, uint32_t *updated, BUN maxupdated)
+log_commit(logger *lg, logged_range *pending, uint32_t *updated, BUN 
maxupdated)
 {
TRC_DEBUG(WAL, "commit");
 
-   return bm_commit(lg, updated, maxupdated);
+   return bm_commit(lg, pending, updated, maxupdated);
 }
 
 static gdk_return
@@ -1649,19 +1651,25 @@ cleanup_and_swap(logger *lg, int *r, con
lg->catalog_lid = nlids;
lg->cnt = BATcount(lg->catalog_bid);
lg->deleted -= cleanup;
+   for (logged_range *p = lg->pending; p; p = p->next) {
+   p->cnt -= cleanup;
+   p->deleted -= cleanup;
+   }
return rcnt;
 }
 
 /* this function is called with log_lock() held; it releases the lock
  * before returning */
 static gdk_return
-bm_subcommit(logger *lg, uint32_t *updated, BUN maxupdated)
+bm_subcommit(logger *lg, logged_range *pending, uint32_t *updated, BUN 
maxupdated)
 {
+   BUN cnt = pending ? pending->cnt : BATcount(lg->catalog_bid);
+   BUN dcnt = pending ? pending->deleted : BATcount(lg->dcatalog);
BUN p, q;
BAT *catalog_bid = lg->catalog_bid;
BAT *catalog_id = lg->catalog_id;
BAT *dcatalog = lg->dcatalog;
-   BUN nn = 13 + BATcount(catalog_bid);
+   BUN nn = 13 + cnt;
bat *n = GDKmalloc(sizeof(bat) * nn);
bat *r = GDKmalloc(sizeof(bat) * nn);
BUN *sizes = GDKmalloc(sizeof(BUN) * nn);
@@ -1704,27 +1712,31 @@ bm_subcommit(logger *lg, uint32_t *updat
n[i++] = col;
}
/* now commit catalog, so it's also up to date on disk */
-   sizes[i] = lg->cnt;
+   sizes[i] = cnt;
n[i++] = catalog_bid->batCacheid;
-   sizes[i] = lg->cnt;
+   sizes[i] = cnt;
n[i++] = catalog_id->batCacheid;
-   sizes[i] = BATcount(dcatalog);
+   sizes[i] = dcnt;
n[i++] = dcatalog->batCacheid;
 
-   if (cleanup &&
-   (rcnt = cleanup_and_swap(lg, r, bids, lids, cnts, catalog_bid,
-catalog_id, dcatalog, cleanup, updated,
-maxupdated)) < 0) {
-   GDKfree(n);
-   GDKfree(r);
-   GDKfree(sizes);
-   log_unlock(lg);
-   return GDK_FAIL;
+   if (cleanup) {
+   if ((rcnt = cleanup_and_swap(lg, r, bids, lids, cnts,
+catalog_bid, catalog_id, dcatalog,
+cleanup, updated,
+maxupdated)) < 0) {
+   GDKfree(n);
+   GDKfree(r);
+   GDKfree(sizes);
+   log_unlock(lg);
+   return GDK_FAIL;
+   }
+   cnt -= cleanup;
+   dcnt -= cleanup;
}
if (dcatalog != lg->dcatalog) {
-   i = subcommit_list_add(i, n, sizes, 
lg->catalog_bid->batCacheid, BATcount(lg->catalog_bid));
-   i = subcommit_list_add(i, n, sizes, lg->catalog_id->batCacheid, 
BATcount(lg->catalog_bid));
-

MonetDB: default - Simplifies logic of rewrite_replica opt

2023-11-07 Thread stefanos mavros via checkin-list 
Changeset: 75d8a126b11c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/75d8a126b11c
Modified Files:
sql/server/rel_distribute.c
sql/test/rel-optimizers/Tests/All
Branch: default
Log Message:

Simplifies logic of rewrite_replica opt


diffs (54 lines):

diff --git a/sql/server/rel_distribute.c b/sql/server/rel_distribute.c
--- a/sql/server/rel_distribute.c
+++ b/sql/server/rel_distribute.c
@@ -203,14 +203,15 @@ rel_rewrite_replica_(visitor *v, sql_rel
if (!eliminate_remote_or_replica_refs(v, &rel))
return rel;
 
+   /* if we are higher in the tree clear the previous REMOTE prop in the 
visitor state */
+   if (v->data && v->depth <= ((rps*)v->data)->depth) {
+   v->data = NULL;
+   }
+
/* no-leaf nodes: store the REMOTE property uris in the state of the 
visitor
 * leaf nodes: check if they are basetable replicas and proceed with 
the rewrite */
prop *p;
if (!is_basetable(rel->op)) {
-   /* if we are higher in the tree clear the previous REMOTE prop 
in the visitor state */
-   if (v->data && v->depth <= ((rps*)v->data)->depth) {
-   v->data = NULL;
-   }
/* if there is a REMOTE prop set it to the visitor state */
if ((p = find_prop(rel->p, PROP_REMOTE)) != NULL) {
rps *rp = SA_NEW(v->sql->sa, rps);
@@ -226,10 +227,7 @@ rel_rewrite_replica_(visitor *v, sql_rel
if (t && isReplicaTable(t)) {
/* we might have reached a replica table through a 
branch that has
 * no REMOTE property. In this case we have to set the 
v->data */
-   bool no_rmt_branch = false;
if (!v->data && (p = find_prop(rel->p, PROP_REMOTE)) != 
NULL) {
-   no_rmt_branch = true;
-
rps *rp = SA_NEW(v->sql->sa, rps);
rp->depth = v->depth;
rp->rmt = p;
@@ -244,11 +242,6 @@ rel_rewrite_replica_(visitor *v, sql_rel
sql_rel *r = replica_rewrite(v, t, rel->exps);
rel_destroy(rel);
rel = r;
-
-   /* if the whole rel tree branch is local (no upper 
REMOTE prop)
-* clean the visitor state */
-   if (no_rmt_branch)
-   v->data = NULL;
}
}
return rel;
diff --git a/sql/test/rel-optimizers/Tests/All 
b/sql/test/rel-optimizers/Tests/All
--- a/sql/test/rel-optimizers/Tests/All
+++ b/sql/test/rel-optimizers/Tests/All
@@ -5,3 +5,4 @@ join-merge-remote-replica
 replicas-base
 replicas-join
 local-replica
+remote-replica
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: default - Fixes bug with wrong uri in REMOTE prop

2023-11-07 Thread stefanos mavros via checkin-list 
Changeset: c5cf3b7476c4 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/c5cf3b7476c4
Modified Files:
sql/server/rel_distribute.c
Branch: default
Log Message:

Fixes bug with wrong uri in REMOTE prop


diffs (12 lines):

diff --git a/sql/server/rel_distribute.c b/sql/server/rel_distribute.c
--- a/sql/server/rel_distribute.c
+++ b/sql/server/rel_distribute.c
@@ -106,7 +106,7 @@ do_replica_rewrite(mvc *sql, list *exps,
list *uris = sa_list(sql->sa);
tid_uri *tu = SA_NEW(sql->sa, tid_uri);
tu->id = p->base.id;
-   tu->uri = sa_strconcat(sql->sa, sa_strconcat(sql->sa, 
p->s->base.name, "."), p->base.name);
+   tu->uri = p->query;
append(uris, tu);
 
prop *rmt_prop = r->p = prop_create(sql->sa, PROP_REMOTE, r->p);
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: default - Fixes bug with processing replicas in no REMO...

2023-11-07 Thread stefanos mavros via checkin-list 
Changeset: fb62d6ceb03a for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/fb62d6ceb03a
Modified Files:
sql/server/rel_distribute.c
Branch: default
Log Message:

Fixes bug with processing replicas in no REMOTE branches


diffs (67 lines):

diff --git a/sql/server/rel_distribute.c b/sql/server/rel_distribute.c
--- a/sql/server/rel_distribute.c
+++ b/sql/server/rel_distribute.c
@@ -19,6 +19,7 @@ typedef struct rmt_prop_state {
int depth;
prop* rmt;
sql_rel* orig;
+   bool no_rmt_branch_rpl_leaf;
 } rps;
 
 static int
@@ -119,7 +120,8 @@ static sql_rel *
 replica_rewrite(visitor *v, sql_table *t, list *exps)
 {
sql_rel *res = NULL;
-   prop *rp = ((rps*)v->data)->rmt;
+   rps *rpstate = v->data;
+   prop *rp = rpstate->rmt;
sqlid tid = rp->id;
list *uris = rp->value.pval;
 
@@ -138,7 +140,8 @@ replica_rewrite(visitor *v, sql_table *t
 
for (node *m = uris->h; m && !res; m = m->next) {
if (strcmp(((tid_uri*)m->data)->uri, pt->query) 
== 0) {
-   res = do_replica_rewrite(v->sql, exps, 
t, pt, 0);
+   res = do_replica_rewrite(v->sql, exps, 
t, pt,
+
rpstate->no_rmt_branch_rpl_leaf ? true: false);
}
}
}
@@ -214,6 +217,7 @@ rel_rewrite_replica_(visitor *v, sql_rel
rp->depth = v->depth;
rp->rmt = p;
rp->orig = rel;
+   rp->no_rmt_branch_rpl_leaf = false;
v->data = rp;
}
} else {
@@ -222,11 +226,15 @@ rel_rewrite_replica_(visitor *v, sql_rel
if (t && isReplicaTable(t)) {
/* we might have reached a replica table through a 
branch that has
 * no REMOTE property. In this case we have to set the 
v->data */
+   bool no_rmt_branch = false;
if (!v->data && (p = find_prop(rel->p, PROP_REMOTE)) != 
NULL) {
+   no_rmt_branch = true;
+
rps *rp = SA_NEW(v->sql->sa, rps);
rp->depth = v->depth;
rp->rmt = p;
rp->orig = rel;
+   rp->no_rmt_branch_rpl_leaf = true;
v->data = rp;
}
 
@@ -236,6 +244,11 @@ rel_rewrite_replica_(visitor *v, sql_rel
sql_rel *r = replica_rewrite(v, t, rel->exps);
rel_destroy(rel);
rel = r;
+
+   /* if the whole rel tree branch is local (no upper 
REMOTE prop)
+* clean the visitor state */
+   if (no_rmt_branch)
+   v->data = NULL;
}
}
return rel;
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: no_type_bat - cleanup. reduce use of getSignature

2023-11-07 Thread Niels Nes via checkin-list 
Changeset: 2c69c3ca6e57 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/2c69c3ca6e57
Modified Files:
clients/Tests/exports.stable.out
monetdb5/mal/Tests/tst191.maltest
monetdb5/mal/mal.h
monetdb5/mal/mal_client.h
monetdb5/mal/mal_function.c
monetdb5/mal/mal_function.h
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_module.c
monetdb5/mal/mal_prelude.c
monetdb5/mal/mal_resolve.c
monetdb5/mal/mel.h
monetdb5/modules/mal/manual.c
monetdb5/optimizer/Tests/tst4006.maltest
monetdb5/optimizer/opt_macro.c
monetdb5/optimizer/opt_mergetable.c
tools/monetdbe/monetdbe.c
Branch: no_type_bat
Log Message:

cleanup. reduce use of getSignature


diffs (truncated from 612 to 300 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -975,7 +975,6 @@ str getExceptionPlace(const char *);
 enum malexception getExceptionType(const char *);
 int getFltConstant(MalBlkPtr mb, flt val);
 Scenario getFreeScenario(void);
-Symbol getFunctionSymbol(Module scope, InstrPtr p);
 int getHgeConstant(MalBlkPtr mb, hge val);
 int getIntConstant(MalBlkPtr mb, int val);
 int getLngConstant(MalBlkPtr mb, lng val);
diff --git a/monetdb5/mal/Tests/tst191.maltest 
b/monetdb5/mal/Tests/tst191.maltest
--- a/monetdb5/mal/Tests/tst191.maltest
+++ b/monetdb5/mal/Tests/tst191.maltest
@@ -5,11 +5,11 @@ function welcome0(lim:int):str;
 return (w,i);
 end welcome0
 
-statement error
+statement ok
 y:= user.welcome0(1)
 
 query T rowsort
 io.print(y)
 
-NULL
+"hello"
 
diff --git a/monetdb5/mal/mal.h b/monetdb5/mal/mal.h
--- a/monetdb5/mal/mal.h
+++ b/monetdb5/mal/mal.h
@@ -108,14 +108,18 @@ mal_export const char *mal_version(void)
 typedef int malType;
 typedef void (*MALfcn)(void);
 
+#include "mel.h"
+
 typedef struct SYMDEF {
struct SYMDEF *peer;/* where to look next */
struct SYMDEF *skip;/* skip to next different symbol */
const char *name;
int kind;   /* what kind of symbol 
*/
struct MALBLK *def; /* the details of the MAL fcn */
+   mel_func *func;
 } *Symbol, SymRecord;
 
+
 typedef struct VARRECORD {
char name[IDLENGTH];/* use the space for the full name */
char kind;  /* Could be either _, X 
or C to stamp the variable type */
@@ -141,7 +145,7 @@ typedef struct VARRECORD {
  * mal_profiler.c)
  */
 
-typedef struct {
+typedef struct INSTR {
bte token;  /* instruction type */
bit barrier;/* flow of control modifier 
takes:
   BARRIER, 
LEAVE, REDO, EXIT, CATCH, RAISE */
diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h
--- a/monetdb5/mal/mal_client.h
+++ b/monetdb5/mal/mal_client.h
@@ -44,7 +44,6 @@ typedef struct CLIENT_INPUT {
struct CLIENT_INPUT *next;
 } ClientInput;
 
-struct CLIENT;
 typedef str (*init_client)(struct CLIENT *, const char *, const char *,
   const char *);
 typedef str (*exit_client)(struct CLIENT *);
diff --git a/monetdb5/mal/mal_function.c b/monetdb5/mal/mal_function.c
--- a/monetdb5/mal/mal_function.c
+++ b/monetdb5/mal/mal_function.c
@@ -64,28 +64,6 @@ newFunction(const char *mod, const char 
return newFunctionArgs(mod, nme, kind, MAXARG);
 }
 
-/*
- * Optimizers may be interested in the function definition
- * for obtaining properties. Rather than polution of the
- * instruction record with a scope reference, we use a lookup function until it
- * becomes a performance hindrance.
- */
-Symbol
-getFunctionSymbol(Module scope, InstrPtr p)
-{
-   Module m;
-   Symbol s;
-
-   for (m = findModule(scope, getModuleId(p)); m; m = m->link)
-   if (idcmp(m->name, getModuleId(p)) == 0) {
-   s = m->space[getSymbolIndex(getFunctionId(p))];
-   for (; s; s = s->peer)
-   if (getSignature(s)->fcn == p->fcn)
-   return s;
-   }
-   return 0;
-}
-
 int
 getPC(MalBlkPtr mb, InstrPtr p)
 {
@@ -287,30 +265,21 @@ getBarrierEnvelop(MalBlkPtr mb)
 static void
 replaceTypeVar(MalBlkPtr mb, InstrPtr p, int v, malType t)
 {
-   int j, i, x, y;
-
-   for (j = 0; j < mb->stop; j++) {
+   for (int j = 0; j < mb->stop; j++) {
p = getInstrPtr(mb, j);
-   if (p->polymorphic)
-   for (i = 0; i < p->argc; i++)
-   if (isPolymorphic(x = getArgType(mb, p, i))) {
+   if (p->polymorphic) {
+   for (int i = 0; i < p->argc; i++) {
+

MonetDB: default - Changes remote name in tests

2023-11-07 Thread stefanos mavros via checkin-list 
Changeset: 8966fc2ced21 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/8966fc2ced21
Modified Files:
sql/test/rel-optimizers/Tests/replicas-base.test
sql/test/rel-optimizers/Tests/replicas-join.test
Branch: default
Log Message:

Changes remote name in tests


diffs (69 lines):

diff --git a/sql/test/rel-optimizers/Tests/replicas-base.test 
b/sql/test/rel-optimizers/Tests/replicas-base.test
--- a/sql/test/rel-optimizers/Tests/replicas-base.test
+++ b/sql/test/rel-optimizers/Tests/replicas-base.test
@@ -5,10 +5,10 @@ statement ok
 insert into foo_local values (1, 'hello'), (2, 'world'), (3, '!!')
 
 statement ok
-create remote table foo_r1 (n int, m text) on 
'mapi:monetdb://localhost:50002/node2'
+create remote table foo_r2 (n int, m text) on 
'mapi:monetdb://localhost:50002/node2'
 
 statement ok
-create remote table foo_r2 (n int, m text) on 
'mapi:monetdb://localhost:50003/node3'
+create remote table foo_r3 (n int, m text) on 
'mapi:monetdb://localhost:50003/node3'
 
 
 statement ok
@@ -18,10 +18,10 @@ statement ok
 insert into bar_local values (10, 'alice'), (2, 'bob'), (3, 'tom'), (4, 
'jerry')
 
 statement ok
-create remote table bar_r1 (n int, m text) on 
'mapi:monetdb://localhost:50002/node2'
+create remote table bar_r2 (n int, m text) on 
'mapi:monetdb://localhost:50002/node2'
 
 statement ok
-create remote table bar_r2 (n int, m text) on 
'mapi:monetdb://localhost:50003/node3'
+create remote table bar_r3 (n int, m text) on 
'mapi:monetdb://localhost:50003/node3'
 
 statement ok
 create remote table buz_rmt (l int) on 'mapi:monetdb://localhost:50002/node2'
diff --git a/sql/test/rel-optimizers/Tests/replicas-join.test 
b/sql/test/rel-optimizers/Tests/replicas-join.test
--- a/sql/test/rel-optimizers/Tests/replicas-join.test
+++ b/sql/test/rel-optimizers/Tests/replicas-join.test
@@ -5,10 +5,10 @@ statement ok
 alter table foo_rpl add table foo_local
 
 statement ok
-alter table foo_rpl add table foo_r1
+alter table foo_rpl add table foo_r2
 
 statement ok
-alter table foo_rpl add table foo_r2
+alter table foo_rpl add table foo_r3
 
 statement ok
 create replica table bar_rpl (n int, m text)
@@ -17,10 +17,10 @@ statement ok
 alter table bar_rpl add table bar_local
 
 statement ok
-alter table bar_rpl add table bar_r1
+alter table bar_rpl add table bar_r2
 
 statement ok
-alter table bar_rpl add table bar_r2
+alter table bar_rpl add table bar_r3
 
 query T nosort
 plan select * from foo_rpl
@@ -50,8 +50,8 @@ table (
 | project (
 | | crossproduct (
 | | | crossproduct (
-| | | | REMOTE("sys"."foo_r1") [ "foo_r1"."n" as "foo_rpl"."n", "foo_r1"."m" 
as "foo_rpl"."m", "foo_r1"."%TID%" NOT NULL UNIQUE as "foo_rpl"."%TID%" ],
-| | | | REMOTE("sys"."bar_r1") [ "bar_r1"."n" as "bar_rpl"."n", "bar_r1"."m" 
as "bar_rpl"."m", "bar_r1"."%TID%" NOT NULL UNIQUE as "bar_rpl"."%TID%" ]
+| | | | REMOTE("sys"."foo_r2") [ "foo_r2"."n" as "foo_rpl"."n", "foo_r2"."m" 
as "foo_rpl"."m", "foo_r2"."%TID%" NOT NULL UNIQUE as "foo_rpl"."%TID%" ],
+| | | | REMOTE("sys"."bar_r2") [ "bar_r2"."n" as "bar_rpl"."n", "bar_r2"."m" 
as "bar_rpl"."m", "bar_r2"."%TID%" NOT NULL UNIQUE as "bar_rpl"."%TID%" ]
 | | | ) [  ],
 | | | REMOTE("sys"."buz_rmt") [ "buz_rmt"."l" ]
 | | ) [  ]
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: literal_features - add tests for various unique constra...

2023-11-07 Thread Yunus Koning via checkin-list 
Changeset: 82cfc16811b2 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/82cfc16811b2
Added Files:
sql/test/2023/Tests/unique_nulls_distinct.test
Modified Files:
sql/test/2023/Tests/All
Branch: literal_features
Log Message:

add tests for various unique constraints


diffs (136 lines):

diff --git a/sql/test/2023/Tests/All b/sql/test/2023/Tests/All
--- a/sql/test/2023/Tests/All
+++ b/sql/test/2023/Tests/All
@@ -1,3 +1,4 @@
 literals
 btrim
 any_value
+unique_nulls_distinct
diff --git a/sql/test/2023/Tests/unique_nulls_distinct.test 
b/sql/test/2023/Tests/unique_nulls_distinct.test
new file mode 100644
--- /dev/null
+++ b/sql/test/2023/Tests/unique_nulls_distinct.test
@@ -0,0 +1,123 @@
+
+statement ok
+CREATE TABLE und1 (i1 int, i2 int, UNIQUE (i1, i2))
+
+statement ok
+CREATE TABLE und2 (i1 int, i2 int, UNIQUE NULLS DISTINCT (i1, i2))
+
+statement ok
+CREATE TABLE unnd1 (i1 int, i2 int, UNIQUE NULLS NOT DISTINCT (i1, i2))
+
+statement ok
+INSERT INTO und1 VALUES (NULL, 10)
+
+statement ok
+INSERT INTO und1 VALUES (NULL, 10), (NULL, 10)
+
+statement ok
+INSERT INTO und1 VALUES (20, 10)
+
+statement error
+INSERT INTO und1 VALUES (20, 10)
+
+statement error
+INSERT INTO und1 VALUES (30, 10), (30, 10)
+
+statement ok
+INSERT INTO und2 VALUES (NULL, 10)
+
+statement ok
+INSERT INTO und2 VALUES (NULL, 10), (NULL, 10)
+
+statement ok
+INSERT INTO und2 VALUES (20, 10)
+
+statement error
+INSERT INTO und2 VALUES (20, 10)
+
+statement error
+INSERT INTO und2 VALUES (30, 10), (30, 10)
+
+statement error
+INSERT INTO unnd1 VALUES (NULL, 10), (NULL, 10)
+
+statement ok
+INSERT INTO unnd1 VALUES (NULL, 10)
+
+statement error
+INSERT INTO unnd1 VALUES (NULL, 10)
+
+statement ok
+INSERT INTO unnd1 VALUES (20, 10)
+
+statement error
+INSERT INTO unnd1 VALUES (20, 10)
+
+statement error
+INSERT INTO unnd1 VALUES (30, 10), (30, 10)
+
+statement ok
+CREATE TABLE und3 (i1 int UNIQUE)
+
+statement ok
+CREATE TABLE und4 (i1 int UNIQUE NULLS DISTINCT )
+
+statement ok
+CREATE TABLE unnd2 (i1 int UNIQUE NULLS NOT DISTINCT)
+
+statement ok
+INSERT INTO und3 VALUES (NULL)
+
+statement ok
+INSERT INTO und3 VALUES (NULL), (NULL)
+
+statement ok
+INSERT INTO und3 VALUES (10)
+
+statement error
+INSERT INTO und3 VALUES (10)
+
+statement error
+INSERT INTO und3 VALUES (10), (20)
+
+statement error
+INSERT INTO und3 VALUES (30), (30)
+
+statement ok
+INSERT INTO und4 VALUES (NULL)
+
+statement ok
+INSERT INTO und4 VALUES (NULL), (NULL)
+
+statement ok
+INSERT INTO und4 VALUES (10)
+
+statement error
+INSERT INTO und4 VALUES (10)
+
+statement error
+INSERT INTO und4 VALUES (10), (20)
+
+statement error
+INSERT INTO und4 VALUES (30), (30)
+
+statement error
+INSERT INTO unnd2 VALUES (NULL), (NULL)
+
+statement ok
+INSERT INTO unnd2 VALUES (NULL)
+
+statement error
+INSERT INTO unnd2 VALUES (NULL)
+
+statement ok
+INSERT INTO unnd2 VALUES (10)
+
+statement error
+INSERT INTO unnd2 VALUES (10)
+
+statement error
+INSERT INTO unnd2 VALUES (10), (20)
+
+statement error
+INSERT INTO unnd2 VALUES (30), (30)
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: literal_features - Also introduce explicit UNIQUE NULLS...

2023-11-07 Thread Yunus Koning via checkin-list 
Changeset: c096265d9f8b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/c096265d9f8b
Modified Files:
sql/server/sql_parser.y
Branch: literal_features
Log Message:

Also introduce explicit UNIQUE NULLS DISTINCT syntax


diffs (20 lines):

diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -2144,6 +2144,7 @@ column_constraint_type:
 NOT sqlNULL{ $$ = _symbol_create( SQL_NOT_NULL, NULL); }
  |  sqlNULL{ $$ = _symbol_create( SQL_NULL, NULL); }
  |  UNIQUE { $$ = _symbol_create( SQL_UNIQUE, NULL ); }
+ |  UNIQUE NULLS DISTINCT  { $$ = _symbol_create( SQL_UNIQUE, NULL ); }
  |  UNIQUE NULLS NOT DISTINCT  { $$ = _symbol_create( 
SQL_UNIQUE_NULLS_NOT_DISTINCT, NULL ); }
  |  PRIMARY KEY{ $$ = _symbol_create( SQL_PRIMARY_KEY, NULL ); }
  |  REFERENCES qname opt_column_list opt_match opt_ref_action
@@ -2161,6 +2162,8 @@ column_constraint_type:
 table_constraint_type:
 UNIQUE column_commalist_parens
{ $$ = _symbol_create_list( SQL_UNIQUE, $2); }
+ |  UNIQUE NULLS DISTINCT column_commalist_parens
+   { $$ = _symbol_create_list( SQL_UNIQUE, $4); }
  |  UNIQUE NULLS NOT DISTINCT column_commalist_parens
{ $$ = _symbol_create_list( 
SQL_UNIQUE_NULLS_NOT_DISTINCT, $5); }
  |  PRIMARY KEY column_commalist_parens
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: default - Adds new tests with only local replica tables

2023-11-07 Thread stefanos mavros via checkin-list 
Changeset: 9f0980f1fd15 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/9f0980f1fd15
Added Files:
sql/test/rel-optimizers/Tests/local-replica.reqtests
sql/test/rel-optimizers/Tests/local-replica.test
Modified Files:
sql/test/rel-optimizers/Tests/All
Branch: default
Log Message:

Adds new tests with only local replica tables


diffs (61 lines):

diff --git a/sql/test/rel-optimizers/Tests/All 
b/sql/test/rel-optimizers/Tests/All
--- a/sql/test/rel-optimizers/Tests/All
+++ b/sql/test/rel-optimizers/Tests/All
@@ -4,3 +4,4 @@ groupjoin
 join-merge-remote-replica
 replicas-base
 replicas-join
+local-replica
diff --git a/sql/test/rel-optimizers/Tests/local-replica.reqtests 
b/sql/test/rel-optimizers/Tests/local-replica.reqtests
new file mode 100644
--- /dev/null
+++ b/sql/test/rel-optimizers/Tests/local-replica.reqtests
@@ -0,0 +1,1 @@
+replicas-base
diff --git a/sql/test/rel-optimizers/Tests/local-replica.test 
b/sql/test/rel-optimizers/Tests/local-replica.test
new file mode 100644
--- /dev/null
+++ b/sql/test/rel-optimizers/Tests/local-replica.test
@@ -0,0 +1,42 @@
+
+statement ok
+create replica table foo_rpl_loc (n int, m text)
+
+statement ok
+alter table foo_rpl_loc add table foo_local
+
+statement ok
+create replica table bar_rpl_loc (n int, m text)
+
+statement ok
+alter table bar_rpl_loc add table bar_local
+
+query T nosort
+plan select * from foo_rpl_loc
+
+project (
+| table("sys"."foo_local") [ "foo_local"."n" as "foo_rpl_loc"."n", 
"foo_local"."m" as "foo_rpl_loc"."m", "foo_local"."%TID%" NOT NULL UNIQUE as 
"foo_rpl_loc"."%TID%" ]
+) [ "foo_rpl_loc"."n", "foo_rpl_loc"."m" ]
+
+query T nosort
+plan select * from foo_rpl_loc, bar_rpl_loc
+
+project (
+| crossproduct (
+| | table("sys"."foo_local") [ "foo_local"."n" UNIQUE as "foo_rpl_loc"."n", 
"foo_local"."m" as "foo_rpl_loc"."m", "foo_local"."%TID%" NOT NULL UNIQUE as 
"foo_rpl_loc"."%TID%" ],
+| | table("sys"."bar_local") [ "bar_local"."n" as "bar_rpl_loc"."n", 
"bar_local"."m" as "bar_rpl_loc"."m", "bar_local"."%TID%" NOT NULL UNIQUE as 
"bar_rpl_loc"."%TID%" ]
+| ) [  ]
+) [ "foo_rpl_loc"."n", "foo_rpl_loc"."m", "bar_rpl_loc"."n", "bar_rpl_loc"."m" 
]
+
+query T nosort
+plan select * from foo_rpl_loc, buz_rmt
+
+project (
+| crossproduct (
+| | table("sys"."foo_local") [ "foo_local"."n" UNIQUE as "foo_rpl_loc"."n", 
"foo_local"."m" as "foo_rpl_loc"."m", "foo_local"."%TID%" NOT NULL UNIQUE as 
"foo_rpl_loc"."%TID%" ],
+| | table (
+| | | REMOTE("sys"."buz_rmt") [ "buz_rmt"."l" ] REMOTE 
mapi:monetdb://localhost:50002/node2
+| | ) [ "buz_rmt"."l" ]
+| ) [  ]
+) [ "foo_rpl_loc"."n", "foo_rpl_loc"."m", "buz_rmt"."l" ]
+
___
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org

MonetDB: literal_features - SQL23: introduce UNIQUE NULLS NOT DI...

2023-11-07 Thread Yunus Koning via checkin-list 
Changeset: f1b3e33b1168 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/f1b3e33b1168
Modified Files:
sql/backends/monet5/rel_bin.c
sql/include/sql_catalog.h
sql/server/rel_schema.c
sql/server/sql_parser.y
sql/server/sql_partition.c
sql/server/sql_tokens.h
Branch: literal_features
Log Message:

SQL23: introduce UNIQUE NULLS NOT DISTINCT constraint


diffs (138 lines):

diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -4643,7 +4643,9 @@ insert_check_ukey(backend *be, list *ins
stmt_add_column_predicate(be, c->c);
 
col = stmt_col(be, c->c, dels, dels->partition);
-   if ((k->type == ukey) && stmt_has_null(col)) {
+   if (k->type == unndkey)
+   s = stmt_uselect(be, col, cs, 
cmp_equal, s, 0, 1);
+   else if ((k->type == ukey) && 
stmt_has_null(col)) {
stmt *nn = stmt_selectnonil(be, col, s);
s = stmt_uselect(be, col, cs, 
cmp_equal, nn, 0, 0);
} else {
@@ -4668,7 +4670,7 @@ insert_check_ukey(backend *be, list *ins
list_append(lje, col);
list_append(rje, cs);
}
-   s = releqjoin(be, lje, rje, NULL, 1 /* hash used */, 0, 
0);
+   s = releqjoin(be, lje, rje, NULL, 1 /* hash used */, 0, 
k->type == unndkey? 1: 0);
s = stmt_result(be, s, 0);
}
s = stmt_binop(be, stmt_aggr(be, s, NULL, NULL, cnt, 1, 0, 1), 
stmt_atom_lng(be, 0), NULL, ne);
@@ -4732,12 +4734,12 @@ insert_check_ukey(backend *be, list *ins
s = stmt_project(be, nn, s);
}
if (h->nrcols) {
-   s = stmt_join(be, s, h, 0, cmp_equal, 0, 0, false);
+   s = stmt_join(be, s, h, 0, cmp_equal, 0, k->type == 
unndkey? 1: 0, false);
/* s should be empty */
s = stmt_result(be, s, 0);
s = stmt_aggr(be, s, NULL, NULL, cnt, 1, 0, 1);
} else {
-   s = stmt_uselect(be, s, h, cmp_equal, NULL, 0, 0);
+   s = stmt_uselect(be, s, h, cmp_equal, NULL, 0, k->type 
== unndkey? 1: 0);
/* s should be empty */
s = stmt_aggr(be, s, NULL, NULL, cnt, 1, 0, 1);
}
@@ -4844,7 +4846,7 @@ sql_insert_key(backend *be, list *insert
 *  insert values
 *  insert fkey/pkey index
 */
-   if (k->type == pkey || k->type == ukey) {
+   if (k->type == pkey || k->type == ukey || k->type == unndkey) {
return insert_check_ukey(be, inserts, k, idx_inserts);
} else {/* foreign keys */
return insert_check_fkey(be, inserts, k, idx_inserts, pin);
diff --git a/sql/include/sql_catalog.h b/sql/include/sql_catalog.h
--- a/sql/include/sql_catalog.h
+++ b/sql/include/sql_catalog.h
@@ -524,8 +524,9 @@ typedef struct sql_subfunc {
 
 typedef enum key_type {
pkey,
-   ukey,
-   fkey
+   ukey, /* default behavior is that NULLS are distinct, e.g. there can be 
multiple null values in a column with regular UNIQUE constraint */
+   fkey,
+   unndkey /* NULLS are not distinct, i.e. NULLS act as regular values for 
uniqueness checks */
 } key_type;
 
 typedef struct sql_kc {
diff --git a/sql/server/rel_schema.c b/sql/server/rel_schema.c
--- a/sql/server/rel_schema.c
+++ b/sql/server/rel_schema.c
@@ -366,8 +366,9 @@ column_constraint_type(mvc *sql, const c
}
switch (s->token) {
case SQL_UNIQUE:
+   case SQL_UNIQUE_NULLS_NOT_DISTINCT:
case SQL_PRIMARY_KEY: {
-   key_type kt = (s->token == SQL_UNIQUE) ? ukey : pkey;
+   key_type kt = (s->token == SQL_UNIQUE) ? ukey : (s->token == 
SQL_UNIQUE_NULLS_NOT_DISTINCT) ? unndkey : pkey;
sql_key *k;
const char *ns = name;
 
@@ -828,8 +829,9 @@ table_constraint_type(mvc *sql, const ch
 
switch (s->token) {
case SQL_UNIQUE:
+   case SQL_UNIQUE_NULLS_NOT_DISTINCT:
case SQL_PRIMARY_KEY: {
-   key_type kt = (s->token == SQL_PRIMARY_KEY ? pkey : ukey);
+   key_type kt = (s->token == SQL_PRIMARY_KEY ? pkey : s->token == 
SQL_UNIQUE ? ukey : unndkey);
dnode *nms = s->data.lval->h;
sql_key *k;
const char *ns = name;
diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -2144,6 +2144,7 @@ column_constrain

MonetDB: literal_features - merge with default

2023-11-07 Thread Yunus Koning via checkin-list 
Changeset: e274dae813b9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/e274dae813b9
Branch: literal_features
Log Message:

merge with default


diffs (truncated from 696 to 300 lines):

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -16,7 +16,6 @@ jobs:
 strategy:
   fail-fast: false  # don't stop other jobs
   matrix:
-branch: [ master ]
 os: [ ubuntu-latest, macos-latest, windows-latest ]
 c_compiler: [ gcc, clang, cl ]
 include:
@@ -42,13 +41,11 @@ jobs:
   - os: ubuntu-latest
 c_compiler: cl
 runs-on: ${{ matrix.os }}
-env:
-  CTEST: ${{ runner.os == 'Windows' && 'RUN_TESTS' || 'test' }}
 steps:
   - name: Checkout
 uses: actions/checkout@v3
 with:
-  ref: ${{ matrix.branch }}
+  ref: ${{ github.ref }}
 
   - name: install pymonetdb cryptography
 run: pip3 install pymonetdb cryptography
@@ -106,13 +103,22 @@ jobs:
   - name: ctest 
 run: |
   cd build 
-  cmake --build . --target ${{ CTEST }}
+  cmake --build . --target ${{ runner.os == 'Windows' && 'RUN_TESTS' 
|| 'test' }}
+if: runner.os != 'Windows'
 
   - name: mtest 
 run: |
   PATH=$HOME/MDB/bin:$PATH $HOME/MDB/bin/Mtest.py -r --debug=0 --ci 
--no-html --TSTTRGBASE=.
 if: runner.os != 'Windows'
 
+  - name: ctest 
+shell: pwsh
+run: |
+  $env:PATH = 
'C:\MDB\lib;C:\MDB\lib\monetdb5;C:\MDB\bin;C:\vcpkg\installed\x64-windows\bin;C:\vcpkg\installed\x64-windows\debug\bin;'
 + $env:PATH
+  cd build 
+  cmake --build . --target ${{ runner.os == 'Windows' && 'RUN_TESTS' 
|| 'test' }}
+if: runner.os == 'Windows'
+
   - name: mtest 
 shell: pwsh
 run: |
@@ -125,5 +131,5 @@ jobs:
   - name: Publish mtest results
 uses: actions/upload-artifact@v3
 with:
-  name: mtest-${{ matrix.branch }}-${{ matrix.os }}-${{ 
matrix.c_compiler }}
+  name: mtest-${{ github.sha }}-${{ matrix.os }}-${{ matrix.c_compiler 
}}
   path: mtests.tar
diff --git a/sql/backends/monet5/sql_gencode.c 
b/sql/backends/monet5/sql_gencode.c
--- a/sql/backends/monet5/sql_gencode.c
+++ b/sql/backends/monet5/sql_gencode.c
@@ -354,7 +354,8 @@ static int
Client c = MCgetClient(m->clientid);
MalBlkPtr curBlk = 0;
InstrPtr curInstr = 0, p, o;
-   sqlid table_id = prp->id;
+   tid_uri *tu = ((list*)prp->value.pval)->h->data;
+   sqlid table_id = tu->id;
node *n;
int i, q, v, res = -1, added_to_cache = 0, *lret, *rret;
size_t len = 1024, nr, pwlen = 0;
@@ -389,6 +390,7 @@ static int
 
sql_table *rt = sql_trans_find_table(m->session->tr, table_id);
const char *uri = mapiuri_uri(rt->query, m->sa);
+   assert(strcmp(tu->uri, uri) == 0);
if (!rt) {
sql_error(m, 10, SQLSTATE(HY013) MAL_MALLOC_FAIL);
goto cleanup;
@@ -928,8 +930,12 @@ static int
Symbol symbackup = c->curprg;
exception_buffer ebsave = m->sa->eb;
 
-   if (prp->id == 0) {
-   sql_error(m, 003, SQLSTATE(42000) "Missing property on the 
input relation");
+   if (list_empty(prp->value.pval)) {
+   sql_error(m, 003, SQLSTATE(42000) "Missing REMOTE property on 
the input relation");
+   goto bailout;
+   }
+   if (list_length(prp->value.pval) != 1) {
+   sql_error(m, 003, SQLSTATE(42000) "REMOTE property on the input 
relation is NOT unique");
goto bailout;
}
if (strlen(mod) >= IDLENGTH) {
diff --git a/sql/server/rel_distribute.c b/sql/server/rel_distribute.c
--- a/sql/server/rel_distribute.c
+++ b/sql/server/rel_distribute.c
@@ -15,6 +15,12 @@
 #include "rel_remote.h"
 #include "sql_privileges.h"
 
+typedef struct rmt_prop_state {
+   int depth;
+   prop* rmt;
+   sql_rel* orig;
+} rps;
+
 static int
 has_remote_or_replica( sql_rel *rel )
 {
@@ -66,7 +72,7 @@ has_remote_or_replica( sql_rel *rel )
 }
 
 static sql_rel *
-rewrite_replica(mvc *sql, list *exps, sql_table *t, sql_table *p, int 
remote_prop)
+do_replica_rewrite(mvc *sql, list *exps, sql_table *t, sql_table *p, int 
remote_prop)
 {
node *n, *m;
sql_rel *r = rel_basetable(sql, p, t->base.name);
@@ -96,11 +102,15 @@ rewrite_replica(mvc *sql, list *exps, sq
 
/* set_remote() */
if (remote_prop && p && isRemote(p)) {
-   sqlid id = p->base.id;
-   char *local_name = sa_strconcat(sql->sa, sa_strconcat(sql->sa, 
p->s->base.name, "."), p->base.name);
-   prop *p = r->p = prop_create(sql->sa, PROP_REMOTE, r->p);
-   p->id = id;
-   p->value.pval = local_name;
+   list *uris = sa_list(sql->sa);
+   tid_uri *tu = SA_NEW(sql->sa,

MonetDB: literal_features - merge with default

2023-11-07 Thread Yunus Koning via checkin-list 
Changeset: b21b6ac059ba for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b21b6ac059ba
Branch: literal_features
Log Message:

merge with default


diffs (truncated from 22292 to 300 lines):

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -3,7 +3,6 @@ name: MonetDB build and test
 on:
   push:
 branches: 
-  - '*'
   - 'branches/*'
   pull_request:
   # Allows you to run this workflow manually from the Actions tab
@@ -18,40 +17,113 @@ jobs:
   fail-fast: false  # don't stop other jobs
   matrix:
 branch: [ master ]
-os: [ ubuntu-latest, macos-latest ]
+os: [ ubuntu-latest, macos-latest, windows-latest ]
+c_compiler: [ gcc, clang, cl ]
+include:
+  - os: windows-latest
+c_compiler: cl
+  - os: macos-latest
+c_compiler: clang
+  - os: macos-latest
+c_compiler: gcc-12
+  - os: ubuntu-latest
+c_compiler: gcc
+  - os: ubuntu-latest
+c_compiler: clang
+exclude:
+  - os: windows-latest
+c_compiler: gcc
+  - os: windows-latest
+c_compiler: clang
+  - os: macos-latest
+c_compiler: cl
+  - os: macos-latest
+c_compiler: gcc
+  - os: ubuntu-latest
+c_compiler: cl
 runs-on: ${{ matrix.os }}
+env:
+  CTEST: ${{ runner.os == 'Windows' && 'RUN_TESTS' || 'test' }}
 steps:
   - name: Checkout
 uses: actions/checkout@v3
 with:
   ref: ${{ matrix.branch }}
 
+  - name: install pymonetdb cryptography
+run: pip3 install pymonetdb cryptography
+
   - name: make MonetDB on linux
 run: |
   mkdir build
   cd build 
-  cmake ..  -DCMAKE_INSTALL_PREFIX=$HOME/${{ matrix.branch }} 
-DPY3INTEGRATION=OFF -DCMAKE_BUILD_TYPE=Release -DASSERT=OFF -DRINTEGRATION=OFF
+  cmake ..  \
+-DCMAKE_INSTALL_PREFIX=$HOME/MDB \
+-DPY3INTEGRATION=OFF \
+-DRINTEGRATION=OFF \
+-DCMAKE_BUILD_TYPE=Release \
+-DASSERT=OFF \
+-DCMAKE_C_COMPILER=${{ matrix.c_compiler }} \
+-DCMAKE_SUMMARY=ON
   make install -j3
 if: runner.os == 'Linux'
-  -
-name: brew packages
+
+  - name: brew packages
 run: brew install bison
 if: runner.os == 'macOS'
+
   - name: make MonetDB on macos
 run: |
   mkdir build
   cd build 
-  cmake ..  -DCMAKE_INSTALL_PREFIX=$HOME/${{ matrix.branch }} 
-DPY3INTEGRATION=OFF -DCMAKE_BUILD_TYPE=Release -DASSERT=OFF -DRINTEGRATION=OFF 
 \
--DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
+  cmake .. \
+-DCMAKE_INSTALL_PREFIX=$HOME/MDB \
+-DPY3INTEGRATION=OFF \
+-DRINTEGRATION=OFF  \
+-DCMAKE_BUILD_TYPE=Release \
+-DASSERT=OFF \
+-DCMAKE_C_COMPILER=${{ matrix.c_compiler }} \
+-DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison \
+-DCMAKE_SUMMARY=ON
   make install -j3
 if: runner.os == 'macOS'
+
+  - name: choco packages
+run: |
+  choco install winflexbison3
+  vcpkg install libiconv bzip2 libxml2 pcre zlib getopt 
+if: runner.os == 'Windows'
+
+  - name: make MonetDB on Windows
+shell: pwsh
+run: |
+  mkdir build
+  cd build 
+  cmake ..  -DCMAKE_INSTALL_PREFIX=C:\MDB 
-DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake 
-DPY3INTEGRATION=OFF -DRINTEGRATION=OFF  -DCMAKE_BUILD_TYPE=Release 
-DASSERT=OFF -DCMAKE_C_COMPILER=${{ matrix.c_compiler }}  -DCMAKE_SUMMARY=ON
+  cmake --build . --target install
+if: runner.os == 'Windows'
+
   - name: ctest 
 run: |
   cd build 
-  cmake --build . --target test
-  #-
-  #uses: actions/upload-artifact@v3
-  #name: Publish Linux binary wheels
-  #with:
-  #name: monetdbe-linux-wheel-${{ matrix.branch }}-${{ 
matrix.python-version }}
-  #path: dist/*.whl
+  cmake --build . --target ${{ CTEST }}
+
+  - name: mtest 
+run: |
+  PATH=$HOME/MDB/bin:$PATH $HOME/MDB/bin/Mtest.py -r --debug=0 --ci 
--no-html --TSTTRGBASE=.
+if: runner.os != 'Windows'
+
+  - name: mtest 
+shell: pwsh
+run: |
+  $env:PATH = 
'C:\MDB\lib;C:\MDB\lib\monetdb5;C:\MDB\bin;C:\vcpkg\installed\x64-windows\bin;C:\vcpkg\installed\x64-windows\debug\bin;'
 + $env:PATH
+  python C:\MDB\bin\Mtest.py -r --debug=0 --ci --no-html --TSTTRGBASE=.
+if: runner.os == 'Windows'
+
+  - name: Tar files
+run: tar -cvf mtests.tar mTests
+  - name: Publish mtest results
+uses: actions/upload-artifact@v3
+with:
+