Changeset: 5a50ae340f07 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5a50ae340f07
Added Files:
sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.sql
sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.err
sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.out
sql/test/mergetables/Tests/mergedb.Bug-6820.reqtests
sql/test/mergetables/Tests/mergedb_drop.reqtests
sql/test/mergetables/Tests/qlsmith-exist-lateral.reqtests
sql/test/mergetables/Tests/sqlsmith-apply-outer-join-or.sql
sql/test/mergetables/Tests/sqlsmith-apply-outer-join-or.stable.err
sql/test/mergetables/Tests/sqlsmith-apply-outer-join-or.stable.out
sql/test/mergetables/Tests/sqlsmith-exist-lateral.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6426.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6451.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6453.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6455.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6459.reqtests
sql/test/mergetables/Tests/sqlsmith.Bug-6459.stable.out.int128
Modified Files:
gdk/gdk_batop.c
gdk/gdk_heap.c
monetdb5/modules/atoms/mtime.c
sql/server/rel_optimizer.c
sql/test/BugTracker-2017/Tests/All
sql/test/mergetables/Tests/All
sql/test/mergetables/Tests/sqlsmith.Bug-6459.stable.out
Branch: default
Log Message:
Merge with Jul2017 branch.
diffs (truncated from 672 to 300 lines):
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -407,6 +407,113 @@ insert_string_bat(BAT *b, BAT *n, BAT *s
return GDK_FAIL;
}
+static gdk_return
+append_varsized_bat(BAT *b, BAT *n, BAT *s)
+{
+ BATiter ni;
+ BUN start, end, cnt, r;
+ const oid *restrict cand = NULL, *candend = NULL;
+
+ /* only transient bats can use some other bat's vheap */
+ assert(b->batRole == TRANSIENT || b->tvheap->parentid == b->batCacheid);
+ /* make sure the bats use var_t */
+ assert(b->twidth == n->twidth);
+ assert(b->twidth == SIZEOF_VAR_T);
+ if (n->batCount == 0 || (s && s->batCount == 0))
+ return GDK_SUCCEED;
+ CANDINIT(n, s, start, end, cnt, cand, candend);
+ cnt = cand ? (BUN) (candend - cand) : end - start;
+ if (cnt == 0)
+ return GDK_SUCCEED;
+ if (BATcount(b) == 0 &&
+ b->batRole == TRANSIENT &&
+ n->batRestricted == BAT_READ &&
+ b->tvheap != n->tvheap) {
+ /* if b is still empty, in the transient farm, and n
+ * is read-only, we replace b's vheap with a reference
+ * to n's */
+ if (b->tvheap->parentid != b->batCacheid) {
+ BBPunshare(b->tvheap->parentid);
+ } else {
+ HEAPfree(b->tvheap, 1);
+ GDKfree(b->tvheap);
+ }
+ BBPshare(n->tvheap->parentid);
+ b->tvheap = n->tvheap;
+ }
+ if (b->tvheap == n->tvheap) {
+ /* if b and n use the same vheap, we only need to copy
+ * the offsets from n to b */
+ HASHdestroy(b); /* not maintaining, so destroy it */
+ if (cand == NULL) {
+ /* fast memcpy since we copy a consecutive
+ * chunk of memory */
+ memcpy(Tloc(b, BUNlast(b)),
+ Tloc(n, start),
+ cnt * b->twidth);
+ } else {
+ var_t *restrict dst = (var_t *) Tloc(b, BUNlast(b));
+ oid hseq = n->hseqbase;
+ const var_t *restrict src = (const var_t *) Tloc(n, 0);
+ while (cand < candend)
+ *dst++ = src[*cand++ - hseq];
+ }
+ BATsetcount(b, BATcount(b) + cnt);
+ return GDK_SUCCEED;
+ }
+ /* b and n do not share their vheap, so we need to copy data */
+ if (b->tvheap->parentid != b->batCacheid) {
+ /* if b shares its vheap with some other bat, unshare it */
+ Heap *h = GDKzalloc(sizeof(Heap));
+ if (h == NULL)
+ return GDK_FAIL;
+ h->parentid = b->batCacheid;
+ h->farmid = BBPselectfarm(b->batRole, b->ttype, varheap);
+ if (b->tvheap->filename) {
+ const char *nme = BBP_physical(b->batCacheid);
+ h->filename = GDKfilepath(NOFARM, NULL, nme, "theap");
+ if (h->filename == NULL) {
+ GDKfree(h);
+ return GDK_FAIL;
+ }
+ }
+ if (HEAPcopy(h, b->tvheap) != GDK_SUCCEED) {
+ HEAPfree(h, 1);
+ GDKfree(h);
+ return GDK_FAIL;
+ }
+ BBPunshare(b->tvheap->parentid);
+ b->tvheap = h;
+ }
+ /* copy data from n to b */
+ ni = bat_iterator(n);
+ r = BUNlast(b);
+ if (cand) {
+ oid hseq = n->hseqbase;
+ while (cand < candend) {
+ const void *t = BUNtvar(ni, *cand - hseq);
+ bunfastapp_nocheck(b, r, t, Tsize(b));
+ HASHins(b, r, t);
+ r++;
+ cand++;
+ }
+ } else {
+ while (start < end) {
+ const void *t = BUNtvar(ni, start);
+ bunfastapp_nocheck(b, r, t, Tsize(b));
+ HASHins(b, r, t);
+ r++;
+ start++;
+ }
+ }
+ return GDK_SUCCEED;
+
+ bunins_failed:
+ if (b->tunique)
+ BBPunfix(s->batCacheid);
+ return GDK_FAIL;
+}
+
/* Append the contents of BAT n (subject to the optional candidate
* list s) to BAT b. If b is empty, b will get the seqbase of s if it
* was passed in, and else the seqbase of n. */
@@ -421,9 +528,7 @@ BATappend(BAT *b, BAT *n, BAT *s, bit fo
return GDK_SUCCEED;
}
assert(b->batCacheid > 0);
- /* almost: assert(!isVIEW(b)); */
- assert(b->theap.parentid == 0 &&
- (b->tvheap == NULL || b->tvheap->parentid == b->batCacheid ||
b->ttype == TYPE_str));
+ assert(b->theap.parentid == 0);
ALIGNapp(b, "BATappend", force, GDK_FAIL);
BATcompatible(b, n, GDK_FAIL, "BATappend");
@@ -605,9 +710,14 @@ BATappend(BAT *b, BAT *n, BAT *s, bit fo
BBPunfix(s->batCacheid);
return GDK_FAIL;
}
+ } else if (ATOMvarsized(b->ttype)) {
+ if (append_varsized_bat(b, n, s) != GDK_SUCCEED) {
+ if (b->tunique)
+ BBPunfix(s->batCacheid);
+ return GDK_FAIL;
+ }
} else {
- if (!ATOMvarsized(b->ttype) &&
- BATatoms[b->ttype].atomFix == NULL &&
+ if (BATatoms[b->ttype].atomFix == NULL &&
b->ttype != TYPE_void &&
n->ttype != TYPE_void &&
cand == NULL) {
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -1024,7 +1024,8 @@ HEAP_malloc(Heap *heap, size_t nbytes)
size_t newsize;
assert(heap->free + MAX(heap->free, nbytes) <= VAR_MAX);
- newsize = (size_t) roundup_8(heap->free + MAX(heap->free,
nbytes));
+ newsize = MIN(heap->free, (size_t) 1 << 20);
+ newsize = (size_t) roundup_8(heap->free + MAX(newsize, nbytes));
assert(heap->free <= VAR_MAX);
block = (size_t) heap->free; /* current end-of-heap */
@@ -1032,8 +1033,7 @@ HEAP_malloc(Heap *heap, size_t nbytes)
fprintf(stderr, "#No block found\n");
#endif
- /* Double the size of the heap.
- * TUNE: increase heap by different amount. */
+ /* Increase the size of the heap. */
HEAPDEBUG fprintf(stderr, "#HEAPextend in HEAP_malloc %s "
SZFMT " " SZFMT "\n", heap->filename, heap->size, newsize);
if (HEAPextend(heap, newsize, FALSE) != GDK_SUCCEED)
return 0;
diff --git a/monetdb5/modules/atoms/mtime.c b/monetdb5/modules/atoms/mtime.c
--- a/monetdb5/modules/atoms/mtime.c
+++ b/monetdb5/modules/atoms/mtime.c
@@ -3623,6 +3623,7 @@ MTIMEdate_to_str(str *s, const date *d,
fromdate(*d, &t.tm_mday, &mon, &year);
t.tm_mon = mon - 1;
t.tm_year = year - 1900;
+ (void)mktime(&t); /* corrects the tm_wday etc */
if ((sz = strftime(buf, BUFSIZ, *format, &t)) == 0)
throw(MAL, "mtime.date_to_str", "failed to convert date to
string using format '%s'\n", *format);
*s = GDKmalloc(sz + 1);
@@ -3667,6 +3668,7 @@ MTIMEtime_to_str(str *s, const daytime *
memset(&t, 0, sizeof(struct tm));
fromtime(*d, &t.tm_hour, &t.tm_min, &t.tm_sec, &msec);
(void)msec;
+ (void)mktime(&t); /* corrects the tm_wday etc */
if ((sz = strftime(buf, BUFSIZ, *format, &t)) == 0)
throw(MAL, "mtime.time_to_str", "failed to convert time to
string using format '%s'\n", *format);
*s = GDKmalloc(sz + 1);
@@ -3714,6 +3716,7 @@ MTIMEtimestamp_to_str(str *s, const time
t.tm_mon = mon - 1;
t.tm_year = year - 1900;
fromtime(ts->msecs, &t.tm_hour, &t.tm_min, &t.tm_sec, &msec);
+ (void)mktime(&t); /* corrects the tm_wday etc */
(void)msec;
if ((sz = strftime(buf, BUFSIZ, *format, &t)) == 0)
throw(MAL, "mtime.timestamp_to_str", "failed to convert
timestampt to string using format '%s'\n", *format);
diff --git a/sql/server/rel_optimizer.c b/sql/server/rel_optimizer.c
--- a/sql/server/rel_optimizer.c
+++ b/sql/server/rel_optimizer.c
@@ -8684,9 +8684,12 @@ rel_apply_rewrite(int *changes, mvc *sql
p = rel_projections(sql, rel, NULL, 1, 1);
nl = rel_apply(sql, rel_dup(rel->l), rel_dup(r->l), rel->exps,
rel->flag);
nr = rel_apply(sql, rel_dup(rel->l), rel_dup(r->r), rel->exps,
rel->flag);
+ nl = rel_project(sql->sa, nl, rel_projections(sql, nl, NULL, 1,
1));
+ nr = rel_project(sql->sa, nr, rel_projections(sql, nr, NULL, 1,
1));
l = rel_setop(sql->sa, nl, nr, op_union);
l->flag = r->flag;
- l->exps = list_merge(p, r->exps, (fdup)NULL);
+ l->exps = p; //list_merge(p, r->exps, (fdup)NULL);
+ assert(list_length(nl->exps) == list_length(nr->exps) &&
list_length(nl->exps) == list_length(l->exps));
set_processed(l);
rel_destroy(rel);
(*changes)++;
@@ -8760,11 +8763,17 @@ rel_apply_rewrite(int *changes, mvc *sql
return l;
} else { /* both unused */
int flag = rel->flag;
- list *exps = r->exps;
-
- assert(is_join(r->op));
- r = rel_crossproduct(sql->sa, rel_dup(r->l),
rel_dup(r->r), flag == APPLY_LOJ?op_left:op_join);
- r->exps = exps_copy(sql->sa, exps);
+
+ assert(is_join(r->op) || is_semi(r->op));
+ if (is_join(r->op)) {
+ list *exps = r->exps;
+
+ r = rel_crossproduct(sql->sa, rel_dup(r->l),
rel_dup(r->r), flag == APPLY_LOJ?op_left:op_join);
+ r->exps = exps_copy(sql->sa, exps);
+ } else if (is_semi(r->op)) {
+ assert(flag != APPLY_LOJ);
+ r = rel_dup(r);
+ }
rel_destroy(rel);
(*changes)++;
return r;
diff --git a/sql/test/BugTracker-2017/Tests/All
b/sql/test/BugTracker-2017/Tests/All
--- a/sql/test/BugTracker-2017/Tests/All
+++ b/sql/test/BugTracker-2017/Tests/All
@@ -107,3 +107,4 @@ insert_into_multiple_subqueries.Bug-6448
HAVE_SAMTOOLS?sqlsmith.Bug-6449
HAVE_LIBPY?python_loader_clobbers_default_with_null.Bug-6464
skip_problem_best_effort.Bug-6442
+date_to_str.Bug-6467
diff --git a/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.sql
b/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.sql
@@ -0,0 +1,1 @@
+SELECT date '2017-11-14', EXTRACT(year from date '2017-11-14'),
date_to_str(date '2017-11-14', '%Y %m %d -A:%A -G:%G V:%V w:%w W:%W');
diff --git a/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.err
b/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'date_to_str.Bug-6467` in directory 'sql/test/BugTracker-2017`
itself:
+
+
+# 14:32:50 >
+# 14:32:50 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=30624" "--set"
"mapi_usock=/var/tmp/mtest-29760/.s.monetdb.30624" "--set" "monet_prompt="
"--forcemito"
"--dbpath=/home/niels/scratch/rc-monetdb/Linux-x86_64/var/MonetDB/mTests_sql_test_BugTracker-2017"
+# 14:32:50 >
+
+# builtin opt gdk_dbpath =
/home/niels/scratch/rc-monetdb/Linux-x86_64/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 30624
+# cmdline opt mapi_usock = /var/tmp/mtest-29760/.s.monetdb.30624
+# cmdline opt monet_prompt =
+# cmdline opt gdk_dbpath =
/home/niels/scratch/rc-monetdb/Linux-x86_64/var/MonetDB/mTests_sql_test_BugTracker-2017
+# cmdline opt gdk_debug = 536870922
+
+# 14:32:51 >
+# 14:32:51 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-29760" "--port=30624"
+# 14:32:51 >
+
+
+# 14:32:51 >
+# 14:32:51 > "Done."
+# 14:32:51 >
+
diff --git a/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.out
b/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2017/Tests/date_to_str.Bug-6467.stable.out
@@ -0,0 +1,37 @@
+stdout of test 'date_to_str.Bug-6467` in directory 'sql/test/BugTracker-2017`
itself:
+
+
+# 14:32:50 >
+# 14:32:50 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=30624" "--set"
"mapi_usock=/var/tmp/mtest-29760/.s.monetdb.30624" "--set" "monet_prompt="
"--forcemito"
"--dbpath=/home/niels/scratch/rc-monetdb/Linux-x86_64/var/MonetDB/mTests_sql_test_BugTracker-2017"
+# 14:32:50 >
+
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
