Changeset: b63323e589e3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b63323e589e3
Added Files:
sql/test/Tests/hot-snapshot-bz2.py
sql/test/Tests/hot-snapshot-bz2.stable.err
sql/test/Tests/hot-snapshot-bz2.stable.out
sql/test/Tests/hot-snapshot-gz.py
sql/test/Tests/hot-snapshot-gz.stable.err
sql/test/Tests/hot-snapshot-gz.stable.out
sql/test/Tests/hot-snapshot-lz4.py
sql/test/Tests/hot-snapshot-lz4.stable.err
sql/test/Tests/hot-snapshot-lz4.stable.out
sql/test/Tests/hot-snapshot-xz.py
sql/test/Tests/hot-snapshot-xz.stable.err
sql/test/Tests/hot-snapshot-xz.stable.out
sql/test/Tests/hot_snapshot_compression.py
Modified Files:
common/stream/stream.c
sql/server/rel_optimizer.c
sql/server/rel_unnest.c
sql/storage/store.c
sql/test/Tests/All
sql/test/mergetables/Tests/sqlsmith-exists2.stable.out
Branch: default
Log Message:
Merge with Jun2020 branch.
diffs (truncated from 670 to 300 lines):
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -447,6 +447,8 @@ mnstr_write(stream *restrict s, const vo
assert(!s->readonly);
if (s->errnr)
return -1;
+ if (cnt == 0)
+ return 0;
return s->write(s, buf, elmsize, cnt);
}
@@ -1766,7 +1768,7 @@ stream_lz4write(stream *restrict s, cons
lz4->total_processing += ret;
}
- if(lz4->total_processing == lz4->ring_buffer_size) {
+ if (lz4->total_processing > 0) {
real_written = fwrite((void *)lz4->ring_buffer, 1,
lz4->total_processing, lz4->fp);
if (real_written == 0) {
s->errnr = MNSTR_WRITE_ERROR;
@@ -1780,52 +1782,6 @@ stream_lz4write(stream *restrict s, cons
return (ssize_t) (total_written / elmsize);
}
-static void
-stream_lz4close(stream *s)
-{
- lz4_stream *lz4 = s->stream_data.p;
-
- if (lz4) {
- if (!s->readonly) {
- size_t ret, real_written;
-
- if (lz4->total_processing > 0 && lz4->total_processing
< lz4->ring_buffer_size) { /* compress remaining */
- real_written = fwrite(lz4->ring_buffer, 1,
lz4->total_processing, lz4->fp);
- if (real_written == 0) {
- s->errnr = MNSTR_WRITE_ERROR;
- return ;
- }
- lz4->total_processing = 0;
- } /* finish compression */
- ret = LZ4F_compressEnd(lz4->context.comp_context,
lz4->ring_buffer, lz4->ring_buffer_size, NULL);
- if(LZ4F_isError(ret)) {
- s->errnr = MNSTR_WRITE_ERROR;
- return ;
- }
- assert(ret < LZ4DECOMPBUFSIZ);
- lz4->total_processing = ret;
-
- real_written = fwrite(lz4->ring_buffer, 1,
lz4->total_processing, lz4->fp);
- if (real_written == 0) {
- s->errnr = MNSTR_WRITE_ERROR;
- return ;
- }
- lz4->total_processing = 0;
-
- fflush(lz4->fp);
- }
- if(!s->readonly) {
- (void)
LZ4F_freeCompressionContext(lz4->context.comp_context);
- } else {
- (void)
LZ4F_freeDecompressionContext(lz4->context.dec_context);
- }
- fclose(lz4->fp);
- free(lz4->ring_buffer);
- free(lz4);
- }
- s->stream_data.p = NULL;
-}
-
static int
stream_lz4flush(stream *s)
{
@@ -1862,6 +1818,25 @@ stream_lz4flush(stream *s)
return 0;
}
+static void
+stream_lz4close(stream *s)
+{
+ lz4_stream *lz4 = s->stream_data.p;
+
+ if (lz4) {
+ stream_lz4flush(s);
+ if(!s->readonly) {
+ (void)
LZ4F_freeCompressionContext(lz4->context.comp_context);
+ } else {
+ (void)
LZ4F_freeDecompressionContext(lz4->context.dec_context);
+ }
+ fclose(lz4->fp);
+ free(lz4->ring_buffer);
+ free(lz4);
+ }
+ s->stream_data.p = NULL;
+}
+
static stream *
open_lz4stream(const char *restrict filename, const char *restrict flags)
{
diff --git a/sql/server/rel_optimizer.c b/sql/server/rel_optimizer.c
--- a/sql/server/rel_optimizer.c
+++ b/sql/server/rel_optimizer.c
@@ -1640,6 +1640,7 @@ rel_push_count_down(mvc *sql, sql_rel *r
exp_label(sql->sa, e, ++sql->label);
cnt = exp_ref(sql, e);
gbl = rel_groupby(sql, rel_dup(srel), NULL);
+ set_processed(gbl);
rel_groupby_add_aggr(sql, gbl, e);
append(args, cnt);
}
@@ -1652,6 +1653,7 @@ rel_push_count_down(mvc *sql, sql_rel *r
exp_label(sql->sa, e, ++sql->label);
cnt = exp_ref(sql, e);
gbr = rel_groupby(sql, rel_dup(srel), NULL);
+ set_processed(gbr);
rel_groupby_add_aggr(sql, gbr, e);
append(args, cnt);
}
@@ -1664,6 +1666,7 @@ rel_push_count_down(mvc *sql, sql_rel *r
rel_destroy(rel);
rel = rel_project(sql->sa, cp, append(new_exp_list(sql->sa),
nce));
+ set_processed(rel);
(*changes)++;
}
@@ -4379,10 +4382,12 @@ gen_push_groupby_down(mvc *sql, sql_rel
else
cr = j->l = rel_groupby(sql, cr, gbe);
cr->exps = list_merge(cr->exps, aggrs, (fdup)NULL);
+ set_processed(cr);
if (!is_project(cl->op))
cl = rel_project(sql->sa, cl,
rel_projections(sql, cl, NULL, 1, 1));
cl->exps = list_merge(cl->exps, aliases, (fdup)NULL);
+ set_processed(cl);
if (!left)
j->l = cl;
else
@@ -5340,7 +5345,7 @@ find_simple_projection_for_join2semi(sql
sql_rel *res = NULL;
sql_exp *found = NULL;
- if (is_groupby(rel->op) || find_prop(e->p,
PROP_HASHCOL))
+ if (is_groupby(rel->op) || need_distinct(rel) ||
find_prop(e->p, PROP_HASHCOL))
return true;
found = rel_find_exp_and_corresponding_rel(rel->l, e,
&res); /* grouping column on inner relation */
@@ -5348,7 +5353,7 @@ find_simple_projection_for_join2semi(sql
if (find_prop(found->p, PROP_HASHCOL)) /*
primary key always unique */
return true;
if (found->type == e_column && found->card <=
CARD_AGGR) {
- if (!is_groupby(res->op) &&
list_length(res->exps) != 1)
+ if (!(is_groupby(res->op) ||
need_distinct(res)) && list_length(res->exps) != 1)
return false;
for (node *n = res->exps->h ; n ; n =
n->next) { /* must be the single column in the group by expression list */
sql_exp *e = n->data;
@@ -6113,6 +6118,7 @@ rel_groupby_distinct(mvc *sql, sql_rel *
arg->h->data = darg;
l = rel->l = rel_groupby(sql, rel->l, gbe);
l->exps = exps;
+ set_processed(l);
rel->r = ngbe;
rel->exps = nexps;
set_nodistinct(distinct);
diff --git a/sql/server/rel_unnest.c b/sql/server/rel_unnest.c
--- a/sql/server/rel_unnest.c
+++ b/sql/server/rel_unnest.c
@@ -1999,8 +1999,10 @@ rel_union_exps(mvc *sql, sql_exp **l, li
if (exp_has_rel(ve))
sq = exp_rel_get_rel(sql->sa, ve); /* get subquery */
- else
+ else {
sq = rel_project(sql->sa, NULL,
append(sa_list(sql->sa), ve));
+ set_processed(sq);
+ }
if (is_tuple) { /* cast each one */
for (node *m=sq->exps->h, *o = ((list *)(*l)->f)->h; m
&& o; m = m->next, o = o->next) {
r = m->data;
@@ -2024,6 +2026,7 @@ rel_union_exps(mvc *sql, sql_exp **l, li
if (exp_has_rel(ve)) {
ve = exp_rel_update_exp(sql, ve);
sq = rel_project(sql->sa, sq,
append(sa_list(sql->sa), ve));
+ set_processed(sq);
}
exp_set_freevar(sql, ve, sq);
}
@@ -2033,6 +2036,7 @@ rel_union_exps(mvc *sql, sql_exp **l, li
} else {
u = rel_setop(sql->sa, u, sq, op_union);
rel_set_exps(u, exps);
+ set_processed(u);
}
exps = rel_projections(sql, sq, NULL, 1/*keep names */, 1);
}
@@ -2142,6 +2146,7 @@ rewrite_anyequal(mvc *sql, sql_rel *rel,
} else if (has_label(re)) {
re = exp_ref(sql, re);
}
+ set_processed(rsq);
}
if (is_project(rel->op) || (is_select(rel->op) &&
(depth > 0 || (!is_tuple && rsq && rel_has_freevar(sql, rsq) &&
!is_anyequal(sf))))) {
@@ -2216,6 +2221,7 @@ rewrite_anyequal(mvc *sql, sql_rel *rel,
le = rel_groupby_add_aggr(sql, lsq, a);
if (exp_name(e))
exp_prop_alias(sql->sa, le, e);
+ set_processed(lsq);
return le;
} else {
if (lsq)
@@ -2342,6 +2348,7 @@ rewrite_compare(mvc *sql, sql_rel *rel,
list_hash_clear(rsq->exps);
re = exp_ref(sql, re);
}
+ set_processed(rsq);
}
int is_cnt = 0;
@@ -2377,6 +2384,7 @@ rewrite_compare(mvc *sql, sql_rel *rel,
}
re = exp_aggr1(sql->sa, re, a, 0, 1,
CARD_AGGR, has_nil(re));
re = rel_groupby_add_aggr(sql, rsq, re);
+ set_processed(rsq);
} else if (rsq && exp_card(re) > CARD_ATOM) {
sql_subfunc *zero_or_one =
sql_bind_func(sql->sa, NULL, compare_aggr_op(op, quantifier), exp_subtype(re),
NULL, F_AGGR);
@@ -2384,6 +2392,7 @@ rewrite_compare(mvc *sql, sql_rel *rel,
re = exp_aggr1(sql->sa, re,
zero_or_one, 0, 0, CARD_AGGR, has_nil(re));
re = rel_groupby_add_aggr(sql, rsq, re);
+ set_processed(rsq);
}
if (rsq)
(void)rewrite_inner(sql, rel, rsq,
is_cnt?op_left:op_join);
@@ -2607,6 +2616,7 @@ rewrite_exists(mvc *sql, sql_rel *rel, s
return NULL;
if (exp_name(e))
exp_prop_alias(sql->sa, le, e);
+ set_processed(sq);
} else { /* rewrite into semi/anti join */
(void)rewrite_inner(sql, rel, sq,
is_exists(sf)?op_semi:op_anti);
return exp_atom_bool(sql->sa, 1);
@@ -2631,6 +2641,7 @@ rewrite_ifthenelse(mvc *sql, sql_rel *re
sql_rel *r = rel->l = rel_project(sql->sa, rel->l,
rel_projections(sql, rel->l, NULL, 1, 1));
e->ref = 0;
+ set_processed(r);
append(r->exps, e);
return exp_ref(sql, e);
}
@@ -2671,6 +2682,7 @@ rewrite_ifthenelse(mvc *sql, sql_rel *re
}
lsq = rel_project(sql->sa, lsq,
append(sa_list(sql->sa), then_exp));
exp_set_freevar(sql, cond, lsq);
+ set_processed(lsq);
lsq = rel_select(sql->sa, lsq, cond);
if (exp_has_rel(else_exp)) {
exp_set_freevar(sql, else_exp, rsq);
@@ -2684,11 +2696,13 @@ rewrite_ifthenelse(mvc *sql, sql_rel *re
cond = exp_copy(sql, cond);
exp_set_freevar(sql, cond, rsq);
cond = rel_unop_(sql, rsq, cond, NULL, "not",
card_value);
+ set_processed(rsq);
rsq = rel_select(sql->sa, rsq, cond);
usq = rel_setop(sql->sa, lsq, rsq, op_union);
rel_set_exps(usq, append(sa_list(sql->sa), exp_ref(sql,
e)));
if (single)
set_single(usq);
+ set_processed(usq);
e = exp_rel(sql, usq);
} else
if (!e->used && (has_nil(cond) || (inner &&
is_outerjoin(inner->op))) && (cond->type != e_func || !is_isnull_func(nf))) {
@@ -2751,6 +2765,7 @@ rewrite_remove_xp_project(mvc *sql, sql_
if (is_atom(t->type) && !exp_name(t)) { /* atom
with out alias cannot be used later */
rel = rel_project(sql->sa, rel->l,
rel_projections(sql, rel->l, NULL, 1, 1));
list_merge(rel->exps, r->exps,
(fdup)NULL);
+ set_processed(rel);
(*changes)++;
}
}
@@ -2819,6 +2834,7 @@ rewrite_fix_count(mvc *sql, sql_rel *rel
if (rel_changes) { /* add project */
exps = list_merge(rel_projections(sql, rel->l,
NULL, 1, 1), rexps, (fdup)NULL);
rel = rel_project(sql->sa, rel, exps);
+ set_processed(rel);
r->used = 1;
(*changes)++;
}
@@ -2902,7 +2918,9 @@ rewrite_groupings(mvc *sql, sql_rel *rel
append(pexps, ne);
}
nrel->exps = exps;
+ set_processed(nrel);
nrel = rel_project(sql->sa, nrel, pexps);
+ set_processed(nrel);
if (!unions)
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
