Changeset: 7c6d965023b1 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/7c6d965023b1
Modified Files:
monetdb5/mal/mal_pipelines.c
monetdb5/modules/mal/heapn.c
sql/backends/monet5/rel_physical.c
sql/backends/monet5/rel_pphash.c
sql/server/rel_optimize_sel.c
sql/server/rel_optimizer.c
sql/server/rel_statistics.c
Branch: default
Log Message:
when rewriting order of joins, sometimes the counts/estimates where not
properly updated, leading to huge hash tables.
diffs (truncated from 418 to 300 lines):
diff --git a/monetdb5/mal/mal_pipelines.c b/monetdb5/mal/mal_pipelines.c
--- a/monetdb5/mal/mal_pipelines.c
+++ b/monetdb5/mal/mal_pipelines.c
@@ -284,6 +284,11 @@ runMALpipelines(Client cntxt, MalBlkPtr
if (!s)
throw(MAL, "pipelines", SQLSTATE(HY013) MAL_MALLOC_FAIL);
MT_lock_set(&pipelineLock);
+ if (ATOMIC_GET(&exiting)) {
+ MT_lock_unset(&pipelineLock);
+ return MAL_SUCCEED;
+ }
+
bool profiler = cntxt->sqlprofiler;
*s = (Pipelines) {
.mb = mb,
@@ -332,9 +337,13 @@ runMALpipelines(Client cntxt, MalBlkPtr
}
if (j) {
/* here we should wait */
- MT_lock_unset(&pipelineLock);
+ MT_lock_unset(&pipelineLock);
MT_sleep_ms(100);
- MT_lock_set(&pipelineLock);
+ MT_lock_set(&pipelineLock);
+ if (ATOMIC_GET(&exiting)) {
+ MT_lock_unset(&pipelineLock);
+ return MAL_SUCCEED;
+ }
}
}
@@ -345,6 +354,10 @@ runMALpipelines(Client cntxt, MalBlkPtr
MT_sema_destroy(&s->s);
MT_lock_destroy(&s->l);
MT_lock_set(&pipelineLock);
+ if (ATOMIC_GET(&exiting)) {
+ MT_lock_unset(&pipelineLock);
+ return MAL_SUCCEED;
+ }
bool has_sink = (s->sink != 0);
str err = s->error;
if (err) {
diff --git a/monetdb5/modules/mal/heapn.c b/monetdb5/modules/mal/heapn.c
--- a/monetdb5/modules/mal/heapn.c
+++ b/monetdb5/modules/mal/heapn.c
@@ -1275,6 +1275,7 @@ extern void BATswap_heaps(BAT *u, BAT *b
\
for(; i<cnt; i++) \
ri[pi[i]] = bi[si[i]]; \
+ handled = true; \
}
#define aproject(T,w,Toff) \
@@ -1284,6 +1285,7 @@ extern void BATswap_heaps(BAT *u, BAT *b
\
for(; i<cnt; i++) \
ri[pi[i]] = bi[si[i]]; \
+ handled = true; \
}
#define aproject_(T) \
@@ -1300,6 +1302,7 @@ extern void BATswap_heaps(BAT *u, BAT *b
err = 1; \
} \
bat_iterator_end(&bi); \
+ handled = true; \
}
static str
@@ -1402,6 +1405,7 @@ HEAPproject(Client ctx, bat *rid, bat *p
oid *si = Tloc(S, 0);
size_t i = 0;
BUN cnt = BATcount(P);
+ bool handled = false;
project(bte)
project(sht)
@@ -1415,7 +1419,7 @@ HEAPproject(Client ctx, bat *rid, bat *p
project(flt)
project(dbl)
project(uuid)
- if (local_storage) {
+ if (local_storage && !handled) {
// aproject_(str)
if (ATOMstorage(tt) == TYPE_str) { \
BATiter bi = bat_iterator(b); \
@@ -1433,13 +1437,15 @@ HEAPproject(Client ctx, bat *rid, bat *p
err = 1; \
} \
bat_iterator_end(&bi); \
+ handled = true; \
}
- } else {
+ } else if(!handled) {
aproject(str,1,uint8_t)
aproject(str,2,uint16_t)
aproject(str,4,uint32_t)
aproject(str,8,var_t)
}
+ assert(handled);
if (!err) {
if (!private)
pipeline_lock2(r);
diff --git a/sql/backends/monet5/rel_physical.c
b/sql/backends/monet5/rel_physical.c
--- a/sql/backends/monet5/rel_physical.c
+++ b/sql/backends/monet5/rel_physical.c
@@ -451,6 +451,8 @@ find_payload_exps(mvc *sql, list **exps_
else
exps = list_merge(list_dup(exps, NULL), p->attr, NULL);
}
+ if (is_topn(p->op) && p->l)
+ p = p->l;
if (is_select(p->op) || is_join(p->op))
exps = NULL;
if (exps_hsh) {
diff --git a/sql/backends/monet5/rel_pphash.c b/sql/backends/monet5/rel_pphash.c
--- a/sql/backends/monet5/rel_pphash.c
+++ b/sql/backends/monet5/rel_pphash.c
@@ -803,9 +803,13 @@ rel2bin_oahash_cart(backend *be, sql_rel
assert(stmts_ht->type == st_list && stmts_prb_res->type == st_list);
stmt *rowrepeat = stmts_ht->op4.lval->h->data;
- stmt *setrepeat = stmts_prb_res->op4.lval->h->data;
- assert(rowrepeat && setrepeat); /* must find */
+ stmt *setrepeat =
!list_empty(stmts_prb_res->op4.lval)?stmts_prb_res->op4.lval->h->data:NULL;
+ prop *count = find_prop(rel->p, PROP_COUNT);
+ (void)count;
+ assert(rowrepeat && (setrepeat || (count && !count->value.lval))); /*
must find */
+ if (!setrepeat)
+ setrepeat = stmt_bat_new(be, sql_fetch_localtype(TYPE_bte), 0);
/* Check for single: either rel->r is single or rel->l is empty. */
if (rel->single && list_empty(rel->exps)) {
if (rowrepeat->nrcols > 0) {
diff --git a/sql/server/rel_optimize_sel.c b/sql/server/rel_optimize_sel.c
--- a/sql/server/rel_optimize_sel.c
+++ b/sql/server/rel_optimize_sel.c
@@ -1306,6 +1306,8 @@ push_up_join_exps( mvc *sql, sql_rel *re
if (rel_is_ref(rel))
return NULL;
+ if (is_single(rel))
+ return NULL;
switch(rel->op) {
case op_join: {
sql_rel *rl = rel->l;
@@ -1614,6 +1616,7 @@ find_fk( mvc *sql, list *rels, list *exp
i->p = NULL;
je = exp_compare(sql->sa, i, t,
cmp_equal);
}
+ t->p = prop_create(sql->sa,
PROP_HASHCOL, NULL);
/* Remove all join expressions */
for (n = eje->h; n; n = n->next) {
@@ -4038,7 +4041,7 @@ rel_join_order_(visitor *v, sql_rel *rel
}
if (is_join(rel->op))
rel = reorder_join(v, rel);
- if (rel && v->opt >= 0)
+ if (rel_is_ref(rel) && rel && v->opt >= 0)
rel->opt = v->opt;
return rel;
}
diff --git a/sql/server/rel_optimizer.c b/sql/server/rel_optimizer.c
--- a/sql/server/rel_optimizer.c
+++ b/sql/server/rel_optimizer.c
@@ -108,6 +108,11 @@ rel_unionize_mt_tables_munion(visitor *v
/* create the munion */
sql_rel *mu = rel_setop_n_ary(v->sql->sa, rels, op_munion);
rel_setop_n_ary_set_exps(v->sql, mu, rel_projections(v->sql, mt, NULL,
1, 1), true);
+ for(node *n = mu->exps->h; n; n = n->next) {
+ sql_exp *e = n->data;
+ e->unique = 0;
+ }
+
set_processed(mu);
return mu;
@@ -413,8 +418,11 @@ merge_table_prune_and_unionize(visitor *
if (tables->cnt == 1) {
nrel = rel_wrap_select_around_mt_child(v,
tables->h->data, info);
} else {
- //nrel = rel_unionize_mt_tables_balanced(v,
mt_rel, tables, info);
nrel = rel_setop_n_ary(v->sql->sa, tables,
op_munion);
+ for(node *n = nrel->exps->h; n; n = n->next) {
+ sql_exp *e = n->data;
+ e->unique = 0;
+ }
}
} else if (mvc_debug_on(v->sql, 32)) {
for (node *n = tables->h; n ; n = n->next) {
@@ -433,6 +441,10 @@ merge_table_prune_and_unionize(visitor *
if (nrel) {
nrel = rel_setop_n_ary(v->sql->sa,
append(append(sa_list(v->sql->sa), nrel), next), op_munion);
rel_setop_n_ary_set_exps(v->sql, nrel,
rel_projections(v->sql, mt_rel, NULL, 1, 1), true);
+ for(node *n = nrel->exps->h; n; n =
n->next) {
+ sql_exp *e = n->data;
+ e->unique = 0;
+ }
set_processed(nrel);
} else {
nrel = next;
diff --git a/sql/server/rel_statistics.c b/sql/server/rel_statistics.c
--- a/sql/server/rel_statistics.c
+++ b/sql/server/rel_statistics.c
@@ -1297,9 +1297,8 @@ rel_get_statistics(visitor *v, global_pr
run_optimizer
bind_get_statistics(visitor *v, global_props *gp)
{
- bool pipeline_mode = MT_thread_get_qry_ctx()->pipeline_mode;
(void) v;
- return (!pipeline_mode && gp->opt_level == 1 /*&&
!gp->cnt[op_insert]*/) ? rel_get_statistics : NULL;
+ return (gp->opt_level == 1) ? rel_get_statistics : NULL;
}
@@ -1590,6 +1589,177 @@ bind_final_optimization_loop(visitor *v,
(flag & push_select_up) && (flag &
optimize_select_and_joins_topdown) && (flag & optimize_projections) ?
rel_final_optimization_loop : NULL;
}
+static sql_rel *
+rel_estimate(visitor *v, sql_rel *rel)
+{
+ /* Don't prune updates as pruning will possibly result in removing the
joins which therefore cannot be used for constraint checking */
+ sql_rel *l = rel->l, *r = rel->r;
+
+ if (is_join(rel->op) || is_semi(rel->op)) {
+ rel->l = rel_estimate(v, rel->l);
+ rel->r = rel_estimate(v, rel->r);
+ }
+ switch (rel->op) {
+ case op_join:
+ case op_left:
+ case op_right:
+ case op_full:
+ {
+ /* propagate row count */
+ BUN lv = get_rel_count(l), rv = get_rel_count(r),
r_uniques_estimate = BUN_MAX, join_idx_estimate = BUN_MAX;
+
+ //assert (lv < 1000000000 && rv < 1000000000);
+ if (!list_empty(rel->exps) && !is_single(rel) && rel->op ==
op_join) {
+ for (node *n = rel->exps->h ; n ; n = n->next) {
+ sql_exp *e = n->data, *el = e->l, *er = e->r;
+ BUN uniques_estimate = BUN_MAX;
+
+ if (find_prop(e->p, PROP_JOINIDX)) {
+ prop *lsp = find_prop(l->p,
PROP_SELECTIVITY), *rsp = find_prop(r->p, PROP_SELECTIVITY);
+ dbl ls = lsp && is_join(l->op) ?
lsp->value.dval : 1.0;
+ dbl rs = rsp && is_join(r->op) ?
rsp->value.dval : 1.0;
+ join_idx_estimate = lv>rv?lv:rv;
+ join_idx_estimate =
(BUN)(join_idx_estimate * ls * rs);
+ if (join_idx_estimate < 1)
+ join_idx_estimate = 1;
+ dbl s =
((dbl)join_idx_estimate/(lv*rv));
+ prop *sp = prop_create(v->sql->sa,
PROP_SELECTIVITY, (prop *) e->p);
+ sp->value.dval = s;
+ e->p = sp;
+ } else if (e->type == e_cmp && e->flag ==
cmp_equal) {
+ /* if one of the sides is unique, the
cardinality will be at most the cardinality of the other side */
+ if (!is_semantics(e) || !has_nil(el) ||
!has_nil(er)) {
+ BUN lu = lv, ru = rv;
+ prop *p = NULL;
+ if ((p = find_prop(el->p,
PROP_NUNIQUES)))
+ lu = (BUN)
p->value.dval;
+ if ((p = find_prop(er->p,
PROP_NUNIQUES)))
+ ru = (BUN)
p->value.dval;
+ if (is_unique(el) || lu > lv) {
+ BUN ncount =
(is_right(rel->op) || is_full(rel->op) || !lv || lu < lv) ? MAX(lv, rv) :
rv/(lu/lv);
+ if (ncount == 0)
+ ncount =
MAX(lv, rv);
+ uniques_estimate =
MIN(uniques_estimate, ncount);
+ } else if (is_unique(er) || ru
> rv) {
+ BUN ncount =
(is_left(rel->op) || is_full(rel->op) || !rv || ru < rv) ? MAX(lv, rv) :
lv/(ru/rv);
+ if (ncount == 0)
+ ncount =
MAX(lv, rv);
+ uniques_estimate =
MIN(uniques_estimate, ncount);
+ }
+ if (uniques_estimate < BUN_MAX)
{
+ if (uniques_estimate <
r_uniques_estimate)
+
r_uniques_estimate = uniques_estimate;
+ dbl s =
((dbl)uniques_estimate/(lv*rv));
+ prop *sp =
prop_create(v->sql->sa, PROP_SELECTIVITY, (prop *) e->p);
+ sp->value.dval = s;
+ e->p = sp;
+ } else {
+ dbl u = (dbl)MAX(lu,
ru);
+ dbl s = u/(lv*rv);
+ //dbl s = 1/u;
+ prop *sp =
prop_create(v->sql->sa, PROP_SELECTIVITY, (prop *) e->p);
+ sp->value.dval = s;
+ e->p = sp;
+ }
+ }
+ }
+ }
+ }
+ if (is_single(rel) || is_left(rel->op)) {
+ set_count_prop(v->sql->sa, rel, lv);
+ } else if (is_right(rel->op)) {
+ set_count_prop(v->sql->sa, rel, rv);
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
