Changeset: 16361d60ba35 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=16361d60ba35
Modified Files:
gdk/gdk_aggr.c
gdk/gdk_calc.h
monetdb5/modules/kernel/00_aggr_hge.mal
monetdb5/modules/kernel/00_aggr_hge.mal.sh
monetdb5/modules/kernel/aggr.c
monetdb5/modules/kernel/aggr.mal
monetdb5/modules/kernel/aggr.mal.sh
monetdb5/modules/kernel/algebra.c
monetdb5/modules/kernel/algebra.h
monetdb5/modules/kernel/algebra.mal
sql/backends/monet5/sql_upgrades.c
sql/scripts/39_analytics.sql
sql/scripts/39_analytics_hge.sql
Branch: statistics-analytics
Log Message:
Implementing missing covar_samp and covar_pop aggregates
diffs (truncated from 937 to 300 lines):
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -3151,6 +3151,76 @@ BATcalcvariance_sample(dbl *avgp, BAT *b
"BATcalcvariance_sample");
}
+#define AGGR_COVARIANCE_SINGLE(TYPE) \
+ do { \
+ TYPE x, y; \
+ for (i = 0; i < cnt; i++) { \
+ x = ((const TYPE *) v1)[i]; \
+ y = ((const TYPE *) v2)[i]; \
+ if (is_##TYPE##_nil(x) || is_##TYPE##_nil(y)) \
+ continue; \
+ n++; \
+ delta1 = (dbl) x - mean1; \
+ mean1 += delta1 / n; \
+ delta2 = (dbl) y - mean2; \
+ mean2 += delta2 / n; \
+ m2 += delta1 * ((dbl) y - mean2); \
+ } \
+ } while (0)
+
+static dbl
+calccovariance(const void *restrict v1, const void *restrict v2, BUN cnt, int
tp, bool issample, const char *func)
+{
+ BUN n = 0, i;
+ dbl mean1 = 0, mean2 = 0, m2 = 0, delta1, delta2;
+
+ switch (tp) {
+ case TYPE_bte:
+ AGGR_COVARIANCE_SINGLE(bte);
+ break;
+ case TYPE_sht:
+ AGGR_COVARIANCE_SINGLE(sht);
+ break;
+ case TYPE_int:
+ AGGR_COVARIANCE_SINGLE(int);
+ break;
+ case TYPE_lng:
+ AGGR_COVARIANCE_SINGLE(lng);
+ break;
+#ifdef HAVE_HGE
+ case TYPE_hge:
+ AGGR_COVARIANCE_SINGLE(hge);
+ break;
+#endif
+ case TYPE_flt:
+ AGGR_COVARIANCE_SINGLE(flt);
+ break;
+ case TYPE_dbl:
+ AGGR_COVARIANCE_SINGLE(dbl);
+ break;
+ default:
+ GDKerror("%s: type (%s) not supported.\n", func, ATOMname(tp));
+ return dbl_nil;
+ }
+ if (n <= (BUN) issample)
+ return dbl_nil;
+ return m2 / (n - issample);
+}
+
+dbl
+BATcalccovariance_population(BAT *b1, BAT *b2)
+{
+ return calccovariance((const void *) Tloc(b1, 0), (const void *)
Tloc(b2, 0),
+ BATcount(b1), b1->ttype,
false, "BATcalccovariance_population");
+}
+
+dbl
+BATcalccovariance_sample(BAT *b1, BAT *b2)
+{
+ return calccovariance((const void *) Tloc(b1, 0), (const void *)
Tloc(b2, 0),
+ BATcount(b1), b1->ttype,
true, "BATcalccovariance_sample");
+}
+
#define AGGR_STDEV(TYPE) \
do { \
const TYPE *restrict vals = (const TYPE *) Tloc(b, 0); \
@@ -3382,3 +3452,179 @@ BATgroupvariance_population(BAT *b, BAT
return dogroupstdev(NULL, b, g, e, s, tp, skip_nils, false, true,
"BATgroupvariance_population");
}
+
+#define AGGR_COVARIANCE(TYPE) \
+ do { \
+ const TYPE *restrict vals1 = (const TYPE *) Tloc(b1, 0);
\
+ const TYPE *restrict vals2 = (const TYPE *) Tloc(b2, 0);
\
+ while (ncand > 0) { \
+ ncand--; \
+ i = canditer_next(&ci) - b1->hseqbase; \
+ if (gids == NULL || \
+ (gids[i] >= min && gids[i] <= max)) { \
+ if (gids) \
+ gid = gids[i] - min; \
+ else \
+ gid = (oid) i; \
+ if (is_##TYPE##_nil(vals1[i]) ||
is_##TYPE##_nil(vals2[i])) { \
+ if (!skip_nils) \
+ cnts[gid] = BUN_NONE; \
+ } else if (cnts[gid] != BUN_NONE) { \
+ cnts[gid]++; \
+ delta1[gid] = (dbl) vals1[i] -
mean1[gid]; \
+ mean1[gid] += delta1[gid] / cnts[gid]; \
+ delta2[gid] = (dbl) vals2[i] -
mean2[gid]; \
+ mean2[gid] += delta2[gid] / cnts[gid]; \
+ m2[gid] += delta1[gid] * ((dbl)
vals2[i] - mean2[gid]); \
+ } \
+ } \
+ } \
+ for (i = 0; i < ngrp; i++) { \
+ if (cnts[i] == 0 || cnts[i] == BUN_NONE) { \
+ dbls[i] = dbl_nil; \
+ mean1[i] = dbl_nil; \
+ mean2[i] = dbl_nil; \
+ nils++; \
+ } else if (cnts[i] == 1) { \
+ dbls[i] = issample ? dbl_nil : 0; \
+ nils2++; \
+ } else { \
+ dbls[i] = m2[i] / (cnts[i] - issample); \
+ } \
+ } \
+ } while (0)
+
+static BAT *
+dogroupcovariance(BAT *b1, BAT *b2, BAT *g, BAT *e, BAT *s, int tp,
+ bool skip_nils, bool issample, const char
*func)
+{
+ const oid *restrict gids;
+ oid gid, min, max;
+ BUN i, ngrp, nils = 0, nils2 = 0, ncand;
+ BUN *restrict cnts = NULL;
+ dbl *restrict dbls, *restrict mean1, *restrict mean2, *restrict delta1,
*restrict delta2, *restrict m2;
+ BAT *bn = NULL;
+ struct canditer ci;
+ const char *err;
+
+ assert(tp == TYPE_dbl && BATcount(b1) == BATcount(b2) && b1->ttype ==
b2->ttype && BATtdense(b1) == BATtdense(b2));
+ (void) tp;
+
+ if ((err = BATgroupaggrinit(b1, g, e, s, &min, &max, &ngrp, &ci,
&ncand)) != NULL) {
+ GDKerror("%s: %s\n", func, err);
+ return NULL;
+ }
+ if (g == NULL) {
+ GDKerror("%s: b1, b2 and g must be aligned\n", func);
+ return NULL;
+ }
+
+ if (BATcount(b1) == 0 || ngrp == 0)
+ return BATconstant(ngrp == 0 ? 0 : min, TYPE_dbl, &dbl_nil,
ngrp, TRANSIENT);
+
+ if ((e == NULL ||
+ (BATcount(e) == BATcount(b1) && (e->hseqbase == b1->hseqbase ||
e->hseqbase == b2->hseqbase))) &&
+ (BATtdense(g) || (g->tkey && g->tnonil)) &&
+ (issample || (b1->tnonil && b2->tnonil))) {
+ /* trivial: singleton groups, so all results are equal
+ * to zero (population) or nil (sample) */
+ dbl v = issample ? dbl_nil : 0;
+ return BATconstant(ngrp == 0 ? 0 : min, TYPE_dbl, &v, ngrp,
TRANSIENT);
+ }
+
+ delta1 = GDKmalloc(ngrp * sizeof(dbl));
+ delta2 = GDKmalloc(ngrp * sizeof(dbl));
+ m2 = GDKzalloc(ngrp * sizeof(dbl));
+ cnts = GDKzalloc(ngrp * sizeof(BUN));
+ mean1 = GDKzalloc(ngrp * sizeof(dbl));
+ mean2 = GDKzalloc(ngrp * sizeof(dbl));
+
+ if (mean1 == NULL || mean2 == NULL || delta1 == NULL || delta2 == NULL
|| m2 == NULL || cnts == NULL)
+ goto alloc_fail;
+
+ bn = COLnew(min, TYPE_dbl, ngrp, TRANSIENT);
+ if (bn == NULL)
+ goto alloc_fail;
+ dbls = (dbl *) Tloc(bn, 0);
+
+ if (!g || BATtdense(g))
+ gids = NULL;
+ else
+ gids = (const oid *) Tloc(g, 0);
+
+ switch (b1->ttype) {
+ case TYPE_bte:
+ AGGR_COVARIANCE(bte);
+ break;
+ case TYPE_sht:
+ AGGR_COVARIANCE(sht);
+ break;
+ case TYPE_int:
+ AGGR_COVARIANCE(int);
+ break;
+ case TYPE_lng:
+ AGGR_COVARIANCE(lng);
+ break;
+#ifdef HAVE_HGE
+ case TYPE_hge:
+ AGGR_COVARIANCE(hge);
+ break;
+#endif
+ case TYPE_flt:
+ AGGR_COVARIANCE(flt);
+ break;
+ case TYPE_dbl:
+ AGGR_COVARIANCE(dbl);
+ break;
+ default:
+ BBPreclaim(bn);
+ GDKfree(mean1);
+ GDKfree(mean2);
+ GDKfree(delta1);
+ GDKfree(delta2);
+ GDKfree(m2);
+ GDKfree(cnts);
+ GDKerror("%s: type (%s) not supported.\n", func,
ATOMname(b1->ttype));
+ return NULL;
+ }
+ GDKfree(mean1);
+ GDKfree(mean2);
+
+ if (issample)
+ nils += nils2;
+ GDKfree(delta1);
+ GDKfree(delta2);
+ GDKfree(m2);
+ GDKfree(cnts);
+ BATsetcount(bn, ngrp);
+ bn->tkey = ngrp <= 1;
+ bn->tsorted = ngrp <= 1;
+ bn->trevsorted = ngrp <= 1;
+ bn->tnil = nils != 0;
+ bn->tnonil = nils == 0;
+ return bn;
+alloc_fail:
+ BBPreclaim(bn);
+ GDKfree(mean1);
+ GDKfree(mean2);
+ GDKfree(delta1);
+ GDKfree(delta2);
+ GDKfree(m2);
+ GDKfree(cnts);
+ GDKerror("%s: cannot allocate enough memory.\n", func);
+ return NULL;
+}
+
+BAT *
+BATgroupcovariance_sample(BAT *b1, BAT *b2, BAT *g, BAT *e, BAT *s, int tp,
bool skip_nils, bool abort_on_error)
+{
+ (void) abort_on_error;
+ return dogroupcovariance(b1, b2, g, e, s, tp, skip_nils, true,
"BATgroupcovariance_sample");
+}
+
+BAT *
+BATgroupcovariance_population(BAT *b1, BAT *b2, BAT *g, BAT *e, BAT *s, int
tp, bool skip_nils, bool abort_on_error)
+{
+ (void) abort_on_error;
+ return dogroupcovariance(b1, b2, g, e, s, tp, skip_nils, false,
"BATgroupcovariance_population");
+}
diff --git a/gdk/gdk_calc.h b/gdk/gdk_calc.h
--- a/gdk/gdk_calc.h
+++ b/gdk/gdk_calc.h
@@ -154,6 +154,10 @@ gdk_export dbl BATcalcvariance_populatio
gdk_export dbl BATcalcvariance_sample(dbl *avgp, BAT *b);
gdk_export BAT *BATgroupvariance_sample(BAT *b, BAT *g, BAT *e, BAT *s, int
tp, bool skip_nils, bool abort_on_error);
gdk_export BAT *BATgroupvariance_population(BAT *b, BAT *g, BAT *e, BAT *s,
int tp, bool skip_nils, bool abort_on_error);
+gdk_export dbl BATcalccovariance_sample(BAT *b1, BAT *b2);
+gdk_export dbl BATcalccovariance_population(BAT *b1, BAT *b2);
+gdk_export BAT *BATgroupcovariance_sample(BAT *b1, BAT *b2, BAT *g, BAT *e,
BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupcovariance_population(BAT *b1, BAT *b2, BAT *g, BAT
*e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
gdk_export BAT *BATgroupstr_group_concat(BAT *b, BAT *g, BAT *e, BAT *s, bool
skip_nils, bool abort_on_error, const char *separator);
gdk_export gdk_return BATstr_group_concat(ValPtr res, BAT *b, BAT *s, bool
skip_nils, bool abort_on_error, bool nil_if_empty, const char *separator);
diff --git a/monetdb5/modules/kernel/00_aggr_hge.mal
b/monetdb5/modules/kernel/00_aggr_hge.mal
--- a/monetdb5/modules/kernel/00_aggr_hge.mal
+++ b/monetdb5/modules/kernel/00_aggr_hge.mal
@@ -210,3 +210,27 @@ command subvariancep(b:bat[:hge],g:bat[:
address AGGRsubvariancepcand_dbl
comment "Grouped variance (population/biased) aggregate with candidates list";
+command covariance(b:bat[:hge],c:bat[:hge],g:bat[:oid],e:bat[:any_1])
:bat[:dbl]
+address AGGRcovariance
+comment "Covariance sample aggregate";
+
+command
subcovariance(b:bat[:hge],c:bat[:hge],g:bat[:oid],e:bat[:any_1],skip_nils:bit,abort_on_error:bit)
:bat[:dbl]
+address AGGRsubcovariance
+comment "Grouped covariance sample aggregate";
+
+command
subcovariance(b:bat[:hge],c:bat[:hge],g:bat[:oid],e:bat[:any_1],s:bat[:oid],skip_nils:bit,abort_on_error:bit)
:bat[:dbl]
+address AGGRsubcovariancecand
+comment "Grouped covariance sample aggregate with candidate list";
+
+command covariancep(b:bat[:hge],c:bat[:hge],g:bat[:oid],e:bat[:any_1])
:bat[:dbl]
+address AGGRcovariancep
+comment "Covariance population aggregate";
+
+command
subcovariancep(b:bat[:hge],c:bat[:hge],g:bat[:oid],e:bat[:any_1],skip_nils:bit,abort_on_error:bit)
:bat[:dbl]
+address AGGRsubcovariancep
+comment "Grouped covariance population aggregate";
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
