I wrote: > Before v12, stddev_pop() had the following behavior with just a > single input value: > ... > As of v12, though, all three cases produce 0. I am not sure what > to think about that with respect to an infinity input, but I'm > quite sure I don't like it for NaN input.
While I'm still not sure whether there's an academic argument that zero is a reasonable stddev value for a single input that is Inf, it seems to me that backwards compatibility is a sufficient reason for going back to producing NaN for that. Hence, attached are some proposed patches. 0001 just adds test cases demonstrating the current behavior; then 0002 makes the proposed code change. It's easy to check that the test case results after 0002 match what v11 produces. 0003 deals with a different problem which I noted in [1]: the numeric variants of var_samp and stddev_samp also do the wrong thing for a single special input. Their disease is that they produce NaN for a single NaN input, where it seems more sensible to produce NULL. At least, NULL is what we get for the same case with the float aggregates, so we have to change one or the other set of functions if we want consistency. I propose back-patching 0001/0002 as far as v12, since the failure to match the old outputs seems like a pretty clear bug/regression. However, I'd be content to apply 0003 only to HEAD. That misbehavior is very ancient, and the lack of complaints suggests that few people really care about this fine point. regards, tom lane [1] https://www.postgresql.org/message-id/606717.1591924582%40sss.pgh.pa.us
diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index d659013e41..0a6884d382 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -127,7 +127,79 @@ SELECT var_samp(b::numeric) FROM aggtest; -- population variance is defined for a single tuple, sample variance -- is not -SELECT var_pop(1.0), var_samp(2.0); +SELECT var_pop(1.0::float8), var_samp(2.0::float8); + var_pop | var_samp +---------+---------- + 0 | +(1 row) + +SELECT stddev_pop(3.0::float8), stddev_samp(4.0::float8); + stddev_pop | stddev_samp +------------+------------- + 0 | +(1 row) + +SELECT var_pop('inf'::float8), var_samp('inf'::float8); + var_pop | var_samp +---------+---------- + 0 | +(1 row) + +SELECT stddev_pop('inf'::float8), stddev_samp('inf'::float8); + stddev_pop | stddev_samp +------------+------------- + 0 | +(1 row) + +SELECT var_pop('nan'::float8), var_samp('nan'::float8); + var_pop | var_samp +---------+---------- + 0 | +(1 row) + +SELECT stddev_pop('nan'::float8), stddev_samp('nan'::float8); + stddev_pop | stddev_samp +------------+------------- + 0 | +(1 row) + +SELECT var_pop(1.0::float4), var_samp(2.0::float4); + var_pop | var_samp +---------+---------- + 0 | +(1 row) + +SELECT stddev_pop(3.0::float4), stddev_samp(4.0::float4); + stddev_pop | stddev_samp +------------+------------- + 0 | +(1 row) + +SELECT var_pop('inf'::float4), var_samp('inf'::float4); + var_pop | var_samp +---------+---------- + 0 | +(1 row) + +SELECT stddev_pop('inf'::float4), stddev_samp('inf'::float4); + stddev_pop | stddev_samp +------------+------------- + 0 | +(1 row) + +SELECT var_pop('nan'::float4), var_samp('nan'::float4); + var_pop | var_samp +---------+---------- + 0 | +(1 row) + +SELECT stddev_pop('nan'::float4), stddev_samp('nan'::float4); + stddev_pop | stddev_samp +------------+------------- + 0 | +(1 row) + +SELECT var_pop(1.0::numeric), var_samp(2.0::numeric); var_pop | var_samp ---------+---------- 0 | @@ -139,6 +211,18 @@ SELECT stddev_pop(3.0::numeric), stddev_samp(4.0::numeric); 0 | (1 row) +SELECT var_pop('nan'::numeric), var_samp('nan'::numeric); + var_pop | var_samp +---------+---------- + NaN | NaN +(1 row) + +SELECT stddev_pop('nan'::numeric), stddev_samp('nan'::numeric); + stddev_pop | stddev_samp +------------+------------- + NaN | NaN +(1 row) + -- verify correct results for null and NaN inputs select sum(null::int4) from generate_series(1,3); sum @@ -299,6 +383,25 @@ SELECT corr(b, a) FROM aggtest; 0.139634516517873 (1 row) +-- check single-tuple behavior +SELECT covar_pop(1::float8,2::float8), covar_samp(3::float8,4::float8); + covar_pop | covar_samp +-----------+------------ + 0 | +(1 row) + +SELECT covar_pop(1::float8,'inf'::float8), covar_samp(3::float8,'inf'::float8); + covar_pop | covar_samp +-----------+------------ + 0 | +(1 row) + +SELECT covar_pop(1::float8,'nan'::float8), covar_samp(3::float8,'nan'::float8); + covar_pop | covar_samp +-----------+------------ + 0 | +(1 row) + -- test accum and combine functions directly CREATE TABLE regr_test (x float8, y float8); INSERT INTO regr_test VALUES (10,150),(20,250),(30,350),(80,540),(100,200); diff --git a/src/test/regress/sql/aggregates.sql b/src/test/regress/sql/aggregates.sql index 2a066f5a3a..044d515507 100644 --- a/src/test/regress/sql/aggregates.sql +++ b/src/test/regress/sql/aggregates.sql @@ -39,8 +39,22 @@ SELECT var_samp(b::numeric) FROM aggtest; -- population variance is defined for a single tuple, sample variance -- is not -SELECT var_pop(1.0), var_samp(2.0); +SELECT var_pop(1.0::float8), var_samp(2.0::float8); +SELECT stddev_pop(3.0::float8), stddev_samp(4.0::float8); +SELECT var_pop('inf'::float8), var_samp('inf'::float8); +SELECT stddev_pop('inf'::float8), stddev_samp('inf'::float8); +SELECT var_pop('nan'::float8), var_samp('nan'::float8); +SELECT stddev_pop('nan'::float8), stddev_samp('nan'::float8); +SELECT var_pop(1.0::float4), var_samp(2.0::float4); +SELECT stddev_pop(3.0::float4), stddev_samp(4.0::float4); +SELECT var_pop('inf'::float4), var_samp('inf'::float4); +SELECT stddev_pop('inf'::float4), stddev_samp('inf'::float4); +SELECT var_pop('nan'::float4), var_samp('nan'::float4); +SELECT stddev_pop('nan'::float4), stddev_samp('nan'::float4); +SELECT var_pop(1.0::numeric), var_samp(2.0::numeric); SELECT stddev_pop(3.0::numeric), stddev_samp(4.0::numeric); +SELECT var_pop('nan'::numeric), var_samp('nan'::numeric); +SELECT stddev_pop('nan'::numeric), stddev_samp('nan'::numeric); -- verify correct results for null and NaN inputs select sum(null::int4) from generate_series(1,3); @@ -81,6 +95,11 @@ SELECT regr_slope(b, a), regr_intercept(b, a) FROM aggtest; SELECT covar_pop(b, a), covar_samp(b, a) FROM aggtest; SELECT corr(b, a) FROM aggtest; +-- check single-tuple behavior +SELECT covar_pop(1::float8,2::float8), covar_samp(3::float8,4::float8); +SELECT covar_pop(1::float8,'inf'::float8), covar_samp(3::float8,'inf'::float8); +SELECT covar_pop(1::float8,'nan'::float8), covar_samp(3::float8,'nan'::float8); + -- test accum and combine functions directly CREATE TABLE regr_test (x float8, y float8); INSERT INTO regr_test VALUES (10,150),(20,250),(30,350),(80,540),(100,200);
diff --git a/src/backend/utils/adt/float.c b/src/backend/utils/adt/float.c index 2101d58674..6a717f19bb 100644 --- a/src/backend/utils/adt/float.c +++ b/src/backend/utils/adt/float.c @@ -2925,6 +2925,17 @@ float8_accum(PG_FUNCTION_ARGS) Sxx = get_float8_nan(); } } + else + { + /* + * At the first input, we normally can leave Sxx as 0. However, if + * the first input is Inf or NaN, we'd better force Sxx to NaN; + * otherwise we will falsely report variance zero when there are no + * more inputs. + */ + if (isnan(newval) || isinf(newval)) + Sxx = get_float8_nan(); + } /* * If we're invoked as an aggregate, we can cheat and modify our first @@ -2999,6 +3010,17 @@ float4_accum(PG_FUNCTION_ARGS) Sxx = get_float8_nan(); } } + else + { + /* + * At the first input, we normally can leave Sxx as 0. However, if + * the first input is Inf or NaN, we'd better force Sxx to NaN; + * otherwise we will falsely report variance zero when there are no + * more inputs. + */ + if (isnan(newval) || isinf(newval)) + Sxx = get_float8_nan(); + } /* * If we're invoked as an aggregate, we can cheat and modify our first @@ -3225,6 +3247,19 @@ float8_regr_accum(PG_FUNCTION_ARGS) Sxy = get_float8_nan(); } } + else + { + /* + * At the first input, we normally can leave Sxx et al as 0. However, + * if the first input is Inf or NaN, we'd better force the dependent + * sums to NaN; otherwise we will falsely report variance zero when + * there are no more inputs. + */ + if (isnan(newvalX) || isinf(newvalX)) + Sxx = Sxy = get_float8_nan(); + if (isnan(newvalY) || isinf(newvalY)) + Syy = Sxy = get_float8_nan(); + } /* * If we're invoked as an aggregate, we can cheat and modify our first diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index 0a6884d382..e4ffa5ee42 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -142,25 +142,25 @@ SELECT stddev_pop(3.0::float8), stddev_samp(4.0::float8); SELECT var_pop('inf'::float8), var_samp('inf'::float8); var_pop | var_samp ---------+---------- - 0 | + NaN | (1 row) SELECT stddev_pop('inf'::float8), stddev_samp('inf'::float8); stddev_pop | stddev_samp ------------+------------- - 0 | + NaN | (1 row) SELECT var_pop('nan'::float8), var_samp('nan'::float8); var_pop | var_samp ---------+---------- - 0 | + NaN | (1 row) SELECT stddev_pop('nan'::float8), stddev_samp('nan'::float8); stddev_pop | stddev_samp ------------+------------- - 0 | + NaN | (1 row) SELECT var_pop(1.0::float4), var_samp(2.0::float4); @@ -178,25 +178,25 @@ SELECT stddev_pop(3.0::float4), stddev_samp(4.0::float4); SELECT var_pop('inf'::float4), var_samp('inf'::float4); var_pop | var_samp ---------+---------- - 0 | + NaN | (1 row) SELECT stddev_pop('inf'::float4), stddev_samp('inf'::float4); stddev_pop | stddev_samp ------------+------------- - 0 | + NaN | (1 row) SELECT var_pop('nan'::float4), var_samp('nan'::float4); var_pop | var_samp ---------+---------- - 0 | + NaN | (1 row) SELECT stddev_pop('nan'::float4), stddev_samp('nan'::float4); stddev_pop | stddev_samp ------------+------------- - 0 | + NaN | (1 row) SELECT var_pop(1.0::numeric), var_samp(2.0::numeric); @@ -393,13 +393,13 @@ SELECT covar_pop(1::float8,2::float8), covar_samp(3::float8,4::float8); SELECT covar_pop(1::float8,'inf'::float8), covar_samp(3::float8,'inf'::float8); covar_pop | covar_samp -----------+------------ - 0 | + NaN | (1 row) SELECT covar_pop(1::float8,'nan'::float8), covar_samp(3::float8,'nan'::float8); covar_pop | covar_samp -----------+------------ - 0 | + NaN | (1 row) -- test accum and combine functions directly
diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index 553e261ed0..eea4239854 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -5172,11 +5172,21 @@ numeric_stddev_internal(NumericAggState *state, vsumX, vsumX2, vNminus1; - const NumericVar *comp; + int64 totCount; int rscale; - /* Deal with empty input and NaN-input cases */ - if (state == NULL || (state->N + state->NaNcount) == 0) + /* + * Sample stddev and variance are undefined when N <= 1; population stddev + * is undefined when N == 0. Return NULL in either case (note that NaNs + * count as normal inputs for this purpose). + */ + if (state == NULL || (totCount = state->N + state->NaNcount) == 0) + { + *is_null = true; + return NULL; + } + + if (sample && totCount <= 1) { *is_null = true; return NULL; @@ -5184,9 +5194,13 @@ numeric_stddev_internal(NumericAggState *state, *is_null = false; + /* + * Deal with NaN inputs. + */ if (state->NaNcount > 0) return make_result(&const_nan); + /* OK, normal calculation applies */ init_var(&vN); init_var(&vsumX); init_var(&vsumX2); @@ -5195,21 +5209,6 @@ numeric_stddev_internal(NumericAggState *state, accum_sum_final(&(state->sumX), &vsumX); accum_sum_final(&(state->sumX2), &vsumX2); - /* - * Sample stddev and variance are undefined when N <= 1; population stddev - * is undefined when N == 0. Return NULL in either case. - */ - if (sample) - comp = &const_one; - else - comp = &const_zero; - - if (cmp_var(&vN, comp) <= 0) - { - *is_null = true; - return NULL; - } - init_var(&vNminus1); sub_var(&vN, &const_one, &vNminus1); diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index e4ffa5ee42..3bd184ae29 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -214,13 +214,13 @@ SELECT stddev_pop(3.0::numeric), stddev_samp(4.0::numeric); SELECT var_pop('nan'::numeric), var_samp('nan'::numeric); var_pop | var_samp ---------+---------- - NaN | NaN + NaN | (1 row) SELECT stddev_pop('nan'::numeric), stddev_samp('nan'::numeric); stddev_pop | stddev_samp ------------+------------- - NaN | NaN + NaN | (1 row) -- verify correct results for null and NaN inputs