Since session rates are accounted per thread group, the "rate-limit
sessions" check sums, for each group, the remaining budget computed
against the full configured limit. A frontend therefore keeps accepting
connections as long as any group stays below the limit:
- with evenly loaded groups, the frontend accepts up to the limit
multiplied by the number of thread groups;
- when some groups receive no traffic, e.g. with "bind ... thread
1/all" and several thread groups, the limit is never enforced.
No explicit thread group setting is needed to be affected. Since 3.3,
the default "performance" cpu-policy creates one thread group per CCX
when neither "nbthread" nor "thread-groups" is set, and binds are
spread over all groups. On machines with several CCX, a frontend with
no thread-related setting may thus accept several times its configured
session rate. Setting "thread-groups 1" works around the issue.
With "rate-limit sessions 100" and three thread groups, a local test
measured about 270 sessions/s on a bind using all groups, and more than
21000 sessions/s on a bind restricted to the first group.
The limit is meant to apply to the frontend as a whole. Let's sum the
per-group session totals first and compare the result with the limit,
both for the number of sessions that may still be accepted and for the
delay before the next one. Two helpers, proxy_fe_sess_remain() and
proxy_fe_sess_delay(), now do this for all callers. The same test then
measures 100 sessions/s in both cases.
A reg-test is provided: with a bind restricted to the first of two
thread groups and a limit of one session per second, the second session
must wait for the rate to drop and see fe_sess_rate at 1, not 2.
This was introduced in 3.3 by commit 16eb0fab3 ("MAJOR: counters:
dispatch counters over thread groups"), so this must be backported to
3.3.
---
include/haproxy/proxy.h | 39 +++++++++++++++++
.../proxy/rate_limit_sessions_tgroups.vtc | 42 +++++++++++++++++++
src/listener.c | 22 +++-------
src/proxy.c | 2 +-
4 files changed, 87 insertions(+), 18 deletions(-)
create mode 100644 reg-tests/proxy/rate_limit_sessions_tgroups.vtc
diff --git a/include/haproxy/proxy.h b/include/haproxy/proxy.h
index da09c46d3..d834f9112 100644
--- a/include/haproxy/proxy.h
+++ b/include/haproxy/proxy.h
@@ -210,6 +210,45 @@ static inline void proxy_inc_fe_sess_ctr(struct listener
*l, struct proxy *fe)
update_freq_ctr(&fe->fe_counters._sess_per_sec,
1));
}
+/* Returns the total number of sessions accepted by frontend <fe> over the
+ * current and last one-second periods, summed over all thread groups. Dividing
+ * it by the period gives the frontend-wide session rate.
+ */
+static inline ullong proxy_fe_sess_total(const struct proxy *fe)
+{
+ return COUNTERS_SHARED_TOTAL_ARG2(fe->fe_counters.shared.tg,
sess_per_sec,
+ freq_ctr_total, MS_TO_TICKS(1000), 0);
+}
+
+/* Returns the number of new sessions frontend <fe> may still accept while
+ * respecting its "rate-limit sessions" setting across all thread groups, or 0
+ * if the limit was reached. <fe>->fe_sps_lim must not be zero.
+ */
+static inline uint proxy_fe_sess_remain(const struct proxy *fe)
+{
+ uint avg = div64_32(proxy_fe_sess_total(fe), MS_TO_TICKS(1000));
+
+ return (avg < fe->fe_sps_lim) ? fe->fe_sps_lim - avg : 0;
+}
+
+/* Returns the wait time in ms before frontend <fe> may accept a new session
+ * while respecting its "rate-limit sessions" setting across all thread groups.
+ * It returns zero if it can accept immediately, otherwise at least one ms.
+ * <fe>->fe_sps_lim must not be zero.
+ */
+static inline uint proxy_fe_sess_delay(const struct proxy *fe)
+{
+ ullong total = proxy_fe_sess_total(fe);
+ ullong limit = (ullong)fe->fe_sps_lim * MS_TO_TICKS(1000);
+ uint wait;
+
+ if (total < limit)
+ return 0;
+
+ wait = div64_32(total - limit, fe->fe_sps_lim);
+ return MAX(wait, 1);
+}
+
/* increase the number of cumulated HTTP sessions on the designated frontend.
* <http_ver> must be the HTTP version for such requests.
*/
diff --git a/reg-tests/proxy/rate_limit_sessions_tgroups.vtc
b/reg-tests/proxy/rate_limit_sessions_tgroups.vtc
new file mode 100644
index 000000000..f73f1b3aa
--- /dev/null
+++ b/reg-tests/proxy/rate_limit_sessions_tgroups.vtc
@@ -0,0 +1,42 @@
+varnishtest "rate-limit sessions with multiple thread groups"
+
+# The session rate limit of a frontend must apply to the frontend as a whole
+# even when its sessions are accounted over several thread groups. Here the
+# listener only runs on the first of two thread groups, so with a per-group
+# check the idle second group would let the second session in immediately and
+# fe_sess_rate would report 2. With a frontend-wide check, the second session
+# waits in the backlog until the rate drops below 1 and sees a rate of 1.
+
+feature cmd "$HAPROXY_PROGRAM -cc 'feature(THREAD)'"
+feature ignore_unknown_macro
+
+haproxy h1 -conf {
+ global
+ nbthread 2
+ thread-groups 2
+
+ defaults
+ mode http
+ timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+ timeout client "${HAPROXY_TEST_TIMEOUT-5s}"
+ timeout server "${HAPROXY_TEST_TIMEOUT-5s}"
+
+ frontend fe
+ bind "fd@${fe}" thread 1/all
+ rate-limit sessions 1
+ http-request return status 200 hdr x-sess-rate %[fe_sess_rate]
+} -start
+
+client c1 -connect ${h1_fe_sock} {
+ txreq
+ rxresp
+ expect resp.status == 200
+ expect resp.http.x-sess-rate == 1
+} -run
+
+client c2 -connect ${h1_fe_sock} {
+ txreq
+ rxresp
+ expect resp.status == 200
+ expect resp.http.x-sess-rate == 1
+} -run
diff --git a/src/listener.c b/src/listener.c
index 1c46303d8..e5a0eaa75 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -1400,23 +1400,11 @@ void listener_accept(struct listener *l)
}
#endif
if (p && p->fe_sps_lim) {
- int max = 0;
- int it;
-
- for (it = 0; (it < global.nbtgroups && p->fe_counters.shared.tg
&& p->fe_counters.shared.tg[it]); it++)
- max +=
freq_ctr_remain(&p->fe_counters.shared.tg[it]->sess_per_sec, p->fe_sps_lim, 0);
+ int max = proxy_fe_sess_remain(p);
if (unlikely(!max)) {
- unsigned int min_wait = 0;
-
- for (it = 0; it < global.nbtgroups; it++) {
- unsigned int cur_wait =
next_event_delay(&p->fe_counters.shared.tg[it]->sess_per_sec, p->fe_sps_lim, 0);
- if (!it || cur_wait < min_wait)
- min_wait = cur_wait;
- }
-
/* frontend accept rate limit was reached */
- expire = tick_add(now_ms, min_wait);
+ expire = tick_add(now_ms, proxy_fe_sess_delay(p));
goto limit_proxy;
}
@@ -1904,7 +1892,7 @@ void listener_accept(struct listener *l)
dequeue_all_listeners();
if (p && !MT_LIST_ISEMPTY(&p->listener_queue) &&
- (!p->fe_sps_lim ||
COUNTERS_SHARED_TOTAL_ARG2(p->fe_counters.shared.tg, sess_per_sec,
freq_ctr_remain, p->fe_sps_lim, 0) > 0))
+ (!p->fe_sps_lim || proxy_fe_sess_remain(p) > 0))
dequeue_proxy_listeners(p, 0);
}
return;
@@ -1963,14 +1951,14 @@ void listener_release(struct listener *l)
dequeue_all_listeners();
if (fe && !MT_LIST_ISEMPTY(&fe->listener_queue) &&
- (!fe->fe_sps_lim ||
COUNTERS_SHARED_TOTAL_ARG2(fe->fe_counters.shared.tg, sess_per_sec,
freq_ctr_remain, fe->fe_sps_lim, 0) > 0))
+ (!fe->fe_sps_lim || proxy_fe_sess_remain(fe) > 0))
dequeue_proxy_listeners(fe, 0);
else if (fe) {
unsigned int wait;
int expire = TICK_ETERNITY;
if (fe->task && fe->fe_sps_lim &&
- (wait =
COUNTERS_SHARED_TOTAL_ARG2(fe->fe_counters.shared.tg, sess_per_sec,
next_event_delay, fe->fe_sps_lim, 0))) {
+ (wait = proxy_fe_sess_delay(fe))) {
/* we're blocking because a limit was reached on the
number of
* requests/s on the frontend. We want to re-check
ASAP, which
* means in 1 ms before estimated expiration date,
because the
diff --git a/src/proxy.c b/src/proxy.c
index 00d8925ab..fc1785b99 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -3955,7 +3955,7 @@ struct task *manage_proxy(struct task *t, void *context,
unsigned int state)
goto out;
if (p->fe_sps_lim &&
- (wait = COUNTERS_SHARED_TOTAL_ARG2(p->fe_counters.shared.tg,
sess_per_sec, next_event_delay, p->fe_sps_lim, 0))) {
+ (wait = proxy_fe_sess_delay(p))) {
/* we're blocking because a limit was reached on the number of
* requests/s on the frontend. We want to re-check ASAP, which
--
2.55.0
