This is an automated email from the ASF dual-hosted git repository.
moonming pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 2b6d519a0 fix(prometheus): count stream bytes of slots created after
the first read (#14101)
2b6d519a0 is described below
commit 2b6d519a01e446b5b1d98168cedf9a7a4daa40b6
Author: AlinsRan <[email protected]>
AuthorDate: Mon Nov 9 11:28:27 2026 +0800
fix(prometheus): count stream bytes of slots created after the first read
(#14101)
* fix(prometheus): count stream bytes of slots created after the first read
Stream metrics zone slots are created on demand by the first session that
carries their labels. publish_stream_bytes only took a baseline the first
time it saw a slot, so the bytes of every session that ended before a new
service's slot was first read were never counted, and per listen_addr
totals summed over services came out low.
Record in the metrics dict that the zone has been read once. After that, a
slot without a baseline is new and is counted from zero; only the first
read after start or after the dict is flushed takes baselines without
counting.
* test(prometheus): compare the replayed slot against the zone total
* fix(prometheus): keep the read marker unset while a baseline is missing
* fix(prometheus): count a stream slot without a baseline from zero
Drop the read marker. The zone keeps its totals across a reload and so do
the baselines, which live in the same dict as the counters, so a slot
without a baseline has had nothing counted: it is new, or the process has
just started. Counting it from zero also counts the traffic between the
start and the first read, which the first read used to only baseline.
Tests check that the svc-a slot exists before comparing against it, and
cover a read of an emptied dict.
---
apisix/plugins/prometheus/exporter.lua | 17 ++---
docs/en/latest/plugins/prometheus.md | 3 +
docs/zh/latest/plugins/prometheus.md | 2 +
t/stream-plugin/prometheus-metrics-service.t | 105 +++++++++++++++++++++++----
4 files changed, 100 insertions(+), 27 deletions(-)
diff --git a/apisix/plugins/prometheus/exporter.lua
b/apisix/plugins/prometheus/exporter.lua
index 42467761e..32dabac1c 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -404,18 +404,11 @@ local function publish_stream_bytes(dict, listen_addr,
service, service_id, dire
local key = STREAM_PUBLISHED_PREFIX .. listen_addr .. "\31" .. service ..
"\31"
.. service_id .. "\31" .. field
- local published = dict:get(key)
- if not published then
- -- The zone counts from when nginx started while the counter outlives a
- -- reload, so the first sight of a slot only takes a baseline;
- -- replaying the whole total into a counter that survived would double
- -- it. The cost is that traffic before the first read is not counted.
- local ok, err = dict:set(key, total)
- if not ok then
- core.log.error("failed to baseline stream bandwidth for ", key, ":
", err)
- end
- return
- end
+ -- Baselines outlive a reload like the zone, so a slot without one has had
+ -- nothing counted: it is new, or the process just started. The cost is
+ -- that a baseline evicted from a full dict replays the total of its slot
+ -- once.
+ local published = dict:get(key) or 0
if total == published then
return
diff --git a/docs/en/latest/plugins/prometheus.md
b/docs/en/latest/plugins/prometheus.md
index 97da21a7b..632ae25dd 100644
--- a/docs/en/latest/plugins/prometheus.md
+++ b/docs/en/latest/plugins/prometheus.md
@@ -274,6 +274,9 @@ Under plain forwarding `downstream`/`ingress` tracks
`upstream`/`egress`, and
`upstream`/`ingress` tracks `downstream`/`egress`; a lasting mismatch is itself
a signal that one side stopped reading.
+A Service is counted from its first session, including sessions that end
+before the next `refresh_interval`.
+
### Labels for `apisix_bandwidth`
The following labels are used to differentiate `apisix_bandwidth` metrics.
diff --git a/docs/zh/latest/plugins/prometheus.md
b/docs/zh/latest/plugins/prometheus.md
index 434847b7c..48cd6a4ab 100644
--- a/docs/zh/latest/plugins/prometheus.md
+++ b/docs/zh/latest/plugins/prometheus.md
@@ -262,6 +262,8 @@ UDP 没有关闭、FIN 或重置信号，因此只会出现其中一部分状态
在普通转发中，`downstream`/`ingress` 通常对应 `upstream`/`egress`，`upstream`/`ingress`
通常对应 `downstream`/`egress`；持续不一致可能表示某一侧停止读取。
+每个 Service 从其第一个会话起计入，包括在下一个 `refresh_interval` 之前就已结束的会话。
+
### `apisix_bandwidth` 的标签
以下标签用于区分 `apisix_bandwidth` 指标。
diff --git a/t/stream-plugin/prometheus-metrics-service.t
b/t/stream-plugin/prometheus-metrics-service.t
index 9a9c46e6b..9ac703023 100644
--- a/t/stream-plugin/prometheus-metrics-service.t
+++ b/t/stream-plugin/prometheus-metrics-service.t
@@ -106,8 +106,8 @@ _EOC_
end
-- Opens a session on 1985 and scrapes while it is live. A scrape runs
- -- first so that the bandwidth baselines are taken before the session
- -- exists: the first read of the zone only baselines what it finds.
+ -- first so that the totals already in the zone are published before the
+ -- session exists.
function _G.scrape_live_session()
local body, err = scrape()
if not body then
@@ -154,6 +154,14 @@ _EOC_
'listen_addr="0.0.0.0:1985",service="svc-a",service_id="svc-a",'
.. 'type="ingress",side="downstream"')
end
+ -- what the zone holds for that series, nil when the slot does not exist
+ function _G.svc_a_zone_ingress()
+ for _, entry in ipairs(require("resty.apisix.stream.metrics").dump())
do
+ if entry.listen_addr == "0.0.0.0:1985" and entry.labels[1] ==
"svc-a" then
+ return entry.downstream_ingress
+ end
+ end
+ end
function _G.active_on(body, service, service_id)
return series_value(body, "apisix_stream_active_connections",
@@ -224,9 +232,9 @@ __DATA__
=== TEST 2: a live session is split out under its service_id
The session is labelled in preread, so the service series carries it and the
-unlabelled series of the same listen_addr does not. Like any zone slot, the
-service's slot only takes a baseline the first time it is read, so its bytes
-are counted from its second session on.
+unlabelled series of the same listen_addr does not. A slot without a
+baseline is counted from zero, so the service's first session is counted in
+full even though it ends before its slot is first read.
--- config
location /probe {
content_by_lua_block {
@@ -261,7 +269,7 @@ GET /probe
--- stream_enable
--- timeout: 20
--- response_body
-first session bandwidth=no-series
+first session bandwidth=5
service live=1
unlabelled live=0
service bandwidth=counted
@@ -384,7 +392,7 @@ the id label by the earlier cases stays, at 0.
ngx.sleep(1.5)
- -- the first session under the name only baselines its new slot
+ -- the name gets a new slot, counted from its first session
local body, err = scrape_live_session()
if not body then
ngx.say(err)
@@ -430,11 +438,9 @@
qr/apisix_stream_status\{code="200",listen_addr="0\.0\.0\.0:1985",service="Order
-=== TEST 9: a slot that lost its baseline rebaselines instead of replaying
-A baseline can go missing while the rest of the dict stays, when it is
-evicted or could not be written. Counting that slot from zero would land its
-whole lifetime total in one interval; it has to take a new baseline instead,
-and later traffic is counted as usual.
+=== TEST 9: a slot that lost its baseline is counted from zero
+There is no telling a slot whose baseline was evicted from a new one, so it
+replays its total once, and later traffic is counted as usual.
--- config
location /probe {
content_by_lua_block {
@@ -468,15 +474,22 @@ and later traffic is counted as usual.
end
local before = tonumber(svc_a_ingress(body))
- ngx.shared["prometheus-metrics"]:delete(
-
"stream_bytes_published:0.0.0.0:1985\31svc-a\31svc-a\31downstream_ingress")
+ local dict = ngx.shared["prometheus-metrics"]
+ local key = "stream_bytes_published:0.0.0.0:1985\31svc-a\31svc-a"
+ .. "\31downstream_ingress"
+ ngx.say("baseline dropped: ", dict:get(key) ~= nil)
+ dict:delete(key)
body, err = scrape()
if not body then
ngx.say(err)
return
end
- ngx.say("after the lost baseline: ", tonumber(svc_a_ingress(body))
== before)
+ local zone = svc_a_zone_ingress()
+ ngx.say("zone holds traffic: ", (zone or 0) > 0)
+ local replayed = before + (zone or 0)
+ before = tonumber(svc_a_ingress(body))
+ ngx.say("after the lost baseline: ", before == replayed)
body, err = scrape_live_session()
if not body then
@@ -493,6 +506,8 @@ GET /probe
--- stream_enable
--- timeout: 20
--- response_body
+baseline dropped: true
+zone holds traffic: true
after the lost baseline: true
after one more session: +5
@@ -590,3 +605,63 @@ GET /probe
--- response_body
status under the first name: 1
status under the new name: no-series
+
+
+
+=== TEST 11: an emptied metric dict counts the zone in full
+Flushing the dict leaves no counter and no baseline, the state the dict starts
+from. A slot then has had nothing counted, so the next read counts its whole
+total, sessions that ended before it included.
+--- config
+ location /probe {
+ content_by_lua_block {
+ -- the previous case moved the route to another service
+ local t = require("lib.test_admin").test
+ local code = t("/apisix/admin/stream_routes/1", ngx.HTTP_PUT, [[{
+ "plugins": {
+ "prometheus": {}
+ },
+ "service_id": "svc-a"
+ }]])
+ if code > 300 then
+ ngx.say("stream route: ", code)
+ return
+ end
+ ngx.sleep(1.5)
+
+ local body, err = scrape_live_session()
+ if not body then
+ ngx.say(err)
+ return
+ end
+
+ ngx.shared["prometheus-metrics"]:flush_all()
+
+ body, err = scrape()
+ if not body then
+ ngx.say(err)
+ return
+ end
+ local zone = svc_a_zone_ingress()
+ ngx.say("zone holds traffic: ", (zone or 0) > 0)
+ ngx.say("first read: ", tonumber(svc_a_ingress(body)) == zone)
+
+ local before = tonumber(svc_a_ingress(body))
+ body, err = scrape_live_session()
+ if not body then
+ ngx.say(err)
+ return
+ end
+ ngx.say("after one more session: +", tonumber(svc_a_ingress(body))
- before)
+
+ ngx.sleep(1.5)
+ }
+ }
+--- request
+GET /probe
+--- stream_enable
+--- timeout: 20
+--- response_body
+zone holds traffic: true
+first read: true
+after one more session: +5
