This is an automated email from the ASF dual-hosted git repository.
shreemaan-abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new f6d0ee804 fix(prometheus): truncate llm model-name labels to bound
cardinality (#13637)
f6d0ee804 is described below
commit f6d0ee8040ccaf5da9f3f3b05a6ede8a86a66a9a
Author: Shreemaan Abhishek <[email protected]>
AuthorDate: Thu Jul 2 18:16:32 2026 +0800
fix(prometheus): truncate llm model-name labels to bound cardinality
(#13637)
---
apisix/plugins/prometheus/exporter.lua | 54 +++++++++++++++++++-------
docs/en/latest/plugins/prometheus.md | 2 +
docs/zh/latest/plugins/prometheus.md | 2 +
t/plugin/prometheus-ai-proxy.t | 71 ++++++++++++++++++++++++++++++++++
4 files changed, 116 insertions(+), 13 deletions(-)
diff --git a/apisix/plugins/prometheus/exporter.lua
b/apisix/plugins/prometheus/exporter.lua
index 90c5d810d..de3e4a8bf 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -27,6 +27,7 @@ local C = ffi.C
local pcall = pcall
local select = select
local type = type
+local tostring = tostring
local prometheus
local prometheus_bkp
local router = require("apisix.router")
@@ -45,6 +46,7 @@ local latency_details =
require("apisix.utils.log-util").latency_details_in_ms
local xrpc = require("apisix.stream.xrpc")
local unpack = unpack
local next = next
+local str_sub = string.sub
local process = require("ngx.process")
local tonumber = tonumber
local shdict_prometheus_cache = ngx.shared["prometheus-cache"]
@@ -64,6 +66,25 @@ local DEFAULT_BUCKETS = {1, 2, 5, 10, 20, 50, 100, 200, 500,
1000, 2000, 5000, 1
-- sizes) with the upper bound raised to 1M to cover large-context models.
local DEFAULT_TOKEN_BUCKETS = {1, 10, 50, 100, 200, 500, 1000, 2000, 5000,
10000,
20000, 50000, 100000, 200000, 500000, 1000000}
+
+-- Max byte length for model-name labels. Model names are client-supplied, so
+-- cap them to stop arbitrarily long strings from exhausting the metrics shm.
+-- Non-scalar values become "<non-scalar>" so pointer strings never enter the
index.
+local MAX_MODEL_LABEL_LEN = 128
+
+local function model_to_label(val)
+ if val == nil then
+ return nil
+ end
+ local t = type(val)
+ if t == "string" then
+ return str_sub(val, 1, MAX_MODEL_LABEL_LEN)
+ elseif t == "number" then
+ return str_sub(tostring(val), 1, MAX_MODEL_LABEL_LEN)
+ else
+ return "<non-scalar>"
+ end
+end
-- Default refresh interval
local DEFAULT_REFRESH_INTERVAL = 15
@@ -435,11 +456,17 @@ function _M.http_log(conf, ctx)
local response_source = core.response.get_response_source(ctx)
+ -- Truncate model names before they become label values.
vars.request_llm_model /
+ -- vars.llm_model keep their full values for other consumers; only the
metrics path
+ -- is capped.
+ local request_llm_model_label = model_to_label(vars.request_llm_model)
+ local llm_model_label = model_to_label(vars.llm_model)
+
metrics.status:inc(1,
get_enabled_label_values_for_metric("http_status",
disabled_label_metric_map,
vars.status, route_id, matched_uri, matched_host,
service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
response_source,
unpack(extra_labels("http_status", ctx))))
@@ -450,21 +477,21 @@ function _M.http_log(conf, ctx)
metrics.latency:observe(latency,
get_enabled_label_values_for_metric("http_latency",
disabled_label_metric_map,
"request", route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(latency_extra_label_values)))
if upstream_latency then
metrics.latency:observe(upstream_latency,
get_enabled_label_values_for_metric("http_latency",
disabled_label_metric_map,
"upstream", route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(latency_extra_label_values)))
end
metrics.latency:observe(apisix_latency,
get_enabled_label_values_for_metric("http_latency",
disabled_label_metric_map,
"apisix", route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(latency_extra_label_values)))
local bandwidth_extra_label_values = extra_labels("bandwidth", ctx)
@@ -472,13 +499,13 @@ function _M.http_log(conf, ctx)
metrics.bandwidth:inc(vars.request_length,
get_enabled_label_values_for_metric("bandwidth",
disabled_label_metric_map,
"ingress", route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(bandwidth_extra_label_values)))
metrics.bandwidth:inc(vars.bytes_sent,
get_enabled_label_values_for_metric("bandwidth",
disabled_label_metric_map,
"egress", route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(bandwidth_extra_label_values)))
if vars.request_type == "ai_stream" or vars.request_type == "ai_chat" then
@@ -491,7 +518,7 @@ function _M.http_log(conf, ctx)
metrics.llm_latency:observe(tonumber(vars.apisix_upstream_response_time),
get_enabled_label_values_for_metric("llm_latency",
disabled_label_metric_map,
"total", route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label,
llm_model_label,
unpack(extra_labels("llm_latency", ctx))))
-- type="ttft": time to first token, only streaming exposes a real
one.
@@ -499,7 +526,7 @@ function _M.http_log(conf, ctx)
metrics.llm_latency:observe(tonumber(llm_time_to_first_token),
get_enabled_label_values_for_metric("llm_latency",
disabled_label_metric_map,
"ttft", route_id, service_id, consumer_name,
balancer_ip,
- vars.request_type, vars.request_llm_model,
vars.llm_model,
+ vars.request_type, request_llm_model_label,
llm_model_label,
unpack(extra_labels("llm_latency", ctx))))
end
end
@@ -507,27 +534,27 @@ function _M.http_log(conf, ctx)
metrics.llm_prompt_tokens:inc(tonumber(vars.llm_prompt_tokens),
get_enabled_label_values_for_metric("llm_prompt_tokens",
disabled_label_metric_map,
route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(extra_labels("llm_prompt_tokens", ctx))))
metrics.llm_prompt_tokens_dist:observe(tonumber(vars.llm_prompt_tokens),
get_enabled_label_values_for_metric("llm_prompt_tokens_dist",
disabled_label_metric_map,
route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(extra_labels("llm_prompt_tokens_dist", ctx))))
metrics.llm_completion_tokens:inc(tonumber(vars.llm_completion_tokens),
get_enabled_label_values_for_metric("llm_completion_tokens",
disabled_label_metric_map,
route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(extra_labels("llm_completion_tokens", ctx))))
metrics.llm_completion_tokens_dist:observe(tonumber(vars.llm_completion_tokens),
get_enabled_label_values_for_metric("llm_completion_tokens_dist",
disabled_label_metric_map,
route_id, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, request_llm_model_label, llm_model_label,
unpack(extra_labels("llm_completion_tokens_dist", ctx))))
end
end
@@ -931,7 +958,8 @@ local function inc_llm_active_connections(ctx, value)
get_enabled_label_values_for_metric("llm_active_connections",
disabled_label_metric_map,
route_name, route_id, matched_uri,
matched_host, service_name, service_id, consumer_name, balancer_ip,
- vars.request_type, vars.request_llm_model, vars.llm_model,
+ vars.request_type, model_to_label(vars.request_llm_model),
+ model_to_label(vars.llm_model),
unpack(extra_labels("llm_active_connections", ctx)))
)
end
diff --git a/docs/en/latest/plugins/prometheus.md
b/docs/en/latest/plugins/prometheus.md
index 194d32042..e31ff269e 100644
--- a/docs/en/latest/plugins/prometheus.md
+++ b/docs/en/latest/plugins/prometheus.md
@@ -106,6 +106,8 @@ Collapsing a label's value to `""` keeps the label
registered in the metric sche
See [Reduce Metric Cardinality by Disabling
Labels](#reduce-metric-cardinality-by-disabling-labels) for an example.
+The `request_llm_model` and `llm_model` label values are derived from
client-supplied model names. To bound cardinality, APISIX truncates each of
these label values to 128 bytes before recording. If you do not need per-model
breakdowns, list `request_llm_model` and `llm_model` under `disabled_labels`
for the LLM metrics to collapse them to a single empty-valued series.
+
## Metrics
There are different types of metrics in Prometheus. To understand their
differences, see [metrics
types](https://prometheus.io/docs/concepts/metric_types/).
diff --git a/docs/zh/latest/plugins/prometheus.md
b/docs/zh/latest/plugins/prometheus.md
index d7eed001a..f647b5d6a 100644
--- a/docs/zh/latest/plugins/prometheus.md
+++ b/docs/zh/latest/plugins/prometheus.md
@@ -106,6 +106,8 @@ plugin_attr:
示例请参见[通过禁用标签降低指标基数](#通过禁用标签降低指标基数)。
+`request_llm_model` 与 `llm_model` 标签值来源于客户端提供的模型名称。为了限制基数，APISIX
在记录前会将这两个标签值截断为 128 字节。如果你不需要按模型细分，可将 `request_llm_model` 和 `llm_model` 列入 LLM
指标的 `disabled_labels`，从而将其折叠为一条空值时间序列。
+
## 指标
Prometheus
中有不同类型的指标。要了解它们之间的区别，请参见[指标类型](https://prometheus.io/docs/concepts/metric_types/)。
diff --git a/t/plugin/prometheus-ai-proxy.t b/t/plugin/prometheus-ai-proxy.t
index 9a641825b..0e58f1c2f 100644
--- a/t/plugin/prometheus-ai-proxy.t
+++ b/t/plugin/prometheus-ai-proxy.t
@@ -432,3 +432,74 @@
qr/apisix_llm_latency_bucket\{type="ttft",.*route_id="4",.*,node="openai-gpt4".*
GET /apisix/prometheus/metrics
--- response_body eval
qr/apisix_llm_latency_count\{type="total",.*route_id="4",.*,node="openai-gpt4".*request_type="ai_stream",request_llm_model="gpt-3",llm_model="gpt-4"\}
1/
+
+
+
+=== TEST 21: send a chat request whose model name exceeds the 128-byte label
cap
+--- request eval
+"POST /chat\n" .
+qq#{"messages":[{"role":"user","content":"What is 1+1?"}], "model": "@{[ 'a' x
200 ]}"}#
+--- more_headers
+X-AI-Fixture: prometheus/chat-basic.json
+--- error_code: 200
+
+
+
+=== TEST 22: request_llm_model label is truncated to 128 bytes (cardinality
DoS guard)
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_llm_prompt_tokens\{.*request_llm_model="a{128}",llm_model="gpt-4"\}/
+--- response_body_unlike eval
+qr/request_llm_model="a{129}"/
+
+
+
+=== TEST 23: disable request_llm_model / llm_model labels via plugin_metadata
disabled_labels
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local _, body = t("/apisix/admin/plugin_metadata/prometheus",
+ ngx.HTTP_PUT,
+ [[{
+ "disabled_labels": {
+ "llm_prompt_tokens": ["request_llm_model", "llm_model"]
+ }
+ }]]
+ )
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 24: send a chat request with a distinct model
+--- request
+POST /chat
+{"messages":[{"role":"user","content":"What is 1+1?"}], "model":
"distinct-model-aaa"}
+--- more_headers
+X-AI-Fixture: prometheus/chat-basic.json
+--- error_code: 200
+
+
+
+=== TEST 25: send another chat request with a different distinct model
+--- request
+POST /chat
+{"messages":[{"role":"user","content":"What is 1+1?"}], "model":
"distinct-model-bbb"}
+--- more_headers
+X-AI-Fixture: prometheus/chat-basic.json
+--- error_code: 200
+
+
+
+=== TEST 26: disabled_labels collapses distinct client models to one
empty-valued series
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_llm_prompt_tokens\{.*request_llm_model="",llm_model=""\}/
+--- response_body_unlike eval
+qr/apisix_llm_prompt_tokens\{.*request_llm_model="distinct-model-/
