This is an automated email from the ASF dual-hosted git repository.
nic-6443 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new e14c6e9371 feat(prometheus): export per-process timers, connection
slots and Lua memory (#14034)
e14c6e9371 is described below
commit e14c6e9371edc084cc51321d4bee78646fa8367d
Author: Nic <[email protected]>
AuthorDate: Fri Oct 16 09:58:41 2026 +0800
feat(prometheus): export per-process timers, connection slots and Lua
memory (#14034)
---
apisix/cli/ngx_tpl.lua | 9 +
apisix/plugins/prometheus/exporter.lua | 32 +-
apisix/plugins/prometheus/process_metrics.lua | 442 ++++++++++++++++++++++++++
docs/en/latest/plugins/prometheus.md | 17 +
docs/zh/latest/plugins/prometheus.md | 17 +
t/cli/test_prometheus_reload.sh | 106 ++++++
t/cli/test_prometheus_stream.sh | 39 +++
t/plugin/prometheus-process-metrics.t | 210 ++++++++++++
8 files changed, 866 insertions(+), 6 deletions(-)
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index e7262a3541..d872ff3b56 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -90,6 +90,15 @@ http {
.. [=[$prefix/deps/lib/lua/5.1/?.so;;]=]
.. [=[{*lua_cpath*};";
+ # the same limits as the stream subsystem, which nginx_timers_max
+ # reports for every Lua VM
+ {% if max_pending_timers then %}
+ lua_max_pending_timers {* max_pending_timers *};
+ {% end %}
+ {% if max_running_timers then %}
+ lua_max_running_timers {* max_running_timers *};
+ {% end %}
+
{% if enabled_stream_plugins["prometheus"] then %}
init_by_lua_block {
diff --git a/apisix/plugins/prometheus/exporter.lua
b/apisix/plugins/prometheus/exporter.lua
index 7fe666c7a7..4ccc3f56b7 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -45,6 +45,7 @@ local get_protos =
require("apisix.plugins.grpc-transcode.proto").protos
local service_fetch = require("apisix.http.service").get
local latency_details = require("apisix.utils.log-util").latency_details_in_ms
local xrpc = require("apisix.stream.xrpc")
+local process_metrics = require("apisix.plugins.prometheus.process_metrics")
local unpack = unpack
local next = next
local str_sub = string.sub
@@ -500,12 +501,22 @@ local function collect_stream_zone_metrics()
end
+local function get_refresh_interval()
+ local attr = plugin.plugin_attr("prometheus")
+ if attr and attr.refresh_interval then
+ return attr.refresh_interval
+ end
+ return DEFAULT_REFRESH_INTERVAL
+end
+
+
function _M.http_init(prometheus_enabled_in_stream)
-- todo: support hot reload, we may need to update the lua-prometheus
-- library
if ngx.get_phase() ~= "init" and ngx.get_phase() ~= "init_worker" then
if prometheus_bkp then
prometheus = prometheus_bkp
+ process_metrics.start(get_refresh_interval())
end
return
end
@@ -695,6 +706,8 @@ function _M.http_init(prometheus_enabled_in_stream)
ai_cache_embedding_latency_buckets,
ai_cache_embedding_latency_exptime)
+ process_metrics.init(prometheus)
+
if prometheus_enabled_in_stream then
init_stream_metrics()
end
@@ -703,6 +716,7 @@ end
function _M.stream_init()
if ngx.get_phase() ~= "init" and ngx.get_phase() ~= "init_worker" then
+ process_metrics.start(get_refresh_interval())
return
end
@@ -722,6 +736,9 @@ function _M.stream_init()
prometheus = base_prometheus.init("prometheus-metrics", metric_prefix)
init_stream_metrics()
+
+ process_metrics.init(prometheus)
+ process_metrics.start(get_refresh_interval())
end
@@ -1186,6 +1203,9 @@ end
local function collect(yieldable)
+ -- the other processes report their own values on a timer
+ process_metrics.refresh()
+
-- collect ngx.shared.DICT status
shared_dict_status()
@@ -1291,16 +1311,14 @@ end
local function init_exporter_timer()
+ local refresh_interval = get_refresh_interval()
+
+ process_metrics.start(refresh_interval)
+
if process.type() ~= "privileged agent" then
return
end
- local refresh_interval = DEFAULT_REFRESH_INTERVAL
- local attr = plugin.plugin_attr("prometheus")
- if attr and attr.refresh_interval then
- refresh_interval = attr.refresh_interval
- end
-
local cache_exptime = refresh_interval * 2
exporter_timer(false, false, cache_exptime)
@@ -1457,6 +1475,8 @@ end
function _M.destroy()
+ process_metrics.stop()
+
if prometheus ~= nil then
prometheus_bkp = prometheus
prometheus = nil
diff --git a/apisix/plugins/prometheus/process_metrics.lua
b/apisix/plugins/prometheus/process_metrics.lua
new file mode 100644
index 0000000000..ff4ddfb4c3
--- /dev/null
+++ b/apisix/plugins/prometheus/process_metrics.lua
@@ -0,0 +1,442 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- Per-process OpenResty internals: Lua timers, connection slots and Lua VM
+-- memory. These values only exist inside the process they describe, while the
+-- exposition is rendered by a single process, so every process writes its own
+-- gauges into the shared metric dict from a timer, and the process rendering
+-- the exposition refreshes its own values right before it does.
+local core = require("apisix.core")
+local ffi = require("ffi")
+local process = require("ngx.process")
+local C = ffi.C
+local ngx = ngx
+local timer_at = ngx.timer.at
+local collectgarbage = collectgarbage
+local tonumber = tonumber
+local tostring = tostring
+local pcall = pcall
+local ipairs = ipairs
+local pairs = pairs
+local select = select
+
+
+-- The prefix of struct ngx_cycle_s up to write_events. Its layout is the same
+-- in every nginx core shipped by the supported runtimes (1.21.4 to 1.29.x);
+-- the self check in check_cycle_layout() is what keeps a future change from
+-- turning into a wrong number.
+ffi.cdef[[
+typedef struct {
+ uintptr_t log_level;
+ void *file;
+ uintptr_t connection;
+ long disk_full_time;
+ void *handler;
+ void *data;
+ void *writer;
+ void *wdata;
+ char *action;
+ void *next;
+} apisix_prom_ngx_log_t;
+
+typedef struct {
+ void *elts;
+ uintptr_t nelts;
+ size_t size;
+ uintptr_t nalloc;
+ void *pool;
+} apisix_prom_ngx_array_t;
+
+typedef struct {
+ void *elts;
+ uintptr_t nelts;
+ void *next;
+} apisix_prom_ngx_list_part_t;
+
+typedef struct {
+ void *last;
+ apisix_prom_ngx_list_part_t part;
+ size_t size;
+ uintptr_t nalloc;
+ void *pool;
+} apisix_prom_ngx_list_t;
+
+typedef struct {
+ uintptr_t key;
+ void *left;
+ void *right;
+ void *parent;
+ unsigned char color;
+ unsigned char data;
+} apisix_prom_ngx_rbtree_node_t;
+
+typedef struct {
+ void *root;
+ void *sentinel;
+ void *insert;
+} apisix_prom_ngx_rbtree_t;
+
+typedef struct {
+ void *data;
+ void *read;
+ void *write;
+} apisix_prom_ngx_connection_head_t;
+
+typedef struct {
+ void *conf_ctx;
+ void *pool;
+ void *log;
+ apisix_prom_ngx_log_t new_log;
+ uintptr_t log_use_stderr;
+ void *files;
+ void *free_connections;
+ uintptr_t free_connection_n;
+ void *modules;
+ uintptr_t modules_n;
+ uintptr_t modules_used;
+ void *reusable_connections_queue[2];
+ uintptr_t reusable_connections_n;
+ long connections_reuse_time;
+ apisix_prom_ngx_array_t listening;
+ apisix_prom_ngx_array_t paths;
+ apisix_prom_ngx_array_t config_dump;
+ apisix_prom_ngx_rbtree_t config_dump_rbtree;
+ apisix_prom_ngx_rbtree_node_t config_dump_sentinel;
+ apisix_prom_ngx_list_t open_files;
+ apisix_prom_ngx_list_t shared_memory;
+ uintptr_t connection_n;
+ uintptr_t files_n;
+ apisix_prom_ngx_connection_head_t *connections;
+ void *read_events;
+ void *write_events;
+} apisix_prom_ngx_cycle_t;
+
+extern apisix_prom_ngx_cycle_t *ngx_cycle;
+]]
+
+
+-- nginx's own defaults for lua_max_running_timers / lua_max_pending_timers,
+-- in effect when the configuration leaves them out
+local DEFAULT_MAX_RUNNING_TIMERS = 256
+local DEFAULT_MAX_PENDING_TIMERS = 1024
+local DEFAULT_INTERVAL = 15
+
+-- Shared by every process, so a layout mismatch found by one worker keeps the
+-- connection gauges out of the exposition everywhere, including the
+-- privileged agent, whose connection count cannot be checked against config.
+local CONNECTIONS_VERDICT_KEY = "process_metrics:connections_readable"
+
+local _M = {}
+
+local gauges
+local registry
+local dict
+local process_label
+local no_extra_labels = {}
+local extra_label_names = no_extra_labels
+local extra_label_values
+local subsystem = ngx.config.subsystem
+local verdict_logged = false
+
+local timer_running = false
+local stopped = true
+local interval = DEFAULT_INTERVAL
+local retired_cleanups = 0
+
+
+local function get_process_label()
+ if process.type() == "privileged agent" then
+ return "privileged-agent"
+ end
+
+ return "worker-" .. tostring(ngx.worker.id() or 0)
+end
+
+
+-- The CLI refuses to start without nginx_config, but a forced re-read of
+-- config.yaml at runtime (plugin reload) can still drop it: `nginx_config: ~`
+-- deletes the default table while merging.
+local function nginx_config()
+ return core.config.local_conf().nginx_config or {}
+end
+
+
+local function timers_max()
+ local conf = nginx_config()
+ return conf.max_running_timers or DEFAULT_MAX_RUNNING_TIMERS,
+ conf.max_pending_timers or DEFAULT_MAX_PENDING_TIMERS
+end
+
+
+local function read_cycle()
+ local cycle = C.ngx_cycle
+ local connection_n = tonumber(cycle.connection_n)
+ local free_n = tonumber(cycle.free_connection_n)
+
+ -- every connection slot is wired to its own pair of events at startup, so
+ -- this pins down the offsets right after connection_n as well
+ local conns = cycle.connections
+ if connection_n == 0 or free_n > connection_n or conns == nil
+ or conns[0].read ~= cycle.read_events or conns[0].write ~=
cycle.write_events
+ then
+ return nil
+ end
+
+ return connection_n, free_n
+end
+
+
+local function check_cycle_layout()
+ local ok, connection_n = pcall(read_cycle)
+ if not ok or not connection_n then
+ return false, "unexpected nginx cycle layout"
+ end
+
+ local event = nginx_config().event
+ local expected = event and tonumber(event.worker_connections)
+ if connection_n ~= expected then
+ return false, "worker connection slots read as " ..
tostring(connection_n)
+ .. ", but worker_connections is " .. tostring(expected)
+ end
+
+ return true
+end
+
+
+-- Only a worker can tell whether the numbers read through FFI are right: its
+-- connection slots must match worker_connections from the configuration.
+local function connections_readable()
+ local readable = dict:get(CONNECTIONS_VERDICT_KEY)
+ if readable ~= nil then
+ return readable
+ end
+
+ if process.type() == "privileged agent" then
+ -- no worker has checked yet
+ return false
+ end
+
+ local ok, err = check_cycle_layout()
+ dict:set(CONNECTIONS_VERDICT_KEY, ok)
+ if not ok then
+ gauges.connections:reset()
+ gauges.connections_max:reset()
+ if not verdict_logged then
+ verdict_logged = true
+ core.log.warn("process connection metrics are disabled: ", err)
+ end
+ end
+
+ return ok
+end
+
+
+local function with_extra_labels(labels)
+ for _, name in ipairs(extra_label_names) do
+ labels[#labels + 1] = name
+ end
+ return labels
+end
+
+
+local function label_values(extra, ...)
+ local values = {...}
+ local n = select("#", ...)
+ for i, value in ipairs(extra) do
+ values[n + i] = value
+ end
+ return values
+end
+
+
+-- extra_labels, when given, appends the same labels to every gauge here:
+-- {names = {...}, values = function() return {...} end}
+function _M.init(prometheus_registry, extra_labels)
+ registry = prometheus_registry
+ dict = registry.dict
+ if extra_labels then
+ extra_label_names = extra_labels.names
+ extra_label_values = extra_labels.values
+ end
+
+ gauges = {
+ timers = registry:gauge("nginx_timers",
+ "Number of Lua timers per process and subsystem, by state",
+ with_extra_labels({"process", "subsystem", "state"})),
+ timers_max = registry:gauge("nginx_timers_max",
+ "Configured maximum number of Lua timers per process, by state",
+ with_extra_labels({"state"})),
+ connections = registry:gauge("nginx_process_connections",
+ "Number of connection slots in use per nginx process",
+ with_extra_labels({"process"})),
+ connections_max = registry:gauge("nginx_process_connections_max",
+ "Number of connection slots per nginx process",
+ with_extra_labels({"process"})),
+ lua_memory = registry:gauge("nginx_lua_memory_bytes",
+ "Memory allocated by the Lua VM per process and subsystem",
+ with_extra_labels({"process", "subsystem"})),
+ }
+
+ -- the verdict describes the binary and the configuration of the processes
+ -- starting now, so each worker checks again rather than trusting the one
+ -- left behind in the shared dict by a reload
+ if subsystem == "http" and process.type() ~= "privileged agent" then
+ dict:delete(CONNECTIONS_VERDICT_KEY)
+ end
+end
+
+
+local function report()
+ process_label = process_label or get_process_label()
+ local extra = extra_label_values and extra_label_values() or
no_extra_labels
+
+ local timers = gauges.timers
+ timers:set(ngx.timer.running_count(),
+ label_values(extra, process_label, subsystem, "running"))
+ timers:set(ngx.timer.pending_count(),
+ label_values(extra, process_label, subsystem, "pending"))
+
+ gauges.lua_memory:set(collectgarbage("count") * 1024,
+ label_values(extra, process_label, subsystem))
+
+ -- the remaining values are per process rather than per Lua VM, so only
+ -- the http VM reports them; it is present in every process that exports
+ if subsystem ~= "http" then
+ return
+ end
+
+ local max_running, max_pending = timers_max()
+ gauges.timers_max:set(max_running, label_values(extra, "running"))
+ gauges.timers_max:set(max_pending, label_values(extra, "pending"))
+
+ if not connections_readable() then
+ return
+ end
+
+ local connection_n, free_n = read_cycle()
+ if not connection_n then
+ return
+ end
+
+ gauges.connections:set(connection_n - free_n, label_values(extra,
process_label))
+ gauges.connections_max:set(connection_n, label_values(extra,
process_label))
+end
+
+
+-- Writes this process's values now. A failure here is logged rather than
+-- raised, so it can never cost the caller the rest of the exposition.
+function _M.refresh()
+ if not gauges then
+ return
+ end
+
+ local ok, err = pcall(report)
+ if not ok then
+ core.log.error("failed to report process metrics: ", err)
+ end
+end
+
+
+-- A reload with fewer workers leaves the series of the workers that are gone
+-- in the shared dict, where nothing overwrites them any more. Expiring them is
+-- not an option: some nginx-lua-prometheus versions do not renew the key index
+-- of a series that is set again, so a live series would expire as well.
+local function remove_retired_workers()
+ local names = {}
+ for _, gauge in pairs(gauges) do
+ names[gauge.name] = true
+ end
+
+ local worker_count = ngx.worker.count()
+ local key_index = registry.key_index
+ for _, key in ipairs(key_index:list()) do
+ local name = key:match("^([^{]+){")
+ local id = name and names[name] and
key:match('process="worker%-(%d+)"')
+ if id and tonumber(id) >= worker_count then
+ key_index:remove(key)
+ dict:delete(key)
+ end
+ end
+end
+
+
+local function tick(premature)
+ if premature or stopped then
+ timer_running = false
+ if stopped then
+ core.log.info("process metrics stopped")
+ end
+ return
+ end
+
+ if retired_cleanups > 0 then
+ retired_cleanups = retired_cleanups - 1
+ local ok, err = pcall(remove_retired_workers)
+ if not ok then
+ core.log.error("failed to remove the metrics of retired workers:
", err)
+ end
+ end
+
+ _M.refresh()
+
+ local ok, err = timer_at(interval, tick)
+ if not ok then
+ timer_running = false
+ core.log.error("failed to schedule process metrics: ", err)
+ end
+end
+
+
+-- Idempotent: calling it again while reporting only updates the interval.
+function _M.start(report_interval)
+ if not gauges then
+ return
+ end
+
+ interval = report_interval or DEFAULT_INTERVAL
+ stopped = false
+
+ -- one process is enough, and the privileged agent always exists. The
second
+ -- pass, one interval later, catches a sample that an exiting worker was
+ -- still writing while the first one ran.
+ if subsystem == "http" and process.type() == "privileged agent" then
+ retired_cleanups = 2
+ end
+
+ if timer_running then
+ return
+ end
+
+ timer_running = true
+ local ok, err = timer_at(0, tick)
+ if not ok then
+ timer_running = false
+ core.log.error("failed to start process metrics: ", err)
+ return
+ end
+
+ core.log.info("process metrics started, reporting every ", interval, "s")
+end
+
+
+-- The pending timer sees this when it fires and does not schedule another.
+function _M.stop()
+ stopped = true
+end
+
+
+return _M
diff --git a/docs/en/latest/plugins/prometheus.md
b/docs/en/latest/plugins/prometheus.md
index e0f16dad9e..b793cf0709 100644
--- a/docs/en/latest/plugins/prometheus.md
+++ b/docs/en/latest/plugins/prometheus.md
@@ -170,6 +170,11 @@ The following table lists the core metrics registered
directly by the `prometheu
| apisix_node_info | gauge | Information of the
APISIX node, such as host name and the current APISIX version.
|
| apisix_shared_dict_capacity_bytes | gauge | The total capacity of an
[NGINX shared
dictionary](https://github.com/openresty/lua-nginx-module#ngxshareddict).
|
| apisix_shared_dict_free_space_bytes | gauge | The remaining space in
an [NGINX shared
dictionary](https://github.com/openresty/lua-nginx-module#ngxshareddict).
|
+| apisix_nginx_timers | gauge | Number of running and
pending Lua timers of each NGINX process, per subsystem.
|
+| apisix_nginx_timers_max | gauge | Configured maximum
number of running and pending Lua timers per process
(`nginx_config.max_running_timers` and `nginx_config.max_pending_timers`).
|
+| apisix_nginx_process_connections | gauge | Number of connection
slots in use by each NGINX process, including listening sockets, client
connections and outgoing cosocket connections.
|
+| apisix_nginx_process_connections_max | gauge | Number of connection
slots of each NGINX process: `worker_connections` for workers, and 512 for the
privileged agent.
|
+| apisix_nginx_lua_memory_bytes | gauge | Memory allocated by the
Lua VM of each NGINX process, per subsystem.
|
| apisix_upstream_status | gauge | Health check status of
upstream nodes, available if health checks are configured on the upstream. A
value of `1` represents healthy and `0` represents unhealthy.
|
| apisix_stream_connection_total | counter | Total number of
connections handled per Stream Route.
|
| apisix_stream_active_connections | gauge | Number of Stream
sessions currently being proxied, per listening address. Covers TCP connections
and UDP sessions alike. |
@@ -440,6 +445,18 @@ The following labels are used to differentiate
`apisix_upstream_status` metrics.
| ip | IP address of the upstream node.
|
| port | Port number of the node.
|
+### Labels for the NGINX process metrics
+
+`apisix_nginx_timers`, `apisix_nginx_timers_max`,
`apisix_nginx_process_connections`, `apisix_nginx_process_connections_max` and
`apisix_nginx_lua_memory_bytes` describe the internals of each NGINX process,
so a resource running out in a single process, such as the privileged agent,
shows up before it causes errors. Every process writes its own values every
`refresh_interval`, and the process building the exposition refreshes its own
values first.
+
+| Name | Description
|
+| --------- |
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
+| process | The process the value belongs to: `worker-<id>` for a worker
process, or `privileged-agent`.
|
+| subsystem | `http` or `stream`. Each subsystem has a Lua VM of its own, with
its own timers and memory. Only on `apisix_nginx_timers` and
`apisix_nginx_lua_memory_bytes`. |
+| state | `running` or `pending`. Only on `apisix_nginx_timers` and
`apisix_nginx_timers_max`.
|
+
+The connection metrics are read from NGINX's internal cycle structure. When a
worker finds that the number of connection slots it reads does not match
`worker_connections` in the configuration, APISIX logs a warning and leaves
both connection metrics out for every process instead of exporting wrong
numbers.
+
## Examples
The examples below demonstrate how you can work with the `prometheus` Plugin
for different scenarios.
diff --git a/docs/zh/latest/plugins/prometheus.md
b/docs/zh/latest/plugins/prometheus.md
index 13128a44ac..33b513950f 100644
--- a/docs/zh/latest/plugins/prometheus.md
+++ b/docs/zh/latest/plugins/prometheus.md
@@ -170,6 +170,11 @@ Prometheus 中有不同类型的指标。要了解它们之间的区别，请参
| apisix_node_info | gauge | APISIX 节点的信息，例如主机名和当前的 APISIX 版本号。
|
| apisix_shared_dict_capacity_bytes | gauge | [NGINX
共享字典](https://github.com/openresty/lua-nginx-module#ngxshareddict) 的总容量。
|
| apisix_shared_dict_free_space_bytes | gauge | [NGINX
共享字典](https://github.com/openresty/lua-nginx-module#ngxshareddict) 中剩余的空间。
|
+| apisix_nginx_timers | gauge | 每个 NGINX 进程中运行中和等待中的 Lua
定时器数量，按子系统区分。
|
+| apisix_nginx_timers_max | gauge | 每个进程允许的运行中和等待中 Lua
定时器的上限（`nginx_config.max_running_timers` 和 `nginx_config.max_pending_timers`）。
|
+| apisix_nginx_process_connections | gauge | 每个 NGINX
进程已占用的连接槽位数，包括监听套接字、客户端连接和 cosocket 发起的出站连接。
|
+| apisix_nginx_process_connections_max | gauge | 每个 NGINX 进程的连接槽位总数：worker 进程为
`worker_connections`，privileged agent 为 512。
|
+| apisix_nginx_lua_memory_bytes | gauge | 每个 NGINX 进程中 Lua VM 分配的内存，按子系统区分。
|
| apisix_upstream_status | gauge | 上游节点的健康检查状态，如果在上游配置了健康检查，则可用。值为 `1`
表示健康，`0` 表示不健康。
|
| apisix_stream_connection_total | counter | 每个 Stream Route 处理的总连接数。
|
| apisix_stream_active_connections | gauge | 当前正在代理的 Stream 会话数，按监听地址区分。同时覆盖
TCP 连接与 UDP 会话。
|
@@ -428,6 +433,18 @@ APISIX 延迟
| ip | 上游节点的 IP 地址。
|
| port | 节点的端口号。
|
+### NGINX 进程指标的标签
+
+`apisix_nginx_timers`、`apisix_nginx_timers_max`、`apisix_nginx_process_connections`、`apisix_nginx_process_connections_max`
和 `apisix_nginx_lua_memory_bytes` 描述每个 NGINX 进程的内部状态，因此单个进程（例如 privileged
agent）的资源即将耗尽时，可以在它引发错误之前被发现。每个进程每隔 `refresh_interval`
写入一次自己的数值，生成指标输出的进程会先刷新自己的数值。
+
+| 名称 | 描述
|
+| --------- |
----------------------------------------------------------------------------------------------------------------------------
|
+| process | 数值所属的进程：worker 进程为 `worker-<id>`，privileged agent 为
`privileged-agent`。 |
+| subsystem | `http` 或 `stream`。每个子系统有独立的 Lua VM，定时器和内存也各自独立。仅用于
`apisix_nginx_timers` 和 `apisix_nginx_lua_memory_bytes`。 |
+| state | `running` 或 `pending`。仅用于 `apisix_nginx_timers` 和
`apisix_nginx_timers_max`。 |
+
+连接相关指标读取自 NGINX 内部的 cycle 结构。如果某个 worker 读到的连接槽位数与配置中的 `worker_connections`
不一致，APISIX 会记录一条警告，并在所有进程中都不导出这两个连接指标，而不是导出错误的数值。
+
## 示例
以下示例演示如何在不同场景中使用 `prometheus` 插件。
diff --git a/t/cli/test_prometheus_reload.sh b/t/cli/test_prometheus_reload.sh
index 7a8b1a1f81..f4ee9f9e45 100755
--- a/t/cli/test_prometheus_reload.sh
+++ b/t/cli/test_prometheus_reload.sh
@@ -89,3 +89,109 @@ if curl -i http://127.0.0.1:9091/apisix/prometheus/metrics
| grep "{}" > /dev/nu
echo "failed: metrics should not contain '{}' when prometheus is enabled"
exit 1
fi
+
+# The per-process metrics follow the plugin through hot reloads: each process
+# runs exactly one reporting timer while prometheus is enabled and none while
+# it is disabled.
+make stop
+sleep 0.5
+rm logs/error.log || true
+
+write_conf() {
+ echo "
+deployment:
+ role: traditional
+ role_traditional:
+ config_provider: etcd
+ admin:
+ admin_key: null
+apisix:
+ node_listen: 1984
+nginx_config:
+ error_log_level: info
+ worker_processes: $1
+plugins:
+ - $2
+plugin_attr:
+ prometheus:
+ refresh_interval: 1
+" > conf/config.yaml
+}
+
+# wait_for <description> <command...>: polls for up to 20 seconds
+wait_for() {
+ local desc=$1
+ shift
+ local deadline=$(( $(date +%s) + 20 ))
+ { set +x; } 2>/dev/null
+ while [ "$(date +%s)" -lt "$deadline" ]; do
+ if "$@"; then
+ set -x
+ return 0
+ fi
+ sleep 0.5
+ done
+ set -x
+ echo "failed: $desc"
+ exit 1
+}
+
+metrics() {
+ curl -s --connect-timeout 1 --max-time 2
http://127.0.0.1:9091/apisix/prometheus/metrics || true
+}
+
+# 2 workers and the privileged agent, http only
+started_is() { [ "$(grep -c 'process metrics started' logs/error.log)" -eq
"$1" ]; }
+stopped_is() { [ "$(grep -c 'process metrics stopped' logs/error.log)" -eq
"$1" ]; }
+reports_workers() {
+ local out
+ out="$(metrics)"
+ echo "$out" | grep -qE
'apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="http"\} [0-9]' \
+ && echo "$out" | grep -qE
'apisix_nginx_lua_memory_bytes\{process="worker-1",subsystem="http"\} [0-9]' \
+ && echo "$out" | grep -qE
'apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="http"\}
[0-9]'
+}
+exports_nothing() { metrics | grep -q "{}"; }
+
+write_conf 2 prometheus
+make run
+
+wait_for "every process reports when prometheus is enabled" reports_workers
+wait_for "one reporting timer per process" started_is 3
+
+write_conf 2 ip-restriction
+curl -s -o /dev/null http://127.0.0.1:9090/v1/plugins/reload -XPUT
+
+wait_for "every reporting timer stops when prometheus is disabled" stopped_is 3
+wait_for "nothing is exported when prometheus is disabled" exports_nothing
+
+write_conf 2 prometheus
+curl -s -o /dev/null http://127.0.0.1:9090/v1/plugins/reload -XPUT
+
+wait_for "reporting resumes when prometheus is enabled again" started_is 6
+wait_for "every process reports again" reports_workers
+
+# reloading with prometheus still enabled keeps the running timer
+curl -s -o /dev/null http://127.0.0.1:9090/v1/plugins/reload -XPUT
+sleep 3
+if ! started_is 6 || ! stopped_is 3; then
+ echo "failed: a plugin reload must not start a second reporting timer"
+ exit 1
+fi
+
+echo "passed: process metrics follow the plugin through hot reloads"
+
+# Scaling down must not leave the series of the workers that are gone.
+write_conf 1 prometheus
+make reload
+
+no_retired_worker() {
+ local out
+ out="$(metrics)"
+ echo "$out" | grep -qE
'apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="http"\} [0-9]' \
+ && ! echo "$out" | grep -q 'process="worker-1"'
+}
+wait_for "the series of a retired worker are removed" no_retired_worker
+
+make stop
+
+echo "passed: the series of retired workers are removed"
diff --git a/t/cli/test_prometheus_stream.sh b/t/cli/test_prometheus_stream.sh
index fcbfb5a2ee..c9b59f7a9c 100755
--- a/t/cli/test_prometheus_stream.sh
+++ b/t/cli/test_prometheus_stream.sh
@@ -124,3 +124,42 @@ if ! echo "$out" | grep "apisix_node_info{hostname=" >
/dev/null; then
fi
echo "passed: prometheus works when only stream is enabled"
+
+# the http subsystem exists only to export metrics here, yet every process
+# still reports its own Lua VMs and connection slots
+ok=0
+deadline=$(( $(date +%s) + 20 ))
+{ set +x; } 2>/dev/null
+while [ "$(date +%s)" -lt "$deadline" ]; do
+ out="$(curl -s --connect-timeout 1 --max-time 2
http://127.0.0.1:9091/apisix/prometheus/metrics || true)"
+ if echo "$out" | grep -qE
'apisix_nginx_timers\{process="worker-0",subsystem="stream",state="running"\}
[0-9]+$' \
+ && echo "$out" | grep -qE
'apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="http"\}
[1-9]' \
+ && echo "$out" | grep -qE
'apisix_nginx_process_connections_max\{process="privileged-agent"\} 512$'; then
+ ok=1
+ break
+ fi
+ sleep 0.5
+done
+set -x
+if [ "$ok" -ne 1 ]; then
+ echo "failed: process metrics are missing when only stream is enabled"
+ exit 1
+fi
+
+echo "passed: process metrics work when only stream is enabled"
+
+# the export-only http block runs on the same timer limits as the stream
+# subsystem, so the reported limits hold for every Lua VM
+http_block="$(sed -n '/^http {/,/^}/p' conf/nginx.conf)"
+if ! echo "$http_block" | grep -q "lua_max_pending_timers 16384;" \
+ || ! echo "$http_block" | grep -q "lua_max_running_timers 4096;"; then
+ echo "failed: the stream-only http block does not carry the timer limits"
+ exit 1
+fi
+
+if ! echo "$out" | grep -qE 'apisix_nginx_timers_max\{state="pending"\}
16384$'; then
+ echo "failed: nginx_timers_max does not report the configured limit"
+ exit 1
+fi
+
+echo "passed: the stream-only http block carries the timer limits"
diff --git a/t/plugin/prometheus-process-metrics.t
b/t/plugin/prometheus-process-metrics.t
new file mode 100644
index 0000000000..059343fb00
--- /dev/null
+++ b/t/plugin/prometheus-process-metrics.t
@@ -0,0 +1,210 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request && !defined $block->stream_request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ # t::APISIX renders worker_connections 128 into nginx.conf, so the
+ # configuration has to say the same for the connection gauges to pass
+ # their self check
+ if (!defined $block->extra_yaml_config) {
+ $block->set_value("extra_yaml_config", <<_EOC_);
+stream_plugins:
+ - prometheus
+nginx_config:
+ max_running_timers: 1234
+ max_pending_timers: 5678
+ event:
+ worker_connections: 128
+plugin_attr:
+ prometheus:
+ refresh_interval: 0.1
+_EOC_
+ }
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: pre-create public API route
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/metrics',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "public-api": {}
+ },
+ "uri": "/apisix/prometheus/metrics"
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: every process reports its timers, connection slots and Lua memory
+--- config
+ location /t {
+ content_by_lua_block {
+ ngx.sleep(0.5)
+ local http = require("resty.http")
+ local httpc = http.new()
+ local res = assert(httpc:request_uri(
+ "http://127.0.0.1:1984/apisix/prometheus/metrics";))
+ local patterns = {
+
[[apisix_nginx_timers\{process="worker-0",subsystem="http",state="running"\}
\d+\n]],
+
[[apisix_nginx_timers\{process="worker-0",subsystem="http",state="pending"\}
\d+]],
+
[[apisix_nginx_timers\{process="privileged-agent",subsystem="http",state="running"\}
\d+]],
+ [[apisix_nginx_timers_max\{state="running"\} 1234\n]],
+ [[apisix_nginx_timers_max\{state="pending"\} 5678\n]],
+ [[apisix_nginx_process_connections\{process="worker-0"\}
[1-9]\d*\n]],
+
[[apisix_nginx_process_connections\{process="privileged-agent"\} [1-9]\d*\n]],
+ [[apisix_nginx_process_connections_max\{process="worker-0"\}
128\n]],
+
[[apisix_nginx_process_connections_max\{process="privileged-agent"\} 512\n]],
+
[[apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="http"\}
[1-9]\d*]],
+
[[apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="http"\}
[1-9]\d*]],
+ }
+ for _, p in ipairs(patterns) do
+ if not ngx.re.find(res.body, p, "jo") then
+ ngx.say("missing ", p)
+ end
+ end
+ ngx.say("done")
+ }
+ }
+--- response_body
+done
+
+
+
+=== TEST 3: connection slots held by cosockets show up in the worker's gauge
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+
+ local function worker_connections()
+ ngx.sleep(0.5)
+ local httpc = http.new()
+ local res = assert(httpc:request_uri(
+ "http://127.0.0.1:1984/apisix/prometheus/metrics";))
+ local m = ngx.re.match(res.body,
+ [[apisix_nginx_process_connections\{process="worker-0"\}
(\d+)]], "jo")
+ return tonumber(m[1])
+ end
+
+ local before = worker_connections()
+
+ local socks = {}
+ for i = 1, 20 do
+ local sock = ngx.socket.tcp()
+ assert(sock:connect("127.0.0.1", 1980))
+ socks[i] = sock
+ end
+
+ local held = worker_connections()
+
+ for _, sock in ipairs(socks) do
+ sock:close()
+ end
+
+ local released = worker_connections()
+
+ ngx.say("held: ", held - before >= 20)
+ ngx.say("released: ", held - released >= 20)
+ }
+ }
+--- response_body
+held: true
+released: true
+
+
+
+=== TEST 4: connection gauges stay out when the slots do not match
worker_connections
+--- extra_yaml_config
+plugin_attr:
+ prometheus:
+ refresh_interval: 0.1
+--- config
+ location /t {
+ content_by_lua_block {
+ ngx.sleep(0.5)
+ local http = require("resty.http")
+ local httpc = http.new()
+ local res = assert(httpc:request_uri(
+ "http://127.0.0.1:1984/apisix/prometheus/metrics";))
+ ngx.say(ngx.re.find(res.body, "apisix_nginx_process_connections",
"jo") == nil)
+ ngx.say(ngx.re.find(res.body,
+
[[apisix_nginx_timers\{process="worker-0",subsystem="http",state="pending"\}]],
+ "jo") ~= nil)
+ }
+ }
+--- response_body
+true
+true
+--- error_log
+process connection metrics are disabled: worker connection slots read as 128,
but worker_connections is 10620
+
+
+
+=== TEST 5: the stream Lua VM of every process reports as well
+--- stream_server_config
+ content_by_lua_block {
+ ngx.req.socket():receive()
+ ngx.sleep(0.5)
+ local http = require("resty.http")
+ local httpc = http.new()
+ local res = assert(httpc:request_uri(
+ "http://127.0.0.1:1984/apisix/prometheus/metrics";))
+ local patterns = {
+
[[apisix_nginx_timers\{process="worker-0",subsystem="stream",state="running"\}
\d+\n]],
+
[[apisix_nginx_timers\{process="worker-0",subsystem="stream",state="pending"\}
\d+]],
+
[[apisix_nginx_timers\{process="privileged-agent",subsystem="stream",state="pending"\}
\d+]],
+
[[apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="stream"\}
[1-9]\d*]],
+
[[apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="stream"\}
[1-9]\d*]],
+ }
+ for _, p in ipairs(patterns) do
+ if not ngx.re.find(res.body, p, "jo") then
+ ngx.say("missing ", p)
+ end
+ end
+ ngx.say("done")
+ }
+--- stream_request
+ping
+--- stream_response
+done
