This is an automated email from the ASF dual-hosted git repository. AlinsRan pushed a commit to branch feat/toolset-plugin in repository https://gitbox.apache.org/repos/asf/apisix.git
commit 8cc56dc9eab86bde65654fcaa73eace8e134a44f Author: AlinsRan <[email protected]> AuthorDate: Wed May 27 04:08:42 2026 +0800 feat: add toolset plugin The toolset plugin is a diagnostics and observability framework that hosts multiple lightweight sub-plugins, each independently configured via plugin_attr and dynamically loaded/unloaded at runtime. Sub-plugins included: - trace: instruments APISIX request phases and logs a timing table for sampled requests, supports host/path filtering, sampling rate, trace header detection, and minimum timespan threshold - table_count: periodically measures and logs the entry count of specified Lua module tables, useful for monitoring memory growth Co-authored-by: Copilot <[email protected]> --- apisix/cli/config.lua | 16 + apisix/plugins/toolset/config.lua | 17 + apisix/plugins/toolset/init.lua | 148 ++++++++ apisix/plugins/toolset/src/table-count/init.lua | 105 ++++++ apisix/plugins/toolset/src/trace.lua | 445 ++++++++++++++++++++++++ conf/config.yaml.example | 18 +- docs/en/latest/config.json | 1 + docs/en/latest/plugins/toolset.md | 159 +++++++++ 8 files changed, 908 insertions(+), 1 deletion(-) diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua index c42ecbdee..375025734 100644 --- a/apisix/cli/config.lua +++ b/apisix/cli/config.lua @@ -340,6 +340,22 @@ local _M = { ["server-info"] = { report_ttl = 60 }, + toolset = { + trace = { + rate = 1, + hosts = {}, + paths = {}, + gen_uid = false, + vars = {}, + timespan_threshold = 0 + }, + table_count = { + lua_modules = {}, + interval = 5, + depth = 10, + scopes = {"worker", "privileged agent"} + } + }, ["dubbo-proxy"] = { upstream_multiplex_count = 32 }, diff --git a/apisix/plugins/toolset/config.lua b/apisix/plugins/toolset/config.lua new file mode 100644 index 000000000..350616386 --- /dev/null +++ b/apisix/plugins/toolset/config.lua @@ -0,0 +1,17 @@ +return { + trace = { + rate = 1, -- allow only 1 request per 100 requests + hosts = {}, -- only the requests carrying these host headers will be traced + paths = {}, -- only these request_uris will be traced + gen_uid = false, -- adds a UID to the trace if none of the traceable headers are found + vars = {}, -- add these nginx or inbuilt variables to trace table + timespan_threshold = 0 -- requests taking longer than this value (in seconds) will be traced + }, + table_count = { + lua_modules = {}, -- change it + interval = 5, + depth = 10, -- when it is not passed, default depth will be 1 + -- optional, default is all APISIX processes + scopes = {"worker", "privileged agent"} + } +} diff --git a/apisix/plugins/toolset/init.lua b/apisix/plugins/toolset/init.lua new file mode 100644 index 000000000..66efed913 --- /dev/null +++ b/apisix/plugins/toolset/init.lua @@ -0,0 +1,148 @@ +local pairs = pairs +local core = require("apisix.core") +local ngx = ngx +local cache = core.table.new(0, 32) +local stop_timer = false +local load, unload = "load", "unload" +local package = package +local pcall = pcall +local require = require +local string = string + +local _M = { + version = 0.1, + priority = 22901, + name = "toolset", + schema = {}, + scope = "global", +} + + +local function get_plugin_config() + -- clear cache to reload + package.loaded["apisix.plugins.toolset.config"] = nil + local loaded, plugins_config = pcall(require, "apisix.plugins.toolset.config") + if loaded and plugins_config == true then + core.log.warn("empty plugin config file") + return nil + end + if not loaded then + core.log.error("failed to load plugin config: ", plugins_config) + return nil + end + return plugins_config +end + + +local function is_config_changed(plugin_name, plugin_config) + if core.table.deep_eq(cache[plugin_name], plugin_config) then + return false + end + return true +end + + +local function is_config_empty(plugin_config) + return plugin_config == nil or core.table.deep_eq(plugin_config, {}) +end + + +local function perform_operation_for_plugin(plugin_name, plugin_config, operation) + if operation == load then + local loaded, plugin = pcall(require, "apisix.plugins.toolset.src." + .. string.gsub(plugin_name, "_", "-")) + if not loaded then + core.log.warn("could not load plugin because it was not found: ", plugin_name) + return + end + core.log.warn("initializing sub plugin for toolset plugin: ", plugin_name) + plugin.init() + cache[plugin_name] = plugin_config + elseif operation == unload then + local loaded, plugin = pcall(require, "apisix.plugins.toolset.src." .. + string.gsub(plugin_name, "_", "-")) + if not loaded then + core.log.warn("could not unload plugin because it was not found: ", plugin_name) + return + end + core.log.warn("destroying sub plugin for toolset plugin: ", plugin_name) + plugin.destroy() + cache[plugin_name] = nil + end +end + + +local function sync() + core.log.info("syncing toolset plugin") + local plugin_configs = get_plugin_config() + local processed_plugins = {} + if plugin_configs then + for plugin_name, plugin_config in pairs(plugin_configs) do + processed_plugins[plugin_name] = true + -- checks if the config is different from cache + if is_config_changed(plugin_name, plugin_config) then + if is_config_empty(plugin_config) then + -- allow executing even with empty config. + -- Assuming the plugin will run with default values + core.log.warn("empty config found for ", plugin_name,".Running with default values") + end + core.log.warn("config changed. reloading plugin: ", plugin_name) + local ok, err = pcall(perform_operation_for_plugin, plugin_name, plugin_config, load) + if not ok then + core.log.error("toolset plugin load raised: ", err) + end + end + end + end + + for plugin_name, plugin_config in pairs(cache) do + if not processed_plugins[plugin_name] then + core.log.warn("plugin config unloaded: ", plugin_name) + local ok, err = pcall(perform_operation_for_plugin, plugin_name, plugin_config, unload) + if not ok then + core.log.error("toolset plugin unload raised: ", err) + end + end + end + if not stop_timer then + local ok, err = ngx.timer.at(1, sync) + if not ok then + core.log.error("failed to create timer for running toolset ", err) + end + end +end + + +function _M.init() + core.log.info("initializing toolset plugin") + local plugins_config = get_plugin_config() + if plugins_config then + for plugin_name, plugin_config in pairs(plugins_config) do + if is_config_empty(plugin_config) then + -- allow executing even with empty config. + -- Assuming the plugin will run with default values + core.log.warn("empty config found for ", plugin_name,".Running with default values") + end + perform_operation_for_plugin(plugin_name, plugin_config, load) + end + end + ngx.timer.at(1, sync) +end + + +function _M.destroy() + local plugin_configs = get_plugin_config() + if plugin_configs then + for plugin_name, plugin_config in pairs(plugin_configs) do + perform_operation_for_plugin(plugin_name, plugin_config, unload) + end + + end + for plugin_name, plugin_config in pairs(cache) do + perform_operation_for_plugin(plugin_name, plugin_config, unload) + end + + stop_timer = true +end + +return _M diff --git a/apisix/plugins/toolset/src/table-count/init.lua b/apisix/plugins/toolset/src/table-count/init.lua new file mode 100644 index 000000000..ade2f5ab8 --- /dev/null +++ b/apisix/plugins/toolset/src/table-count/init.lua @@ -0,0 +1,105 @@ +local core = require("apisix.core") +local ngx = require("ngx") +local process = require("ngx.process") + +local pairs = pairs +local ipairs = ipairs +local type = type +local timer = ngx.timer +local require = require +local package = package + +local plugin_name = "table-count" + +local schema = {} +local stop = false +-- only one run of init() function should be running at a time. +-- when init() is reloaded the run number is incremented. It also helps in debugging. +local current_run = 0 + +local _M = { + version = 0.1, + priority = 22902, + name = plugin_name, + schema = schema, + scope = "global", +} + +local function tab_item_count(tab, cache,depth) + if depth == 0 then + core.log.warn("out of depth..skipping count") + return + end + depth = depth - 1 + cache = cache or {} + local count = 0 + for _, value in pairs(tab) do + if cache[value] then + core.log.warn("circular reference detected..skipping count") + goto continue + end + if type(value) == "table" and not cache[value] then + cache[value] = true + local tab_count = tab_item_count(value, cache,depth) + if tab_count then + count = count + tab_count + 1 + end + else + count = count + 1 + end + ::continue:: + end + return count +end + +function _M.init() + package.loaded["apisix.plugins.toolset.config"] = nil + local config = require("apisix.plugins.toolset.config").table_count + if config.lua_modules == nil or #config.lua_modules == 0 then + core.log.warn("no lua_modules provided for table count") + return + end + if not config.scopes then + core.log.warn("no scope provided. Running for all scopes") + goto continue + end + if #config.scopes ~= 0 then + for _,scope in ipairs(config.scopes) do + if process.type() == scope then + goto continue + end + end + return + end + ::continue:: + -- Extract configuration values + current_run = current_run + 1 + local interval = config.interval or 5 + local run_count + run_count = function(run_no) + local depth = config.depth or 1 + for _, package_name in ipairs(config.lua_modules) do + local package = require(package_name) + local count = tab_item_count(package, {},depth) + core.log.warn("package ", package_name, " table count is: ", count," for loaded: ",run_no) + end + if stop or run_no ~= current_run then + return + end + local ok, err = timer.at(interval, run_count,current_run) + if not ok then + core.log.error("failed to create timer for running table count ", err) + end + end + + local ok, err = timer.at(0, run_count,current_run) + if not ok then + core.log.error("failed to create timer for running table count ", err) + end +end + +function _M.destroy() + stop = true +end + +return _M diff --git a/apisix/plugins/toolset/src/trace.lua b/apisix/plugins/toolset/src/trace.lua new file mode 100644 index 000000000..2c7f48ab6 --- /dev/null +++ b/apisix/plugins/toolset/src/trace.lua @@ -0,0 +1,445 @@ +local require = require +local apisix = require("apisix") +local core = require("apisix.core") +local uuid = require("resty.jit-uuid") + +local conf_path = "apisix.plugins.toolset.config" + +local ngx = ngx +local pairs = pairs +local type = type +local package = package +local tostring = tostring +local format = string.format +local floor = math.floor +local gsub = ngx.re.gsub +local m_random = math.random +local m_randomseed = math.randomseed +local t_remove = table.remove +local re_match = ngx.re.match +local counter = 1 + +local old_http_access_phase +local old_match_route +local old_http_log_phase +local old_http_balancer_phase +local old_http_header_filter_phase +local old_http_body_filter_phase +local old_resolve + +local schema = {} + +local PHASE_UPSTREAM = "upstream (req + response)" +local PHASE_CLIENT = "response" + +local suffix = [[ ++----------+---------------------------+----------+-------------------------+ +]] +local prefix = [[ + ++----------+---------------------------+----------+-------------------------+ +| Role | Phase | Timespan | Start time | +]] .. suffix + +local trace_headers = { + "x-request-id", -- request id header + "sw8", -- skywalking + "traceparent", -- opentelemetry + "x-b3-traceid", -- zipkin +} +local plugin_name = "trace" + +local _M = { + version = 0.1, + priority = 22901, + name = plugin_name, + schema = schema, + scope = "global", +} + +local function nspaces(n) + return (" "):rep(n) +end + +local function add_entry(phase, timespan, curtime) + core.log.info("add entry for: ", phase) + local role + local tpl = [[ +| %s| %s| %s| %s | +]] + if phase == PHASE_UPSTREAM then + role = "Upstream " + elseif phase == PHASE_CLIENT then + role = "Client " + else + role = "APISIX " + end + + -- add spaces around the text for table formatting + phase = phase .. nspaces(26 - #phase) + timespan = timespan .. nspaces(9 - #tostring(timespan)) + ngx.ctx.trace_log = ngx.ctx.trace_log .. format(tpl, role, phase, timespan, curtime) +end + + +local function timespan(raw) + if raw == 0 then + return "0ms" + end + local factor = 1000 -- 1000ms in 1s + local unit = "ms" + if raw >= 1 then -- if greater than 1s don't convert to ms + factor = 1 + unit = "s" + end + return floor(raw * factor + 0.5) .. unit +end + + +local function localtime_msec(now) + local lt = ngx.localtime() + local msec = now * 1000 - floor(now) * 1000 + if msec > 0 then + return lt .. "." .. msec + end + return lt .. ".000" +end + + +local function match(incoming, conf) + conf = gsub(conf, "\\*", ".*") + conf = "^" .. conf .. "$" + core.log.info("matching: ", incoming, " against: ", conf) + + local matches = re_match(incoming, "^" .. conf .. "$", "jo") + if not matches then + return nil + end + return matches[0] +end + + +local unique_random +do + local numbers = {} + for i = 1, 100 do + numbers[i] = i + end + unique_random = function() + m_randomseed(ngx.now()) + while true do + local index = m_random(100) + local num = numbers[index] + if num then + t_remove(numbers, index) + return num + end + end + end +end + + +local function incr_counter() + counter = counter + 1 + if counter > 99 then + counter = 0 + end +end + + +local function preprocess(trace_conf, ctx) + if not trace_conf.rate or type(trace_conf.rate) ~= "number" then + ctx.trace = true -- trace all reqs if rate isn't defined + return + end + if trace_conf.rate == 1 then + ctx.trace = counter == 1 -- trace only first request + incr_counter() + return + end + core.log.info("trace_conf.rate: ", trace_conf.rate) + local rand = unique_random() + if rand <= trace_conf.rate then + ctx.trace = true + end + core.log.info("random number: ", rand) + incr_counter() +end + + +local function check(trace_conf, uri_or_host) + for _, val in pairs(trace_conf) do + if match(uri_or_host, val) == uri_or_host then + return true + end + end + return false +end + + +local function check_host(trace_conf) + local req_host = core.request.header(ngx.ctx, "host") + if (trace_conf.hosts and #trace_conf.hosts > 0) and (req_host and #req_host > 0) then + return check(trace_conf.hosts, req_host) + end + -- pass host check if hosts field is not defined in config.lua + return trace_conf.hosts ~= nil +end + + +local function check_uri(trace_conf) + if trace_conf.paths and #trace_conf.paths > 0 then + return check(trace_conf.paths, ngx.ctx.api_ctx.var.request_uri) + end + -- pass uri check if paths field is not defined in config.lua + return true +end + + +local function prepend(ctx, field, val) + ctx.trace_log = "\n" .. field .. ": " .. val .. ctx.trace_log +end + + +local function add_headers(ctx) + local count = 0 + for _, header_field in pairs(trace_headers) do + local val = core.request.header(ctx, header_field) + if val and #val > 0 then + prepend(ctx, header_field, val) + count = count + 1 + end + end + return count +end + + +local function add_vars(ctx, vars) + local count = 0 + if vars and #vars > 0 then + for _, var in pairs(vars) do + local val = ngx.var[var] + if val and #val > 0 then + prepend(ctx, var, val) + count = count + 1 + end + end + end + return count +end + + +function _M.init() + package.loaded[conf_path] = false + local trace_conf = require(conf_path).trace + core.log.info("trace_conf: ", core.json.encode(trace_conf)) + + local conf = core.config.local_conf() + local router_name = "radixtree_uri" + if conf and conf.apisix and conf.apisix.router then + router_name = conf.apisix.router.http or router_name + end + + local dns = require("apisix.core.dns.client") + if dns then + if not old_resolve then + old_resolve = dns.resolve + end + + dns.resolve = function (...) + local match_start = ngx.now() + ngx.ctx.dns_lt = localtime_msec(match_start) + local ret = old_resolve(...) + ngx.update_time() + + ngx.ctx.dns_resolve_timespan = ngx.now() - match_start + return ret + end + end + + local router = require("apisix.http.router." .. router_name) + if not old_match_route then + old_match_route = router.match + end + router.match = function(...) + local match_start = ngx.now() + ngx.ctx.match_lt = localtime_msec(match_start) + + old_match_route(...) + ngx.update_time() + + ngx.ctx.match_timespan = ngx.now() - match_start + end + + if not old_http_access_phase then + old_http_access_phase = apisix.http_access_phase + end + apisix.http_access_phase = function(...) + ngx.ctx.trace = false + preprocess(trace_conf, ngx.ctx) + if not ngx.ctx.trace then + old_http_access_phase(...) + else + ngx.ctx.trace_log = prefix + + local access_start = ngx.now() + ngx.ctx.req_start = access_start + ngx.ctx.access_lt = localtime_msec(access_start) + + old_http_access_phase(...) + + local host_pass = check_host(trace_conf) + local path_pass = check_uri(trace_conf) + + core.log.info("path check: ", path_pass, ". host check: ", host_pass) + ngx.ctx.trace = path_pass or host_pass + ngx.update_time() + + ngx.ctx.access_timespan = ngx.now() - access_start + end + end + + if not old_http_balancer_phase then + old_http_balancer_phase = apisix.http_balancer_phase + end + apisix.http_balancer_phase = function(...) + if not ngx.ctx.trace then + old_http_balancer_phase(...) + else + local num_headers = add_headers(ngx.ctx) + local num_vars = add_vars(ngx.ctx, trace_conf.vars) + -- if no vars or headers were added add a uuid + if (num_headers + num_vars) < 1 and trace_conf.gen_uid then + ngx.ctx.trace_log = "\n" .. "uuid: " .. uuid() .. ngx.ctx.trace_log + end + + local balancer_start = ngx.now() + ngx.ctx.balancer_lt = localtime_msec(balancer_start) + + old_http_balancer_phase(...) + ngx.update_time() + + ngx.ctx.balancer_timespan = ngx.now() - balancer_start + ngx.update_time() + ngx.ctx.upstream_start = ngx.now() + ngx.ctx.upstream_lt = localtime_msec(ngx.ctx.upstream_start) + end + end + + if not old_http_header_filter_phase then + old_http_header_filter_phase = apisix.http_header_filter_phase + end + apisix.http_header_filter_phase = function(...) + if not ngx.ctx.trace then + old_http_header_filter_phase(...) + else + local header_filter_start = ngx.now() + ngx.ctx.upstream_end = header_filter_start + ngx.ctx.header_filter_start = localtime_msec(header_filter_start) + + old_http_header_filter_phase(...) + ngx.update_time() + + ngx.ctx.header_filter_timespan = ngx.now() - header_filter_start + end + end + + if not old_http_body_filter_phase then + old_http_body_filter_phase = apisix.http_body_filter_phase + end + apisix.http_body_filter_phase = function(...) + local body_filter_start = ngx.now() + if not ngx.ctx.trace then + old_http_body_filter_phase(...) + else + if not ngx.ctx.bf_timespan then + ngx.ctx.bf_timespan = 0 + ngx.ctx.bf_lt = localtime_msec(body_filter_start) + end + + old_http_body_filter_phase(...) + ngx.update_time() + + ngx.ctx.bf_end = ngx.now() + ngx.ctx.bf_timespan = ngx.ctx.bf_timespan + (ngx.ctx.bf_end - body_filter_start) + ngx.ctx.response_lt = localtime_msec(ngx.ctx.bf_end) + end + end + + if not old_http_log_phase then + old_http_log_phase = apisix.http_log_phase + end + apisix.http_log_phase = function(...) + if not ngx.ctx.trace then + old_http_log_phase(...) + else + local log_start = ngx.now() + local log_lt = localtime_msec(log_start) + + old_http_log_phase(...) + ngx.update_time() + local log_end = ngx.now() + + local premature = false + -- when route match fails access_timespan = nil + if not ngx.ctx.access_timespan then + ngx.ctx.access_timespan = 0 + ngx.ctx.balancer_timespan = 0 + premature = true + end + + local upstream_timespan = 0 + if not premature then + upstream_timespan = ngx.ctx.upstream_end - ngx.ctx.upstream_start + end + + local client_timespan = log_start - ngx.ctx.bf_end + local log_timespan = log_end - log_start + local total_time = ngx.ctx.access_timespan + ngx.ctx.balancer_timespan + upstream_timespan + + ngx.ctx.header_filter_timespan + ngx.ctx.bf_timespan + client_timespan + + log_timespan + + if total_time >= (trace_conf.timespan_threshold or 0) then + add_entry("access", timespan(ngx.ctx.access_timespan), ngx.ctx.access_lt) + add_entry("\\_match_route", timespan(ngx.ctx.match_timespan), ngx.ctx.match_lt) + if ngx.ctx.dns_resolve_timespan then + add_entry("\\_dns_resolve", timespan(ngx.ctx.dns_resolve_timespan), ngx.ctx.dns_lt) + end + if not premature then + add_entry("balancer", timespan(ngx.ctx.balancer_timespan), ngx.ctx.balancer_lt) + add_entry(PHASE_UPSTREAM, + timespan(upstream_timespan), ngx.ctx.upstream_lt) + end + add_entry("header_filter", timespan(ngx.ctx.header_filter_timespan), + ngx.ctx.header_filter_start) + add_entry("body_filter", timespan(ngx.ctx.bf_timespan), ngx.ctx.bf_lt) + if not premature then + add_entry(PHASE_CLIENT, timespan(client_timespan), ngx.ctx.response_lt) + end + add_entry("log", timespan(log_timespan), log_lt) + core.log.warn("trace: ", ngx.ctx.trace_log .. suffix) + end + end + ngx.ctx.trace_log = "" -- clear trace + ngx.ctx.bf_timespan = nil -- clear body_filter timespan + end +end + +function _M.destroy() + local conf = core.config.local_conf() + local router_name = "radixtree_uri" + if conf and conf.apisix and conf.apisix.router then + router_name = conf.apisix.router.http or router_name + end + + local router = require("apisix.http.router." .. router_name) + router.match = old_match_route + + apisix.http_access_phase = old_http_access_phase + apisix.http_balancer_phase = old_http_balancer_phase + apisix.http_header_filter_phase = old_http_header_filter_phase + apisix.http_body_filter_phase = old_http_body_filter_phase + apisix.http_log_phase = old_http_log_phase +end + +return _M diff --git a/conf/config.yaml.example b/conf/config.yaml.example index 6023c83bc..ab15e1a8e 100644 --- a/conf/config.yaml.example +++ b/conf/config.yaml.example @@ -471,8 +471,9 @@ graphql: plugins: # plugin list (sorted by priority) - real-ip # priority: 23000 - - ai # priority: 22900 #- exit-transformer # priority: 22950, disabled by default + #- toolset # priority: 22901, disabled by default + - ai # priority: 22900 - client-control # priority: 22000 - proxy-control # priority: 21990 - request-id # priority: 12015 @@ -665,6 +666,21 @@ plugin_attr: # Plugin attributes server-info: # Plugin: server-info report_ttl: 60 # Set the TTL in seconds for server info in etcd. # Maximum: 86400. Minimum: 3. + # toolset: # Plugin: toolset + # trace: # Sub-plugin: trace - instruments APISIX phases and logs timing info + # rate: 1 # Allow only 1 request per 100 requests to be traced + # hosts: [] # Only trace requests with these host headers (empty = all) + # paths: [] # Only trace requests with these URIs (empty = all) + # gen_uid: false # Add a UID to the trace when no traceable headers are found + # vars: [] # Additional nginx/inbuilt variables to include in trace output + # timespan_threshold: 0 # Only log traces for requests taking longer than this (in seconds) + # table_count: # Sub-plugin: table_count - periodically logs table sizes of Lua modules + # lua_modules: [] # List of Lua module names to measure (e.g. ["apisix.router"]) + # interval: 5 # Interval in seconds between measurements + # depth: 10 # Maximum depth for recursive table counting + # scopes: # APISIX process scopes to run in (default: all) + # - worker + # - privileged agent dubbo-proxy: # Plugin: dubbo-proxy upstream_multiplex_count: 32 # Set the maximum number of connections that can be multiplexed over # a single network connection between the Dubbo Proxy and the upstream diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index 115448b95..188736485 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -94,6 +94,7 @@ "plugins/brotli", "plugins/real-ip", "plugins/server-info", + "plugins/toolset", "plugins/ext-plugin-pre-req", "plugins/ext-plugin-post-req", "plugins/ext-plugin-post-resp", diff --git a/docs/en/latest/plugins/toolset.md b/docs/en/latest/plugins/toolset.md new file mode 100644 index 000000000..940b57f91 --- /dev/null +++ b/docs/en/latest/plugins/toolset.md @@ -0,0 +1,159 @@ +--- +title: toolset +keywords: + - Apache APISIX + - API Gateway + - Plugin + - Toolset + - toolset + - trace + - table-count +description: This document contains information about the Apache APISIX toolset Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `toolset` Plugin is a diagnostics and observability framework that hosts multiple lightweight sub-plugins. Each sub-plugin is configured in `config.yaml` under the `plugin_attr.toolset` key and is dynamically loaded or unloaded at runtime without restarting APISIX. The `toolset` plugin itself has no per-route schema and always operates at the global scope. + +### Sub-plugins + +| Sub-plugin | Description | +|---------------|------------------------------------------------------------------------------------------------| +| `trace` | Instruments APISIX request phases and emits a timing table to the error log for matching requests. | +| `table_count` | Periodically measures and logs the item count of specified Lua module tables. | + +## Attributes + +The `toolset` Plugin is configured through `plugin_attr` in `config.yaml` and has no route-level attributes. + +### trace + +| Name | Type | Required | Default | Description | +|-----------------------|---------|----------|---------|--------------------------------------------------------------------------------------------------------------| +| rate | integer | False | 1 | Sampling rate as N-out-of-100. `1` traces 1 request per 100; set to `100` to trace every request. | +| hosts | array | False | `[]` | Allowlist of `Host` header values (glob patterns supported). Empty means all hosts pass. | +| paths | array | False | `[]` | Allowlist of request URI patterns (glob patterns supported). Empty means all paths pass. | +| gen_uid | boolean | False | false | When `true`, generates a UUID for traces where no standard trace header (`x-request-id`, `traceparent`, etc.) is found. | +| vars | array | False | `[]` | Additional nginx or APISIX variables to prepend to the trace output. | +| timespan_threshold | number | False | 0 | Minimum total request duration (in seconds) required before emitting the trace log. `0` logs all traces. | + +### table_count + +| Name | Type | Required | Default | Description | +|--------------|---------|----------|-----------------------------------|---------------------------------------------------------------------------------------| +| lua_modules | array | True | | List of Lua module paths to measure (e.g. `["apisix.router"]`). | +| interval | integer | False | 5 | Interval in seconds between measurements. | +| depth | integer | False | 10 | Maximum recursion depth when counting table entries. `0` disables recursive counting. | +| scopes | array | False | `["worker", "privileged agent"]` | APISIX process types in which the sub-plugin runs. | + +## Enable Plugin + +The `toolset` Plugin must be added to the `plugins` list in `config.yaml`. All sub-plugin configuration is placed under `plugin_attr.toolset`: + +```yaml +plugins: + - toolset + +plugin_attr: + toolset: + trace: + rate: 10 + hosts: + - "*.example.com" + paths: + - "/api/*" + gen_uid: true + vars: + - remote_addr + timespan_threshold: 0.5 + table_count: + lua_modules: + - apisix.router + interval: 10 + depth: 5 + scopes: + - worker +``` + +## Example usage + +### Tracing slow requests + +The following configuration traces up to 10% of requests to `*.example.com` whose total processing time exceeds 500ms: + +```yaml +plugin_attr: + toolset: + trace: + rate: 10 + hosts: + - "*.example.com" + timespan_threshold: 0.5 +``` + +When a request meets the criteria, APISIX writes a table similar to the following to the error log at `WARN` level: + +``` ++----------+---------------------------+----------+-------------------------+ +| Role | Phase | Timespan | Start time | ++----------+---------------------------+----------+-------------------------+ +| APISIX | access | 3ms | 2024-01-01 12:00:00.123 | +| APISIX | \_match_route | 1ms | 2024-01-01 12:00:00.124 | +| APISIX | balancer | 1ms | 2024-01-01 12:00:00.125 | +| Upstream | upstream (req + response) | 520ms | 2024-01-01 12:00:00.126 | +| APISIX | header_filter | 0ms | 2024-01-01 12:00:00.646 | +| APISIX | body_filter | 0ms | 2024-01-01 12:00:00.646 | +| Client | response | 1ms | 2024-01-01 12:00:00.647 | +| APISIX | log | 0ms | 2024-01-01 12:00:00.648 | ++----------+---------------------------+----------+-------------------------+ +``` + +### Monitoring router table growth + +The following configuration measures the item count of the `apisix.router` Lua module every 30 seconds in worker processes: + +```yaml +plugin_attr: + toolset: + table_count: + lua_modules: + - apisix.router + interval: 30 + depth: 5 + scopes: + - worker +``` + +Results are written to the error log at `WARN` level: + +``` +package apisix.router table count is: 1234 for loaded: 1 +``` + +## Disable Plugin + +Remove `toolset` from the `plugins` list in `config.yaml` and reload APISIX: + +```yaml +plugins: + # - toolset # remove or comment out +```
