This is an automated email from the ASF dual-hosted git repository.
AlinsRan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 89e627384 feat(control-api): report the health checks a plugin owns
(#13899)
89e627384 is described below
commit 89e627384571c78f73e188c340a3b19861986c69
Author: AlinsRan <[email protected]>
AuthorDate: Tue Sep 1 15:45:44 2026 +0800
feat(control-api): report the health checks a plugin owns (#13899)
---
apisix/control/v1.lua | 103 ++++-
apisix/plugin.lua | 3 +
apisix/plugins/ai-proxy-multi.lua | 37 +-
docs/en/latest/control-api.md | 64 +++
docs/en/latest/plugins/ai-proxy-multi.md | 2 +
docs/zh/latest/control-api.md | 60 +++
docs/zh/latest/plugins/ai-proxy-multi.md | 2 +
t/control/healthcheck-ai-proxy-multi.t | 660 +++++++++++++++++++++++++++++++
8 files changed, 924 insertions(+), 7 deletions(-)
diff --git a/apisix/control/v1.lua b/apisix/control/v1.lua
index b224b6838..c73c8db5d 100644
--- a/apisix/control/v1.lua
+++ b/apisix/control/v1.lua
@@ -26,6 +26,7 @@ local healthcheck_manager =
require("apisix.healthcheck_manager")
local get_upstreams = upstream_mod.upstreams
local collectgarbage = collectgarbage
local ipairs = ipairs
+local pairs = pairs
local pcall = pcall
local setmetatable = setmetatable
local str_format = string.format
@@ -77,12 +78,14 @@ function _M.schema()
end
local healthcheck
-local function extra_checker_info(value)
+-- `value` is anything get_healthchecker_name() accepts: a resource config, or
a
+-- bare {resource_key = ...} for a checker a plugin owns.
+local function get_checker_nodes(value)
if not healthcheck then
healthcheck = require("resty.healthcheck")
end
- local name = healthcheck_manager.get_healthchecker_name(value.value)
+ local name = healthcheck_manager.get_healthchecker_name(value)
local nodes, err = healthcheck.get_target_list(name,
"upstream-healthcheck")
if err then
core.log.error("healthcheck.get_target_list failed: ", err)
@@ -92,9 +95,14 @@ local function extra_checker_info(value)
-- so keep the field a JSON array to report `[]` instead of `{}` then
setmetatable(nodes, core.json.array_mt)
end
+ return nodes
+end
+
+
+local function extra_checker_info(value)
return {
name = value.key,
- nodes = nodes,
+ nodes = get_checker_nodes(value.value),
}
end
@@ -108,6 +116,41 @@ local function get_checker_type(checks)
end
+-- A plugin can run active health checks of its own on nodes that belong to no
+-- upstream -- ai-proxy-multi probes every LLM instance and skips the unhealthy
+-- ones when it picks a target. Those checkers are keyed by the resource key
plus
+-- a JSON path, a layout only the plugin knows, so ask the plugin for them
+-- instead of guessing. What a checker stands for is the plugin's business too:
+-- it names itself in `meta`, reported verbatim.
+local function add_plugin_healthcheck_info(infos, value)
+ local plugins = value.value.plugins
+ if not plugins then
+ return
+ end
+
+ for name, plugin_conf in pairs(plugins) do
+ -- a disabled plugin never runs, so its checkers never exist; and its
+ -- config is kept even when check_schema() rejects it, so it must not
be
+ -- handed to the plugin either
+ if not plugin.check_disable(plugin_conf) then
+ local plugin_obj = plugin.get(name)
+ if plugin_obj and plugin_obj.list_healthcheck_targets then
+ local targets =
plugin_obj.list_healthcheck_targets(plugin_conf, value.key)
+ for _, target in ipairs(targets or {}) do
+ core.table.insert(infos, {
+ name = target.resource_path,
+ plugin = name,
+ meta = target.meta,
+ type = get_checker_type(target.checks),
+ nodes = get_checker_nodes({resource_key =
target.resource_path}),
+ })
+ end
+ end
+ end
+ end
+end
+
+
local function iter_and_add_healthcheck_info(infos, values)
if not values then
return
@@ -120,6 +163,7 @@ local function iter_and_add_healthcheck_info(infos, values)
info.type = get_checker_type(checks)
core.table.insert(infos, info)
end
+ add_plugin_healthcheck_info(infos, value)
end
end
@@ -241,15 +285,50 @@ local function iter_and_find_healthcheck_info(values,
src_type, src_id)
end
+-- Every checker a resource owns, in the same entry shape the /v1/healthcheck
+-- listing uses. A resource can own more than one -- its upstream plus one per
+-- plugin instance -- which the single object returned by
+-- /v1/healthcheck/{src_type}/{src_id} cannot express, so this is a
sub-resource
+-- of its own rather than a new field on that object. A resource with no health
+-- check at all is not an error here: it owns an empty set of checkers.
+local function iter_and_find_resource_checkers(values, src_type, src_id)
+ if not values then
+ return nil, str_format("%s[%s] not found", src_type, src_id)
+ end
+
+ for _, value in core.config_util.iterate_values(values) do
+ if value.value.id == src_id then
+ local infos = core.table.new(1, 0)
+ local checks = value.value.checks or
+ (value.value.upstream and value.value.upstream.checks)
+ if checks then
+ local info = extra_checker_info(value)
+ info.type = get_checker_type(checks)
+ core.table.insert(infos, info)
+ end
+ add_plugin_healthcheck_info(infos, value)
+ -- an empty result must still serialize as [], not {}
+ return setmetatable(infos, core.json.array_mt)
+ end
+ end
+
+ return nil, str_format("%s[%s] not found", src_type, src_id)
+end
+
+
function _M.get_health_checker()
local uri_segs = core.utils.split_uri(ngx_var.uri)
core.log.info("healthcheck uri: ", core.json.delay_encode(uri_segs))
- local src_type, src_id = uri_segs[4], uri_segs[5]
+ local src_type, src_id, sub_res = uri_segs[4], uri_segs[5], uri_segs[6]
if not src_id then
return 404, {error_msg = str_format("missing src id for src type %s",
src_type)}
end
+ if sub_res and (sub_res ~= "checkers" or uri_segs[7]) then
+ return 400, {error_msg = str_format("invalid sub resource %s",
sub_res)}
+ end
+
local values
if src_type == "routes" then
values = get_routes()
@@ -263,6 +342,22 @@ function _M.get_health_checker()
return 400, {error_msg = str_format("invalid src type %s", src_type)}
end
+ if sub_res then
+ local infos, err = iter_and_find_resource_checkers(values, src_type,
src_id)
+ if not infos then
+ return 404, {error_msg = err}
+ end
+ local out, err = try_render_html({stats = infos})
+ if out then
+ core.response.set_header("Content-Type", "text/html")
+ return 200, out
+ end
+ if err then
+ return 503, {error_msg = err}
+ end
+ return 200, infos
+ end
+
local info, err = iter_and_find_healthcheck_info(values, src_type, src_id)
if not info then
return 404, {error_msg = err}
diff --git a/apisix/plugin.lua b/apisix/plugin.lua
index 6f988fd7d..5a7a405a0 100644
--- a/apisix/plugin.lua
+++ b/apisix/plugin.lua
@@ -162,6 +162,9 @@ local function check_disable(plugin_conf)
return plugin_conf._meta.disable
end
+-- exposed for callers that must not act on a plugin config which never runs,
+-- such as the control API reporting the health checkers a plugin owns
+_M.check_disable = check_disable
local PLUGIN_TYPE_HTTP = 1
local PLUGIN_TYPE_STREAM = 2
diff --git a/apisix/plugins/ai-proxy-multi.lua
b/apisix/plugins/ai-proxy-multi.lua
index 6ce28e793..19aba41a3 100644
--- a/apisix/plugins/ai-proxy-multi.lua
+++ b/apisix/plugins/ai-proxy-multi.lua
@@ -597,6 +597,16 @@ local function create_server_picker(conf, ups_tab,
checkers)
end
+-- Identity of the health checker of the i-th instance: the parent resource key
+-- plus the JSON path of the instance. The health check manager parses it back
+-- to reach construct_upstream(), so both ends must agree on this exact layout.
+local function instance_resource_path(resource_key, index)
+ -- json path is 0 indexed so we need to decrement the index
+ return resource_key .. "#plugins['" .. plugin_name .. "'].instances["
+ .. index - 1 .. "]"
+end
+
+
local function get_instance_conf(instances, name)
for _, ins in ipairs(instances) do
if ins.name == name then
@@ -616,9 +626,8 @@ local function pick_target(ctx, conf, ups_tab)
for i, instance in ipairs(conf.instances) do
if instance.checks then
resolve_endpoint(instance)
- -- json path is 0 indexed so we need to decrement i
- local resource_path = conf._meta.parent.resource_key ..
- "#plugins['ai-proxy-multi'].instances[" ..
i-1 .. "]"
+ local resource_path = instance_resource_path(
+ conf._meta.parent.resource_key, i)
local resource_version = conf._meta.parent.resource_version
if instance._nodes_ver then
resource_version = resource_version .. instance._nodes_ver
@@ -982,6 +991,28 @@ local function retry_on_error(ctx, conf, code, body)
return code
end
+-- Each instance can run its own active health check, whose checker is keyed by
+-- instance_resource_path(). Nothing outside the plugin can guess that layout,
so
+-- expose it for the control API, which reports the health status of every
+-- configured checker (control/v1.lua). `meta` describes the checker to whoever
+-- reports it -- what an instance is is the plugin's business, so the control
API
+-- passes it through instead of knowing about it.
+function _M.list_healthcheck_targets(conf, resource_key)
+ local targets = {}
+ for i, instance in ipairs(conf.instances or {}) do
+ if instance.checks then
+ core.table.insert(targets, {
+ resource_path = instance_resource_path(resource_key, i),
+ checks = instance.checks,
+ meta = {instance = instance.name},
+ })
+ end
+ end
+
+ return targets
+end
+
+
function _M.construct_upstream(instance)
if not instance then
return nil, "instance configuration is nil"
diff --git a/docs/en/latest/control-api.md b/docs/en/latest/control-api.md
index 6f594c402..a945b2db9 100644
--- a/docs/en/latest/control-api.md
+++ b/docs/en/latest/control-api.md
@@ -145,6 +145,38 @@ Each of the returned objects contain the following fields:
* nodes[i].counter.tcp_failure: tcp connect/read/write failures count.
* nodes[i].counter.timeout_failure: timeout count.
+A plugin can run active health checks of its own, on nodes that belong to no
+upstream. [ai-proxy-multi](./plugins/ai-proxy-multi.md) does this: it probes
+every LLM instance and skips the unhealthy ones when it picks a target. Such a
+checker is reported with two extra fields:
+
+```json
+{
+ "name": "/apisix/routes/1#plugins['ai-proxy-multi'].instances[0]",
+ "plugin": "ai-proxy-multi",
+ "meta": {
+ "instance": "openai"
+ },
+ "type": "http",
+ "nodes": [
+ {
+ "ip": "52.86.68.46",
+ "port": 443,
+ "status": "healthy",
+ "counter": {
+ "http_failure": 0,
+ "success": 2,
+ "timeout_failure": 0,
+ "tcp_failure": 0
+ }
+ }
+ ]
+}
+```
+
+* plugin: name of the plugin owning the health checker. Absent for an upstream
health checker.
+* meta: filled by the plugin to describe what the checker stands for. Its
content is plugin specific; `ai-proxy-multi` reports the instance name.
+
You can also use `/v1/healthcheck/$src_type/$src_id` to get the health status
of specific nodes.
For example, `GET /v1/healthcheck/upstreams/1` returns:
@@ -193,6 +225,38 @@ If you use browser to access the control API URL, then you
will get the HTML out
+### GET /v1/healthcheck/{src_type}/{src_id}/checkers
+
+Returns every health checker one resource owns, as an array of the entries
+described above. A resource can own more than one -- its upstream plus one per
+plugin instance -- which `GET /v1/healthcheck/$src_type/$src_id` cannot
express,
+since it returns a single object.
+
+For example, `GET /v1/healthcheck/routes/1/checkers` returns:
+
+```json
+[
+ {
+ "name": "/apisix/routes/1",
+ "type": "http",
+ "nodes": [...]
+ },
+ {
+ "name": "/apisix/routes/1#plugins['ai-proxy-multi'].instances[0]",
+ "plugin": "ai-proxy-multi",
+ "meta": {
+ "instance": "openai"
+ },
+ "type": "http",
+ "nodes": [...]
+ }
+]
+```
+
+A resource with no health check at all is not an error here: it owns an empty
+set of checkers, and the endpoint returns `[]`. `404` is returned only when the
+resource itself does not exist.
+
### POST /v1/gc
Introduced in [v2.8](https://github.com/apache/apisix/releases/tag/2.8).
diff --git a/docs/en/latest/plugins/ai-proxy-multi.md
b/docs/en/latest/plugins/ai-proxy-multi.md
index 4f0dc647a..425771e30 100644
--- a/docs/en/latest/plugins/ai-proxy-multi.md
+++ b/docs/en/latest/plugins/ai-proxy-multi.md
@@ -2630,6 +2630,8 @@ kubectl apply -f ai-proxy-multi-ic.yaml
For verification, the behaviours should be consistent with the verification in
[active health checks](../tutorials/health-check.md).
+The status these checks produce is reported by the [Control
API](../control-api.md): `GET /v1/healthcheck` lists one entry per instance,
and `GET /v1/healthcheck/routes/{id}/checkers` returns every checker the Route
owns. Each entry carries the instance name in `meta.instance`.
+
### Include LLM Information in Access Log
The following example demonstrates how you can log LLM request related
information in the gateway's access log to improve analytics and audit. The
following variables are available:
diff --git a/docs/zh/latest/control-api.md b/docs/zh/latest/control-api.md
index 9040ce0e8..f7751e737 100644
--- a/docs/zh/latest/control-api.md
+++ b/docs/zh/latest/control-api.md
@@ -141,6 +141,36 @@ APISIX 中一些插件添加了自己的 control API。如果你对他们感兴
* nodes[i].counter.tcp_failure: TCP 连接或读写的失败计数器。
* nodes[i].counter.timeout_failure: 超时计数器。
+插件也可以对不属于任何上游的节点进行主动健康检查。[ai-proxy-multi](./plugins/ai-proxy-multi.md)
+就是这样：它探测每个 LLM 实例，并在选择目标时跳过不健康的实例。这类检查器会额外带上两个字段：
+
+```json
+{
+ "name": "/apisix/routes/1#plugins['ai-proxy-multi'].instances[0]",
+ "plugin": "ai-proxy-multi",
+ "meta": {
+ "instance": "openai"
+ },
+ "type": "http",
+ "nodes": [
+ {
+ "ip": "52.86.68.46",
+ "port": 443,
+ "status": "healthy",
+ "counter": {
+ "http_failure": 0,
+ "success": 2,
+ "timeout_failure": 0,
+ "tcp_failure": 0
+ }
+ }
+ ]
+}
+```
+
+* plugin: 拥有该健康检查器的插件名。上游自身的健康检查器没有该字段。
+* meta: 由插件填写，用于描述该检查器代表什么，内容由插件自行定义；`ai-proxy-multi` 在其中报告实例名。
+
用户也可以通过 `/v1/healthcheck/$src_type/$src_id` 来获取指定 health checker 的状态。
例如，`GET /v1/healthcheck/upstreams/1` 返回：
@@ -188,6 +218,36 @@ APISIX 中一些插件添加了自己的 control API。如果你对他们感兴
+### GET /v1/healthcheck/{src_type}/{src_id}/checkers
+
+以数组形式返回某个资源拥有的全部健康检查器，数组元素与上文描述的 entry 结构一致。
+一个资源可能拥有多个健康检查器——它自身的上游，加上每个插件实例各一个——而
+`GET /v1/healthcheck/$src_type/$src_id` 返回的是单个对象，无法表达这种情况。
+
+例如，`GET /v1/healthcheck/routes/1/checkers` 返回：
+
+```json
+[
+ {
+ "name": "/apisix/routes/1",
+ "type": "http",
+ "nodes": [...]
+ },
+ {
+ "name": "/apisix/routes/1#plugins['ai-proxy-multi'].instances[0]",
+ "plugin": "ai-proxy-multi",
+ "meta": {
+ "instance": "openai"
+ },
+ "type": "http",
+ "nodes": [...]
+ }
+]
+```
+
+资源没有配置任何健康检查在这里不算错误：它拥有一个空的检查器集合，接口返回 `[]`。
+只有资源本身不存在时才返回 `404`。
+
### POST /v1/gc
引入自 2.8 版本
diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md
b/docs/zh/latest/plugins/ai-proxy-multi.md
index e5141d016..dd50b0259 100644
--- a/docs/zh/latest/plugins/ai-proxy-multi.md
+++ b/docs/zh/latest/plugins/ai-proxy-multi.md
@@ -2416,6 +2416,8 @@ kubectl apply -f ai-proxy-multi-ic.yaml
为了验证，行为应与[主动健康检查](../tutorials/health-check.md)中的验证一致。
+这些健康检查产生的状态可以通过[控制接口](../control-api.md)获取：`GET /v1/healthcheck` 会为每个实例返回一个
entry，`GET /v1/healthcheck/routes/{id}/checkers` 则返回该路由拥有的全部检查器。每个 entry 通过
`meta.instance` 携带实例名。
+
### 发送请求日志到日志记录器
以下示例演示了如何记录请求和响应信息（包括 LLM 模型、令牌和负载），并将其推送到日志记录器。在继续之前，您应该先设置一个日志记录器，例如
Kafka。有关更多信息，请参阅 [`kafka-logger`](./kafka-logger.md)。
diff --git a/t/control/healthcheck-ai-proxy-multi.t
b/t/control/healthcheck-ai-proxy-multi.t
new file mode 100644
index 000000000..ed271c0a3
--- /dev/null
+++ b/t/control/healthcheck-ai-proxy-multi.t
@@ -0,0 +1,660 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+no_shuffle();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $http_config = $block->http_config // <<_EOC_;
+ server {
+ server_name openai;
+ listen 127.0.0.1:16724;
+
+ default_type 'application/json';
+
+ location /v1/chat/completions {
+ content_by_lua_block {
+
ngx.say([[{"choices":[{"message":{"content":"ok","role":"assistant"}}]}]])
+ }
+ }
+
+ location /status {
+ content_by_lua_block {
+ ngx.say("ok")
+ }
+ }
+ }
+_EOC_
+
+ $block->set_value("http_config", $http_config);
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: report the active health check status of each ai-proxy-multi
instance
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local http = require("resty.http")
+
+ local checks = {
+ active = {
+ timeout = 1,
+ http_path = "/status",
+ healthy = {
+ interval = 1,
+ successes = 1,
+ },
+ unhealthy = {
+ interval = 1,
+ http_failures = 1,
+ tcp_failures = 1,
+ timeouts = 1,
+ },
+ },
+ }
+
+ local route = {
+ uri = "/ai",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ fallback_strategy =
"instance_health_and_rate_limiting",
+ instances = {
+ {
+ name = "openai-healthy",
+ provider = "openai",
+ weight = 1,
+ priority = 1,
+ auth = {header = {Authorization = "Bearer
token"}},
+ options = {model = "gpt-4"},
+ override = {endpoint =
"http://127.0.0.1:16724"},
+ checks = checks,
+ },
+ {
+ name = "openai-unhealthy",
+ provider = "openai",
+ weight = 1,
+ priority = 0,
+ auth = {header = {Authorization = "Bearer
token"}},
+ options = {model = "gpt-4"},
+ override = {endpoint =
"http://127.0.0.1:16725"},
+ checks = checks,
+ },
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+
+ local code, body = t.test("/apisix/admin/routes/1", ngx.HTTP_PUT,
+ json.encode(route))
+ assert(code < 300, body)
+
+ -- the checkers are created on demand, so the route has to be hit
once
+ local httpc = http.new()
+ local res, err = httpc:request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port .. "/ai",
+ {
+ method = "POST",
+ body = json.encode({messages = {{role = "user", content =
"hi"}}}),
+ headers = {["Content-Type"] = "application/json"},
+ }
+ )
+ assert(res, err)
+ assert(res.status == 200, "unexpected status: " ..
tostring(res.status))
+
+ ngx.sleep(3)
+
+ local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET)
+ assert(code == 200, body)
+ res = json.decode(res)
+
+ local infos = {}
+ for _, info in ipairs(res) do
+ if info.plugin and info.name:find("/routes/1#", 1, true) then
+ table.insert(infos, info)
+ end
+ end
+ table.sort(infos, function(a, b) return a.name < b.name end)
+
+ for _, info in ipairs(infos) do
+ ngx.say(info.name, " plugin=", info.plugin, " instance=",
info.meta.instance,
+ " type=", info.type, " nodes=", #info.nodes,
+ " ", info.nodes[1].ip, ":", info.nodes[1].port,
+ " ", info.nodes[1].status)
+ end
+ }
+ }
+--- response_body
+/apisix/routes/1#plugins['ai-proxy-multi'].instances[0] plugin=ai-proxy-multi
instance=openai-healthy type=http nodes=1 127.0.0.1:16724 healthy
+/apisix/routes/1#plugins['ai-proxy-multi'].instances[1] plugin=ai-proxy-multi
instance=openai-unhealthy type=http nodes=1 127.0.0.1:16725 unhealthy
+--- timeout: 10
+
+
+
+=== TEST 2: plugin instance checkers coexist with upstream checkers
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local http = require("resty.http")
+
+ local checks = {
+ active = {
+ timeout = 1,
+ http_path = "/status",
+ healthy = {
+ interval = 1,
+ successes = 1,
+ },
+ unhealthy = {
+ interval = 1,
+ http_failures = 1,
+ tcp_failures = 1,
+ timeouts = 1,
+ },
+ },
+ }
+
+ local route = {
+ uri = "/ai",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ fallback_strategy =
"instance_health_and_rate_limiting",
+ instances = {
+ {
+ name = "openai-first",
+ provider = "openai",
+ weight = 1,
+ priority = 1,
+ auth = {header = {Authorization = "Bearer
token"}},
+ options = {model = "gpt-4"},
+ override = {endpoint =
"http://127.0.0.1:16724"},
+ checks = checks,
+ },
+ {
+ name = "openai-second",
+ provider = "openai",
+ weight = 1,
+ priority = 0,
+ auth = {header = {Authorization = "Bearer
token"}},
+ options = {model = "gpt-4"},
+ override = {endpoint =
"http://127.0.0.1:16724"},
+ checks = checks,
+ },
+ },
+ ssl_verify = false,
+ },
+ },
+ }
+
+ local code, body = t.test("/apisix/admin/routes/1", ngx.HTTP_PUT,
+ json.encode(route))
+ assert(code < 300, body)
+
+ code, body = t.test("/apisix/admin/routes/2", ngx.HTTP_PUT,
json.encode({
+ uri = "/hello",
+ upstream = {
+ type = "roundrobin",
+ nodes = {["127.0.0.1:1980"] = 1},
+ checks = checks,
+ },
+ }))
+ assert(code < 300, body)
+
+ for _, uri in ipairs({"/ai", "/hello"}) do
+ local httpc = http.new()
+ local res, err = httpc:request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port .. uri,
+ {
+ method = "POST",
+ body = json.encode({messages = {{role = "user",
content = "hi"}}}),
+ headers = {["Content-Type"] = "application/json"},
+ }
+ )
+ assert(res, err)
+ end
+
+ ngx.sleep(3)
+
+ local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET)
+ assert(code == 200, body)
+ res = json.decode(res)
+
+ local infos = {}
+ for _, info in ipairs(res) do
+ if info.name:find("/routes/1", 1, true)
+ or info.name:find("/routes/2", 1, true) then
+ table.insert(infos, info)
+ end
+ end
+ table.sort(infos, function(a, b) return a.name < b.name end)
+
+ for _, info in ipairs(infos) do
+ ngx.say(info.name, " plugin=", tostring(info.plugin),
+ " nodes=", #info.nodes)
+ end
+ }
+ }
+--- response_body
+/apisix/routes/1#plugins['ai-proxy-multi'].instances[0] plugin=ai-proxy-multi
nodes=1
+/apisix/routes/1#plugins['ai-proxy-multi'].instances[1] plugin=ai-proxy-multi
nodes=1
+/apisix/routes/2 plugin=nil nodes=1
+--- timeout: 10
+
+
+
+=== TEST 3: disable_upstream_healthcheck stops probing the plugin instances too
+--- yaml_config
+apisix:
+ disable_upstream_healthcheck: true
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local http = require("resty.http")
+
+ local httpc = http.new()
+ local res, err = httpc:request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port .. "/ai",
+ {
+ method = "POST",
+ body = json.encode({messages = {{role = "user", content =
"hi"}}}),
+ headers = {["Content-Type"] = "application/json"},
+ }
+ )
+ assert(res, err)
+
+ ngx.sleep(3)
+
+ -- the entries stay listed, but no checker was ever created, so
every
+ -- node list is empty -- the same as for a plain upstream checker
+ local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET)
+ assert(code == 200, body)
+ res = json.decode(res)
+ local probed = 0
+ for _, info in ipairs(res) do
+ probed = probed + #info.nodes
+ end
+ ngx.say("probed nodes: ", probed)
+
+ local code, body, res =
t.test("/v1/healthcheck/routes/1/checkers", ngx.HTTP_GET)
+ assert(code == 200, body)
+ res = json.decode(res)
+ probed = 0
+ for _, info in ipairs(res) do
+ probed = probed + #info.nodes
+ end
+ ngx.say("checkers: ", #res, " probed nodes: ", probed)
+ }
+ }
+--- response_body
+probed nodes: 0
+checkers: 2 probed nodes: 0
+--- timeout: 10
+
+
+
+=== TEST 4: instances are listed before the checkers have probed anything
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+
+ local checks = {
+ active = {
+ timeout = 1,
+ http_path = "/status",
+ healthy = {
+ interval = 1,
+ successes = 1,
+ },
+ unhealthy = {
+ interval = 1,
+ http_failures = 1,
+ tcp_failures = 1,
+ timeouts = 1,
+ },
+ },
+ }
+
+ local instances = {}
+ for _, name in ipairs({"openai-a", "openai-b"}) do
+ table.insert(instances, {
+ name = name,
+ provider = "openai",
+ weight = 1,
+ auth = {header = {Authorization = "Bearer token"}},
+ options = {model = "gpt-4"},
+ override = {endpoint = "http://127.0.0.1:16724"},
+ checks = checks,
+ })
+ end
+
+ local code, body = t.test("/apisix/admin/routes/3", ngx.HTTP_PUT,
json.encode({
+ uri = "/ai-untouched",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ instances = instances,
+ ssl_verify = false,
+ },
+ },
+ }))
+ assert(code < 300, body)
+
+ -- give the route time to reach the worker, but never send a
request
+ ngx.sleep(1)
+
+ local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET)
+ assert(code == 200, body)
+ res = json.decode(res)
+
+ local infos = {}
+ for _, info in ipairs(res) do
+ if info.name:find("/routes/3#", 1, true) then
+ table.insert(infos, info)
+ end
+ end
+ table.sort(infos, function(a, b) return a.name < b.name end)
+
+ for _, info in ipairs(infos) do
+ ngx.say(info.name, " instance=", info.meta.instance,
+ " type=", info.type, " nodes=", #info.nodes)
+ end
+ }
+ }
+--- response_body
+/apisix/routes/3#plugins['ai-proxy-multi'].instances[0] instance=openai-a
type=http nodes=0
+/apisix/routes/3#plugins['ai-proxy-multi'].instances[1] instance=openai-b
type=http nodes=0
+--- timeout: 10
+
+
+
+=== TEST 5: the checkers sub-resource lists every checker of an AI route
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local http = require("resty.http")
+
+ local checks = {
+ active = {
+ timeout = 1,
+ http_path = "/status",
+ healthy = {
+ interval = 1,
+ successes = 1,
+ },
+ unhealthy = {
+ interval = 1,
+ http_failures = 1,
+ tcp_failures = 1,
+ timeouts = 1,
+ },
+ },
+ }
+
+ local code, body = t.test("/apisix/admin/routes/4", ngx.HTTP_PUT,
json.encode({
+ uri = "/ai4",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ fallback_strategy =
"instance_health_and_rate_limiting",
+ instances = {
+ {
+ name = "openai-up",
+ provider = "openai",
+ weight = 1,
+ priority = 1,
+ auth = {header = {Authorization = "Bearer
token"}},
+ options = {model = "gpt-4"},
+ override = {endpoint =
"http://127.0.0.1:16724"},
+ checks = checks,
+ },
+ {
+ name = "openai-down",
+ provider = "openai",
+ weight = 1,
+ priority = 0,
+ auth = {header = {Authorization = "Bearer
token"}},
+ options = {model = "gpt-4"},
+ override = {endpoint =
"http://127.0.0.1:16725"},
+ checks = checks,
+ },
+ },
+ ssl_verify = false,
+ },
+ },
+ }))
+ assert(code < 300, body)
+
+ local httpc = http.new()
+ local res, err = httpc:request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port .. "/ai4",
+ {
+ method = "POST",
+ body = json.encode({messages = {{role = "user", content =
"hi"}}}),
+ headers = {["Content-Type"] = "application/json"},
+ }
+ )
+ assert(res, err)
+ assert(res.status == 200, "unexpected status: " ..
tostring(res.status))
+
+ ngx.sleep(3)
+
+ local code, body, res =
t.test("/v1/healthcheck/routes/4/checkers", ngx.HTTP_GET)
+ assert(code == 200, body)
+ res = json.decode(res)
+ table.sort(res, function(a, b) return a.name < b.name end)
+
+ for _, info in ipairs(res) do
+ ngx.say(info.name, " plugin=", info.plugin,
+ " instance=", info.meta.instance,
+ " type=", info.type,
+ " ", info.nodes[1].ip, ":", info.nodes[1].port,
+ " ", info.nodes[1].status)
+ end
+ }
+ }
+--- response_body
+/apisix/routes/4#plugins['ai-proxy-multi'].instances[0] plugin=ai-proxy-multi
instance=openai-up type=http 127.0.0.1:16724 healthy
+/apisix/routes/4#plugins['ai-proxy-multi'].instances[1] plugin=ai-proxy-multi
instance=openai-down type=http 127.0.0.1:16725 unhealthy
+--- timeout: 10
+
+
+
+=== TEST 6: the checkers sub-resource reports a plain upstream checker too
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+ local http = require("resty.http")
+
+ local code, body = t.test("/apisix/admin/routes/5", ngx.HTTP_PUT,
json.encode({
+ uri = "/hello5",
+ upstream = {
+ type = "roundrobin",
+ nodes = {["127.0.0.1:1980"] = 1},
+ checks = {
+ active = {
+ timeout = 1,
+ http_path = "/status",
+ healthy = {
+ interval = 1,
+ successes = 1,
+ },
+ unhealthy = {
+ interval = 1,
+ http_failures = 1,
+ tcp_failures = 1,
+ timeouts = 1,
+ },
+ },
+ },
+ },
+ }))
+ assert(code < 300, body)
+
+ local httpc = http.new()
+ local res, err = httpc:request_uri(
+ "http://127.0.0.1:"; .. ngx.var.server_port .. "/hello5",
+ {method = "GET"}
+ )
+ assert(res, err)
+
+ ngx.sleep(3)
+
+ local code, body, res =
t.test("/v1/healthcheck/routes/5/checkers", ngx.HTTP_GET)
+ assert(code == 200, body)
+ res = json.decode(res)
+
+ for _, info in ipairs(res) do
+ ngx.say(info.name, " plugin=", tostring(info.plugin),
+ " meta=", tostring(info.meta),
+ " type=", info.type, " nodes=", #info.nodes)
+ end
+
+ -- the single-object endpoint keeps returning exactly one checker
+ local code, body, res = t.test("/v1/healthcheck/routes/5",
ngx.HTTP_GET)
+ assert(code == 200, body)
+ res = json.decode(res)
+ ngx.say("single: ", res.name, " nodes=", #res.nodes)
+ }
+ }
+--- response_body
+/apisix/routes/5 plugin=nil meta=nil type=http nodes=1
+single: /apisix/routes/5 nodes=1
+--- timeout: 10
+
+
+
+=== TEST 7: a resource without health checks owns an empty checker set
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+
+ local code, body = t.test("/apisix/admin/routes/6", ngx.HTTP_PUT,
json.encode({
+ uri = "/hello6",
+ upstream = {
+ type = "roundrobin",
+ nodes = {["127.0.0.1:1980"] = 1},
+ },
+ }))
+ assert(code < 300, body)
+
+ local function trim(s)
+ return (tostring(s):gsub("%s+$", ""))
+ end
+
+ local code, body, res =
t.test("/v1/healthcheck/routes/6/checkers", ngx.HTTP_GET)
+ ngx.say("no checks: ", code, " ", trim(res))
+
+ -- the single-object endpoint still reports this as an error
+ local code, body = t.test("/v1/healthcheck/routes/6", ngx.HTTP_GET)
+ ngx.say("single: ", code, " ", trim(body))
+
+ local code, body = t.test("/v1/healthcheck/routes/404/checkers",
ngx.HTTP_GET)
+ ngx.say("missing: ", code, " ", trim(body))
+
+ local code, body = t.test("/v1/healthcheck/routes/6/nodes",
ngx.HTTP_GET)
+ ngx.say("bad sub resource: ", code, " ", trim(body))
+ }
+ }
+--- response_body
+no checks: 200 []
+single: 404 {"error_msg":"no checker for routes[6]"}
+missing: 404 {"error_msg":"routes[404] not found"}
+bad sub resource: 400 {"error_msg":"invalid sub resource nodes"}
+--- timeout: 10
+
+
+
+=== TEST 8: a disabled plugin owns no checkers
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ local json = require("toolkit.json")
+
+ -- a disabled plugin is kept even when its configuration does not
pass
+ -- check_schema(), so `instances` here is not even an array
+ local code, body = t.test("/apisix/admin/routes/7", ngx.HTTP_PUT,
json.encode({
+ uri = "/ai7",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ _meta = {disable = true},
+ instances = "not an array",
+ },
+ },
+ upstream = {
+ type = "roundrobin",
+ nodes = {["127.0.0.1:1980"] = 1},
+ },
+ }))
+ assert(code < 300, body)
+
+ ngx.sleep(1)
+
+ local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET)
+ ngx.say("list: ", code)
+
+ local code, body, res =
t.test("/v1/healthcheck/routes/7/checkers", ngx.HTTP_GET)
+ ngx.say("checkers: ", code, " ", (tostring(res):gsub("%s+$", "")))
+ }
+ }
+--- response_body
+list: 200
+checkers: 200 []
+--- timeout: 10
+
+
+
+=== TEST 9: clean up
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin")
+ for _, id in ipairs({1, 2, 3, 4, 5, 6, 7}) do
+ local code, body = t.test("/apisix/admin/routes/" .. id,
ngx.HTTP_DELETE)
+ assert(code < 300, body)
+ end
+ ngx.say("passed")
+ }
+ }
+--- response_body
+passed
