This is an automated email from the ASF dual-hosted git repository.
shreemaan-abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 7ea42d45dc feat(ai-proxy): send LLM requests through
ngx_http_ffi_client (#13778)
7ea42d45dc is described below
commit 7ea42d45dcb0c3a7f3e263338d0ef816d0873705
Author: Shreemaan Abhishek <[email protected]>
AuthorDate: Tue Aug 18 13:56:15 2026 +0800
feat(ai-proxy): send LLM requests through ngx_http_ffi_client (#13778)
---
.requirements | 2 +-
apisix/cli/config.lua | 3 +
apisix/plugins/ai-transport/http.lua | 42 +-
apisix/utils/http.lua | 125 ++++++
ci/linux-install-openresty.sh | 6 +-
conf/config.yaml.example | 3 +
docs/en/latest/plugins/ai-proxy-multi.md | 17 +
docs/en/latest/plugins/ai-proxy.md | 17 +
t/plugin/ai-transport-http.t | 690 +++++++++++++++++++++++++++++++
9 files changed, 899 insertions(+), 6 deletions(-)
diff --git a/.requirements b/.requirements
index 8bd4cf8ae8..9c76f3ee01 100644
--- a/.requirements
+++ b/.requirements
@@ -17,5 +17,5 @@
APISIX_PACKAGE_NAME=apisix
-APISIX_RUNTIME=1.3.14
+APISIX_RUNTIME=1.3.16
APISIX_DASHBOARD_COMMIT=c8d3466d3c36386d3888efbc8250cd8183c77298
diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua
index ab34d8469a..4315794f21 100644
--- a/apisix/cli/config.lua
+++ b/apisix/cli/config.lua
@@ -309,6 +309,9 @@ local _M = {
},
stream_plugins = { "ip-restriction", "limit-conn", "mqtt-proxy", "syslog",
"traffic-split" },
plugin_attr = {
+ ["ai-proxy"] = {
+ http_client = "ngx_http_ffi_client"
+ },
["log-rotate"] = {
timeout = 10000,
interval = 3600,
diff --git a/apisix/plugins/ai-transport/http.lua
b/apisix/plugins/ai-transport/http.lua
index a2075aa3f4..ee72fd7a33 100644
--- a/apisix/plugins/ai-transport/http.lua
+++ b/apisix/plugins/ai-transport/http.lua
@@ -19,7 +19,7 @@
-- Provides HTTP client lifecycle management for AI provider requests.
local core = require("apisix.core")
-local http = require("resty.http")
+local http_client = require("apisix.utils.http")
local ngx_now = ngx.now
local pairs = pairs
local ipairs = ipairs
@@ -28,8 +28,41 @@ local type = type
local str_lower = string.lower
local tostring = tostring
+local attr_schema = {
+ type = "object",
+ properties = {
+ http_client = http_client.client_schema,
+ },
+}
+
local _M = {}
+local client_name
+
+
+--- Which client this transport should use.
+-- `plugin_attr.ai-proxy.http_client` names it; the shared module owns the
+-- names, validation and loading. Read on first request, because local_conf is
+-- not readable while this module is still loading.
+local function resolve_client_name()
+ if client_name then
+ return client_name
+ end
+
+ local local_conf = core.config.local_conf()
+ local attr = core.table.try_read_attr(local_conf, "plugin_attr",
"ai-proxy") or {}
+
+ local ok, err = core.schema.check(attr_schema, attr)
+ if not ok then
+ core.log.error("invalid plugin_attr.ai-proxy: ", err)
+ return nil, "invalid plugin_attr.ai-proxy: " .. err
+ end
+
+ client_name = attr.http_client or http_client.DEFAULT_CLIENT
+
+ return client_name
+end
+
--- Map network errors to HTTP status codes.
-- Cosocket timers report "timeout"; OS errno (ETIMEDOUT) and the resolver
@@ -100,7 +133,12 @@ end
-- @return string|nil Error message
-- @return table|nil Upstream metadata on failure (for recording failed
attempts)
function _M.request(params, timeout)
- local httpc, err = http.new()
+ local name, name_err = resolve_client_name()
+ if not name then
+ return nil, "failed to create http client: " .. name_err
+ end
+
+ local httpc, err = http_client.new(name)
if not httpc then
return nil, "failed to create http client: " .. (err or "unknown")
end
diff --git a/apisix/utils/http.lua b/apisix/utils/http.lua
new file mode 100644
index 0000000000..d63be0e13f
--- /dev/null
+++ b/apisix/utils/http.lua
@@ -0,0 +1,125 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--- Outbound HTTP client selection.
+-- Shared by any module that makes outbound HTTP calls, so the choice between
+-- `ngx_http_ffi_client` and `lua-resty-http` is made in one place rather than
+-- per plugin. The caller decides where the preference comes from (its own
+-- config key) and passes the name in.
+
+local core = require("apisix.core")
+local pcall = pcall
+local require = require
+local tostring = tostring
+local type = type
+
+local FFI_CLIENT = "ngx_http_ffi_client"
+local LUA_RESTY_HTTP = "lua-resty-http"
+
+-- the client name used in configuration is not the module name
+local CLIENT_MODULES = {
+ [FFI_CLIENT] = "resty.ngx_http_ffi_client",
+ [LUA_RESTY_HTTP] = "resty.http",
+}
+
+local loaded = {}
+
+
+local _M = {
+ version = 0.1,
+ FFI_CLIENT = FFI_CLIENT,
+ LUA_RESTY_HTTP = LUA_RESTY_HTTP,
+ DEFAULT_CLIENT = FFI_CLIENT,
+}
+
+
+--- Schema fragment for a client-name config field.
+-- Callers embed this in their own attribute schema so every module validates
+-- the name the same way.
+_M.client_schema = {
+ type = "string",
+ enum = {FFI_CLIENT, LUA_RESTY_HTTP},
+ default = FFI_CLIENT,
+}
+
+
+--- Load the module for a client name.
+-- `ngx_http_ffi_client` is a C client with the same object API as
+-- lua-resty-http and around half its outbound CPU cost, and it exists only
+-- when the gateway runtime was built with the module. A name that cannot be
+-- loaded is an error, never a silent switch to the other client.
+-- Cached per name once loaded, so a failure is retried rather than remembered.
+local function load_client(name)
+ local cached = loaded[name]
+ if cached then
+ return cached
+ end
+
+ local module_name = CLIENT_MODULES[name]
+ if not module_name then
+ return nil, "unknown http client: " .. tostring(name)
+ end
+
+ local ok, mod = pcall(require, module_name)
+ if not ok or type(mod) ~= "table" then
+ core.log.error(module_name, " is not available: ", mod)
+ return nil, module_name .. " is not available: " .. tostring(mod)
+ end
+
+ -- Cosockets have their names resolved by apisix/patch.lua, which routes
+ -- them through core.resolver and so honours dns_resolver, /etc/hosts and
+ -- the search domains. The C client dials from C and never touches a
+ -- cosocket, so without this it would see only nginx's `resolver`. Handing
+ -- it the same resolver keeps every outbound name on one set of rules.
+ -- A client too old to take one is an error rather than a client quietly
+ -- resolving names by different rules than the rest of the gateway.
+ if name == FFI_CLIENT then
+ if type(mod.set_resolver) ~= "function" then
+ core.log.error(module_name, " does not support set_resolver, ",
+ "the runtime is older than the pinned one")
+ return nil, module_name .. " does not support set_resolver, "
+ .. "the runtime is older than the pinned one"
+ end
+
+ mod.set_resolver(core.resolver.parse_domain)
+ end
+
+ loaded[name] = mod
+
+ return mod
+end
+
+
+--- Create an HTTP client.
+-- @tparam string|nil name client name; defaults to DEFAULT_CLIENT
+-- @treturn table|nil the client
+-- @treturn string|nil error message
+function _M.new(name)
+ name = name or _M.DEFAULT_CLIENT
+
+ local mod, err = load_client(name)
+ if not mod then
+ return nil, err
+ end
+
+ -- The Lua half of `ngx_http_ffi_client` loads even when the C module is
+ -- not compiled into the runtime; new() is what reports that.
+ return mod.new()
+end
+
+
+return _M
diff --git a/ci/linux-install-openresty.sh b/ci/linux-install-openresty.sh
index 07f05e82b8..ffdab29ad5 100755
--- a/ci/linux-install-openresty.sh
+++ b/ci/linux-install-openresty.sh
@@ -61,7 +61,7 @@ else
sudo apt-get -y update --fix-missing
sudo apt-get install -y build-essential gcc g++ cpanminus libxml2-dev
libxslt-dev
- if [ "$APISIX_RUNTIME" != "1.3.14" ]; then
+ if [ "$APISIX_RUNTIME" != "1.3.16" ]; then
echo "Please update the apisix-runtime-debug checksum for
APISIX_RUNTIME=$APISIX_RUNTIME" >&2
exit 1
fi
@@ -69,11 +69,11 @@ else
case "$ARCH" in
x86_64|amd64)
DEB_ARCH="amd64"
-
EXPECTED_SHA256="2d2350347c982e4467ff9326b5b93fcb9af2089b33b02bcd426e84a5adacf6f2"
+
EXPECTED_SHA256="a56f0adc9bf6f6a491f7548df4f8e45fa3df3dd5e209d4a2ba5341b66eb7e060"
;;
arm64|aarch64)
DEB_ARCH="arm64"
-
EXPECTED_SHA256="495320e6377b96ab8d8a80a980a845e346c88160e2798ba46113a4da9042af4d"
+
EXPECTED_SHA256="b645ee4f5ea36d26aaacb1b1c8278d89756b0b8448701ec561ab982b4768204a"
;;
*)
echo "Unsupported architecture: $ARCH" >&2
diff --git a/conf/config.yaml.example b/conf/config.yaml.example
index 26954e2797..6f6c791ea1 100644
--- a/conf/config.yaml.example
+++ b/conf/config.yaml.example
@@ -659,6 +659,9 @@ stream_plugins: # stream plugin list
(sorted by priority)
# protocols:
# - name: pingpong
plugin_attr: # Plugin attributes
+ ai-proxy: # Plugin: ai-proxy, ai-proxy-multi
+ http_client: ngx_http_ffi_client # HTTP client the AI plugins use to reach
the
+ # LLM upstream: ngx_http_ffi_client or lua-resty-http.
log-rotate: # Plugin: log-rotate
timeout: 10000 # maximum wait time for a log rotation(unit: millisecond)
interval: 3600 # Set the log rotate interval in seconds.
diff --git a/docs/en/latest/plugins/ai-proxy-multi.md
b/docs/en/latest/plugins/ai-proxy-multi.md
index dc58ca565f..ea26402360 100644
--- a/docs/en/latest/plugins/ai-proxy-multi.md
+++ b/docs/en/latest/plugins/ai-proxy-multi.md
@@ -152,6 +152,23 @@ By default, `ai-proxy-multi` forwards the incoming client
request headers to the
Because the LLM upstream is often a third-party service, be aware that any
header the client sends (for example `Authorization`, `Cookie`, or internal
application headers) is forwarded to that provider unless it is overridden by
`auth.header`. If the client should not expose certain headers to the LLM
provider, strip them before the request reaches `ai-proxy-multi`, for example
with the [`proxy-rewrite`](./proxy-rewrite.md) plugin.
+## Upstream HTTP Client
+
+Requests to the LLM upstream go through `ngx_http_ffi_client`, a C HTTP client
that costs around half the outbound CPU time of `lua-resty-http`. Both clients
behave the same on the wire.
+
+`plugin_attr.ai-proxy.http_client` in `config.yaml` names the client:
+
+```yaml
+plugin_attr:
+ ai-proxy:
+ http_client: ngx_http_ffi_client # or lua-resty-http
+```
+
+- `ngx_http_ffi_client` (default): the C client. It requires an APISIX runtime
built with the module, which the runtime pinned in `.requirements` is. On a
runtime without it, the request fails and the error names the missing module;
the plugin never silently switches clients.
+- `lua-resty-http`: the Lua client, on every runtime.
+
+The setting covers `ai-proxy`, `ai-proxy-multi`, and `ai-request-rewrite`,
which share the same transport.
+
## Upstream Error Responses
When the selected LLM upstream returns a `429` or `5xx` status,
`ai-proxy-multi` reads the upstream error body before deciding whether to fall
back:
diff --git a/docs/en/latest/plugins/ai-proxy.md
b/docs/en/latest/plugins/ai-proxy.md
index b93702ed00..c9f132bbb6 100644
--- a/docs/en/latest/plugins/ai-proxy.md
+++ b/docs/en/latest/plugins/ai-proxy.md
@@ -145,6 +145,23 @@ By default, `ai-proxy` forwards the incoming client
request headers to the confi
Because the LLM upstream is often a third-party service, be aware that any
header the client sends (for example `Authorization`, `Cookie`, or internal
application headers) is forwarded to that provider unless it is overridden by
`auth.header`. If the client should not expose certain headers to the LLM
provider, strip them before the request reaches `ai-proxy`, for example with
the [`proxy-rewrite`](./proxy-rewrite.md) plugin.
+## Upstream HTTP Client
+
+Requests to the LLM upstream go through `ngx_http_ffi_client`, a C HTTP client
that costs around half the outbound CPU time of `lua-resty-http`. Both clients
behave the same on the wire.
+
+`plugin_attr.ai-proxy.http_client` in `config.yaml` names the client:
+
+```yaml
+plugin_attr:
+ ai-proxy:
+ http_client: ngx_http_ffi_client # or lua-resty-http
+```
+
+- `ngx_http_ffi_client` (default): the C client. It requires an APISIX runtime
built with the module, which the runtime pinned in `.requirements` is. On a
runtime without it, the request fails and the error names the missing module;
the plugin never silently switches clients.
+- `lua-resty-http`: the Lua client, on every runtime.
+
+The setting covers `ai-proxy`, `ai-proxy-multi`, and `ai-request-rewrite`,
which share the same transport.
+
## Upstream Error Responses
When the LLM upstream returns a `429` or `5xx` status, `ai-proxy` reads the
upstream error body and returns it to the client together with the upstream
status code and `Content-Type`, so provider-side error details (such as
rate-limit information or validation errors) are not discarded.
diff --git a/t/plugin/ai-transport-http.t b/t/plugin/ai-transport-http.t
index b475164939..58e23a6cf7 100644
--- a/t/plugin/ai-transport-http.t
+++ b/t/plugin/ai-transport-http.t
@@ -33,6 +33,10 @@ run_tests;
__DATA__
=== TEST 1: AI transport encodes upstream request body with sorted keys and
preserves empty arrays
+--- extra_yaml_config
+plugin_attr:
+ ai-proxy:
+ http_client: lua-resty-http
--- config
location /t {
content_by_lua_block {
@@ -96,6 +100,10 @@ __DATA__
=== TEST 2: AI transport falls back to cjson when rapidjson encode fails
+--- extra_yaml_config
+plugin_attr:
+ ai-proxy:
+ http_client: lua-resty-http
--- config
location /t {
content_by_lua_block {
@@ -197,6 +205,10 @@ rapidjson nested empty table: \{\}
=== TEST 4: AI transport preserves JSON null values from cjson decode
+--- extra_yaml_config
+plugin_attr:
+ ai-proxy:
+ http_client: lua-resty-http
--- config
location /t {
content_by_lua_block {
@@ -242,6 +254,10 @@ failed to encode AI request body with rapidjson:
=== TEST 5: AI transport preserves manually constructed arrays
+--- extra_yaml_config
+plugin_attr:
+ ai-proxy:
+ http_client: lua-resty-http
--- config
location /t {
content_by_lua_block {
@@ -292,6 +308,10 @@ failed to encode AI request body with rapidjson:
=== TEST 6: connect timeout ("Operation timed out") maps to 504
+--- extra_yaml_config
+plugin_attr:
+ ai-proxy:
+ http_client: lua-resty-http
--- config
location /t {
content_by_lua_block {
@@ -353,3 +373,673 @@ connect: operation timed out => 504
connect: Operation timed out => 504
request: connection refused => 500
request: connection reset by peer => 500
+
+
+
+=== TEST 8: ngx_http_ffi_client is the default client
+--- config
+ location /t {
+ content_by_lua_block {
+ local orig_http = package.loaded["resty.http"]
+ local orig_ffi = package.loaded["resty.ngx_http_ffi_client"]
+ local orig_transport =
package.loaded["apisix.plugins.ai-transport.http"]
+
+ package.loaded["resty.http"] = {
+ new = function()
+ ngx.say("lua-resty-http client created")
+ return {
+ set_timeout = function() end,
+ connect = function() return true end,
+ request = function() return {headers = {}, status =
200} end,
+ }
+ end,
+ }
+
+ package.loaded["resty.ngx_http_ffi_client"] = {
+ set_resolver = function() end,
+ new = function()
+ return {
+ set_timeout = function() end,
+ connect = function() return 1 end,
+ request = function(_, params)
+ ngx.say("ffi client request: ", params.body)
+ return {headers = {}, status = 200}
+ end,
+ }
+ end,
+ }
+
+ package.loaded["apisix.plugins.ai-transport.http"] = nil
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ host = "127.0.0.1",
+ port = 80,
+ path = "/",
+ body = {model = "m"},
+ }, 1000)
+ ngx.say("status: ", res and res.status or err)
+
+ package.loaded["resty.http"] = orig_http
+ package.loaded["resty.ngx_http_ffi_client"] = orig_ffi
+ package.loaded["apisix.plugins.ai-transport.http"] = orig_transport
+ }
+ }
+--- response_body
+ffi client request: {"model":"m"}
+status: 200
+
+
+
+=== TEST 9: a runtime without the C module fails the request
+--- config
+ location /t {
+ content_by_lua_block {
+ local orig_http = package.loaded["resty.http"]
+ local orig_ffi = package.loaded["resty.ngx_http_ffi_client"]
+ local orig_transport =
package.loaded["apisix.plugins.ai-transport.http"]
+
+ package.loaded["resty.http"] = {
+ new = function()
+ ngx.say("lua-resty-http client created")
+ return {
+ set_timeout = function() end,
+ connect = function() return true end,
+ request = function() return {headers = {}, status =
200} end,
+ }
+ end,
+ }
+
+ -- the Lua half loads even when the C module is not built in
+ package.loaded["resty.ngx_http_ffi_client"] = {
+ set_resolver = function() end,
+ new = function()
+ return nil, "ngx_http_ffi_client_module is not loaded"
+ end,
+ }
+
+ package.loaded["apisix.plugins.ai-transport.http"] = nil
+ local transport = require("apisix.plugins.ai-transport.http")
+ for _ = 1, 2 do
+ local res, err = transport.request({
+ host = "127.0.0.1",
+ port = 80,
+ path = "/",
+ body = {model = "m"},
+ }, 1000)
+ ngx.say("status: ", res and res.status or err)
+ end
+
+ package.loaded["resty.http"] = orig_http
+ package.loaded["resty.ngx_http_ffi_client"] = orig_ffi
+ package.loaded["apisix.plugins.ai-transport.http"] = orig_transport
+ }
+ }
+--- response_body
+status: failed to create http client: ngx_http_ffi_client_module is not loaded
+status: failed to create http client: ngx_http_ffi_client_module is not loaded
+
+
+
+=== TEST 10: plugin_attr.ai-proxy.http_client selects lua-resty-http
+--- extra_yaml_config
+plugin_attr:
+ ai-proxy:
+ http_client: lua-resty-http
+--- config
+ location /t {
+ content_by_lua_block {
+ local orig_http = package.loaded["resty.http"]
+ local orig_ffi = package.loaded["resty.ngx_http_ffi_client"]
+ local orig_transport =
package.loaded["apisix.plugins.ai-transport.http"]
+
+ package.loaded["resty.http"] = {
+ new = function()
+ return {
+ set_timeout = function() end,
+ connect = function() return true end,
+ request = function(_, params)
+ ngx.say("lua-resty-http request: ", params.body)
+ return {headers = {}, status = 200}
+ end,
+ }
+ end,
+ }
+
+ package.loaded["resty.ngx_http_ffi_client"] = {
+ new = function()
+ return {
+ set_timeout = function() end,
+ connect = function() return 1 end,
+ request = function()
+ ngx.say("ffi client request")
+ return {headers = {}, status = 200}
+ end,
+ }
+ end,
+ }
+
+ package.loaded["apisix.plugins.ai-transport.http"] = nil
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ host = "127.0.0.1",
+ port = 80,
+ path = "/",
+ body = {model = "m"},
+ }, 1000)
+ ngx.say("status: ", res and res.status or err)
+
+ package.loaded["resty.http"] = orig_http
+ package.loaded["resty.ngx_http_ffi_client"] = orig_ffi
+ package.loaded["apisix.plugins.ai-transport.http"] = orig_transport
+ }
+ }
+--- response_body
+lua-resty-http request: {"model":"m"}
+status: 200
+
+
+
+=== TEST 11: the lua-resty-http path reaches a real upstream
+--- extra_yaml_config
+plugin_attr:
+ ai-proxy:
+ http_client: lua-resty-http
+--- config
+ location = /mock-llm {
+ content_by_lua_block {
+ ngx.req.read_body()
+ ngx.header["Content-Type"] = "application/json"
+ ngx.print('{"echo":', ngx.req.get_body_data(), '}')
+ }
+ }
+
+ location /t {
+ content_by_lua_block {
+ local orig_ffi = package.loaded["resty.ngx_http_ffi_client"]
+
+ -- lua-resty-http stays real; only the client that must not be
+ -- picked is stubbed, so a regression in the selection shows up
+ package.loaded["resty.ngx_http_ffi_client"] = {
+ new = function()
+ ngx.log(ngx.ERR, "unexpected ngx_http_ffi_client
selection")
+ return nil, "unexpected ngx_http_ffi_client selection"
+ end,
+ }
+
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ method = "POST",
+ scheme = "http",
+ host = "127.0.0.1",
+ port = 1984,
+ path = "/mock-llm",
+ headers = {["content-type"] = "application/json"},
+ body = {model = "m"},
+ }, 2000)
+
+ package.loaded["resty.ngx_http_ffi_client"] = orig_ffi
+
+ if not res then
+ ngx.say("err: ", err)
+ return
+ end
+
+ ngx.say("status: ", res.status)
+ ngx.say("content-type: ", res.headers["Content-Type"])
+ ngx.say("body: ", res:read_body())
+ transport.set_keepalive(res, 60000, 30)
+ }
+ }
+--- response_body
+status: 200
+content-type: application/json
+body: {"echo":{"model":"m"}}
+--- no_error_log
+[error]
+unexpected ngx_http_ffi_client selection
+
+
+
+=== TEST 12: a module that loads but is not a table fails the request
+--- config
+ location /t {
+ content_by_lua_block {
+ local orig_http = package.loaded["resty.http"]
+ local orig_ffi = package.loaded["resty.ngx_http_ffi_client"]
+ local orig_transport =
package.loaded["apisix.plugins.ai-transport.http"]
+
+ package.loaded["resty.http"] = {
+ new = function()
+ ngx.say("lua-resty-http client created")
+ return {
+ set_timeout = function() end,
+ connect = function() return true end,
+ request = function() return {headers = {}, status =
200} end,
+ }
+ end,
+ }
+
+ -- require() returns this instead of the module table
+ package.loaded["resty.ngx_http_ffi_client"] = "not a module"
+
+ package.loaded["apisix.plugins.ai-transport.http"] = nil
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ host = "127.0.0.1",
+ port = 80,
+ path = "/",
+ body = {model = "m"},
+ }, 1000)
+ ngx.say("status: ", res and res.status or err)
+
+ package.loaded["resty.http"] = orig_http
+ package.loaded["resty.ngx_http_ffi_client"] = orig_ffi
+ package.loaded["apisix.plugins.ai-transport.http"] = orig_transport
+ }
+ }
+--- response_body
+status: failed to create http client: resty.ngx_http_ffi_client is not
available: not a module
+--- error_log
+resty.ngx_http_ffi_client is not available: not a module
+
+
+
+=== TEST 13: an unknown plugin_attr.ai-proxy.http_client fails the request
+--- extra_yaml_config
+plugin_attr:
+ ai-proxy:
+ http_client: curl
+--- config
+ location /t {
+ content_by_lua_block {
+ local orig_http = package.loaded["resty.http"]
+ local orig_transport =
package.loaded["apisix.plugins.ai-transport.http"]
+
+ package.loaded["resty.http"] = {
+ new = function()
+ ngx.say("lua-resty-http client created")
+ return {
+ set_timeout = function() end,
+ connect = function() return true end,
+ request = function() return {headers = {}, status =
200} end,
+ }
+ end,
+ }
+
+ package.loaded["apisix.plugins.ai-transport.http"] = nil
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ host = "127.0.0.1",
+ port = 80,
+ path = "/",
+ body = {model = "m"},
+ }, 1000)
+ ngx.say("status: ", res and res.status or err)
+
+ package.loaded["resty.http"] = orig_http
+ package.loaded["apisix.plugins.ai-transport.http"] = orig_transport
+ }
+ }
+--- response_body
+status: failed to create http client: invalid plugin_attr.ai-proxy: property
"http_client" validation failed: matches none of the enum values
+--- error_log
+invalid plugin_attr.ai-proxy: property "http_client" validation failed
+
+
+
+=== TEST 14: the C client is given the gateway's resolver
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local orig_ffi = package.loaded["resty.ngx_http_ffi_client"]
+ local orig_utils = package.loaded["apisix.utils.http"]
+
+ local installed
+ package.loaded["resty.ngx_http_ffi_client"] = {
+ set_resolver = function(fn)
+ installed = fn
+ end,
+ new = function()
+ return {}
+ end,
+ }
+
+ package.loaded["apisix.utils.http"] = nil
+ local http_client = require("apisix.utils.http")
+ http_client.new(http_client.FFI_CLIENT)
+
+ -- the C client dials on its own, so what it resolves names with
has
+ -- to be the resolver every other socket in the gateway uses
+ ngx.say("gateway resolver installed: ",
+ installed == core.resolver.parse_domain)
+ ngx.say("resolves through it: ", installed("localhost"))
+
+ package.loaded["resty.ngx_http_ffi_client"] = orig_ffi
+ package.loaded["apisix.utils.http"] = orig_utils
+ }
+ }
+--- response_body
+gateway resolver installed: true
+resolves through it: 127.0.0.1
+--- no_error_log
+[error]
+
+
+
+=== TEST 15: a module whose loader raises fails the request
+--- config
+ location /t {
+ content_by_lua_block {
+ local orig_http = package.loaded["resty.http"]
+ local orig_ffi = package.loaded["resty.ngx_http_ffi_client"]
+ local orig_preload = package.preload["resty.ngx_http_ffi_client"]
+ local orig_transport =
package.loaded["apisix.plugins.ai-transport.http"]
+
+ package.loaded["resty.http"] = {
+ new = function()
+ ngx.say("lua-resty-http client created")
+ return {
+ set_timeout = function() end,
+ connect = function() return true end,
+ request = function() return {headers = {}, status =
200} end,
+ }
+ end,
+ }
+
+ -- an unloaded module whose loader raises: this is what a require()
+ -- failure looks like, as opposed to one that loads the wrong thing
+ package.loaded["resty.ngx_http_ffi_client"] = nil
+ package.preload["resty.ngx_http_ffi_client"] = function()
+ error("simulated loader failure", 0)
+ end
+
+ package.loaded["apisix.plugins.ai-transport.http"] = nil
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ host = "127.0.0.1",
+ port = 80,
+ path = "/",
+ body = {model = "m"},
+ }, 1000)
+ ngx.say("status: ", res and res.status or err)
+
+ package.loaded["resty.http"] = orig_http
+ package.loaded["resty.ngx_http_ffi_client"] = orig_ffi
+ package.preload["resty.ngx_http_ffi_client"] = orig_preload
+ package.loaded["apisix.plugins.ai-transport.http"] = orig_transport
+ }
+ }
+--- response_body
+status: failed to create http client: resty.ngx_http_ffi_client is not
available: simulated loader failure
+--- error_log
+resty.ngx_http_ffi_client is not available: simulated loader failure
+
+
+
+=== TEST 16: the C client carries a buffered request to a real upstream
+--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 |
grep -q ngx_http_ffi_client")) != 0
+--- config
+ location = /mock-buffered {
+ content_by_lua_block {
+ ngx.req.read_body()
+ ngx.header["Content-Type"] = "application/json"
+ ngx.print('{"echo":', ngx.req.get_body_data(), '}')
+ }
+ }
+
+ location /t {
+ content_by_lua_block {
+ -- nothing stubbed: this is the real C client on the default
setting
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ method = "POST",
+ scheme = "http",
+ host = "127.0.0.1",
+ port = 1984,
+ path = "/mock-buffered",
+ headers = {["content-type"] = "application/json"},
+ body = {model = "m"},
+ }, 2000)
+ if not res then
+ ngx.say("err: ", err)
+ return
+ end
+ ngx.say("status: ", res.status)
+ ngx.say("content-type: ", res.headers["Content-Type"])
+ ngx.say("body: ", res:read_body())
+ transport.set_keepalive(res, 60000, 30)
+ }
+ }
+--- response_body
+status: 200
+content-type: application/json
+body: {"echo":{"model":"m"}}
+--- no_error_log
+[error]
+
+
+
+=== TEST 17: the C client streams an SSE response through body_reader
+--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 |
grep -q ngx_http_ffi_client")) != 0
+--- config
+ location = /mock-sse {
+ content_by_lua_block {
+ ngx.header["Content-Type"] = "text/event-stream"
+ for i = 1, 3 do
+ ngx.print("data: {\"n\":", i, "}\n\n")
+ ngx.flush(true)
+ end
+ ngx.print("data: [DONE]\n\n")
+ ngx.flush(true)
+ }
+ }
+
+ location /t {
+ content_by_lua_block {
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ method = "POST",
+ scheme = "http",
+ host = "127.0.0.1",
+ port = 1984,
+ path = "/mock-sse",
+ headers = {["content-type"] = "application/json"},
+ body = {model = "m", stream = true},
+ }, 2000)
+ if not res then
+ ngx.say("err: ", err)
+ return
+ end
+ ngx.say("status: ", res.status)
+ ngx.say("content-type: ", res.headers["Content-Type"])
+
+ local reader = res.body_reader
+ if not reader then
+ ngx.say("no body_reader")
+ return
+ end
+ local buf = {}
+ while true do
+ local chunk, rerr = reader(4096)
+ if rerr then
+ ngx.say("read err: ", rerr)
+ break
+ end
+ if not chunk then
+ break
+ end
+ buf[#buf + 1] = chunk
+ end
+ local body = table.concat(buf)
+ local n = select(2, body:gsub("data: ", ""))
+ ngx.say("sse events: ", n)
+ ngx.say("saw done: ", body:find("[DONE]", 1, true) ~= nil)
+ }
+ }
+--- response_body
+status: 200
+content-type: text/event-stream
+sse events: 4
+saw done: true
+--- no_error_log
+[error]
+
+
+
+=== TEST 18: the C client reuses a pooled connection across requests
+--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 |
grep -q ngx_http_ffi_client")) != 0
+--- config
+ location = /mock-keepalive {
+ # counts requests served on this connection: 1,2,3 proves one
+ # pooled connection carried all three
+ content_by_lua_block { ngx.print("req ", ngx.var.connection_requests) }
+ }
+
+ location /t {
+ content_by_lua_block {
+ local transport = require("apisix.plugins.ai-transport.http")
+ for i = 1, 3 do
+ local res, err = transport.request({
+ method = "POST",
+ scheme = "http",
+ host = "127.0.0.1",
+ port = 1984,
+ path = "/mock-keepalive",
+ headers = {["content-type"] = "application/json"},
+ body = {model = "m"},
+ }, 2000)
+ if not res then
+ ngx.say(i, ": err ", err)
+ return
+ end
+ local body = res:read_body()
+ transport.set_keepalive(res, 60000, 30)
+ ngx.say(i, ": ", res.status, " ", body)
+ end
+ }
+ }
+--- response_body
+1: 200 req 1
+2: 200 req 2
+3: 200 req 3
+--- no_error_log
+[error]
+
+
+
+=== TEST 19: the C client reaches an upstream named by hostname
+--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 |
grep -q ngx_http_ffi_client")) != 0
+--- config
+ location = /mock-host {
+ content_by_lua_block {
+ ngx.print("host header: ", ngx.var.http_host)
+ }
+ }
+
+ location /t {
+ content_by_lua_block {
+ -- "localhost" only resolves through core.resolver, so this proves
+ -- the real client is going through the resolver it was given
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ method = "POST",
+ scheme = "http",
+ host = "localhost",
+ port = 1984,
+ path = "/mock-host",
+ headers = {["content-type"] = "application/json"},
+ body = {model = "m"},
+ }, 2000)
+ if not res then
+ ngx.say("err: ", err)
+ return
+ end
+ ngx.say("status: ", res.status)
+ ngx.say(res:read_body())
+ }
+ }
+--- response_body
+status: 200
+host header: localhost:1984
+--- no_error_log
+[error]
+
+
+
+=== TEST 20: the C client verifies TLS against the configured trust store
+--- skip_eval: 3: system((($ENV{TEST_NGINX_BINARY} || "nginx") . " -V 2>&1 |
grep -q ngx_http_ffi_client")) != 0
+--- http_config
+ server {
+ listen 21981 ssl;
+ ssl_certificate cert/apisix.crt;
+ ssl_certificate_key cert/apisix.key;
+ server_name test.com;
+ location = /mock-tls {
+ content_by_lua_block { ngx.print("tls ok") }
+ }
+ }
+--- config
+ location /t {
+ content_by_lua_block {
+ -- no per-call CA: the trust store comes from
+ -- lua_ssl_trusted_certificate, as it does for every cosocket
+ local transport = require("apisix.plugins.ai-transport.http")
+ local res, err = transport.request({
+ method = "POST",
+ scheme = "https",
+ host = "127.0.0.1",
+ port = 21981,
+ path = "/mock-tls",
+ ssl_verify = true,
+ ssl_server_name = "test.com",
+ headers = {["content-type"] = "application/json"},
+ body = {model = "m"},
+ }, 2000)
+ if not res then
+ ngx.say("err: ", err)
+ return
+ end
+ ngx.say("status: ", res.status)
+ ngx.say(res:read_body())
+ }
+ }
+--- response_body
+status: 200
+tls ok
+--- no_error_log
+[error]
+
+
+
+=== TEST 21: a client that cannot take a resolver fails the request
+--- config
+ location /t {
+ content_by_lua_block {
+ local orig_ffi = package.loaded["resty.ngx_http_ffi_client"]
+ local orig_utils = package.loaded["apisix.utils.http"]
+
+ -- a runtime older than the pinned one: the module is there, but it
+ -- resolves names by nginx's rules rather than the gateway's
+ package.loaded["resty.ngx_http_ffi_client"] = {
+ new = function()
+ return {}
+ end,
+ }
+
+ package.loaded["apisix.utils.http"] = nil
+ local http_client = require("apisix.utils.http")
+ for _ = 1, 2 do
+ local client, err = http_client.new(http_client.FFI_CLIENT)
+ ngx.say("client: ", client and "created" or err)
+ end
+
+ package.loaded["resty.ngx_http_ffi_client"] = orig_ffi
+ package.loaded["apisix.utils.http"] = orig_utils
+ }
+ }
+--- response_body
+client: resty.ngx_http_ffi_client does not support set_resolver, the runtime
is older than the pinned one
+client: resty.ngx_http_ffi_client does not support set_resolver, the runtime
is older than the pinned one
+--- error_log
+does not support set_resolver
