This is an automated email from the ASF dual-hosted git repository.
shreemaanabhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 88164f47e feat(ai-proxy): support proxying openai compatible LLMs
(#12004)
88164f47e is described below
commit 88164f47ead10965c4c9e9a8a3368ce20f6b022c
Author: Shreemaan Abhishek <shreemaan.abhis...@gmail.com>
AuthorDate: Sat Mar 1 15:00:35 2025 +0545
feat(ai-proxy): support proxying openai compatible LLMs (#12004)
---
apisix/plugins/ai-drivers/deepseek.lua | 2 +-
.../{openai-compatible.lua => openai-base.lua} | 9 +-
apisix/plugins/ai-drivers/openai-compatible.lua | 91 +----
apisix/plugins/ai-drivers/openai.lua | 2 +-
apisix/plugins/ai-proxy/schema.lua | 6 +-
docs/en/latest/plugins/ai-proxy-multi.md | 55 +++
docs/en/latest/plugins/ai-proxy.md | 36 ++
t/plugin/ai-proxy-multi.openai-compatible.t | 326 +++++++++++++++++
t/plugin/ai-proxy.openai-compatible.t | 384 +++++++++++++++++++++
9 files changed, 813 insertions(+), 98 deletions(-)
diff --git a/apisix/plugins/ai-drivers/deepseek.lua
b/apisix/plugins/ai-drivers/deepseek.lua
index ab441c636..19c2e90e3 100644
--- a/apisix/plugins/ai-drivers/deepseek.lua
+++ b/apisix/plugins/ai-drivers/deepseek.lua
@@ -15,7 +15,7 @@
-- limitations under the License.
--
-return require("apisix.plugins.ai-drivers.openai-compatible").new(
+return require("apisix.plugins.ai-drivers.openai-base").new(
{
host = "api.deepseek.com",
path = "/chat/completions",
diff --git a/apisix/plugins/ai-drivers/openai-compatible.lua
b/apisix/plugins/ai-drivers/openai-base.lua
similarity index 95%
copy from apisix/plugins/ai-drivers/openai-compatible.lua
copy to apisix/plugins/ai-drivers/openai-base.lua
index fd5d2163c..a9eb31059 100644
--- a/apisix/plugins/ai-drivers/openai-compatible.lua
+++ b/apisix/plugins/ai-drivers/openai-base.lua
@@ -93,9 +93,14 @@ function _M.request(self, conf, request_table, extra_opts)
request_table[opt] = val
end
end
- params.body = core.json.encode(request_table)
- httpc:set_timeout(conf.keepalive_timeout)
+ local req_json, err = core.json.encode(request_table)
+ if not req_json then
+ return nil, err
+ end
+
+ params.body = req_json
+
local res, err = httpc:request(params)
if not res then
return nil, err
diff --git a/apisix/plugins/ai-drivers/openai-compatible.lua
b/apisix/plugins/ai-drivers/openai-compatible.lua
index fd5d2163c..b6c21cf51 100644
--- a/apisix/plugins/ai-drivers/openai-compatible.lua
+++ b/apisix/plugins/ai-drivers/openai-compatible.lua
@@ -14,94 +14,5 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-local _M = {}
-local mt = {
- __index = _M
-}
-
-local core = require("apisix.core")
-local http = require("resty.http")
-local url = require("socket.url")
-
-local pairs = pairs
-local type = type
-local setmetatable = setmetatable
-
-
-function _M.new(opts)
-
- local self = {
- host = opts.host,
- port = opts.port,
- path = opts.path,
- }
- return setmetatable(self, mt)
-end
-
-
-function _M.request(self, conf, request_table, extra_opts)
- local httpc, err = http.new()
- if not httpc then
- return nil, "failed to create http client to send request to LLM
server: " .. err
- end
- httpc:set_timeout(conf.timeout)
-
- local endpoint = extra_opts.endpoint
- local parsed_url
- if endpoint then
- parsed_url = url.parse(endpoint)
- end
-
- local ok, err = httpc:connect({
- scheme = endpoint and parsed_url.scheme or "https",
- host = endpoint and parsed_url.host or self.host,
- port = endpoint and parsed_url.port or self.port,
- ssl_verify = conf.ssl_verify,
- ssl_server_name = endpoint and parsed_url.host or self.host,
- pool_size = conf.keepalive and conf.keepalive_pool,
- })
-
- if not ok then
- return nil, "failed to connect to LLM server: " .. err
- end
-
- local query_params = extra_opts.query_params
-
- if type(parsed_url) == "table" and parsed_url.query and #parsed_url.query
> 0 then
- local args_tab = core.string.decode_args(parsed_url.query)
- if type(args_tab) == "table" then
- core.table.merge(query_params, args_tab)
- end
- end
-
- local path = (endpoint and parsed_url.path or self.path)
-
- local headers = extra_opts.headers
- headers["Content-Type"] = "application/json"
- local params = {
- method = "POST",
- headers = headers,
- keepalive = conf.keepalive,
- ssl_verify = conf.ssl_verify,
- path = path,
- query = query_params
- }
-
- if extra_opts.model_options then
- for opt, val in pairs(extra_opts.model_options) do
- request_table[opt] = val
- end
- end
- params.body = core.json.encode(request_table)
-
- httpc:set_timeout(conf.keepalive_timeout)
- local res, err = httpc:request(params)
- if not res then
- return nil, err
- end
-
- return res, nil, httpc
-end
-
-return _M
+return require("apisix.plugins.ai-drivers.openai-base").new({})
diff --git a/apisix/plugins/ai-drivers/openai.lua
b/apisix/plugins/ai-drivers/openai.lua
index 785ede193..e922c8b1e 100644
--- a/apisix/plugins/ai-drivers/openai.lua
+++ b/apisix/plugins/ai-drivers/openai.lua
@@ -15,7 +15,7 @@
-- limitations under the License.
--
-return require("apisix.plugins.ai-drivers.openai-compatible").new(
+return require("apisix.plugins.ai-drivers.openai-base").new(
{
host = "api.openai.com",
path = "/v1/chat/completions",
diff --git a/apisix/plugins/ai-proxy/schema.lua
b/apisix/plugins/ai-proxy/schema.lua
index d0ba33fdc..4f756216a 100644
--- a/apisix/plugins/ai-proxy/schema.lua
+++ b/apisix/plugins/ai-proxy/schema.lua
@@ -84,8 +84,7 @@ local model_schema = {
provider = {
type = "string",
description = "Name of the AI service provider.",
- oneOf = { "openai" }, -- add more providers later
-
+ enum = { "openai", "openai-compatible", "deepseek" }, -- add more
providers later
},
name = {
type = "string",
@@ -114,7 +113,7 @@ local provider_schema = {
name = {
type = "string",
description = "Name of the AI service provider.",
- enum = { "openai", "deepseek" }, -- add more providers later
+ enum = { "openai", "deepseek", "openai-compatible" }, -- add
more providers later
},
model = {
@@ -160,7 +159,6 @@ _M.ai_proxy_schema = {
description = "timeout in milliseconds",
},
keepalive = {type = "boolean", default = true},
- keepalive_timeout = {type = "integer", minimum = 1000, default =
60000},
keepalive_pool = {type = "integer", minimum = 1, default = 30},
ssl_verify = {type = "boolean", default = true },
},
diff --git a/docs/en/latest/plugins/ai-proxy-multi.md
b/docs/en/latest/plugins/ai-proxy-multi.md
index 72d8a9cfa..2359d4b9a 100644
--- a/docs/en/latest/plugins/ai-proxy-multi.md
+++ b/docs/en/latest/plugins/ai-proxy-multi.md
@@ -193,3 +193,58 @@ curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
```
In the above configuration `priority` for the deepseek provider is set to `0`.
Which means if `openai` provider is unavailable then `ai-proxy-multi` plugin
will retry sending request to `deepseek` in the second attempt.
+
+### Send request to an OpenAI compatible LLM
+
+Create a route with the `ai-proxy-multi` plugin with `provider.name` set to
`openai-compatible` and the endpoint of the model set to
`provider.override.endpoint` like so:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${ADMIN_API_KEY}" \
+ -d '{
+ "id": "ai-proxy-multi-route",
+ "uri": "/anything",
+ "methods": ["POST"],
+ "plugins": {
+ "ai-proxy-multi": {
+ "providers": [
+ {
+ "name": "openai-compatible",
+ "model": "qwen-plus",
+ "weight": 1,
+ "priority": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$OPENAI_API_KEY"'"
+ }
+ },
+ "override": {
+ "endpoint":
"https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions";
+ }
+ },
+ {
+ "name": "deepseek",
+ "model": "deepseek-chat",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer '"$DEEPSEEK_API_KEY"'"
+ }
+ },
+ "options": {
+ "max_tokens": 512,
+ "temperature": 1.0
+ }
+ }
+ ],
+ "passthrough": false
+ }
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "httpbin.org": 1
+ }
+ }
+ }'
+```
diff --git a/docs/en/latest/plugins/ai-proxy.md
b/docs/en/latest/plugins/ai-proxy.md
index 0f68911bb..0194205d9 100644
--- a/docs/en/latest/plugins/ai-proxy.md
+++ b/docs/en/latest/plugins/ai-proxy.md
@@ -142,3 +142,39 @@ You will receive a response like this:
"usage": { "completion_tokens": 15, "prompt_tokens": 23, "total_tokens": 38 }
}
```
+
+### Send request to an OpenAI compatible LLM
+
+Create a route with the `ai-proxy` plugin with `provider` set to
`openai-compatible` and the endpoint of the model set to `override.endpoint`
like so:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes/1"; -X PUT \
+ -H "X-API-KEY: ${ADMIN_API_KEY}" \
+ -d '{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "auth": {
+ "header": {
+ "Authorization": "Bearer <some-token>"
+ }
+ },
+ "model": {
+ "provider": "openai-compatible",
+ "name": "qwen-plus"
+ },
+ "override": {
+ "endpoint":
"https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions";
+ }
+ }
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "somerandom.com:443": 1
+ },
+ "scheme": "https",
+ "pass_host": "node"
+ }
+ }'
+```
diff --git a/t/plugin/ai-proxy-multi.openai-compatible.t
b/t/plugin/ai-proxy-multi.openai-compatible.t
new file mode 100644
index 000000000..f80be6dc4
--- /dev/null
+++ b/t/plugin/ai-proxy-multi.openai-compatible.t
@@ -0,0 +1,326 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+
+
+my $resp_file = 't/assets/ai-proxy-response.json';
+open(my $fh, '<', $resp_file) or die "Could not open file '$resp_file' $!";
+my $resp = do { local $/; <$fh> };
+close($fh);
+
+print "Hello, World!\n";
+print $resp;
+
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $user_yaml_config = <<_EOC_;
+plugins:
+ - ai-proxy-multi
+_EOC_
+ $block->set_value("extra_yaml_config", $user_yaml_config);
+
+ my $http_config = $block->http_config // <<_EOC_;
+ server {
+ server_name openai;
+ listen 6724;
+
+ default_type 'application/json';
+
+ location /anything {
+ content_by_lua_block {
+ local json = require("cjson.safe")
+
+ if ngx.req.get_method() ~= "POST" then
+ ngx.status = 400
+ ngx.say("Unsupported request method: ",
ngx.req.get_method())
+ end
+ ngx.req.read_body()
+ local body = ngx.req.get_body_data()
+
+ if body ~= "SELECT * FROM STUDENTS" then
+ ngx.status = 503
+ ngx.say("passthrough doesn't work")
+ return
+ end
+ ngx.say('{"foo", "bar"}')
+ }
+ }
+
+ location /v1/chat/completions {
+ content_by_lua_block {
+ local json = require("cjson.safe")
+
+ if ngx.req.get_method() ~= "POST" then
+ ngx.status = 400
+ ngx.say("Unsupported request method: ",
ngx.req.get_method())
+ end
+ ngx.req.read_body()
+ local body, err = ngx.req.get_body_data()
+ body, err = json.decode(body)
+
+ local test_type = ngx.req.get_headers()["test-type"]
+ if test_type == "options" then
+ if body.foo == "bar" then
+ ngx.status = 200
+ ngx.say("options works")
+ else
+ ngx.status = 500
+ ngx.say("model options feature doesn't work")
+ end
+ return
+ end
+
+ local header_auth = ngx.req.get_headers()["authorization"]
+ local query_auth = ngx.req.get_uri_args()["apikey"]
+
+ if header_auth ~= "Bearer token" and query_auth ~=
"apikey" then
+ ngx.status = 401
+ ngx.say("Unauthorized")
+ return
+ end
+
+ if header_auth == "Bearer token" or query_auth == "apikey"
then
+ ngx.req.read_body()
+ local body, err = ngx.req.get_body_data()
+ body, err = json.decode(body)
+
+ if not body.messages or #body.messages < 1 then
+ ngx.status = 400
+ ngx.say([[{ "error": "bad request"}]])
+ return
+ end
+
+ if body.messages[1].content == "write an SQL query to
get all rows from student table" then
+ ngx.print("SELECT * FROM STUDENTS")
+ return
+ end
+
+ ngx.status = 200
+ ngx.say([[$resp]])
+ return
+ end
+
+
+ ngx.status = 503
+ ngx.say("reached the end of the test suite")
+ }
+ }
+
+ location /random {
+ content_by_lua_block {
+ ngx.say("path override works")
+ }
+ }
+ }
+_EOC_
+
+ $block->set_value("http_config", $http_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set route with right auth header
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "providers": [
+ {
+ "name": "openai-compatible",
+ "model": "custom",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "options": {
+ "max_tokens": 512,
+ "temperature": 1.0
+ },
+ "override": {
+ "endpoint":
"http://localhost:6724/v1/chat/completions";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "canbeanything.com": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: send request
+--- request
+POST /anything
+{ "messages": [ { "role": "system", "content": "You are a mathematician" }, {
"role": "user", "content": "What is 1+1?"} ] }
+--- more_headers
+Authorization: Bearer token
+--- error_code: 200
+--- response_body eval
+qr/\{ "content": "1 \+ 1 = 2\.", "role": "assistant" \}/
+
+
+
+=== TEST 3: set route with stream = true (SSE)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "providers": [
+ {
+ "name": "openai-compatible",
+ "model": "custom-instruct",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "options": {
+ "max_tokens": 512,
+ "temperature": 1.0,
+ "stream": true
+ },
+ "override": {
+ "endpoint":
"http://localhost:7737/v1/chat/completions";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "canbeanything.com": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 4: test is SSE works as expected
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local httpc = http.new()
+ local core = require("apisix.core")
+
+ local ok, err = httpc:connect({
+ scheme = "http",
+ host = "localhost",
+ port = ngx.var.server_port,
+ })
+
+ if not ok then
+ ngx.status = 500
+ ngx.say(err)
+ return
+ end
+
+ local params = {
+ method = "POST",
+ headers = {
+ ["Content-Type"] = "application/json",
+ },
+ path = "/anything",
+ body = [[{
+ "messages": [
+ { "role": "system", "content": "some content" }
+ ]
+ }]],
+ }
+
+ local res, err = httpc:request(params)
+ if not res then
+ ngx.status = 500
+ ngx.say(err)
+ return
+ end
+
+ local final_res = {}
+ while true do
+ local chunk, err = res.body_reader() -- will read chunk by
chunk
+ if err then
+ core.log.error("failed to read response chunk: ", err)
+ break
+ end
+ if not chunk then
+ break
+ end
+ core.table.insert_tail(final_res, chunk)
+ end
+
+ ngx.print(#final_res .. final_res[6])
+ }
+ }
+--- response_body_like eval
+qr/6data: \[DONE\]\n\n/
diff --git a/t/plugin/ai-proxy.openai-compatible.t
b/t/plugin/ai-proxy.openai-compatible.t
new file mode 100644
index 000000000..bba9db40c
--- /dev/null
+++ b/t/plugin/ai-proxy.openai-compatible.t
@@ -0,0 +1,384 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+
+
+my $resp_file = 't/assets/ai-proxy-response.json';
+open(my $fh, '<', $resp_file) or die "Could not open file '$resp_file' $!";
+my $resp = do { local $/; <$fh> };
+close($fh);
+
+print "Hello, World!\n";
+print $resp;
+
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $http_config = $block->http_config // <<_EOC_;
+ server {
+ server_name openai;
+ listen 6724;
+
+ default_type 'application/json';
+
+ location /anything {
+ content_by_lua_block {
+ local json = require("cjson.safe")
+
+ if ngx.req.get_method() ~= "POST" then
+ ngx.status = 400
+ ngx.say("Unsupported request method: ",
ngx.req.get_method())
+ end
+ ngx.req.read_body()
+ local body = ngx.req.get_body_data()
+
+ if body ~= "SELECT * FROM STUDENTS" then
+ ngx.status = 503
+ ngx.say("passthrough doesn't work")
+ return
+ end
+ ngx.say('{"foo", "bar"}')
+ }
+ }
+
+ location /v1/chat/completions {
+ content_by_lua_block {
+ local json = require("cjson.safe")
+
+ if ngx.req.get_method() ~= "POST" then
+ ngx.status = 400
+ ngx.say("Unsupported request method: ",
ngx.req.get_method())
+ end
+ ngx.req.read_body()
+ local body, err = ngx.req.get_body_data()
+ body, err = json.decode(body)
+
+ local test_type = ngx.req.get_headers()["test-type"]
+ if test_type == "options" then
+ if body.foo == "bar" then
+ ngx.status = 200
+ ngx.say("options works")
+ else
+ ngx.status = 500
+ ngx.say("model options feature doesn't work")
+ end
+ return
+ end
+
+ local header_auth = ngx.req.get_headers()["authorization"]
+ local query_auth = ngx.req.get_uri_args()["apikey"]
+
+ if header_auth ~= "Bearer token" and query_auth ~=
"apikey" then
+ ngx.status = 401
+ ngx.say("Unauthorized")
+ return
+ end
+
+ if header_auth == "Bearer token" or query_auth == "apikey"
then
+ ngx.req.read_body()
+ local body, err = ngx.req.get_body_data()
+ body, err = json.decode(body)
+
+ if not body.messages or #body.messages < 1 then
+ ngx.status = 400
+ ngx.say([[{ "error": "bad request"}]])
+ return
+ end
+
+ if body.messages[1].content == "write an SQL query to
get all rows from student table" then
+ ngx.print("SELECT * FROM STUDENTS")
+ return
+ end
+
+ ngx.status = 200
+ ngx.say([[$resp]])
+ return
+ end
+
+
+ ngx.status = 503
+ ngx.say("reached the end of the test suite")
+ }
+ }
+
+ location /random {
+ content_by_lua_block {
+ ngx.say("path override works")
+ }
+ }
+ }
+_EOC_
+
+ $block->set_value("http_config", $http_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set route with right auth header
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "model": {
+ "provider": "openai-compatible",
+ "name": "custom",
+ "options": {
+ "max_tokens": 512,
+ "temperature": 1.0
+ }
+ },
+ "override": {
+ "endpoint":
"http://localhost:6724/v1/chat/completions";
+ },
+ "ssl_verify": false
+ }
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "canbeanything.com": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: send request
+--- request
+POST /anything
+{ "messages": [ { "role": "system", "content": "You are a mathematician" }, {
"role": "user", "content": "What is 1+1?"} ] }
+--- more_headers
+Authorization: Bearer token
+--- error_code: 200
+--- response_body eval
+qr/\{ "content": "1 \+ 1 = 2\.", "role": "assistant" \}/
+
+
+
+=== TEST 3: override path
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "model": {
+ "provider": "openai-compatible",
+ "name": "some-model",
+ "options": {
+ "foo": "bar",
+ "temperature": 1.0
+ }
+ },
+ "override": {
+ "endpoint": "http://localhost:6724/random";
+ },
+ "ssl_verify": false
+ }
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "canbeanything.com": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body, actual_body = t("/anything",
+ ngx.HTTP_POST,
+ [[{
+ "messages": [
+ { "role": "system", "content": "You are a
mathematician" },
+ { "role": "user", "content": "What is 1+1?" }
+ ]
+ }]],
+ nil,
+ {
+ ["test-type"] = "path",
+ ["Content-Type"] = "application/json",
+ }
+ )
+
+ ngx.status = code
+ ngx.say(actual_body)
+
+ }
+ }
+--- response_body_chomp
+path override works
+
+
+
+=== TEST 4: set route with stream = true (SSE)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy": {
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "model": {
+ "provider": "openai-compatible",
+ "name": "custom",
+ "options": {
+ "max_tokens": 512,
+ "temperature": 1.0,
+ "stream": true
+ }
+ },
+ "override": {
+ "endpoint":
"http://localhost:7737/v1/chat/completions";
+ },
+ "ssl_verify": false
+ }
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "canbeanything.com": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 5: test is SSE works as expected
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local httpc = http.new()
+ local core = require("apisix.core")
+
+ local ok, err = httpc:connect({
+ scheme = "http",
+ host = "localhost",
+ port = ngx.var.server_port,
+ })
+
+ if not ok then
+ ngx.status = 500
+ ngx.say(err)
+ return
+ end
+
+ local params = {
+ method = "POST",
+ headers = {
+ ["Content-Type"] = "application/json",
+ },
+ path = "/anything",
+ body = [[{
+ "messages": [
+ { "role": "system", "content": "some content" }
+ ]
+ }]],
+ }
+
+ local res, err = httpc:request(params)
+ if not res then
+ ngx.status = 500
+ ngx.say(err)
+ return
+ end
+
+ local final_res = {}
+ while true do
+ local chunk, err = res.body_reader() -- will read chunk by
chunk
+ if err then
+ core.log.error("failed to read response chunk: ", err)
+ break
+ end
+ if not chunk then
+ break
+ end
+ core.table.insert_tail(final_res, chunk)
+ end
+
+ ngx.print(#final_res .. final_res[6])
+ }
+ }
+--- response_body_like eval
+qr/6data: \[DONE\]\n\n/
