This is an automated email from the ASF dual-hosted git repository.
shreemaanabhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 312f1e5bb fix(ai-proxy): remove `passthrough` (#12014)
312f1e5bb is described below
commit 312f1e5bb5646d295578f1589b0db57f2e570b92
Author: Shreemaan Abhishek <[email protected]>
AuthorDate: Mon Mar 3 13:25:05 2025 +0545
fix(ai-proxy): remove `passthrough` (#12014)
---
apisix/plugins/ai-proxy/base.lua | 19 -------
apisix/plugins/ai-proxy/schema.lua | 2 -
docs/en/latest/plugins/ai-proxy-multi.md | 7 +--
docs/en/latest/plugins/ai-proxy.md | 3 +-
t/plugin/ai-proxy-multi.t | 90 +-------------------------------
t/plugin/ai-proxy.openai-compatible.t | 20 -------
t/plugin/ai-proxy.t | 87 +-----------------------------
7 files changed, 7 insertions(+), 221 deletions(-)
diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua
index bd6e945ba..6de6ceb8c 100644
--- a/apisix/plugins/ai-proxy/base.lua
+++ b/apisix/plugins/ai-proxy/base.lua
@@ -20,7 +20,6 @@ local core = require("apisix.core")
local bad_request = ngx.HTTP_BAD_REQUEST
local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR
local schema = require("apisix.plugins.ai-proxy.schema")
-local ngx_req = ngx.req
local ngx_print = ngx.print
local ngx_flush = ngx.flush
@@ -69,24 +68,6 @@ function _M.new(proxy_request_to_llm_func,
get_model_name_func)
return internal_server_error
end
- if conf.passthrough then
- ngx_req.init_body()
- while true do
- local chunk, err = body_reader() -- will read chunk by chunk
- if err then
- core.log.error("failed to read response chunk: ", err)
- break
- end
- if not chunk then
- break
- end
- ngx_req.append_body(chunk)
- end
- ngx_req.finish_body()
- keepalive_or_close(conf, httpc)
- return
- end
-
if request_table.stream then
while true do
local chunk, err = body_reader() -- will read chunk by chunk
diff --git a/apisix/plugins/ai-proxy/schema.lua
b/apisix/plugins/ai-proxy/schema.lua
index 8fe8e2d53..a2c25e924 100644
--- a/apisix/plugins/ai-proxy/schema.lua
+++ b/apisix/plugins/ai-proxy/schema.lua
@@ -149,7 +149,6 @@ _M.ai_proxy_schema = {
properties = {
auth = auth_schema,
model = model_schema,
- passthrough = { type = "boolean", default = false },
timeout = {
type = "integer",
minimum = 1,
@@ -193,7 +192,6 @@ _M.ai_proxy_multi_schema = {
default = { algorithm = "roundrobin" }
},
providers = provider_schema,
- passthrough = { type = "boolean", default = false },
timeout = {
type = "integer",
minimum = 1,
diff --git a/docs/en/latest/plugins/ai-proxy-multi.md
b/docs/en/latest/plugins/ai-proxy-multi.md
index bc449e138..bf0f459b7 100644
--- a/docs/en/latest/plugins/ai-proxy-multi.md
+++ b/docs/en/latest/plugins/ai-proxy-multi.md
@@ -70,7 +70,6 @@ Proxying requests to OpenAI is supported now. Other LLM
services will be support
| provider.options.top_p | No | number | Defines the top-p
probability mass (0 - 1) for nucleus sampling.
| |
| provider.options.stream | No | boolean | Enables streaming
responses via SSE.
| |
| provider.override.endpoint | No | string | Custom host
override for the AI provider.
| |
-| passthrough | No | boolean | If true, requests
are forwarded without processing.
| false |
| timeout | No | integer | Request timeout in
milliseconds (1-60000).
| 3000 |
| keepalive | No | boolean | Enables keepalive
connections.
| true |
| keepalive_timeout | No | integer | Timeout for
keepalive connections (minimum 1000ms).
| 60000 |
@@ -120,8 +119,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
"temperature": 1.0
}
}
- ],
- "passthrough": false
+ ]
}
},
"upstream": {
@@ -179,8 +177,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
"temperature": 1.0
}
}
- ],
- "passthrough": false
+ ]
}
},
"upstream": {
diff --git a/docs/en/latest/plugins/ai-proxy.md
b/docs/en/latest/plugins/ai-proxy.md
index ab910368b..e93703f9a 100644
--- a/docs/en/latest/plugins/ai-proxy.md
+++ b/docs/en/latest/plugins/ai-proxy.md
@@ -63,7 +63,6 @@ Proxying requests to OpenAI is supported now. Other LLM
services will be support
| model.options.top_p | No | Number | Top-p probability
mass. Range: 0 - 1 |
| model.options.stream | No | Boolean | Stream response by
SSE. |
| override.endpoint | No | String | Override the endpoint
of the AI provider |
-| passthrough | No | Boolean | If enabled, the
response from LLM will be sent to the upstream. Default: false |
| timeout | No | Integer | Timeout in
milliseconds for requests to LLM. Range: 1 - 60000. Default: 3000 |
| keepalive | No | Boolean | Enable keepalive for
requests to LLM. Default: true |
| keepalive_timeout | No | Integer | Keepalive timeout in
milliseconds for requests to LLM. Minimum: 1000. Default: 60000 |
@@ -107,7 +106,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes/1"; -X PUT \
}'
```
-Since `passthrough` is not enabled upstream node can be any arbitrary value
because it won't be contacted.
+Upstream node can be any arbitrary value because it won't be contacted.
Now send a request:
diff --git a/t/plugin/ai-proxy-multi.t b/t/plugin/ai-proxy-multi.t
index 68eed015d..7969dbd81 100644
--- a/t/plugin/ai-proxy-multi.t
+++ b/t/plugin/ai-proxy-multi.t
@@ -52,26 +52,6 @@ _EOC_
default_type 'application/json';
- location /anything {
- content_by_lua_block {
- local json = require("cjson.safe")
-
- if ngx.req.get_method() ~= "POST" then
- ngx.status = 400
- ngx.say("Unsupported request method: ",
ngx.req.get_method())
- end
- ngx.req.read_body()
- local body = ngx.req.get_body_data()
-
- if body ~= "SELECT * FROM STUDENTS" then
- ngx.status = 503
- ngx.say("passthrough doesn't work")
- return
- end
- ngx.say('{"foo", "bar"}')
- }
- }
-
location /v1/chat/completions {
content_by_lua_block {
local json = require("cjson.safe")
@@ -543,73 +523,7 @@ path override works
-=== TEST 14: set route with right auth header
---- config
- location /t {
- content_by_lua_block {
- local t = require("lib.test_admin").test
- local code, body = t('/apisix/admin/routes/1',
- ngx.HTTP_PUT,
- [[{
- "uri": "/anything",
- "plugins": {
- "ai-proxy-multi": {
- "providers": [
- {
- "name": "openai",
- "model": "gpt-35-turbo-instruct",
- "weight": 1,
- "auth": {
- "header": {
- "Authorization": "Bearer token"
- }
- },
- "options": {
- "max_tokens": 512,
- "temperature": 1.0
- },
- "override": {
- "endpoint": "http://localhost:6724";
- }
- }
- ],
- "ssl_verify": false,
- "passthrough": true
- }
- },
- "upstream": {
- "type": "roundrobin",
- "nodes": {
- "127.0.0.1:6724": 1
- }
- }
- }]]
- )
-
- if code >= 300 then
- ngx.status = code
- end
- ngx.say(body)
- }
- }
---- response_body
-passed
-
-
-
-=== TEST 15: send request with wrong method should work
---- request
-POST /anything
-{ "messages": [ { "role": "user", "content": "write an SQL query to get all
rows from student table" } ] }
---- more_headers
-Authorization: Bearer token
---- error_code: 200
---- response_body
-{"foo", "bar"}
-
-
-
-=== TEST 16: set route with stream = true (SSE)
+=== TEST 14: set route with stream = true (SSE)
--- config
location /t {
content_by_lua_block {
@@ -663,7 +577,7 @@ passed
-=== TEST 17: test is SSE works as expected
+=== TEST 15: test is SSE works as expected
--- config
location /t {
content_by_lua_block {
diff --git a/t/plugin/ai-proxy.openai-compatible.t
b/t/plugin/ai-proxy.openai-compatible.t
index bba9db40c..a98161a48 100644
--- a/t/plugin/ai-proxy.openai-compatible.t
+++ b/t/plugin/ai-proxy.openai-compatible.t
@@ -46,26 +46,6 @@ add_block_preprocessor(sub {
default_type 'application/json';
- location /anything {
- content_by_lua_block {
- local json = require("cjson.safe")
-
- if ngx.req.get_method() ~= "POST" then
- ngx.status = 400
- ngx.say("Unsupported request method: ",
ngx.req.get_method())
- end
- ngx.req.read_body()
- local body = ngx.req.get_body_data()
-
- if body ~= "SELECT * FROM STUDENTS" then
- ngx.status = 503
- ngx.say("passthrough doesn't work")
- return
- end
- ngx.say('{"foo", "bar"}')
- }
- }
-
location /v1/chat/completions {
content_by_lua_block {
local json = require("cjson.safe")
diff --git a/t/plugin/ai-proxy.t b/t/plugin/ai-proxy.t
index 445e406f6..8cfd88018 100644
--- a/t/plugin/ai-proxy.t
+++ b/t/plugin/ai-proxy.t
@@ -46,26 +46,6 @@ add_block_preprocessor(sub {
default_type 'application/json';
- location /anything {
- content_by_lua_block {
- local json = require("cjson.safe")
-
- if ngx.req.get_method() ~= "POST" then
- ngx.status = 400
- ngx.say("Unsupported request method: ",
ngx.req.get_method())
- end
- ngx.req.read_body()
- local body = ngx.req.get_body_data()
-
- if body ~= "SELECT * FROM STUDENTS" then
- ngx.status = 503
- ngx.say("passthrough doesn't work")
- return
- end
- ngx.say('{"foo", "bar"}')
- }
- }
-
location /v1/chat/completions {
content_by_lua_block {
local json = require("cjson.safe")
@@ -519,70 +499,7 @@ path override works
-=== TEST 14: set route with right auth header
---- config
- location /t {
- content_by_lua_block {
- local t = require("lib.test_admin").test
- local code, body = t('/apisix/admin/routes/1',
- ngx.HTTP_PUT,
- [[{
- "uri": "/anything",
- "plugins": {
- "ai-proxy": {
- "auth": {
- "header": {
- "Authorization": "Bearer token"
- }
- },
- "model": {
- "provider": "openai",
- "name": "gpt-35-turbo-instruct",
- "options": {
- "max_tokens": 512,
- "temperature": 1.0
- }
- },
- "override": {
- "endpoint": "http://localhost:6724";
- },
- "ssl_verify": false,
- "passthrough": true
- }
- },
- "upstream": {
- "type": "roundrobin",
- "nodes": {
- "127.0.0.1:6724": 1
- }
- }
- }]]
- )
-
- if code >= 300 then
- ngx.status = code
- end
- ngx.say(body)
- }
- }
---- response_body
-passed
-
-
-
-=== TEST 15: send request with wrong method should work
---- request
-POST /anything
-{ "messages": [ { "role": "user", "content": "write an SQL query to get all
rows from student table" } ] }
---- more_headers
-Authorization: Bearer token
---- error_code: 200
---- response_body
-{"foo", "bar"}
-
-
-
-=== TEST 16: set route with stream = true (SSE)
+=== TEST 14: set route with stream = true (SSE)
--- config
location /t {
content_by_lua_block {
@@ -633,7 +550,7 @@ passed
-=== TEST 17: test is SSE works as expected
+=== TEST 15: test is SSE works as expected
--- config
location /t {
content_by_lua_block {
