This is an automated email from the ASF dual-hosted git repository.
shreemaanabhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 752d9393d fix(ai-proxy-multi): dead loop when retrying (#12012)
752d9393d is described below
commit 752d9393d21db0aefecf484c81fe4989df2e6e9a
Author: Shreemaan Abhishek <shreemaan.abhis...@gmail.com>
AuthorDate: Mon Mar 3 08:49:51 2025 +0545
fix(ai-proxy-multi): dead loop when retrying (#12012)
---
apisix/plugins/ai-proxy-multi.lua | 4 +--
t/plugin/ai-proxy-multi2.t | 65 +++++++++++++++++++++++++++++++++++++++
2 files changed, 67 insertions(+), 2 deletions(-)
diff --git a/apisix/plugins/ai-proxy-multi.lua
b/apisix/plugins/ai-proxy-multi.lua
index 2ffaa1595..4993270b9 100644
--- a/apisix/plugins/ai-proxy-multi.lua
+++ b/apisix/plugins/ai-proxy-multi.lua
@@ -145,7 +145,6 @@ end
local function pick_target(ctx, conf, ups_tab)
- ctx.ai_balancer_try_count = (ctx.ai_balancer_try_count or 0) + 1
if ctx.ai_balancer_try_count > 1 then
if ctx.server_picker and ctx.server_picker.after_balance then
ctx.server_picker.after_balance(ctx, true)
@@ -172,6 +171,7 @@ end
local function get_load_balanced_provider(ctx, conf, ups_tab, request_table)
+ ctx.ai_balancer_try_count = (ctx.ai_balancer_try_count or 0) + 1
local provider_name, provider_conf
if #conf.providers == 1 then
provider_name = conf.providers[1].name
@@ -215,7 +215,7 @@ local function proxy_request_to_llm(conf, request_table,
ctx)
local ai_driver = require("apisix.plugins.ai-drivers." .. provider)
local res, err, httpc = ai_driver:request(conf, request_table, extra_opts)
if not res then
- if (ctx.balancer_try_count or 0) < 1 then
+ if (ctx.ai_balancer_try_count or 0) < 2 then
core.log.warn("failed to send request to LLM: ", err, ".
Retrying...")
goto retry
end
diff --git a/t/plugin/ai-proxy-multi2.t b/t/plugin/ai-proxy-multi2.t
index 9a77dc5f7..00c1714a3 100644
--- a/t/plugin/ai-proxy-multi2.t
+++ b/t/plugin/ai-proxy-multi2.t
@@ -359,3 +359,68 @@ POST /anything
found query params: {"api_key":"apikey","some_query":"yes"}
--- response_body
passed
+
+
+
+=== TEST 9: set route with unavailable endpoint
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "providers": [
+ {
+ "name": "openai",
+ "model": "gpt-4",
+ "weight": 1,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "options": {
+ "max_tokens": 512,
+ "temperature": 1.0
+ },
+ "override": {
+ "endpoint":
"http://unavailable.endpoint.ehfwuehr:404";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "canbeanything.com": 1
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 10: ai-proxy-multi should retry once and fail
+# i.e it should not attempt to proxy request endlessly
+--- request
+POST /anything
+{ "messages": [ { "role": "system", "content": "You are a mathematician" }, {
"role": "user", "content": "What is 1+1?"} ] }
+--- error_code: 500
+--- error_log
+parse_domain(): failed to parse domain: unavailable.endpoint.ehfwuehr, error:
failed to query the DNS server: dns
+phase_func(): failed to send request to LLM service: failed to connect to LLM
server: failed to parse domain
