This is an automated email from the ASF dual-hosted git repository.
shreemaan-abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 99865026d feat(ai-proxy): support aws bedrock (#13249)
99865026d is described below
commit 99865026dc81b320248cbb34955a4430c1894820
Author: Shreemaan Abhishek <[email protected]>
AuthorDate: Mon Apr 27 20:08:05 2026 +0800
feat(ai-proxy): support aws bedrock (#13249)
---
apisix/plugins/ai-protocols/bedrock-converse.lua | 286 +++++++++++++
apisix/plugins/ai-protocols/init.lua | 5 +-
apisix/plugins/ai-providers/base.lua | 25 ++
apisix/plugins/ai-providers/bedrock.lua | 83 ++++
apisix/plugins/ai-providers/schema.lua | 2 +-
apisix/plugins/ai-proxy-multi.lua | 4 +
apisix/plugins/ai-proxy.lua | 2 +-
apisix/plugins/ai-proxy/schema.lua | 97 +++--
apisix/plugins/ai-transport/auth-aws.lua | 155 +++++++
apisix/plugins/ai-transport/http.lua | 14 +-
apisix/utils/log-sanitize.lua | 3 +-
docs/en/latest/plugins/ai-proxy-multi.md | 331 ++++++++++++++-
docs/en/latest/plugins/ai-proxy.md | 121 +++++-
docs/zh/latest/plugins/ai-proxy-multi.md | 142 ++++++-
docs/zh/latest/plugins/ai-proxy.md | 123 +++++-
t/fixtures/bedrock/converse-basic.json | 22 +
t/lib/server.lua | 132 +++++-
t/plugin/ai-proxy-bedrock-single.t | 429 +++++++++++++++++++
t/plugin/ai-proxy-bedrock.t | 518 +++++++++++++++++++++++
19 files changed, 2429 insertions(+), 65 deletions(-)
diff --git a/apisix/plugins/ai-protocols/bedrock-converse.lua
b/apisix/plugins/ai-protocols/bedrock-converse.lua
new file mode 100644
index 000000000..0e0d2061a
--- /dev/null
+++ b/apisix/plugins/ai-protocols/bedrock-converse.lua
@@ -0,0 +1,286 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--- Bedrock Converse protocol adapter (client-side).
+-- Handles detection and response parsing for the Amazon Bedrock
+-- Converse API format. Non-streaming only in this phase.
+
+local core = require("apisix.core")
+local string_sub = string.sub
+local type = type
+local ipairs = ipairs
+local table = table
+
+local _M = {}
+
+
+--- Detect whether the request matches the Bedrock Converse API format.
+-- Uses URI suffix (/converse) and body (valid JSON table with messages).
+function _M.matches(body, ctx)
+ local uri = ctx.var and ctx.var.uri
+ return uri and string_sub(uri, -9) == "/converse"
+ and type(body) == "table" and type(body.messages) == "table"
+end
+
+
+--- Check whether the request is a streaming request.
+-- Streaming is not supported in this phase.
+function _M.is_streaming(body)
+ return false
+end
+
+
+--- Prepare the outgoing request body for the target provider.
+-- TODO: support streaming. Bedrock uses a separate /converse-stream endpoint
+-- with the AWS EventStream binary protocol, which we don't yet implement.
+-- For now, strip the `stream` field so we only send non-streaming requests
+-- to /converse.
+function _M.prepare_outgoing_request(body)
+ body.stream = nil
+end
+
+
+--- Extract token usage from a non-streaming Bedrock response.
+-- Bedrock format: res_body.usage.inputTokens / outputTokens / totalTokens
+function _M.extract_usage(res_body)
+ if type(res_body) ~= "table" or type(res_body.usage) ~= "table" then
+ return nil, nil
+ end
+ local raw = res_body.usage
+ return {
+ prompt_tokens = raw.inputTokens or 0,
+ completion_tokens = raw.outputTokens or 0,
+ total_tokens = raw.totalTokens
+ or (raw.inputTokens or 0) + (raw.outputTokens or 0),
+ }, raw
+end
+
+
+--- Extract response text from a Bedrock Converse response.
+-- Bedrock format: res_body.output.message.content[].text
+function _M.extract_response_text(res_body)
+ if type(res_body) ~= "table" then
+ return nil
+ end
+ local message = res_body.output and res_body.output.message
+ if type(message) ~= "table" or type(message.content) ~= "table" then
+ return nil
+ end
+ local texts = {}
+ for _, block in ipairs(message.content) do
+ if type(block) == "table" and type(block.text) == "string" then
+ core.table.insert(texts, block.text)
+ end
+ end
+ if #texts > 0 then
+ return table.concat(texts, " ")
+ end
+ return nil
+end
+
+
+--- Extract all text content from a request body for moderation.
+function _M.extract_request_content(body)
+ local contents = {}
+ if type(body.system) == "table" then
+ for _, block in ipairs(body.system) do
+ if type(block) == "table" and type(block.text) == "string" then
+ core.table.insert(contents, block.text)
+ end
+ end
+ end
+ if type(body.messages) == "table" then
+ for _, message in ipairs(body.messages) do
+ if type(message) ~= "table" then
+ goto CONTINUE_MESSAGE
+ end
+ if type(message.content) == "table" then
+ for _, block in ipairs(message.content) do
+ if type(block) == "table" and type(block.text) == "string"
then
+ core.table.insert(contents, block.text)
+ end
+ end
+ end
+ ::CONTINUE_MESSAGE::
+ end
+ end
+ return contents
+end
+
+
+--- Get messages in canonical {role, content} format.
+-- Bedrock content blocks [{text: "..."}] are flattened to plain text.
+function _M.get_messages(body)
+ local messages = {}
+ if type(body.system) == "table" then
+ local texts = {}
+ for _, block in ipairs(body.system) do
+ if type(block) == "table" and type(block.text) == "string" then
+ core.table.insert(texts, block.text)
+ end
+ end
+ if #texts > 0 then
+ core.table.insert(messages, {
+ role = "system",
+ content = table.concat(texts, " "),
+ })
+ end
+ end
+ if type(body.messages) == "table" then
+ for _, message in ipairs(body.messages) do
+ if type(message) ~= "table" then
+ goto CONTINUE
+ end
+ if type(message.content) == "table" then
+ local texts = {}
+ for _, block in ipairs(message.content) do
+ if type(block) == "table" and type(block.text) == "string"
then
+ core.table.insert(texts, block.text)
+ end
+ end
+ if #texts > 0 then
+ core.table.insert(messages, {
+ role = message.role,
+ content = table.concat(texts, " "),
+ })
+ end
+ end
+ ::CONTINUE::
+ end
+ end
+ return messages
+end
+
+
+--- Prepend messages to the request body.
+-- System messages go to body.system; user/assistant messages go to
body.messages.
+function _M.prepend_messages(body, msgs)
+ if not msgs or #msgs == 0 then return end
+
+ local new_system_blocks = {}
+ local new_chat_messages = {}
+ for _, msg in ipairs(msgs) do
+ if msg.role == "system" then
+ core.table.insert(new_system_blocks, {text = msg.content})
+ else
+ core.table.insert(new_chat_messages, {
+ role = msg.role,
+ content = {{text = msg.content}},
+ })
+ end
+ end
+
+ if #new_system_blocks > 0 then
+ if type(body.system) ~= "table" then
+ body.system = {}
+ end
+ local merged_system = {}
+ for _, block in ipairs(new_system_blocks) do
+ core.table.insert(merged_system, block)
+ end
+ for _, block in ipairs(body.system) do
+ core.table.insert(merged_system, block)
+ end
+ body.system = merged_system
+ end
+
+ if #new_chat_messages > 0 then
+ if type(body.messages) ~= "table" then
+ body.messages = {}
+ end
+ local merged_messages = {}
+ for _, msg in ipairs(new_chat_messages) do
+ core.table.insert(merged_messages, msg)
+ end
+ for _, msg in ipairs(body.messages) do
+ core.table.insert(merged_messages, msg)
+ end
+ body.messages = merged_messages
+ end
+end
+
+
+--- Append messages to the request body.
+-- System messages go to body.system; user/assistant messages go to
body.messages.
+function _M.append_messages(body, msgs)
+ if not msgs or #msgs == 0 then return end
+
+ for _, msg in ipairs(msgs) do
+ if msg.role == "system" then
+ if type(body.system) ~= "table" then
+ body.system = {}
+ end
+ core.table.insert(body.system, {text = msg.content})
+ else
+ if type(body.messages) ~= "table" then
+ body.messages = {}
+ end
+ core.table.insert(body.messages, {
+ role = msg.role,
+ content = {{text = msg.content}},
+ })
+ end
+ end
+end
+
+
+--- Get raw request content for logging.
+function _M.get_request_content(body)
+ return body.messages
+end
+
+
+--- Build a non-streaming deny response in Bedrock Converse format.
+function _M.build_deny_response(opts)
+ return core.json.encode({
+ output = {
+ message = {
+ role = "assistant",
+ content = {{text = opts.text}},
+ },
+ },
+ stopReason = "end_turn",
+ usage = opts.usage,
+ })
+end
+
+
+--- Build an empty usage object.
+function _M.empty_usage()
+ return { inputTokens = 0, outputTokens = 0, totalTokens = 0 }
+end
+
+
+--- Build a non-streaming request from system prompt and user content.
+function _M.build_simple_request(system_prompt, user_content, opts)
+ local body = {
+ messages = {{
+ role = "user",
+ content = {{text = user_content}},
+ }},
+ }
+ if system_prompt then
+ body.system = {{text = system_prompt}}
+ end
+ if opts and opts.max_tokens then
+ body.inferenceConfig = { maxTokens = opts.max_tokens }
+ end
+ return body
+end
+
+
+return _M
diff --git a/apisix/plugins/ai-protocols/init.lua
b/apisix/plugins/ai-protocols/init.lua
index 19590bf30..2a20d1b0c 100644
--- a/apisix/plugins/ai-protocols/init.lua
+++ b/apisix/plugins/ai-protocols/init.lua
@@ -33,10 +33,13 @@ local registered = {
["openai-responses"] =
require("apisix.plugins.ai-protocols.openai-responses"),
["openai-embeddings"] =
require("apisix.plugins.ai-protocols.openai-embeddings"),
["anthropic-messages"] =
require("apisix.plugins.ai-protocols.anthropic-messages"),
+ ["bedrock-converse"] =
require("apisix.plugins.ai-protocols.bedrock-converse"),
}
--- Detection order: URL+body first (anthropic, responses), then body-only
(chat, embeddings).
+-- Detection order: URL+body first (bedrock, anthropic, responses),
+-- then body-only (chat, embeddings).
local detection_order = {
+ { name = "bedrock-converse", protocol = registered["bedrock-converse"] },
{ name = "anthropic-messages", protocol = registered["anthropic-messages"]
},
{ name = "openai-responses", protocol = registered["openai-responses"] },
{ name = "openai-chat", protocol = registered["openai-chat"] },
diff --git a/apisix/plugins/ai-providers/base.lua
b/apisix/plugins/ai-providers/base.lua
index e0072f8a0..253e048f6 100644
--- a/apisix/plugins/ai-providers/base.lua
+++ b/apisix/plugins/ai-providers/base.lua
@@ -40,6 +40,7 @@ local deep_merge =
require("apisix.plugins.ai-proxy.merge").deep_merge
local ngx = ngx
local ngx_now = ngx.now
local tonumber = tonumber
+local require = require
local table = table
local pairs = pairs
@@ -156,6 +157,16 @@ function _M.build_request(self, ctx, conf, request_body,
opts)
path = opts.target_path
end
+ if not path then
+ -- Provider's path callback returned nil and override.endpoint did not
+ -- supply one. For providers whose path depends on the model (bedrock,
+ -- vertex-ai), this happens when neither options.model nor body.model
+ -- is set.
+ return nil, "could not resolve upstream path: ensure the route or "
+ .. "request body specifies a model, or that override.endpoint "
+ .. "includes a path", 400
+ end
+
local headers = transport_http.construct_forward_headers(auth.header or
{}, ctx)
if token then
headers["authorization"] = "Bearer " .. token
@@ -207,6 +218,20 @@ function _M.build_request(self, ctx, conf, request_body,
opts)
request_body.model = nil
end
+ -- AWS SigV4 signing (must be last — signs the finalized body)
+ if self.aws_sigv4 and auth.aws then
+ local auth_aws = require("apisix.plugins.ai-transport.auth-aws")
+ local region = opts.conf and opts.conf.region
+ if not region then
+ return nil, "missing region for AWS SigV4 signing "
+ .. "(provider_conf.region required for bedrock)"
+ end
+ local sign_err = auth_aws.sign_request(params, auth.aws, region)
+ if sign_err then
+ return nil, "failed to sign AWS request: " .. sign_err
+ end
+ end
+
return params
end
diff --git a/apisix/plugins/ai-providers/bedrock.lua
b/apisix/plugins/ai-providers/bedrock.lua
new file mode 100644
index 000000000..8fb2f0a1e
--- /dev/null
+++ b/apisix/plugins/ai-providers/bedrock.lua
@@ -0,0 +1,83 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local core = require("apisix.core")
+local str_fmt = string.format
+local ngx_escape_uri = ngx.escape_uri
+
+local host_template = "bedrock-runtime.%s.amazonaws.com"
+local chat_path_template = "/model/%s/converse"
+
+local function get_host(region)
+ return str_fmt(host_template, region)
+end
+
+
+local function get_region(instance_conf)
+ return core.table.try_read_attr(instance_conf, "provider_conf", "region")
+end
+
+
+local function get_node(instance_conf)
+ return {
+ scheme = "https",
+ host = get_host(get_region(instance_conf)),
+ port = 443,
+ }
+end
+
+
+-- Map override.request_body fields to Bedrock Converse API format.
+-- Bedrock uses inferenceConfig.maxTokens (camelCase, nested) for max output
tokens.
+local function rewrite_converse_request_body(body, override, force)
+ if override.max_tokens then
+ body.inferenceConfig = body.inferenceConfig or {}
+ if force or body.inferenceConfig.maxTokens == nil then
+ body.inferenceConfig.maxTokens = override.max_tokens
+ end
+ end
+end
+
+
+return require("apisix.plugins.ai-providers.base").new({
+ get_node = get_node,
+ remove_model = true,
+ aws_sigv4 = true,
+ capabilities = {
+ ["bedrock-converse"] = {
+ host = function(conf)
+ if not conf or not conf.region then
+ return nil
+ end
+ return get_host(conf.region)
+ end,
+ path = function(conf, ctx)
+ local model = ctx and ctx.var.llm_model
+ -- ctx.var.llm_model defaults to "" (empty string), so check
+ -- for empty too — both mean "no model resolved".
+ if not model or model == "" then return nil end
+ -- Encode the model so it stays as a single path segment.
+ -- Required for application inference profile ARNs which
+ -- contain "/" (e.g. "...:application-inference-profile/abc")
+ -- and ":". auth-aws.lua's normalize_and_encode_path is
+ -- idempotent so this pre-encoding is preserved end-to-end.
+ return str_fmt(chat_path_template, ngx_escape_uri(model))
+ end,
+ rewrite_request_body = rewrite_converse_request_body,
+ },
+ },
+})
diff --git a/apisix/plugins/ai-providers/schema.lua
b/apisix/plugins/ai-providers/schema.lua
index d4939821d..d8be7b6eb 100644
--- a/apisix/plugins/ai-providers/schema.lua
+++ b/apisix/plugins/ai-providers/schema.lua
@@ -21,7 +21,7 @@ local _M = {}
_M.providers = {
"openai", "deepseek", "aimlapi", "anthropic",
"openai-compatible", "azure-openai", "openrouter",
- "gemini", "vertex-ai",
+ "gemini", "vertex-ai", "bedrock",
}
return _M
diff --git a/apisix/plugins/ai-proxy-multi.lua
b/apisix/plugins/ai-proxy-multi.lua
index 93c14d9cc..419af6aaa 100644
--- a/apisix/plugins/ai-proxy-multi.lua
+++ b/apisix/plugins/ai-proxy-multi.lua
@@ -116,6 +116,10 @@ function _M.check_schema(conf)
return false, "invalid gcp service_account_json: " .. err
end
end
+ local ok, err = schema.validate_provider_requirements(instance)
+ if not ok then
+ return false, "instance '" .. (instance.name or "?") .. "': " ..
err
+ end
end
local algo = core.table.try_read_attr(conf, "balancer", "algorithm")
local hash_on = core.table.try_read_attr(conf, "balancer", "hash_on")
diff --git a/apisix/plugins/ai-proxy.lua b/apisix/plugins/ai-proxy.lua
index b5e48f4d4..dec03592b 100644
--- a/apisix/plugins/ai-proxy.lua
+++ b/apisix/plugins/ai-proxy.lua
@@ -48,7 +48,7 @@ function _M.check_schema(conf)
return false, "invalid gcp service_account_json: " .. err
end
end
- return true
+ return schema.validate_provider_requirements(conf)
end
diff --git a/apisix/plugins/ai-proxy/schema.lua
b/apisix/plugins/ai-proxy/schema.lua
index 553acc24c..c249b41eb 100644
--- a/apisix/plugins/ai-proxy/schema.lua
+++ b/apisix/plugins/ai-proxy/schema.lua
@@ -58,6 +58,16 @@ local auth_schema = {
},
}
},
+ aws = {
+ type = "object",
+ description = "AWS IAM credentials for SigV4 signing.",
+ properties = {
+ access_key_id = { type = "string", minLength = 1 },
+ secret_access_key = { type = "string", minLength = 1 },
+ session_token = { type = "string", minLength = 1 },
+ },
+ required = { "access_key_id", "secret_access_key" },
+ },
},
additionalProperties = false,
}
@@ -68,7 +78,10 @@ local model_options_schema = {
properties = {
model = {
type = "string",
- description = "Model to execute.",
+ description = "Model to execute. For Bedrock, this can be a model
ID "
+ .. "(e.g., anthropic.claude-3-5-sonnet-20240620-v1:0) or an
inference "
+ .. "profile ARN (e.g., arn:aws:bedrock:us-east-1:123456789012:"
+ .. "application-inference-profile/abc123).",
},
},
additionalProperties = true,
@@ -116,7 +129,17 @@ local override_schema = {
properties = {
endpoint = {
type = "string",
- description = "To be specified to override the endpoint of the AI
Instance",
+ minLength = 1,
+ description = "Override the endpoint of the AI Instance. "
+ .. "Typically used for custom hosts (e.g., AWS "
+ .. "PrivateLink, reverse proxies). You may provide "
+ .. "only the scheme + host, in which case the plugin "
+ .. "computes the provider-specific path, or provide "
+ .. "a full endpoint including path and query, in "
+ .. "which case the plugin uses the supplied path/query. "
+ .. "If your custom path or query contains reserved "
+ .. "characters (e.g., Bedrock inference profile ARNs "
+ .. "containing ':' or '/'), they must be URL-encoded.",
},
llm_options = llm_options_schema,
request_body = request_body_override_schema,
@@ -131,19 +154,16 @@ local override_schema = {
},
}
-local provider_vertex_ai_schema = {
+local provider_conf_schema = {
type = "object",
properties = {
- project_id = {
- type = "string",
- description = "Google Cloud Project ID",
- },
- region = {
- type = "string",
- description = "Google Cloud Region",
- },
+ project_id = { type = "string", description = "GCP project ID
(vertex-ai)" },
+ region = { type = "string", minLength = 1,
+ description = "Region. For vertex-ai: GCP region. "
+ .. "For bedrock: AWS region (required, used for
SigV4)." },
},
- required = { "project_id", "region" },
+ -- No 'required' here -- which fields are required depends on the provider,
+ -- enforced by validate_provider_requirements() in Lua.
}
local ai_instance_schema = {
@@ -175,6 +195,7 @@ local ai_instance_schema = {
auth = auth_schema,
options = model_options_schema,
override = override_schema,
+ provider_conf = provider_conf_schema,
checks = {
type = "object",
properties = {
@@ -184,19 +205,6 @@ local ai_instance_schema = {
}
},
required = {"name", "provider", "auth", "weight"},
- ["if"] = {
- properties = { provider = { enum = { "vertex-ai" } } },
- },
- ["then"] = {
- properties = {
- provider_conf = provider_vertex_ai_schema,
- },
- oneOf = {
- { required = { "provider_conf" } },
- { required = { "override" } },
- },
- },
- ["else"] = {},
},
}
@@ -224,6 +232,7 @@ _M.ai_proxy_schema = {
description = "Type of the AI service instance.",
enum = ai_providers_schema.providers,
},
+ provider_conf = provider_conf_schema,
logging = logging_schema,
auth = auth_schema,
options = model_options_schema,
@@ -262,7 +271,10 @@ _M.ai_proxy_schema = {
override = override_schema,
},
required = {"provider", "auth"},
- encrypt_fields = {"auth.header", "auth.query",
"auth.gcp.service_account_json"},
+ encrypt_fields = {
+ "auth.header", "auth.query", "auth.gcp.service_account_json",
+ "auth.aws.secret_access_key", "auth.aws.session_token",
+ },
}
_M.ai_proxy_multi_schema = {
@@ -348,7 +360,40 @@ _M.ai_proxy_multi_schema = {
"instances.auth.header",
"instances.auth.query",
"instances.auth.gcp.service_account_json",
+ "instances.auth.aws.secret_access_key",
+ "instances.auth.aws.session_token",
},
}
+function _M.validate_provider_requirements(conf)
+ local provider = conf.provider
+ local has_override = conf.override and conf.override.endpoint
+
+ if provider == "vertex-ai" then
+ local pc = conf.provider_conf
+ local has_provider_conf = pc and pc.project_id and pc.region
+ if not has_provider_conf and not has_override then
+ return false, "vertex-ai requires either provider_conf "
+ .. "(project_id + region) or override.endpoint"
+ end
+ end
+
+ if provider == "bedrock" then
+ if not (conf.provider_conf and conf.provider_conf.region) then
+ return false, "bedrock requires provider_conf.region"
+ end
+ if not (conf.auth and conf.auth.aws) then
+ return false, "bedrock requires auth.aws"
+ end
+ -- options.model is intentionally NOT required: clients may pass
`model`
+ -- in the request body when no route-level model is configured. If
+ -- options.model is set, it takes precedence; body.model is only used
+ -- when options.model is absent. If neither is set at request time,
+ -- build_request returns 400.
+ end
+
+ return true
+end
+
+
return _M
diff --git a/apisix/plugins/ai-transport/auth-aws.lua
b/apisix/plugins/ai-transport/auth-aws.lua
new file mode 100644
index 000000000..bd8529a25
--- /dev/null
+++ b/apisix/plugins/ai-transport/auth-aws.lua
@@ -0,0 +1,155 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--- AWS SigV4 signing helper for AI providers.
+-- Signs outgoing HTTP requests using AWS Signature Version 4.
+
+require("resty.aws.config") -- reads env vars before init
+local aws = require("resty.aws")
+local core = require("apisix.core")
+local signer = require("resty.aws.request.sign")
+local ngx = ngx
+local ngx_escape_uri = ngx.escape_uri
+local type = type
+
+local aws_instance
+
+
+-- Decode each non-slash path segment then re-encode N times. Idempotent:
+-- handles both raw paths (e.g. "/model/foo:bar/x") and once-encoded paths
+-- (e.g. "/model/foo%3Abar/x") uniformly, while preserving the exact slash
+-- structure, including empty segments and any trailing slash.
+-- N=1 → once-encoded path (for HTTP wire transport)
+-- N=2 → twice-encoded path (for AWS SigV4 canonical URI)
+local function normalize_and_encode_path(path, n)
+ return (path:gsub("[^/]+", function(segment)
+ local s = ngx.unescape_uri(segment)
+ for _ = 1, n do
+ s = ngx_escape_uri(s)
+ end
+ return s
+ end))
+end
+
+local _M = {}
+
+
+--- Sign an outgoing HTTP request with AWS SigV4.
+-- Must be called AFTER params.body is finalized (SigV4 signs body hash).
+-- After signing, params.body is a JSON string (not a table).
+--
+-- params.path may be either a raw path or once-encoded; this function
+-- normalizes both to the same encoded form for HTTP transport and computes
+-- the double-encoded canonical URI for SigV4.
+--
+-- @param params table HTTP request params {method, host, port, path,
headers, body, query}
+-- @param aws_conf table {access_key_id, secret_access_key, session_token,
+-- endpoint_prefix (optional, defaults to "bedrock")}
+-- @param region string AWS region for SigV4 credential scope
+-- @return nil on success, or error string on failure
+function _M.sign_request(params, aws_conf, region)
+ if type(aws_conf) ~= "table" then
+ return "missing or invalid aws_conf for SigV4 signing"
+ end
+
+ if type(region) ~= "string" or region == "" then
+ return "missing or invalid region for SigV4 signing"
+ end
+
+ -- Validate path: required for canonical URI construction
+ if type(params.path) ~= "string" or params.path == "" then
+ return "missing or invalid path for SigV4 signing"
+ end
+
+ -- Normalize the input path to once-encoded form for HTTP wire transport.
+ -- This is idempotent across raw and once-encoded inputs.
+ params.path = normalize_and_encode_path(params.path, 1)
+
+ -- Serialize body to JSON string (SigV4 signs the exact bytes)
+ if type(params.body) == "table" then
+ local body_str, err = core.json.encode(params.body)
+ if not body_str then
+ return "failed to encode body: " .. (err or "")
+ end
+ params.body = body_str
+ end
+
+ -- Create AWS instance (singleton)
+ if not aws_instance then
+ aws_instance = aws()
+ end
+
+ -- Create Credentials object with :get() method as required by resty.aws
signer
+ local credentials = aws_instance:Credentials({
+ accessKeyId = aws_conf.access_key_id,
+ secretAccessKey = aws_conf.secret_access_key,
+ sessionToken = aws_conf.session_token,
+ })
+
+ -- Build the config object expected by resty.aws.request.sign
+ local config = {
+ region = region,
+ signatureVersion = "v4",
+ endpointPrefix = aws_conf.endpoint_prefix or "bedrock",
+ credentials = credentials,
+ }
+
+ -- Build the request object to sign.
+ -- params.path is already once-encoded for the HTTP wire (e.g., %3A for :).
+ -- For the SigV4 canonical URI, AWS requires reserved characters to be
+ -- encoded twice (e.g., raw ":" → "%3A" on the wire → "%253A" in the
+ -- canonical URI). normalize_and_encode_path with n=2 decodes each segment
+ -- then re-encodes it twice, producing the required double-encoded form.
+ local r = {
+ headers = {},
+ method = params.method or "POST",
+ canonicalURI = normalize_and_encode_path(params.path, 2),
+ host = params.host,
+ port = params.port or 443,
+ body = params.body,
+ query = params.query,
+ }
+
+ local signed, err = signer(config, r)
+ if not signed then
+ return "SigV4 signing failed: " .. (err or "unknown")
+ end
+
+ -- Copy signed auth headers back to params using lowercase keys to match
the
+ -- convention used by construct_forward_headers() in http.lua and avoid
+ -- duplicate headers (e.g. both "Authorization" and "authorization").
+ params.headers = params.headers or {}
+ if signed.headers then
+ if signed.headers["Authorization"] then
+ params.headers["authorization"] = signed.headers["Authorization"]
+ end
+ if signed.headers["X-Amz-Date"] then
+ params.headers["x-amz-date"] = signed.headers["X-Amz-Date"]
+ end
+ if signed.headers["X-Amz-Security-Token"] then
+ params.headers["x-amz-security-token"] =
signed.headers["X-Amz-Security-Token"]
+ end
+ if signed.headers["Host"] then
+ params.headers["host"] = signed.headers["Host"]
+ end
+ end
+
+ return nil
+end
+
+
+return _M
diff --git a/apisix/plugins/ai-transport/http.lua
b/apisix/plugins/ai-transport/http.lua
index f489eab3b..c1e44561d 100644
--- a/apisix/plugins/ai-transport/http.lua
+++ b/apisix/plugins/ai-transport/http.lua
@@ -22,6 +22,7 @@ local core = require("apisix.core")
local http = require("resty.http")
local pairs = pairs
local ipairs = ipairs
+local type = type
local str_lower = string.lower
local _M = {}
@@ -84,9 +85,16 @@ function _M.request(params, timeout)
return nil, "connect: " .. (err or "unknown")
end
- local req_json, err = core.json.encode(params.body)
- if not req_json then
- return nil, "encode body: " .. (err or "unknown")
+ local req_json
+ if type(params.body) == "string" then
+ -- Body already serialized (e.g., by SigV4 signing)
+ req_json = params.body
+ else
+ local err
+ req_json, err = core.json.encode(params.body)
+ if not req_json then
+ return nil, "encode body: " .. (err or "unknown")
+ end
end
params.body = req_json
diff --git a/apisix/utils/log-sanitize.lua b/apisix/utils/log-sanitize.lua
index 5b6344051..9e7ebfc49 100644
--- a/apisix/utils/log-sanitize.lua
+++ b/apisix/utils/log-sanitize.lua
@@ -38,7 +38,8 @@ function _M.redact_params(params)
local lower_k = k:lower()
if lower_k == "authorization" or lower_k == "x-api-key"
or lower_k == "api-key" or lower_k == "cookie"
- or lower_k == "proxy-authorization" then
+ or lower_k == "proxy-authorization"
+ or lower_k == "x-amz-security-token" then
safe_headers[k] = "[REDACTED]"
else
safe_headers[k] = v
diff --git a/docs/en/latest/plugins/ai-proxy-multi.md
b/docs/en/latest/plugins/ai-proxy-multi.md
index 947c46568..ca0a9b239 100644
--- a/docs/en/latest/plugins/ai-proxy-multi.md
+++ b/docs/en/latest/plugins/ai-proxy-multi.md
@@ -7,7 +7,7 @@ keywords:
- ai-proxy-multi
- AI
- LLM
-description: The ai-proxy-multi Plugin extends the capabilities of ai-proxy
with load balancing, retries, fallbacks, and health chekcs, simplifying the
integration with OpenAI, DeepSeek, Azure, AIMLAPI, Anthropic, OpenRouter,
Gemini, Vertex AI, and other OpenAI-compatible APIs.
+description: The ai-proxy-multi Plugin extends the capabilities of ai-proxy
with load balancing, retries, fallbacks, and health checks, simplifying the
integration with OpenAI, DeepSeek, Azure, AIMLAPI, Anthropic, OpenRouter,
Gemini, Vertex AI, Amazon Bedrock, and other OpenAI-compatible APIs.
---
<!--
@@ -38,7 +38,7 @@ import TabItem from '@theme/TabItem';
## Description
-The `ai-proxy-multi` Plugin simplifies access to LLM and embedding models by
transforming Plugin configurations into the designated request format for
OpenAI, DeepSeek, Azure, AIMLAPI, Anthropic, OpenRouter, Gemini, Vertex AI, and
other OpenAI-compatible APIs. It extends the capabilities of
[`ai-proxy`](./ai-proxy.md) with load balancing, retries, fallbacks, and health
checks.
+The `ai-proxy-multi` Plugin simplifies access to LLM and embedding models by
transforming Plugin configurations into the designated request format for
OpenAI, DeepSeek, Azure, AIMLAPI, Anthropic, OpenRouter, Gemini, Vertex AI,
Amazon Bedrock, and other OpenAI-compatible APIs. It extends the capabilities
of [`ai-proxy`](./ai-proxy.md) with load balancing, retries, fallbacks, and
health checks.
In addition, the Plugin also supports logging LLM request information in the
access log, such as token usage, model, time to the first response, and more.
These log entries are also consumed by logging plugins such as `http-logger`
and `kafka-logger`, and do not affect error log.
@@ -50,6 +50,18 @@ In addition, the Plugin also supports logging LLM request
information in the acc
| `messages.role` | String | True | Role of the message (`system`,
`user`, `assistant`).|
| `messages.content` | String | True | Content of the message.
|
+### Bedrock Converse Request Format
+
+When an instance's `provider` is set to `bedrock`, the Plugin expects requests
in the [Bedrock Converse
API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
format. The request URI must end with `/converse` and the body must contain a
`messages` array.
+
+| Name | Type | Required | Description
|
+| ------------------ | ------ | -------- |
----------------------------------------------------------------------------------------------------
|
+| `messages` | Array | True | An array of message objects.
|
+| `messages.role` | String | True | Role of the message (`user`,
`assistant`). |
+| `messages.content` | Array | True | An array of content blocks. Each
block contains a `text` field (e.g., `[{"text": "What is 1+1?"}]`). |
+| `system` | Array | False | Optional system prompt blocks
(e.g., `[{"text": "You are a helpful assistant."}]`). |
+| `inferenceConfig` | Object | False | Optional inference parameters such
as `maxTokens`, `temperature`, `topP`, `stopSequences`, etc. |
+
## Attributes
| Name | Type | Required | Default
| Valid values | Description |
@@ -61,10 +73,10 @@ In addition, the Plugin also supports logging LLM request
information in the acc
| balancer.key | string | False |
| | Used when `type` is `chash`. When
`hash_on` is set to `header` or `cookie`, `key` is required. When `hash_on` is
set to `consumer`, `key` is not required as the consumer name will be used as
the key automatically. |
| instances | array[object] | True |
| | LLM instance configurations. |
| instances.name | string | True |
| | Name of the LLM service instance. |
-| instances.provider | string | True |
| [openai, deepseek, azure-openai, aimlapi, anthropic,
openrouter, gemini, vertex-ai, openai-compatible] | LLM service provider. When
set to `openai`, the Plugin will proxy the request to `api.openai.com`. When
set to `deepseek`, the Plugin will proxy the request to `api.deepseek.com`.
When set to `aimlapi`, the Plugin uses the OpenAI-compatible driver and proxies
the request to `api.aiml [...]
-| instances.provider_conf | object | False |
| | Configuration for the specific
provider. Required when `provider` is set to `vertex-ai` and `override` is not
configured. |
+| instances.provider | string | True |
| [openai, deepseek, azure-openai, aimlapi, anthropic,
openrouter, gemini, vertex-ai, bedrock, openai-compatible] | LLM service
provider. When set to `openai`, the Plugin will proxy the request to
`api.openai.com`. When set to `deepseek`, the Plugin will proxy the request to
`api.deepseek.com`. When set to `aimlapi`, the Plugin uses the
OpenAI-compatible driver and proxies the request to [...]
+| instances.provider_conf | object | False |
| | Configuration for the specific
provider. Required when `provider` is set to `vertex-ai` and `override` is not
configured. Required when `provider` is set to `bedrock`. |
| instances.provider_conf.project_id | string | True |
| | Google Cloud Project ID. |
-| instances.provider_conf.region | string | True |
| | Google Cloud Region. |
+| instances.provider_conf.region | string | True (depending on
provider) | | minLength = 1 (for Bedrock) |
When `provider` is `vertex-ai`, this is the Google Cloud Region. When
`provider` is `bedrock`, this is the AWS region used to construct the Bedrock
endpoint and to sign the request with SigV4 (required, must be non-empty). |
| instances.priority | integer | False | 0
| | Priority of the LLM instance in load
balancing. `priority` takes precedence over `weight`. |
| instances.weight | string | True | 0
| greater or equal to 0 | Weight of the LLM instance in
load balancing. |
| instances.auth | object | True |
| | Authentication configurations. |
@@ -74,13 +86,17 @@ In addition, the Plugin also supports logging LLM request
information in the acc
| instances.auth.gcp.service_account_json | string | False |
| | Content of the GCP service account JSON
file. This can also be configured by setting the `GCP_SERVICE_ACCOUNT`
environment variable. |
| instances.auth.gcp.max_ttl | integer | False |
| minimum = 1 | Maximum TTL (in seconds) for caching
the GCP access token. |
| instances.auth.gcp.expire_early_secs| integer | False | 60
| minimum = 0 | Seconds to expire the access token
before its actual expiration time to avoid edge cases. |
+| instances.auth.aws | object | False |
| | AWS IAM credentials for SigV4 signing
(Bedrock). Required when `provider` is `bedrock`. |
+| instances.auth.aws.access_key_id | string | True |
| minLength = 1 | AWS IAM access key ID. |
+| instances.auth.aws.secret_access_key | string | True |
| minLength = 1 | AWS IAM secret access key. Encrypted
at rest. |
+| instances.auth.aws.session_token | string | False |
| minLength = 1 | AWS session token for temporary
credentials (e.g., from STS assume-role). Encrypted at rest. |
| instances.options | object | False |
| | Model configurations. In addition to
`model`, you can configure additional parameters and they will be forwarded to
the upstream LLM service in the request body. For instance, if you are working
with OpenAI, DeepSeek, or AIMLAPI, you can configure additional parameters such
as `max_tokens`, `temperature`, `top_p`, and `stream`. See your LLM provider's
API documentation f [...]
-| instances.options.model | string | False |
| | Name of the LLM model, such as `gpt-4`
or `gpt-3.5`. See your LLM provider's API documentation for more available
models. |
+| instances.options.model | string | False |
| | Name of the LLM model, such as `gpt-4`
or `gpt-3.5`. See your LLM provider's API documentation for more available
models. For Bedrock, this can be a foundation model ID (e.g.,
`anthropic.claude-3-5-sonnet-20240620-v1:0`), a cross-region inference profile
ID (e.g., `us.anthropic.claude-3-5-sonnet-20240620-v1:0`), or an application
inference profile ARN (e.g., `arn:aws:bed [...]
| logging | object | False |
| | Logging configurations. |
| logging.summaries | boolean | False | false
| | If true, log request LLM model, duration,
request, and response tokens. |
| logging.payloads | boolean | False | false
| | If true, log request and response
payload. |
| instances.override | object | False |
| | Override setting. |
-| instances.override.endpoint | string | False |
| | LLM provider endpoint to replace the
default endpoint with. If not configured, the Plugin uses the default OpenAI
endpoint `https://api.openai.com/v1/chat/completions`. |
+| instances.override.endpoint | string | False |
| | LLM provider endpoint to replace the
default endpoint with. If not configured, the Plugin uses the default OpenAI
endpoint `https://api.openai.com/v1/chat/completions`. When `provider` is
`bedrock`, this can be set to a custom Bedrock endpoint. If the override URL
includes a path containing reserved characters (e.g., Bedrock inference profile
ARNs containing `:` or `/` [...]
| instances.override.llm_options | object | False |
| | Provider-aware LLM options. See
[Provider-aware `max_tokens`
mapping](./ai-proxy.md#provider-aware-max_tokens-mapping) in the `ai-proxy`
documentation. |
| instances.override.llm_options.max_tokens | integer | False |
| ≥ 1 | Maximum number of output tokens.
APISIX automatically maps this to the provider-specific field name. Always
force-overwrites the client value. |
| instances.override.request_body | object | False |
| | Per target-protocol request body
overrides. See [Per-protocol request body
override](./ai-proxy.md#per-protocol-request-body-override) in the `ai-proxy`
documentation. |
@@ -1715,6 +1731,307 @@ If the request is proxied to DeepSeek, you should see a
response similar to the
}
```
+### Load Balance between Amazon Bedrock Instances
+
+The following example demonstrates how you can configure two [Amazon
Bedrock](https://docs.aws.amazon.com/bedrock/) instances in different regions
for load balancing. Each instance authenticates with `auth.aws` and the Plugin
signs the upstream request using AWS SigV4. Requests are sent in [Bedrock
Converse
API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
format and the URI must end with `/converse`.
+
+Save your AWS credentials to environment variables:
+
+```shell
+export AWS_ACCESS_KEY_ID=<your-aws-access-key-id>
+export AWS_SECRET_ACCESS_KEY=<your-aws-secret-access-key>
+```
+
+Create a Route as such:
+
+<Tabs
+groupId="api"
+defaultValue="admin-api"
+values={[
+{label: 'Admin API', value: 'admin-api'},
+{label: 'ADC', value: 'adc'},
+{label: 'Ingress Controller', value: 'aic'}
+]}>
+
+<TabItem value="admin-api">
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-proxy-multi-route",
+ "uri": "/bedrock/converse",
+ "methods": ["POST"],
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock-us-east-1",
+ "provider": "bedrock",
+ "weight": 5,
+ "auth": {
+ "aws": {
+ "access_key_id": "'"$AWS_ACCESS_KEY_ID"'",
+ "secret_access_key": "'"$AWS_SECRET_ACCESS_KEY"'"
+ }
+ },
+ "options": {
+ "model": "anthropic.claude-3-5-sonnet-20240620-v1:0"
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ }
+ },
+ {
+ "name": "bedrock-us-west-2",
+ "provider": "bedrock",
+ "weight": 5,
+ "auth": {
+ "aws": {
+ "access_key_id": "'"$AWS_ACCESS_KEY_ID"'",
+ "secret_access_key": "'"$AWS_SECRET_ACCESS_KEY"'"
+ }
+ },
+ "options": {
+ "model": "us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+ },
+ "provider_conf": {
+ "region": "us-west-2"
+ }
+ }
+ ]
+ }
+ }
+ }'
+```
+
+</TabItem>
+
+<TabItem value="adc">
+
+```yaml title="adc.yaml"
+services:
+ - name: ai-proxy-multi-service
+ routes:
+ - name: ai-proxy-multi-route
+ uris:
+ - /bedrock/converse
+ methods:
+ - POST
+ plugins:
+ ai-proxy-multi:
+ instances:
+ - name: bedrock-us-east-1
+ provider: bedrock
+ weight: 5
+ auth:
+ aws:
+ access_key_id: "${AWS_ACCESS_KEY_ID}"
+ secret_access_key: "${AWS_SECRET_ACCESS_KEY}"
+ options:
+ model: anthropic.claude-3-5-sonnet-20240620-v1:0
+ provider_conf:
+ region: us-east-1
+ - name: bedrock-us-west-2
+ provider: bedrock
+ weight: 5
+ auth:
+ aws:
+ access_key_id: "${AWS_ACCESS_KEY_ID}"
+ secret_access_key: "${AWS_SECRET_ACCESS_KEY}"
+ options:
+ model: us.anthropic.claude-3-5-sonnet-20240620-v1:0
+ provider_conf:
+ region: us-west-2
+```
+
+Synchronize the configuration to the gateway:
+
+```shell
+adc sync -f adc.yaml
+```
+
+</TabItem>
+
+<TabItem value="aic">
+
+<Tabs
+groupId="k8s-api"
+defaultValue="gateway-api"
+values={[
+{label: 'Gateway API', value: 'gateway-api'},
+{label: 'APISIX CRD', value: 'apisix-crd'}
+]}>
+
+<TabItem value="gateway-api">
+
+```yaml title="ai-proxy-multi-ic.yaml"
+apiVersion: apisix.apache.org/v1alpha1
+kind: PluginConfig
+metadata:
+ namespace: aic
+ name: ai-proxy-multi-plugin-config
+spec:
+ plugins:
+ - name: ai-proxy-multi
+ config:
+ instances:
+ - name: bedrock-us-east-1
+ provider: bedrock
+ weight: 5
+ auth:
+ aws:
+ access_key_id: "your-aws-access-key-id"
+ secret_access_key: "your-aws-secret-access-key"
+ options:
+ model: anthropic.claude-3-5-sonnet-20240620-v1:0
+ provider_conf:
+ region: us-east-1
+ - name: bedrock-us-west-2
+ provider: bedrock
+ weight: 5
+ auth:
+ aws:
+ access_key_id: "your-aws-access-key-id"
+ secret_access_key: "your-aws-secret-access-key"
+ options:
+ model: us.anthropic.claude-3-5-sonnet-20240620-v1:0
+ provider_conf:
+ region: us-west-2
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+ namespace: aic
+ name: ai-proxy-multi-route
+spec:
+ parentRefs:
+ - name: apisix
+ rules:
+ - matches:
+ - path:
+ type: Exact
+ value: /bedrock/converse
+ method: POST
+ filters:
+ - type: ExtensionRef
+ extensionRef:
+ group: apisix.apache.org
+ kind: PluginConfig
+ name: ai-proxy-multi-plugin-config
+```
+
+</TabItem>
+
+<TabItem value="apisix-crd">
+
+```yaml title="ai-proxy-multi-ic.yaml"
+apiVersion: apisix.apache.org/v2
+kind: ApisixRoute
+metadata:
+ namespace: aic
+ name: ai-proxy-multi-route
+spec:
+ ingressClassName: apisix
+ http:
+ - name: ai-proxy-multi-route
+ match:
+ paths:
+ - /bedrock/converse
+ methods:
+ - POST
+ plugins:
+ - name: ai-proxy-multi
+ enable: true
+ config:
+ instances:
+ - name: bedrock-us-east-1
+ provider: bedrock
+ weight: 5
+ auth:
+ aws:
+ access_key_id: "your-aws-access-key-id"
+ secret_access_key: "your-aws-secret-access-key"
+ options:
+ model: anthropic.claude-3-5-sonnet-20240620-v1:0
+ provider_conf:
+ region: us-east-1
+ - name: bedrock-us-west-2
+ provider: bedrock
+ weight: 5
+ auth:
+ aws:
+ access_key_id: "your-aws-access-key-id"
+ secret_access_key: "your-aws-secret-access-key"
+ options:
+ model: us.anthropic.claude-3-5-sonnet-20240620-v1:0
+ provider_conf:
+ region: us-west-2
+```
+
+</TabItem>
+
+</Tabs>
+
+Apply the configuration to your cluster:
+
+```shell
+kubectl apply -f ai-proxy-multi-ic.yaml
+```
+
+</TabItem>
+
+</Tabs>
+
+Send a POST request to the Route in [Bedrock
Converse](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
format:
+
+```shell
+curl "http://127.0.0.1:9080/bedrock/converse"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{
+ "messages": [
+ {"role": "user", "content": [{"text": "What is 1+1?"}]}
+ ],
+ "inferenceConfig": {"maxTokens": 256}
+ }'
+```
+
+You should receive a Bedrock Converse response similar to the following:
+
+```json
+{
+ "output": {
+ "message": {
+ "role": "assistant",
+ "content": [
+ {"text": "1 + 1 = 2."}
+ ]
+ }
+ },
+ "stopReason": "end_turn",
+ "usage": {
+ "inputTokens": 14,
+ "outputTokens": 9,
+ "totalTokens": 23
+ },
+ ...
+}
+```
+
+If you need to call an [application inference
profile](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-create.html)
by ARN through `override.endpoint`, the reserved characters in the ARN (`:`
and `/`) must be URL-encoded as `%3A` and `%2F`, for example:
+
+```text
+https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A123456789012%3Aapplication-inference-profile%2Fabc123/converse
+```
+
+:::note
+
+If `auth.aws.session_token` is set, it is used for temporary credentials
(e.g., obtained from AWS STS or an assumed role) and will be added to the
SigV4-signed request automatically. Both `auth.aws.secret_access_key` and
`auth.aws.session_token` are stored encrypted.
+
+Streaming responses (Bedrock `ConverseStream`) are not yet supported by the
Plugin.
+
+:::
+
### Proxy to Embedding Models
The following example demonstrates how you can configure the `ai-proxy-multi`
Plugin to proxy requests and load balance between embedding models.
diff --git a/docs/en/latest/plugins/ai-proxy.md
b/docs/en/latest/plugins/ai-proxy.md
index d9bf7c9e4..5bf7ebdb1 100644
--- a/docs/en/latest/plugins/ai-proxy.md
+++ b/docs/en/latest/plugins/ai-proxy.md
@@ -7,7 +7,7 @@ keywords:
- ai-proxy
- AI
- LLM
-description: The ai-proxy Plugin simplifies access to LLM and embedding models
providers by converting Plugin configurations into the required request format
for OpenAI, DeepSeek, Azure, AIMLAPI, Anthropic, OpenRouter, Gemini, Vertex AI,
and other OpenAI-compatible APIs.
+description: The ai-proxy Plugin simplifies access to LLM and embedding models
providers by converting Plugin configurations into the required request format
for OpenAI, DeepSeek, Azure, AIMLAPI, Anthropic, OpenRouter, Gemini, Vertex AI,
Amazon Bedrock, and other OpenAI-compatible APIs.
---
<!--
@@ -38,7 +38,7 @@ import TabItem from '@theme/TabItem';
## Description
-The `ai-proxy` Plugin simplifies access to LLM and embedding models by
transforming Plugin configurations into the designated request format. It
supports the integration with OpenAI, DeepSeek, Azure, AIMLAPI, Anthropic,
OpenRouter, Gemini, Vertex AI, and other OpenAI-compatible APIs.
+The `ai-proxy` Plugin simplifies access to LLM and embedding models by
transforming Plugin configurations into the designated request format. It
supports the integration with OpenAI, DeepSeek, Azure, AIMLAPI, Anthropic,
OpenRouter, Gemini, Vertex AI, Amazon Bedrock, and other OpenAI-compatible APIs.
In addition, the Plugin also supports logging LLM request information in the
access log, such as token usage, model, time to the first response, and more.
These log entries are also consumed by logging plugins such as `http-logger`
and `kafka-logger`. These options do not affect `error.log`.
@@ -50,14 +50,26 @@ In addition, the Plugin also supports logging LLM request
information in the acc
| `messages.role` | String | True | Role of the message (`system`,
`user`, `assistant`).|
| `messages.content` | String | True | Content of the message.
|
+### Bedrock Converse Request Format
+
+When `provider` is set to `bedrock`, the Plugin expects requests in the
[Bedrock Converse
API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
format. The request URI must end with `/converse` and the body must contain a
`messages` array.
+
+| Name | Type | Required | Description
|
+| ------------------ | ------ | -------- |
----------------------------------------------------------------------------------------------------
|
+| `messages` | Array | True | An array of message objects.
|
+| `messages.role` | String | True | Role of the message (`user`,
`assistant`). |
+| `messages.content` | Array | True | An array of content blocks. Each
block contains a `text` field (e.g., `[{"text": "What is 1+1?"}]`). |
+| `system` | Array | False | Optional system prompt blocks
(e.g., `[{"text": "You are a helpful assistant."}]`). |
+| `inferenceConfig` | Object | False | Optional inference parameters such
as `maxTokens`, `temperature`, `topP`, etc. |
+
## Attributes
| Name | Type | Required | Default | Valid values
| Description |
|--------------------|--------|----------|---------|------------------------------------------|-------------|
-| provider | string | True | | [openai, deepseek,
azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai,
openai-compatible] | LLM service provider. When set to `openai`, the Plugin
will proxy the request to `https://api.openai.com/chat/completions`. When set
to `deepseek`, the Plugin will proxy the request to
`https://api.deepseek.com/chat/completions`. When set to `aimlapi`, the Plugin
uses the OpenAI-compatible driver and proxies the request to `https:// [...]
-| provider_conf | object | False | |
| Configuration for the specific provider. Required when
`provider` is set to `vertex-ai` and `override` is not configured. |
+| provider | string | True | | [openai, deepseek,
azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock,
openai-compatible] | LLM service provider. When set to `openai`, the Plugin
will proxy the request to `https://api.openai.com/chat/completions`. When set
to `deepseek`, the Plugin will proxy the request to
`https://api.deepseek.com/chat/completions`. When set to `aimlapi`, the Plugin
uses the OpenAI-compatible driver and proxies the request to [...]
+| provider_conf | object | False | |
| Configuration for the specific provider. Required when
`provider` is set to `vertex-ai` and `override` is not configured. Required
when `provider` is set to `bedrock`. |
| provider_conf.project_id | string | True | |
| Google Cloud Project ID. |
-| provider_conf.region | string | True | |
| Google Cloud Region. |
+| provider_conf.region | string | True (depending on provider) | |
minLength = 1 (for Bedrock) | When `provider` is `vertex-ai`, this
is the Google Cloud Region. When `provider` is `bedrock`, this is the AWS
region used to construct the Bedrock endpoint and to sign the request with
SigV4 (required, must be non-empty). |
| auth | object | True | |
| Authentication configurations. |
| auth.header | object | False | |
| Authentication headers. At least one of `header` or `query`
must be configured. |
| auth.query | object | False | |
| Authentication query parameters. At least one of `header` or
`query` must be configured. |
@@ -65,13 +77,17 @@ In addition, the Plugin also supports logging LLM request
information in the acc
| auth.gcp.service_account_json | string | False | |
| Content of the GCP service account JSON file. This can also
be configured by setting the `GCP_SERVICE_ACCOUNT` environment variable. |
| auth.gcp.max_ttl | integer | False | | minimum = 1
| Maximum TTL (in seconds) for caching the GCP access token. |
| auth.gcp.expire_early_secs | integer | False | 60 | minimum = 0
| Seconds to expire the access token before its actual
expiration time to avoid edge cases. |
+| auth.aws | object | False | |
| Configuration for AWS authentication. Required when `provider`
is `bedrock`. |
+| auth.aws.access_key_id | string | True | | minLength = 1
| AWS access key ID used for SigV4 signing. |
+| auth.aws.secret_access_key | string | True | | minLength = 1
| AWS secret access key used for SigV4 signing. Stored
encrypted. |
+| auth.aws.session_token | string | False | | minLength = 1
| Optional AWS session token for temporary credentials (e.g.,
from STS or assumed roles). Stored encrypted. |
| options | object | False | |
| Model configurations. In addition to `model`, you can configure
additional parameters and they will be forwarded to the upstream LLM service in
the request body. For instance, if you are working with OpenAI, you can
configure additional parameters such as `temperature`, `top_p`, and `stream`.
See your LLM provider's API documentation for more available options. |
-| options.model | string | False | |
| Name of the LLM model, such as `gpt-4` or `gpt-3.5`. Refer to
the LLM provider's API documentation for available models. |
+| options.model | string | False | |
| Name of the LLM model, such as `gpt-4` or `gpt-3.5`. Refer to
the LLM provider's API documentation for available models. When `provider` is
`bedrock` and `override.endpoint` is not configured, `model` is required and
may be a foundation model ID (e.g.,
`anthropic.claude-3-5-sonnet-20240620-v1:0`), a cross-region inference profile
ID (e.g., `us.anthropic.claude-3-5-sonnet-20240620-v1:0`), or an a [...]
| override | object | False | |
| Override setting. |
-| override.endpoint | string | False | |
| Custom LLM provider endpoint, required when `provider` is
`openai-compatible`. |
+| override.endpoint | string | False | |
| Custom LLM provider endpoint, required when `provider` is
`openai-compatible`. When `provider` is `bedrock`, this can be set to a custom
Bedrock endpoint. If the override URL includes a path containing reserved
characters (e.g., Bedrock inference profile ARNs containing `:` or `/`), those
characters MUST be URL-encoded (`:` → `%3A`, `/` → `%2F`) so the model ID is
preserved as a single path segment. |
| override.llm_options | object | False | |
| Provider-aware LLM options. See [Provider-aware `max_tokens`
mapping](#provider-aware-max_tokens-mapping). |
| override.llm_options.max_tokens | integer | False | | ≥ 1
| Maximum number of output tokens. APISIX automatically
maps this to the provider-specific field name (e.g. `max_completion_tokens` for
OpenAI Chat Completions, `max_output_tokens` for OpenAI Responses API,
`max_tokens` for most other providers). Always force-overwrites the client
value. |
-| override.request_body | object | False | |
| Per target-protocol request body overrides. Keys are target
protocol names (`openai-chat`, `openai-responses`, `openai-embeddings`,
`anthropic-messages`); values are partial request bodies that are deep-merged
into the outgoing body (objects merged recursively, arrays and scalars replaced
wholesale). See [Per-protocol request body
override](#per-protocol-request-body-override). |
+| override.request_body | object | False | |
| Per target-protocol request body overrides. Keys are target
protocol names (`openai-chat`, `openai-responses`, `openai-embeddings`,
`anthropic-messages`, `bedrock-converse`); values are partial request bodies
that are deep-merged into the outgoing body (objects merged recursively, arrays
and scalars replaced wholesale). See [Per-protocol request body
override](#per-protocol-request-body-override). |
| override.request_body_force_override | boolean | False | false |
| When `false` (default), client request body fields
take priority and `override.request_body` values only fill in missing fields.
When `true`, `override.request_body` values forcefully overwrite client fields.
Does not affect `override.llm_options`, which always force-overwrites. |
| logging | object | False | |
| Logging configurations. Does not affect `error.log`. |
| logging.summaries | boolean | False | false |
| If true, logs request LLM model, duration, request, and response
tokens. |
@@ -783,6 +799,95 @@ You should receive a response similar to the following:
}
```
+### Proxy to Amazon Bedrock
+
+The following example demonstrates how you can configure the `ai-proxy` Plugin
to proxy requests to [Amazon Bedrock](https://docs.aws.amazon.com/bedrock/)
using the [Converse
API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html).
The Plugin signs the upstream request using AWS SigV4 with the credentials
configured in `auth.aws`.
+
+Save your AWS credentials to environment variables:
+
+```shell
+export AWS_ACCESS_KEY_ID=<your-aws-access-key-id>
+export AWS_SECRET_ACCESS_KEY=<your-aws-secret-access-key>
+```
+
+Create a Route and configure the `ai-proxy` Plugin as such:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-proxy-route",
+ "uri": "/bedrock/converse",
+ "methods": ["POST"],
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "'"$AWS_ACCESS_KEY_ID"'",
+ "secret_access_key": "'"$AWS_SECRET_ACCESS_KEY"'"
+ }
+ },
+ "options": {
+ "model": "anthropic.claude-3-5-sonnet-20240620-v1:0"
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ }
+ }
+ }
+ }'
+```
+
+Send a POST request to the Route in [Bedrock
Converse](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
format. Note that the URI must end with `/converse`:
+
+```shell
+curl "http://127.0.0.1:9080/bedrock/converse"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{
+ "messages": [
+ {"role": "user", "content": [{"text": "What is 1+1?"}]}
+ ],
+ "inferenceConfig": {"maxTokens": 256}
+ }'
+```
+
+You should receive a Bedrock Converse response similar to the following:
+
+```json
+{
+ "output": {
+ "message": {
+ "role": "assistant",
+ "content": [
+ {"text": "1 + 1 = 2."}
+ ]
+ }
+ },
+ "stopReason": "end_turn",
+ "usage": {
+ "inputTokens": 14,
+ "outputTokens": 9,
+ "totalTokens": 23
+ },
+ ...
+}
+```
+
+If you need to call an [application inference
profile](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-create.html)
by ARN through `override.endpoint`, the reserved characters in the ARN (`:`
and `/`) must be URL-encoded as `%3A` and `%2F`, for example:
+
+```text
+https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A123456789012%3Aapplication-inference-profile%2Fabc123/converse
+```
+
+:::note
+
+If `auth.aws.session_token` is set, it is used for temporary credentials
(e.g., obtained from AWS STS or an assumed role) and will be added to the
SigV4-signed request automatically. Both `auth.aws.secret_access_key` and
`auth.aws.session_token` are stored encrypted.
+
+Streaming responses (Bedrock `ConverseStream`) are not yet supported by the
Plugin.
+
+:::
+
### Proxy to OpenAI Embedding Models
The following example demonstrates how you can configure the `ai-proxy` Plugin
to proxy requests to embedding models. This example will use the OpenAI
embedding model endpoint.
diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md
b/docs/zh/latest/plugins/ai-proxy-multi.md
index f3231e742..358b7da0c 100644
--- a/docs/zh/latest/plugins/ai-proxy-multi.md
+++ b/docs/zh/latest/plugins/ai-proxy-multi.md
@@ -7,7 +7,7 @@ keywords:
- ai-proxy-multi
- AI
- LLM
-description: ai-proxy-multi 插件通过负载均衡、重试、故障转移和健康检查扩展了 ai-proxy 的功能，简化了与
OpenAI、DeepSeek、Azure、AIMLAPI、Anthropic、OpenRouter、Gemini、Vertex AI 和其他 OpenAI
兼容 API 的集成。
+description: ai-proxy-multi 插件通过负载均衡、重试、故障转移和健康检查扩展了 ai-proxy 的功能，简化了与
OpenAI、DeepSeek、Azure、AIMLAPI、Anthropic、OpenRouter、Gemini、Vertex AI、Amazon
Bedrock 和其他 OpenAI 兼容 API 的集成。
---
<!--
@@ -38,7 +38,7 @@ import TabItem from '@theme/TabItem';
## 描述
-`ai-proxy-multi` 插件通过将插件配置转换为
OpenAI、DeepSeek、Azure、AIMLAPI、Anthropic、OpenRouter、Gemini、Vertex AI 和其他 OpenAI
兼容 API 的指定请求格式，简化了对 LLM 和嵌入模型的访问。它通过负载均衡、重试、故障转移和健康检查扩展了
[`ai-proxy`](./ai-proxy.md) 的功能。
+`ai-proxy-multi` 插件通过将插件配置转换为
OpenAI、DeepSeek、Azure、AIMLAPI、Anthropic、OpenRouter、Gemini、Vertex AI、Amazon
Bedrock 和其他 OpenAI 兼容 API 的指定请求格式，简化了对 LLM 和嵌入模型的访问。它通过负载均衡、重试、故障转移和健康检查扩展了
[`ai-proxy`](./ai-proxy.md) 的功能。
此外，该插件还支持在访问日志中记录 LLM 请求信息，如令牌使用量、模型、首次响应时间等。这些日志条目也会被
`http-logger`、`kafka-logger` 等日志插件消费，但不影响 `error.log`。
@@ -50,6 +50,18 @@ import TabItem from '@theme/TabItem';
| `messages.role` | String | 是 | 消息的角色（`system`、`user`、`assistant`）。|
| `messages.content` | String | 是 | 消息的内容。 |
+### Bedrock Converse 请求格式
+
+当某个实例的 `provider` 设置为 `bedrock` 时，插件期望请求采用 [Bedrock Converse
API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
格式。请求 URI 必须以 `/converse` 结尾，且请求体必须包含 `messages` 数组。
+
+| 名称 | 类型 | 必选项 | 描述
|
+| ------------------ | ------ | -------- |
----------------------------------------------------------------------------------------------------
|
+| `messages` | Array | 是 | 消息对象数组。
|
+| `messages.role` | String | 是 | 消息的角色（`user`、`assistant`）。
|
+| `messages.content` | Array | 是 | 内容块数组。每个块包含一个 `text` 字段（例如 `[{"text":
"What is 1+1?"}]`）。 |
+| `system` | Array | 否 | 可选的系统提示块（例如 `[{"text": "You are a
helpful assistant."}]`）。 |
+| `inferenceConfig` | Object | 否 | 可选的推理参数，如
`maxTokens`、`temperature`、`topP` 等。 |
+
## 属性
| 名称 | 类型 | 必选项 | 默认值
| 有效值 | 描述 |
@@ -61,10 +73,10 @@ import TabItem from '@theme/TabItem';
| balancer.key | string | 否 |
| | 当 `type` 为 `chash` 时使用。当 `hash_on` 设置为
`header` 或 `cookie` 时，需要 `key`。当 `hash_on` 设置为 `consumer` 时，不需要
`key`，因为消费者名称将自动用作键。 |
| instances | array[object] | 是 |
| | LLM 实例配置。 |
| instances.name | string | 是 |
| | LLM 服务实例的名称。 |
-| instances.provider | string | 是 |
| [openai, deepseek, azure-openai, aimlapi, anthropic,
openrouter, gemini, vertex-ai, openai-compatible] | LLM 服务提供商。设置为 `openai`
时，插件将代理请求到 `api.openai.com`。设置为 `deepseek` 时，插件将代理请求到 `api.deepseek.com`。设置为
`aimlapi` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `api.aimlapi.com`。设置为 `anthropic`
时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `api.anthropic.com`。设置为 `openrouter` 时，插件使用
OpenAI 兼容驱动程序，默认将请求代理到 `openrouter.ai`。 [...]
-| instances.provider_conf | object | 否 |
| | 特定提供商的配置。当 `provider` 设置为 `vertex-ai` 且未配置
`override` 时必填。 |
+| instances.provider | string | 是 |
| [openai, deepseek, azure-openai, aimlapi, anthropic,
openrouter, gemini, vertex-ai, bedrock, openai-compatible] | LLM 服务提供商。设置为
`openai` 时，插件将代理请求到 `api.openai.com`。设置为 `deepseek` 时，插件将代理请求到
`api.deepseek.com`。设置为 `aimlapi` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到
`api.aimlapi.com`。设置为 `anthropic` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到
`api.anthropic.com`。设置为 `openrouter` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `openro [...]
+| instances.provider_conf | object | 否 |
| | 特定提供商的配置。当 `provider` 设置为 `vertex-ai` 且未配置
`override` 时必填。当 `provider` 设置为 `bedrock` 时必填。 |
| instances.provider_conf.project_id | string | 是 |
| | Google Cloud 项目 ID。 |
-| instances.provider_conf.region | string | 是 |
| | Google Cloud 区域。 |
+| instances.provider_conf.region | string | 视提供商而定 |
| minLength = 1（Bedrock 时） | 当 `provider` 为 `vertex-ai`
时，此项为 Google Cloud 区域。当 `provider` 为 `bedrock` 时，此项为用于构造 Bedrock 端点并使用 SigV4
对请求进行签名的 AWS 区域（必填，不能为空）。 |
| instances.priority | integer | 否 | 0
| | LLM 实例在负载均衡中的优先级。`priority` 优先于 `weight`。 |
| instances.weight | string | 是 | 0
| 大于或等于 0 | LLM 实例在负载均衡中的权重。 |
| instances.auth | object | 是 |
| | 身份验证配置。 |
@@ -74,10 +86,14 @@ import TabItem from '@theme/TabItem';
| instances.auth.gcp.service_account_json | string | 否 |
| | GCP 服务帐户 JSON
文件的内容。也可以通过设置"GCP_SERVICE_ACCOUNT"环境变量来配置。 |
| instances.auth.gcp.max_ttl | integer | 否 |
| minimum = 1 | 用于缓存 GCP 访问令牌的最大 TTL（以秒为单位）。 |
| instances.auth.gcp.expire_early_secs| integer | 否 | 60
| minimum = 0 | 在访问令牌实际过期时间之前使其过期的秒数，以避免边缘情况。 |
+| instances.auth.aws | object | 否 |
| | AWS 身份验证配置。当 `provider` 为 `bedrock` 时必填。 |
+| instances.auth.aws.access_key_id | string | 是 |
| minLength = 1 | 用于 SigV4 签名的 AWS 访问密钥 ID。 |
+| instances.auth.aws.secret_access_key | string | 是 |
| minLength = 1 | 用于 SigV4 签名的 AWS 秘密访问密钥。以加密形式存储。 |
+| instances.auth.aws.session_token | string | 否 |
| minLength = 1 | 可选的 AWS 会话令牌，用于临时凭证（例如来自 STS
或扮演角色获取的凭证）。以加密形式存储。 |
| instances.options | object | 否 |
| | 模型配置。除了 `model` 之外，您还可以配置其他参数，它们将在请求体中转发到上游
LLM 服务。例如，如果您使用 OpenAI、DeepSeek 或 AIMLAPI，可以配置其他参数，如
`max_tokens`、`temperature`、`top_p` 和 `stream`。有关更多可用选项，请参阅您的 LLM 提供商的 API 文档。 |
-| instances.options.model | string | 否 |
| | LLM 模型的名称，如 `gpt-4` 或
`gpt-3.5`。有关更多可用模型，请参阅您的 LLM 提供商的 API 文档。 |
+| instances.options.model | string | 否 |
| | LLM 模型的名称，如 `gpt-4` 或
`gpt-3.5`。有关更多可用模型，请参阅您的 LLM 提供商的 API 文档。当 `provider` 为 `bedrock` 且未配置
`override.endpoint` 时，`model` 为必填项，可以是基础模型 ID（例如
`anthropic.claude-3-5-sonnet-20240620-v1:0`）、跨区域推理配置文件 ID（例如
`us.anthropic.claude-3-5-sonnet-20240620-v1:0`）或应用推理配置文件 ARN（例如
`arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123`）。
|
| instances.override | object | 否 |
| | 覆盖设置。 |
-| instances.override.endpoint | string | 否 |
| | 用于替换默认端点的 LLM 提供商端点。如果未配置，插件使用默认的 OpenAI 端点
`https://api.openai.com/v1/chat/completions`。 |
+| instances.override.endpoint | string | 否 |
| | 用于替换默认端点的 LLM 提供商端点。如果未配置，插件使用默认的 OpenAI 端点
`https://api.openai.com/v1/chat/completions`。当 `provider` 为 `bedrock`
时，可以设置为自定义的 Bedrock 端点。如果覆盖 URL 包含含有保留字符的路径（例如 Bedrock 推理配置文件 ARN 中的 `:` 或
`/`），这些字符必须进行 URL 编码（`:` → `%3A`，`/` → `%2F`），以确保模型 ID 被保留为单个路径段。 |
| instances.override.llm_options | object | 否 |
| | 提供商感知的 LLM 选项。请参阅 `ai-proxy` 文档中的
[`max_tokens` 字段映射](./ai-proxy.md#provider-aware-max_tokens-mapping)。 |
| instances.override.llm_options.max_tokens | integer | 否 |
| ≥ 1 | 最大输出 token 数。APISIX
会自动将该值映射为各上游服务商对应的字段名。始终强制覆盖客户端值。 |
| instances.override.request_body | object | 否 |
| | 按目标协议的请求体覆盖配置。请参阅 `ai-proxy`
文档中的[按协议的请求体覆盖](./ai-proxy.md#per-protocol-request-body-override)。 |
@@ -1717,6 +1733,118 @@ curl "http://127.0.0.1:9080/anything"; -X POST \
}
```
+### 在 Amazon Bedrock 实例之间进行负载均衡
+
+以下示例演示了如何配置位于不同区域的两个 [Amazon Bedrock](https://docs.aws.amazon.com/bedrock/)
实例进行负载均衡。每个实例使用 `auth.aws` 进行身份验证，插件将使用 AWS SigV4 对上游请求进行签名。请求采用 [Bedrock
Converse
API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
格式发送，且 URI 必须以 `/converse` 结尾。
+
+将您的 AWS 凭证保存到环境变量：
+
+```shell
+export AWS_ACCESS_KEY_ID=<your-aws-access-key-id>
+export AWS_SECRET_ACCESS_KEY=<your-aws-secret-access-key>
+```
+
+创建路由：
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-proxy-multi-route",
+ "uri": "/bedrock/converse",
+ "methods": ["POST"],
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock-us-east-1",
+ "provider": "bedrock",
+ "weight": 5,
+ "auth": {
+ "aws": {
+ "access_key_id": "'"$AWS_ACCESS_KEY_ID"'",
+ "secret_access_key": "'"$AWS_SECRET_ACCESS_KEY"'"
+ }
+ },
+ "options": {
+ "model": "anthropic.claude-3-5-sonnet-20240620-v1:0"
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ }
+ },
+ {
+ "name": "bedrock-us-west-2",
+ "provider": "bedrock",
+ "weight": 5,
+ "auth": {
+ "aws": {
+ "access_key_id": "'"$AWS_ACCESS_KEY_ID"'",
+ "secret_access_key": "'"$AWS_SECRET_ACCESS_KEY"'"
+ }
+ },
+ "options": {
+ "model": "us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+ },
+ "provider_conf": {
+ "region": "us-west-2"
+ }
+ }
+ ]
+ }
+ }
+ }'
+```
+
+以 [Bedrock
Converse](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
格式向路由发送 POST 请求：
+
+```shell
+curl "http://127.0.0.1:9080/bedrock/converse"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{
+ "messages": [
+ {"role": "user", "content": [{"text": "What is 1+1?"}]}
+ ],
+ "inferenceConfig": {"maxTokens": 256}
+ }'
+```
+
+您应该收到类似以下的 Bedrock Converse 响应：
+
+```json
+{
+ "output": {
+ "message": {
+ "role": "assistant",
+ "content": [
+ {"text": "1 + 1 = 2."}
+ ]
+ }
+ },
+ "stopReason": "end_turn",
+ "usage": {
+ "inputTokens": 14,
+ "outputTokens": 9,
+ "totalTokens": 23
+ },
+ ...
+}
+```
+
+如果您需要通过 `override.endpoint` 按 ARN
调用[应用推理配置文件](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-create.html)，则
ARN 中的保留字符（`:` 和 `/`）必须分别 URL 编码为 `%3A` 和 `%2F`，例如：
+
+```text
+https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A123456789012%3Aapplication-inference-profile%2Fabc123/converse
+```
+
+:::note
+
+如果设置了 `auth.aws.session_token`，则它将用于临时凭证（例如从 AWS STS 或扮演角色获得的凭证），并将自动添加到 SigV4
签名的请求中。`auth.aws.secret_access_key` 和 `auth.aws.session_token` 都以加密形式存储。
+
+插件目前尚不支持流式响应（Bedrock `ConverseStream`）。
+
+:::
+
### 代理到嵌入模型
以下示例演示了如何配置 `ai-proxy-multi` 插件以代理请求并在嵌入模型之间进行负载均衡。
diff --git a/docs/zh/latest/plugins/ai-proxy.md
b/docs/zh/latest/plugins/ai-proxy.md
index dab32be27..0babeb969 100644
--- a/docs/zh/latest/plugins/ai-proxy.md
+++ b/docs/zh/latest/plugins/ai-proxy.md
@@ -7,7 +7,7 @@ keywords:
- ai-proxy
- AI
- LLM
-description: ai-proxy 插件通过将插件配置转换为所需的请求格式，简化了对 LLM 和嵌入模型提供商的访问，支持
OpenAI、DeepSeek、Azure、AIMLAPI、Anthropic、OpenRouter、Gemini、Vertex AI 和其他 OpenAI
兼容的 API。
+description: ai-proxy 插件通过将插件配置转换为所需的请求格式，简化了对 LLM 和嵌入模型提供商的访问，支持
OpenAI、DeepSeek、Azure、AIMLAPI、Anthropic、OpenRouter、Gemini、Vertex AI、Amazon
Bedrock 和其他 OpenAI 兼容的 API。
---
<!--
@@ -38,7 +38,7 @@ import TabItem from '@theme/TabItem';
## 描述
-`ai-proxy` 插件通过将插件配置转换为指定的请求格式，简化了对 LLM 和嵌入模型的访问。它支持与
OpenAI、DeepSeek、Azure、AIMLAPI、Anthropic、OpenRouter、Gemini、Vertex AI 和其他 OpenAI
兼容的 API 集成。
+`ai-proxy` 插件通过将插件配置转换为指定的请求格式，简化了对 LLM 和嵌入模型的访问。它支持与
OpenAI、DeepSeek、Azure、AIMLAPI、Anthropic、OpenRouter、Gemini、Vertex AI、Amazon
Bedrock 和其他 OpenAI 兼容的 API 集成。
此外，该插件还支持在访问日志中记录 LLM 请求信息，如令牌使用量、模型、首次响应时间等。这些日志条目也会被
`http-logger`、`kafka-logger` 等日志插件消费。这些选项不影响 `error.log`。
@@ -50,14 +50,26 @@ import TabItem from '@theme/TabItem';
| `messages.role` | String | 是 | 消息的角色（`system`、`user`、`assistant`）。|
| `messages.content` | String | 是 | 消息的内容。 |
+### Bedrock Converse 请求格式
+
+当 `provider` 设置为 `bedrock` 时，插件期望请求采用 [Bedrock Converse
API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
格式。请求 URI 必须以 `/converse` 结尾，且请求体必须包含 `messages` 数组。
+
+| 名称 | 类型 | 必选项 | 描述
|
+| ------------------ | ------ | -------- |
----------------------------------------------------------------------------------------------------
|
+| `messages` | Array | 是 | 消息对象数组。
|
+| `messages.role` | String | 是 | 消息的角色（`user`、`assistant`）。
|
+| `messages.content` | Array | 是 | 内容块数组。每个块包含一个 `text` 字段（例如 `[{"text":
"What is 1+1?"}]`）。 |
+| `system` | Array | 否 | 可选的系统提示块（例如 `[{"text": "You are a
helpful assistant."}]`）。 |
+| `inferenceConfig` | Object | 否 | 可选的推理参数，如
`maxTokens`、`temperature`、`topP` 等。 |
+
## 属性
| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述
|
|--------------------|--------|----------|---------|------------------------------------------|-------------|
-| provider | string | 是 | | [openai, deepseek,
azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai,
openai-compatible] | LLM 服务提供商。当设置为 `openai` 时，插件将代理请求到
`https://api.openai.com/chat/completions`。当设置为 `deepseek` 时，插件将代理请求到
`https://api.deepseek.com/chat/completions`。当设置为 `aimlapi` 时，插件使用 OpenAI
兼容驱动程序，默认将请求代理到 `https://api.aimlapi.com/v1/chat/completions`。当设置为 `anthropic`
时，插件将代理请求到 `https://api.anthropic.com/v1/chat/completions`。当设置为 `openrouter`
时，插件 [...]
-| provider_conf | object | 否 | |
| 特定提供商的配置。当 `provider` 设置为 `vertex-ai` 且未配置 `override` 时必需。 |
-| provider_conf.project_id | string | 是 | |
| Google Cloud 项目 ID。 |
-| provider_conf.region | string | 是 | |
| Google Cloud 区域。 |
+| provider | string | 是 | | [openai, deepseek,
azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock,
openai-compatible] | LLM 服务提供商。当设置为 `openai` 时，插件将代理请求到
`https://api.openai.com/chat/completions`。当设置为 `deepseek` 时，插件将代理请求到
`https://api.deepseek.com/chat/completions`。当设置为 `aimlapi` 时，插件使用 OpenAI
兼容驱动程序，默认将请求代理到 `https://api.aimlapi.com/v1/chat/completions`。当设置为 `anthropic`
时，插件将代理请求到 `https://api.anthropic.com/v1/chat/completions`。当设置为 `openrou [...]
+| provider_conf | object | 否 | |
| 特定提供商的配置。当 `provider` 设置为 `vertex-ai` 且未配置 `override` 时必填。当
`provider` 设置为 `bedrock` 时必填。 |
+| provider_conf.project_id | string | 是 | |
| Google Cloud 项目 ID。 |
+| provider_conf.region | string | 视提供商而定 | | minLength = 1（Bedrock 时）
| 当 `provider` 为 `vertex-ai` 时，此项为 Google Cloud 区域。当 `provider` 为
`bedrock` 时，此项为用于构造 Bedrock 端点并使用 SigV4 对请求进行签名的 AWS 区域（必填，不能为空）。 |
| auth | object | 是 | |
| 身份验证配置。 |
| auth.header | object | 否 | |
| 身份验证标头。必须配置 `header` 或 `query` 中的至少一个。 |
| auth.query | object | 否 | |
| 身份验证查询参数。必须配置 `header` 或 `query` 中的至少一个。 |
@@ -65,13 +77,17 @@ import TabItem from '@theme/TabItem';
| auth.gcp.service_account_json | string | 否 | |
| GCP 服务账号 JSON 文件内容。也可以通过设置 `GCP_SERVICE_ACCOUNT` 环境变量来配置。 |
| auth.gcp.max_ttl | integer | 否 | | ≥ 1
| GCP 访问令牌缓存的最大 TTL（秒）。 |
| auth.gcp.expire_early_secs | integer | 否 | 60 | ≥ 0
| 在访问令牌实际过期之前提前过期的秒数，以避免边缘情况。 |
+| auth.aws | object | 否 | |
| AWS 身份验证配置。当 `provider` 为 `bedrock` 时必填。 |
+| auth.aws.access_key_id | string | 是 | | minLength = 1
| 用于 SigV4 签名的 AWS 访问密钥 ID。 |
+| auth.aws.secret_access_key | string | 是 | | minLength = 1
| 用于 SigV4 签名的 AWS 秘密访问密钥。以加密形式存储。 |
+| auth.aws.session_token | string | 否 | | minLength = 1
| 可选的 AWS 会话令牌，用于临时凭证（例如来自 STS 或扮演角色获取的凭证）。以加密形式存储。 |
| options | object | 否 | |
| 模型配置。除了 `model` 之外，你还可以配置其他参数，它们将在请求体中转发到上游 LLM 服务。例如，如果你使用
OpenAI，可以配置其他参数，如 `temperature`、`top_p` 和 `stream`。有关更多可用选项，请参阅你的 LLM 提供商的 API
文档。 |
-| options.model | string | 否 | |
| LLM 模型的名称，如 `gpt-4` 或 `gpt-3.5`。请参阅 LLM 提供商的 API 文档以了解可用模型。 |
+| options.model | string | 否 | |
| LLM 模型的名称，如 `gpt-4` 或 `gpt-3.5`。请参阅 LLM 提供商的 API 文档以了解可用模型。当
`provider` 为 `bedrock` 且未配置 `override.endpoint` 时，`model` 为必填项，可以是基础模型 ID（例如
`anthropic.claude-3-5-sonnet-20240620-v1:0`）、跨区域推理配置文件 ID（例如
`us.anthropic.claude-3-5-sonnet-20240620-v1:0`）或应用推理配置文件 ARN（例如
`arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123`）。
|
| override | object | 否 | |
| 覆盖设置。 |
-| override.endpoint | string | 否 | |
| 自定义 LLM 提供商端点，当 `provider` 为 `openai-compatible` 时必需。 |
+| override.endpoint | string | 否 | |
| 自定义 LLM 提供商端点，当 `provider` 为 `openai-compatible` 时必需。当 `provider` 为
`bedrock` 时，可以设置为自定义的 Bedrock 端点。如果覆盖 URL 包含含有保留字符的路径（例如 Bedrock 推理配置文件 ARN 中的
`:` 或 `/`），这些字符必须进行 URL 编码（`:` → `%3A`，`/` → `%2F`），以确保模型 ID 被保留为单个路径段。 |
| override.llm_options | object | 否 | |
| 提供商感知的 LLM 选项。请参阅 [`max_tokens`
字段映射](#provider-aware-max_tokens-mapping)。 |
| override.llm_options.max_tokens | integer | 否 | | ≥ 1
| 最大输出 token 数。APISIX 会自动将该值映射为各上游服务商对应的字段名（例如 OpenAI Chat
Completions 使用 `max_completion_tokens`、OpenAI Responses API 使用
`max_output_tokens`、其他大多数服务商使用 `max_tokens`）。始终强制覆盖客户端值。 |
-| override.request_body | object | 否 | |
|
按目标协议的请求体覆盖配置。键为目标协议名（`openai-chat`、`openai-responses`、`openai-embeddings`、`anthropic-messages`），值为部分请求体，将深度合并到出站请求体中（对象递归合并，数组和标量整体替换）。请参阅[按协议的请求体覆盖](#per-protocol-request-body-override)。
|
+| override.request_body | object | 否 | |
|
按目标协议的请求体覆盖配置。键为目标协议名（`openai-chat`、`openai-responses`、`openai-embeddings`、`anthropic-messages`、`bedrock-converse`），值为部分请求体，将深度合并到出站请求体中（对象递归合并，数组和标量整体替换）。请参阅[按协议的请求体覆盖](#per-protocol-request-body-override)。
|
| override.request_body_force_override | boolean | 否 | false |
| 为 `false`（默认）时，客户端请求体中的字段优先，`override.request_body`
仅补充缺失字段。为 `true` 时，`override.request_body` 的值强制覆盖客户端请求体中的同名字段。不影响
`override.llm_options`，后者始终强制覆盖。 |
| logging | object | 否 | |
| 日志配置。不影响 `error.log`。 |
| logging.summaries | boolean | 否 | false |
| 如果为 true，记录请求 LLM 模型、持续时间、请求和响应令牌。 |
@@ -783,6 +799,95 @@ curl "http://127.0.0.1:9080/anything"; -X POST \
}
```
+### 代理到 Amazon Bedrock
+
+以下示例演示了如何配置 `ai-proxy` 插件，使用 [Converse
API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
将请求代理到 [Amazon Bedrock](https://docs.aws.amazon.com/bedrock/)。插件将使用在
`auth.aws` 中配置的凭证，通过 AWS SigV4 对上游请求进行签名。
+
+将您的 AWS 凭证保存到环境变量：
+
+```shell
+export AWS_ACCESS_KEY_ID=<your-aws-access-key-id>
+export AWS_SECRET_ACCESS_KEY=<your-aws-secret-access-key>
+```
+
+创建路由并配置 `ai-proxy` 插件：
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes"; -X PUT \
+ -H "X-API-KEY: ${admin_key}" \
+ -d '{
+ "id": "ai-proxy-route",
+ "uri": "/bedrock/converse",
+ "methods": ["POST"],
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "'"$AWS_ACCESS_KEY_ID"'",
+ "secret_access_key": "'"$AWS_SECRET_ACCESS_KEY"'"
+ }
+ },
+ "options": {
+ "model": "anthropic.claude-3-5-sonnet-20240620-v1:0"
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ }
+ }
+ }
+ }'
+```
+
+以 [Bedrock
Converse](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
格式向路由发送 POST 请求。请注意，URI 必须以 `/converse` 结尾：
+
+```shell
+curl "http://127.0.0.1:9080/bedrock/converse"; -X POST \
+ -H "Content-Type: application/json" \
+ -d '{
+ "messages": [
+ {"role": "user", "content": [{"text": "What is 1+1?"}]}
+ ],
+ "inferenceConfig": {"maxTokens": 256}
+ }'
+```
+
+您应该收到类似以下的 Bedrock Converse 响应：
+
+```json
+{
+ "output": {
+ "message": {
+ "role": "assistant",
+ "content": [
+ {"text": "1 + 1 = 2."}
+ ]
+ }
+ },
+ "stopReason": "end_turn",
+ "usage": {
+ "inputTokens": 14,
+ "outputTokens": 9,
+ "totalTokens": 23
+ },
+ ...
+}
+```
+
+如果您需要通过 `override.endpoint` 按 ARN
调用[应用推理配置文件](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-create.html)，则
ARN 中的保留字符（`:` 和 `/`）必须分别 URL 编码为 `%3A` 和 `%2F`，例如：
+
+```text
+https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A123456789012%3Aapplication-inference-profile%2Fabc123/converse
+```
+
+:::note
+
+如果设置了 `auth.aws.session_token`，则它将用于临时凭证（例如从 AWS STS 或扮演角色获得的凭证），并将自动添加到 SigV4
签名的请求中。`auth.aws.secret_access_key` 和 `auth.aws.session_token` 都以加密形式存储。
+
+插件目前尚不支持流式响应（Bedrock `ConverseStream`）。
+
+:::
+
### 代理到 OpenAI 嵌入模型
以下示例演示了如何配置 `ai-proxy` 插件以将请求代理到嵌入模型。此示例将使用 OpenAI 嵌入模型端点。
diff --git a/t/fixtures/bedrock/converse-basic.json
b/t/fixtures/bedrock/converse-basic.json
new file mode 100644
index 000000000..d2055a701
--- /dev/null
+++ b/t/fixtures/bedrock/converse-basic.json
@@ -0,0 +1,22 @@
+{
+ "metrics": {
+ "latencyMs": 252
+ },
+ "output": {
+ "message": {
+ "content": [
+ {
+ "text": "Hello!"
+ }
+ ],
+ "role": "assistant"
+ }
+ },
+ "stopReason": "end_turn",
+ "usage": {
+ "inputTokens": 13,
+ "outputTokens": 5,
+ "serverToolUsage": {},
+ "totalTokens": 18
+ }
+}
diff --git a/t/lib/server.lua b/t/lib/server.lua
index 9a3123d74..97908e214 100644
--- a/t/lib/server.lua
+++ b/t/lib/server.lua
@@ -916,6 +916,126 @@ function _M.aliyun_moderation()
ngx.print(content)
end
+-- Bedrock Converse mock: validates SigV4 headers (algorithm, credential
+-- scope, signed headers, signature length, X-Amz-Date format) and the
+-- request body shape (no `model` field, has `messages`), then serves a
+-- canned response. If the SigV4 signer attached x-amz-security-token, the
+-- token is echoed back in the assistant text so tests can assert
+-- auth.aws.session_token end-to-end.
+function _M.bedrock_converse()
+ local json = require("cjson.safe")
+
+ -- Log raw request URI so tests can assert path encoding (e.g., that ARN
+ -- model IDs are URL-encoded as a single path segment).
+ ngx.log(ngx.WARN, "[test] received uri: ", ngx.var.request_uri)
+
+ if ngx.req.get_method() ~= "POST" then
+ ngx.status = 400
+ ngx.say("Unsupported request method: ", ngx.req.get_method())
+ return
+ end
+
+ local headers = ngx.req.get_headers()
+ local auth_header = headers["authorization"]
+ local amz_date = headers["x-amz-date"]
+ if not auth_header or not amz_date then
+ ngx.status = 403
+ ngx.say(json.encode({message = "Missing Authentication Token"}))
+ return
+ end
+
+ if not auth_header:match("^AWS4%-HMAC%-SHA256 ") then
+ ngx.status = 403
+ ngx.say(json.encode({
+ message = "Authorization header missing AWS4-HMAC-SHA256 algorithm
prefix"
+ }))
+ return
+ end
+
+ -- Strict credential scope:
access_key/<date>/us-east-1/bedrock/aws4_request
+ if not auth_header:match(
+
"Credential=AKIAIOSFODNN7EXAMPLE/%d%d%d%d%d%d%d%d/us%-east%-1/bedrock/aws4_request"
+ ) then
+ ngx.status = 403
+ ngx.say(json.encode({
+ message = "Authorization Credential scope does not match expected "
+ .. "AKIAIOSFODNN7EXAMPLE/<DATE>/us-east-1/bedrock/aws4_request"
+ }))
+ return
+ end
+
+ if not auth_header:match("SignedHeaders=[^,]+") then
+ ngx.status = 403
+ ngx.say(json.encode({
+ message = "Authorization header missing SignedHeaders component"
+ }))
+ return
+ end
+
+ -- Lua patterns don't support {n} quantifiers, so match exactly 64 hex
+ -- chars by repeating %x sixty-four times.
+ local hex64 = string.rep("%x", 64)
+ if not auth_header:match("Signature=" .. hex64) then
+ ngx.status = 403
+ ngx.say(json.encode({
+ message = "Authorization Signature is missing or not 64 hex chars"
+ }))
+ return
+ end
+
+ if not amz_date:match("^%d%d%d%d%d%d%d%dT%d%d%d%d%d%dZ$") then
+ ngx.status = 403
+ ngx.say(json.encode({
+ message = "X-Amz-Date header does not match YYYYMMDDTHHMMSSZ
format"
+ }))
+ return
+ end
+
+ ngx.req.read_body()
+ local body, err = json.decode(ngx.req.get_body_data() or "")
+ if not body then
+ ngx.status = 400
+ ngx.say(json.encode({message = "Invalid JSON: " .. (err or "")}))
+ return
+ end
+
+ -- remove_model = true: bedrock provider must strip `model` from body.
+ if body.model then
+ ngx.status = 400
+ ngx.say(json.encode({
+ message = "model field should not be in request body"
+ }))
+ return
+ end
+
+ if not body.messages or #body.messages < 1 then
+ ngx.status = 400
+ ngx.say(json.encode({message = "messages is required"}))
+ return
+ end
+
+ local fixture_loader = require("lib.fixture_loader")
+ local content, ferr = fixture_loader.load("bedrock/converse-basic.json")
+ if not content then
+ ngx.status = 500
+ ngx.say(ferr)
+ return
+ end
+
+ local session_token = headers["x-amz-security-token"]
+ if session_token then
+ local response = json.decode(content)
+ local text = response.output.message.content[1].text
+ response.output.message.content[1].text = text
+ .. " session_token_seen=" .. session_token
+ content = json.encode(response)
+ end
+
+ ngx.header["Content-Type"] = "application/json"
+ ngx.print(content)
+end
+
+
-- Error endpoints for ai-request-rewrite tests.
function _M.bad_request()
ngx.status = 400
@@ -956,7 +1076,17 @@ end
-- Please add your fake upstream above
function _M.go()
- local action = string.sub(ngx.var.uri, 2)
+ local uri = ngx.var.uri
+
+ -- Bedrock Converse API: /model/<model>/converse where <model> can contain
+ -- ':' and percent-encoded sequences (URL-encoded ARNs), which the path-to-
+ -- function-name conversion below can't represent. Dispatch directly.
+ if uri:match("^/model/.+/converse$") then
+ inject_headers()
+ return _M.bedrock_converse()
+ end
+
+ local action = string.sub(uri, 2)
action = string.gsub(action, "[/\\.-]", "_")
if not action or not _M[action] then
ngx.log(ngx.WARN, "undefined path in test server, uri: ",
ngx.var.request_uri)
diff --git a/t/plugin/ai-proxy-bedrock-single.t
b/t/plugin/ai-proxy-bedrock-single.t
new file mode 100644
index 000000000..349036908
--- /dev/null
+++ b/t/plugin/ai-proxy-bedrock-single.t
@@ -0,0 +1,429 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $main_config = $block->main_config // <<_EOC_;
+ env AWS_EC2_METADATA_DISABLED=true;
+_EOC_
+ $block->set_value("main_config", $main_config);
+
+ my $user_yaml_config = <<_EOC_;
+plugins:
+ - ai-proxy
+ - prometheus
+_EOC_
+ $block->set_value("extra_yaml_config", $user_yaml_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set route with bedrock provider (single ai-proxy)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/single-ai/converse",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse";
+ },
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: send bedrock converse request
+--- request
+POST /single-ai/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/"text"\s*:\s*"Hello!"/
+
+
+
+=== TEST 3: send request with system prompt
+--- request
+POST /single-ai/converse
+{"system":[{"text":"You are a
mathematician"}],"messages":[{"role":"user","content":[{"text":"What is
1+1?"}]}],"inferenceConfig":{"maxTokens":1024}}
+--- error_code: 200
+--- response_body eval
+qr/"text"\s*:\s*"Hello!"/
+
+
+
+=== TEST 4: verify token usage in response
+--- request
+POST /single-ai/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/"inputTokens"\s*:\s*13/
+
+
+
+=== TEST 5: schema validation - missing required auth.aws fields
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/single-ai/converse2",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "AKIAIOSFODNN7EXAMPLE"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ },
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+
+
+
+=== TEST 6: unsupported protocol error - request to non-converse URI
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/3',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/single-ai/bedrock-chat",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse";
+ },
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 7: send request to non-converse URI - should fail with unsupported
protocol
+--- request
+POST /single-ai/bedrock-chat
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 400
+--- response_body eval
+qr/does not support openai-chat protocol/
+
+
+
+=== TEST 8: set route with inference profile ARN as model
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/4',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/single-ai/arn/converse",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/test123"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A123456789012%3Aapplication-inference-profile%2Ftest123/converse";
+ },
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 9: send request with ARN model (passes through SigV4 + URL encoding)
+--- request
+POST /single-ai/arn/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/"text"\s*:\s*"Hello!"/
+--- error_log eval
+qr{\[test\] received uri:
/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A123456789012%3Aapplication-inference-profile%2Ftest123/converse}
+
+
+
+=== TEST 10: set route with session_token in auth.aws
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/5',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/single-ai/session/converse",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ "session_token":
"FwoGZXIvYXdzEXAMPLESESSIONTOKEN"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse";
+ },
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 11: send request with session_token (verify x-amz-security-token
propagation)
+--- request
+POST /single-ai/session/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/session_token_seen=FwoGZXIvYXdzEXAMPLESESSIONTOKEN/
+
+
+
+=== TEST 12: route with default endpoint (no override) passes schema validation
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/6',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/single-ai/default/converse",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ },
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 13: model from request body — no options.model on route
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/8',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/single-ai/body-model-only/converse",
+ "plugins": {
+ "ai-proxy": {
+ "provider": "bedrock",
+ "auth": {
+ "aws": {
+ "access_key_id": "AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "override": {
+ "endpoint": "http://127.0.0.1:1980";
+ },
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 14: send request with body-supplied model (path is built from
body.model)
+--- request
+POST /single-ai/body-model-only/converse
+{"model":"anthropic.claude-3-5-sonnet-20241022-v2:0","messages":[{"role":"user","content":[{"text":"What
is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/"text"\s*:\s*"Hello!"/
+--- error_log eval
+qr{\[test\] received uri:
/model/anthropic\.claude-3-5-sonnet-20241022-v2%3A0/converse}
+
+
+
+=== TEST 15: missing model both on route and in body — clear 400 error
+--- request
+POST /single-ai/body-model-only/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 400
+--- response_body eval
+qr/could not resolve upstream path/
diff --git a/t/plugin/ai-proxy-bedrock.t b/t/plugin/ai-proxy-bedrock.t
new file mode 100644
index 000000000..06be07905
--- /dev/null
+++ b/t/plugin/ai-proxy-bedrock.t
@@ -0,0 +1,518 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $main_config = $block->main_config // <<_EOC_;
+ env AWS_EC2_METADATA_DISABLED=true;
+_EOC_
+ $block->set_value("main_config", $main_config);
+
+ my $user_yaml_config = <<_EOC_;
+plugins:
+ - ai-proxy-multi
+ - prometheus
+_EOC_
+ $block->set_value("extra_yaml_config", $user_yaml_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set route with bedrock provider
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/ai/converse",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock",
+ "provider": "bedrock",
+ "weight": 1,
+ "auth": {
+ "aws": {
+ "access_key_id":
"AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: send bedrock converse request
+--- request
+POST /ai/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/"text"\s*:\s*"Hello!"/
+
+
+
+=== TEST 3: send request with system prompt
+--- request
+POST /ai/converse
+{"system":[{"text":"You are a
mathematician"}],"messages":[{"role":"user","content":[{"text":"What is
1+1?"}]}],"inferenceConfig":{"maxTokens":1024}}
+--- error_code: 200
+--- response_body eval
+qr/"text"\s*:\s*"Hello!"/
+
+
+
+=== TEST 4: verify token usage in response
+--- request
+POST /ai/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/"inputTokens"\s*:\s*13/
+
+
+
+=== TEST 5: schema validation - missing required auth.aws fields
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/2',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/ai/converse2",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock-bad",
+ "provider": "bedrock",
+ "weight": 1,
+ "auth": {
+ "aws": {
+ "access_key_id":
"AKIAIOSFODNN7EXAMPLE"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+
+
+
+=== TEST 6: unsupported protocol error - request to non-converse URI
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/3',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/ai/bedrock-chat",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock",
+ "provider": "bedrock",
+ "weight": 1,
+ "auth": {
+ "aws": {
+ "access_key_id":
"AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 7: send request to non-converse URI - should fail with unsupported
protocol
+--- request
+POST /ai/bedrock-chat
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 400
+--- response_body eval
+qr/does not support openai-chat protocol/
+
+
+
+=== TEST 8: set route with inference profile ARN as model
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/4',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/ai/arn/converse",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock-arn",
+ "provider": "bedrock",
+ "weight": 1,
+ "auth": {
+ "aws": {
+ "access_key_id":
"AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/test123"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A123456789012%3Aapplication-inference-profile%2Ftest123/converse";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 9: send request with ARN model (passes through SigV4 + URL encoding)
+--- request
+POST /ai/arn/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/"text"\s*:\s*"Hello!"/
+--- error_log eval
+qr{\[test\] received uri:
/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A123456789012%3Aapplication-inference-profile%2Ftest123/converse}
+
+
+
+=== TEST 10: set route with session_token in auth.aws
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/5',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/ai/session/converse",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock-session",
+ "provider": "bedrock",
+ "weight": 1,
+ "auth": {
+ "aws": {
+ "access_key_id":
"AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ "session_token":
"FwoGZXIvYXdzEXAMPLESESSIONTOKEN"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 11: send request with session_token (verify x-amz-security-token
propagation)
+--- request
+POST /ai/session/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/session_token_seen=FwoGZXIvYXdzEXAMPLESESSIONTOKEN/
+
+
+
+=== TEST 12: route with default endpoint (no override) passes schema validation
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/6',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/ai/default/converse",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock-default-endpoint",
+ "provider": "bedrock",
+ "weight": 1,
+ "auth": {
+ "aws": {
+ "access_key_id":
"AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "options": {
+ "model":
"anthropic.claude-3-5-sonnet-20241022-v2:0"
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 13: route without options.model passes schema validation
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/7',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/ai/body-model/converse",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock-body-model",
+ "provider": "bedrock",
+ "weight": 1,
+ "auth": {
+ "aws": {
+ "access_key_id":
"AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "override": {
+ "endpoint":
"http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 14: model from request body — no options.model on route
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/8',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/ai/body-model-only/converse",
+ "plugins": {
+ "ai-proxy-multi": {
+ "instances": [
+ {
+ "name": "bedrock-body-only",
+ "provider": "bedrock",
+ "weight": 1,
+ "auth": {
+ "aws": {
+ "access_key_id":
"AKIAIOSFODNN7EXAMPLE",
+ "secret_access_key":
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ },
+ "provider_conf": {
+ "region": "us-east-1"
+ },
+ "override": {
+ "endpoint": "http://127.0.0.1:1980";
+ }
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 15: send request with body-supplied model (path is built from
body.model)
+--- request
+POST /ai/body-model-only/converse
+{"model":"anthropic.claude-3-5-sonnet-20241022-v2:0","messages":[{"role":"user","content":[{"text":"What
is 1+1?"}]}]}
+--- error_code: 200
+--- response_body eval
+qr/"text"\s*:\s*"Hello!"/
+--- error_log eval
+qr{\[test\] received uri:
/model/anthropic\.claude-3-5-sonnet-20241022-v2%3A0/converse}
+
+
+
+=== TEST 16: missing model both on route and in body — clear 400 error
+--- request
+POST /ai/body-model-only/converse
+{"messages":[{"role":"user","content":[{"text":"What is 1+1?"}]}]}
+--- error_code: 400
+--- response_body eval
+qr/could not resolve upstream path/
