moonming commented on code in PR #12036: URL: https://github.com/apache/apisix/pull/12036#discussion_r1996534643
########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | Review Comment: ```suggestion | prompt | Yes | String | The prompt send to LLM service. | ``` ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway Review Comment: ```suggestion - AI Gateway ``` ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. Review Comment: ```suggestion The `ai-request-rewrite` plugin intercepts client requests before they are forwarded to the upstream service. It sends a predefined prompt, along with the original request body, to a specified LLM service. The LLM processes the input and returns a modified request body, which is then used for the upstream request. This allows dynamic transformation of API requests based on AI-generated content. ``` ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | +| provider | Yes | String | Type of the AI service. | +| auth | Yes | Object | Authentication configuration | +| auth.header | No | Object | Authentication headers. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| auth.query | No | Object | Authentication query parameters. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| options | No | Object | Key/value settings for the model | +| options.model | No | String | Model to execute. | Review Comment: need examples of `model` ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | +| provider | Yes | String | Type of the AI service. | +| auth | Yes | Object | Authentication configuration | +| auth.header | No | Object | Authentication headers. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| auth.query | No | Object | Authentication query parameters. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| options | No | Object | Key/value settings for the model | +| options.model | No | String | Model to execute. | +| override.endpoint | No | String | To be specified to override the endpoint of the AI service | +| timeout | No | Integer | Timeout in milliseconds for requests to AI service. Range: 1 - 60000. Default: 3000 | +| keepalive | No | Boolean | Enable keepalive for requests to AI service. Default: true | +| keepalive_timeout | No | Integer | Keepalive timeout in milliseconds for requests to AI service. Minimum: 1000. Default: 60000 | +| keepalive_pool | No | Integer | Keepalive pool size for requests to AI service. Minimum: 1. Default: 30 | +| ssl_verify | No | Boolean | SSL verification for requests to AI service. Default: true | + +## How it works + + + +## Example usage + +Create a route with the `ai-request-rewrite` plugin like: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes/1"; -X PUT \ + -H "X-API-KEY: ${ADMIN_API_KEY}" \ + -d '{ + "uri": "/anything", + "plugins": { + "ai-request-rewrite": { + "prompt": "Given a JSON request body, identify and mask any sensitive information such as credit card numbers, social security numbers, and personal identification numbers (e.g., passport or driver'\''s license numbers). Replace detected sensitive values with a masked format (e.g., \"*** **** **** 1234\") for credit card numbers. Ensure the JSON structure remains unchanged.", + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer <some-token>" + } + }, + "options": { + "model": "gpt-4" + } + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org:80": 1 + } + } + }' +``` + +Now send a request: + +```shell +curl "http://127.0.0.1:9080/anything"; \ + -H "Content-Type: application/json" \ + -d '{ + "name": "John Doe", + "email": "[email protected]", + "credit_card": "4111 1111 1111 1111", + "ssn": "123-45-6789", + "address": "123 Main St" + }' +``` + +The request body for AI Service is as follows: Review Comment: ```suggestion The request body send to the LLM Service is as follows: ``` ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" + }, + keepalive = { + type = "boolean", + default = true + }, + keepalive_pool = { + type = "integer", + minimum = 1, + default = 30 + }, + ssl_verify = { + type = "boolean", + default = true + }, + override = { + type = "object", + properties = { + endpoint = { + type = "string", + description = "To be specified to override " .. + "the endpoint of the AI service provider." + } + } + } + }, + required = {"prompt", "provider", "auth"} +} + +local _M = { + version = 0.1, + priority = 1073, + name = plugin_name, + schema = schema +} + +local function proxy_request_to_llm(conf, request_table, ctx) + local ai_driver = require("apisix.plugins.ai-drivers." .. conf.provider) + + local extra_opts = { + endpoint = core.table.try_read_attr(conf, "override", "endpoint"), + query_params = conf.auth.query or {}, + headers = (conf.auth.header or {}), + model_options = conf.options + } + + local res, err, httpc = ai_driver:request(conf, request_table, extra_opts) + + if not res then return nil, err, nil end + return res, nil, httpc +end + + +local function parse_llm_response(res_body) + local response_table = core.json.decode(res_body) Review Comment: ```suggestion local response_table, err = core.json.decode(res_body) ``` ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" + }, + keepalive = { + type = "boolean", + default = true + }, + keepalive_pool = { + type = "integer", + minimum = 1, + default = 30 + }, + ssl_verify = { + type = "boolean", + default = true + }, + override = { + type = "object", + properties = { + endpoint = { + type = "string", + description = "To be specified to override " .. + "the endpoint of the AI service provider." + } + } + } + }, + required = {"prompt", "provider", "auth"} +} + +local _M = { + version = 0.1, + priority = 1073, + name = plugin_name, + schema = schema +} + +local function proxy_request_to_llm(conf, request_table, ctx) + local ai_driver = require("apisix.plugins.ai-drivers." .. conf.provider) + + local extra_opts = { + endpoint = core.table.try_read_attr(conf, "override", "endpoint"), + query_params = conf.auth.query or {}, + headers = (conf.auth.header or {}), + model_options = conf.options + } + + local res, err, httpc = ai_driver:request(conf, request_table, extra_opts) + + if not res then return nil, err, nil end + return res, nil, httpc +end + + +local function parse_llm_response(res_body) + local response_table = core.json.decode(res_body) + + if not response_table then return nil, "failed to decode llm response" end Review Comment: do we need to log the llm response here? ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" + }, + keepalive = { + type = "boolean", + default = true + }, + keepalive_pool = { + type = "integer", + minimum = 1, + default = 30 + }, + ssl_verify = { + type = "boolean", + default = true + }, + override = { + type = "object", + properties = { + endpoint = { + type = "string", + description = "To be specified to override " .. + "the endpoint of the AI service provider." + } + } + } + }, + required = {"prompt", "provider", "auth"} +} + +local _M = { + version = 0.1, + priority = 1073, + name = plugin_name, + schema = schema +} + +local function proxy_request_to_llm(conf, request_table, ctx) + local ai_driver = require("apisix.plugins.ai-drivers." .. conf.provider) + + local extra_opts = { + endpoint = core.table.try_read_attr(conf, "override", "endpoint"), + query_params = conf.auth.query or {}, + headers = (conf.auth.header or {}), + model_options = conf.options + } + + local res, err, httpc = ai_driver:request(conf, request_table, extra_opts) + + if not res then return nil, err, nil end + return res, nil, httpc +end + + +local function parse_llm_response(res_body) + local response_table = core.json.decode(res_body) + + if not response_table then return nil, "failed to decode llm response" end + + if not response_table.choices then + return nil, "'choices' not in llm response" + end + + return response_table.choices[1].message.content, nil Review Comment: what will happen if response_table.choices[1].message is nil? ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | +| provider | Yes | String | Type of the AI service. | +| auth | Yes | Object | Authentication configuration | +| auth.header | No | Object | Authentication headers. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| auth.query | No | Object | Authentication query parameters. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| options | No | Object | Key/value settings for the model | +| options.model | No | String | Model to execute. | +| override.endpoint | No | String | To be specified to override the endpoint of the AI service | +| timeout | No | Integer | Timeout in milliseconds for requests to AI service. Range: 1 - 60000. Default: 3000 | +| keepalive | No | Boolean | Enable keepalive for requests to AI service. Default: true | +| keepalive_timeout | No | Integer | Keepalive timeout in milliseconds for requests to AI service. Minimum: 1000. Default: 60000 | +| keepalive_pool | No | Integer | Keepalive pool size for requests to AI service. Minimum: 1. Default: 30 | +| ssl_verify | No | Boolean | SSL verification for requests to AI service. Default: true | + +## How it works + + + +## Example usage + +Create a route with the `ai-request-rewrite` plugin like: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes/1"; -X PUT \ + -H "X-API-KEY: ${ADMIN_API_KEY}" \ + -d '{ + "uri": "/anything", + "plugins": { + "ai-request-rewrite": { + "prompt": "Given a JSON request body, identify and mask any sensitive information such as credit card numbers, social security numbers, and personal identification numbers (e.g., passport or driver'\''s license numbers). Replace detected sensitive values with a masked format (e.g., \"*** **** **** 1234\") for credit card numbers. Ensure the JSON structure remains unchanged.", + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer <some-token>" + } + }, + "options": { + "model": "gpt-4" + } + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org:80": 1 + } + } + }' +``` + +Now send a request: + +```shell +curl "http://127.0.0.1:9080/anything"; \ + -H "Content-Type: application/json" \ + -d '{ + "name": "John Doe", + "email": "[email protected]", + "credit_card": "4111 1111 1111 1111", + "ssn": "123-45-6789", + "address": "123 Main St" + }' +``` + +The request body for AI Service is as follows: + +```json +{ + "messages": [ + { + "role": "system", + "content": "Given a JSON request body, identify and mask any sensitive information such as credit card numbers, social security numbers, and personal identification numbers (e.g., passport or driver's license numbers). Replace detected sensitive values with a masked format (e.g., '*** **** **** 1234') for credit card numbers). Ensure the JSON structure remains unchanged." + }, + { + "role": "user", + "content": "{\n\"name\":\"John Doe\",\n\"email\":\"[email protected]\",\n\"credit_card\":\"4111 1111 1111 1111\",\n\"ssn\":\"123-45-6789\",\n\"address\":\"123 Main St\"\n}" + } + ] +} + +``` + +The upstream service will receive a request like this: Review Comment: ```suggestion The LLM processes the input and returns a modified request body, which replace detected sensitive values with a masked format then used for the upstream request: ``` ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" Review Comment: read or send timeout? ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | +| provider | Yes | String | Type of the AI service. | Review Comment: ```suggestion | provider | Yes | String | Name of the LLM service. | ``` ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. Review Comment: need to desc the feature of this plugin. ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | +| provider | Yes | String | Type of the AI service. | +| auth | Yes | Object | Authentication configuration | +| auth.header | No | Object | Authentication headers. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| auth.query | No | Object | Authentication query parameters. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| options | No | Object | Key/value settings for the model | +| options.model | No | String | Model to execute. | +| override.endpoint | No | String | To be specified to override the endpoint of the AI service | +| timeout | No | Integer | Timeout in milliseconds for requests to AI service. Range: 1 - 60000. Default: 3000 | Review Comment: timeout only 3 seconds? the read timeout or connect timeout? ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | +| provider | Yes | String | Type of the AI service. | +| auth | Yes | Object | Authentication configuration | +| auth.header | No | Object | Authentication headers. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| auth.query | No | Object | Authentication query parameters. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| options | No | Object | Key/value settings for the model | +| options.model | No | String | Model to execute. | +| override.endpoint | No | String | To be specified to override the endpoint of the AI service | Review Comment: ditto ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." Review Comment: need a more detail desc and examples ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" + }, + keepalive = { + type = "boolean", + default = true + }, + keepalive_pool = { + type = "integer", + minimum = 1, + default = 30 + }, + ssl_verify = { + type = "boolean", + default = true + }, + override = { + type = "object", + properties = { + endpoint = { + type = "string", + description = "To be specified to override " .. + "the endpoint of the AI service provider." + } + } + } + }, + required = {"prompt", "provider", "auth"} +} + +local _M = { + version = 0.1, + priority = 1073, + name = plugin_name, + schema = schema +} + +local function proxy_request_to_llm(conf, request_table, ctx) + local ai_driver = require("apisix.plugins.ai-drivers." .. conf.provider) + + local extra_opts = { + endpoint = core.table.try_read_attr(conf, "override", "endpoint"), + query_params = conf.auth.query or {}, + headers = (conf.auth.header or {}), + model_options = conf.options + } + + local res, err, httpc = ai_driver:request(conf, request_table, extra_opts) + + if not res then return nil, err, nil end + return res, nil, httpc +end + + +local function parse_llm_response(res_body) + local response_table = core.json.decode(res_body) + + if not response_table then return nil, "failed to decode llm response" end + + if not response_table.choices then + return nil, "'choices' not in llm response" + end + + return response_table.choices[1].message.content, nil Review Comment: ```suggestion return response_table.choices[1].message.content ``` ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | +| provider | Yes | String | Type of the AI service. | Review Comment: And need to give the scope of `provider`: openai, deekseek or others. ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" + }, + keepalive = { + type = "boolean", + default = true + }, + keepalive_pool = { + type = "integer", + minimum = 1, + default = 30 + }, + ssl_verify = { + type = "boolean", + default = true + }, + override = { + type = "object", + properties = { + endpoint = { + type = "string", + description = "To be specified to override " .. + "the endpoint of the AI service provider." + } + } + } + }, + required = {"prompt", "provider", "auth"} +} + +local _M = { + version = 0.1, + priority = 1073, + name = plugin_name, + schema = schema +} + +local function proxy_request_to_llm(conf, request_table, ctx) + local ai_driver = require("apisix.plugins.ai-drivers." .. conf.provider) + + local extra_opts = { + endpoint = core.table.try_read_attr(conf, "override", "endpoint"), + query_params = conf.auth.query or {}, + headers = (conf.auth.header or {}), + model_options = conf.options + } + + local res, err, httpc = ai_driver:request(conf, request_table, extra_opts) + + if not res then return nil, err, nil end + return res, nil, httpc +end + + +local function parse_llm_response(res_body) + local response_table = core.json.decode(res_body) + + if not response_table then return nil, "failed to decode llm response" end + + if not response_table.choices then + return nil, "'choices' not in llm response" + end + + return response_table.choices[1].message.content, nil +end + + +function _M.check_schema(conf) + -- openai-compatible should be used with override.endpoint + if conf.provider == "openai-compatible" then + local override = conf.override + + if not override or not override.endpoint then + return false, "override.endpoint is required for openai-compatible provider" + end + end + + return core.schema.check(schema, conf) +end + + +function _M.access(conf, ctx) + local client_request_table, err = core.request.get_body() + if not client_request_table then + core.log.error("failed to get request body: ", err) + return bad_request, err + end + + -- Prepare request for LLM service + local ai_request_table = { + messages = { + { + role = "system", + content = conf.prompt + }, + { + role = "user", + content = client_request_table + } + }, + stream = false + } + + -- Send request to LLM service + local res, err, httpc = proxy_request_to_llm(conf, ai_request_table, ctx) + if not res then + core.log.error("failed to send request to LLM service: ", err) + return internal_server_error + end + + -- Handle LLM response + if res.status >= 400 then + core.log.error("LLM service returned error status: ", res.status) Review Comment: need to log detail error message ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" + }, + keepalive = { + type = "boolean", + default = true + }, + keepalive_pool = { + type = "integer", + minimum = 1, + default = 30 + }, + ssl_verify = { + type = "boolean", + default = true + }, + override = { + type = "object", + properties = { + endpoint = { + type = "string", + description = "To be specified to override " .. + "the endpoint of the AI service provider." + } + } + } + }, + required = {"prompt", "provider", "auth"} +} + +local _M = { + version = 0.1, + priority = 1073, + name = plugin_name, + schema = schema +} + +local function proxy_request_to_llm(conf, request_table, ctx) + local ai_driver = require("apisix.plugins.ai-drivers." .. conf.provider) + + local extra_opts = { + endpoint = core.table.try_read_attr(conf, "override", "endpoint"), + query_params = conf.auth.query or {}, + headers = (conf.auth.header or {}), + model_options = conf.options + } + + local res, err, httpc = ai_driver:request(conf, request_table, extra_opts) + + if not res then return nil, err, nil end + return res, nil, httpc +end + + +local function parse_llm_response(res_body) + local response_table = core.json.decode(res_body) + + if not response_table then return nil, "failed to decode llm response" end Review Comment: ```suggestion if not response_table then return nil, "failed to decode llm response " .. ", err: " .. err end ``` ########## docs/en/latest/plugins/ai-request-rewrite.md: ########## @@ -0,0 +1,167 @@ +--- +title: ai-request-rewrite +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-request-rewrite +description: This document contains information about the Apache APISIX ai-request-rewrite Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Description + +The `ai-request-rewrite` plugin leverages predefined prompts and AI services to intelligently modify client requests, enabling AI-powered content transformation before forwarding to upstream services. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ------------------------- | ------------ | -------- | ------------------------------------------------------------------------------------ | +| prompt | Yes | String | The prompt send to AI service. | +| provider | Yes | String | Type of the AI service. | +| auth | Yes | Object | Authentication configuration | +| auth.header | No | Object | Authentication headers. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| auth.query | No | Object | Authentication query parameters. Key must match pattern `^[a-zA-Z0-9._-]+$`. | +| options | No | Object | Key/value settings for the model | +| options.model | No | String | Model to execute. | +| override.endpoint | No | String | To be specified to override the endpoint of the AI service | Review Comment: when we need endpoint? ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" + }, + keepalive = { + type = "boolean", + default = true + }, + keepalive_pool = { + type = "integer", + minimum = 1, + default = 30 + }, + ssl_verify = { + type = "boolean", + default = true + }, + override = { + type = "object", + properties = { + endpoint = { + type = "string", + description = "To be specified to override " .. + "the endpoint of the AI service provider." + } + } + } + }, + required = {"prompt", "provider", "auth"} +} + +local _M = { + version = 0.1, + priority = 1073, + name = plugin_name, + schema = schema +} + +local function proxy_request_to_llm(conf, request_table, ctx) + local ai_driver = require("apisix.plugins.ai-drivers." .. conf.provider) + + local extra_opts = { + endpoint = core.table.try_read_attr(conf, "override", "endpoint"), + query_params = conf.auth.query or {}, + headers = (conf.auth.header or {}), + model_options = conf.options + } + + local res, err, httpc = ai_driver:request(conf, request_table, extra_opts) + + if not res then return nil, err, nil end + return res, nil, httpc +end + + +local function parse_llm_response(res_body) + local response_table = core.json.decode(res_body) + + if not response_table then return nil, "failed to decode llm response" end + + if not response_table.choices then + return nil, "'choices' not in llm response" + end + + return response_table.choices[1].message.content, nil +end + + +function _M.check_schema(conf) + -- openai-compatible should be used with override.endpoint + if conf.provider == "openai-compatible" then + local override = conf.override + + if not override or not override.endpoint then + return false, "override.endpoint is required for openai-compatible provider" + end + end + Review Comment: can json-schema do this job? ########## apisix/plugins/ai-request-rewrite.lua: ########## @@ -0,0 +1,212 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx + +local require = require + +local plugin_name = "ai-request-rewrite" + +local bad_request = ngx.HTTP_BAD_REQUEST +local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + +local auth_item_schema = { + type = "object", + patternProperties = { + ["^[a-zA-Z0-9._-]+$"] = { + type = "string" + } + } +} + +local auth_schema = { + type = "object", + properties = { + header = auth_item_schema, + query = auth_item_schema + }, + additionalProperties = false +} + +local model_options_schema = { + description = "Key/value settings for the model", + type = "object", + properties = { + model = { + type = "string", + description = "Model to execute." + } + }, + additionalProperties = true +} + +local schema = { + type = "object", + properties = { + prompt = { + type = "string", + description = "The prompt to rewrite client request." + }, + provider = { + type = "string", + description = "Name of the AI service provider.", + enum = {"openai", "openai-compatible", "deepseek"} -- add more providers later + }, + auth = auth_schema, + options = model_options_schema, + timeout = { + type = "integer", + minimum = 1, + maximum = 60000, + default = 30000, + description = "timeout in milliseconds" + }, + keepalive = { + type = "boolean", + default = true + }, + keepalive_pool = { + type = "integer", + minimum = 1, + default = 30 + }, + ssl_verify = { + type = "boolean", + default = true + }, + override = { + type = "object", + properties = { + endpoint = { + type = "string", + description = "To be specified to override " .. + "the endpoint of the AI service provider." + } + } + } + }, + required = {"prompt", "provider", "auth"} +} + +local _M = { + version = 0.1, + priority = 1073, + name = plugin_name, + schema = schema +} + +local function proxy_request_to_llm(conf, request_table, ctx) + local ai_driver = require("apisix.plugins.ai-drivers." .. conf.provider) + + local extra_opts = { + endpoint = core.table.try_read_attr(conf, "override", "endpoint"), + query_params = conf.auth.query or {}, + headers = (conf.auth.header or {}), + model_options = conf.options + } + + local res, err, httpc = ai_driver:request(conf, request_table, extra_opts) + + if not res then return nil, err, nil end + return res, nil, httpc +end + + +local function parse_llm_response(res_body) + local response_table = core.json.decode(res_body) + + if not response_table then return nil, "failed to decode llm response" end + + if not response_table.choices then + return nil, "'choices' not in llm response" + end + + return response_table.choices[1].message.content, nil +end + + +function _M.check_schema(conf) + -- openai-compatible should be used with override.endpoint + if conf.provider == "openai-compatible" then + local override = conf.override + + if not override or not override.endpoint then + return false, "override.endpoint is required for openai-compatible provider" + end + end + + return core.schema.check(schema, conf) +end + + +function _M.access(conf, ctx) + local client_request_table, err = core.request.get_body() + if not client_request_table then + core.log.error("failed to get request body: ", err) + return bad_request, err + end + + -- Prepare request for LLM service + local ai_request_table = { + messages = { + { + role = "system", + content = conf.prompt + }, + { + role = "user", + content = client_request_table + } + }, + stream = false + } + + -- Send request to LLM service + local res, err, httpc = proxy_request_to_llm(conf, ai_request_table, ctx) + if not res then + core.log.error("failed to send request to LLM service: ", err) + return internal_server_error + end + + -- Handle LLM response + if res.status >= 400 then + core.log.error("LLM service returned error status: ", res.status) + return internal_server_error Review Comment: return 500 or return llm's response status? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
