Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq closed pull request #13308: feat: ai-cache plugin URL: https://github.com/apache/apisix/pull/13308 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on PR #13308: URL: https://github.com/apache/apisix/pull/13308#issuecomment-4443532662 Closing this PR to address @membphis's review properly. Splitting the work into a phased series so the cache-key correctness conversation can be reviewed independently of semantic caching, embedding providers, and multi-protocol support: - **Phase 1**: exact (L1) cache only, **for `openai-chat` protocol only**, with a conservative cache key that accounts for everything affecting LLM output — effective model post-override, protocol, picked upstream instance, full structured `messages` (not concatenated text), whitelisted generation parameters, stream flag, and operator-scoped consumer/vars. All `openai-chat`-compatible providers (Azure OpenAI, DeepSeek, OpenRouter, Together, Fireworks, Ollama, etc.) work transparently. Sliced into 5 small reviewable PRs. - **Phase 2**: additional client protocols (`openai-responses`, `anthropic-messages`, `bedrock-converse`) — one PR each. - **Phase 3+**: semantic (L2) embedding-based cache as its own design track. The full plan including PR breakdown, cache-key invariants, and per-PR test gates is captured in a PRD on my fork: https://github.com/janiussyafiq/apisix/issues/13 The first PR in the series — a small, behavior-preserving `ai-proxy` refactor that extracts the instance-override application logic into helpers (so the cache key can reflect operator-side overrides correctly) — will follow shortly. Thanks for the careful review. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3183834322
##
apisix/plugins/ai-cache.lua:
##
@@ -0,0 +1,297 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local core = require("apisix.core")
+local schema= require("apisix.plugins.ai-cache.schema")
+local exact = require("apisix.plugins.ai-cache.exact")
+local semantic = require("apisix.plugins.ai-cache.semantic")
+local protocols = require("apisix.plugins.ai-protocols")
+local http = require("resty.http")
+local ngx = ngx
+local ngx_time = ngx.time
+local ngx_now = ngx.now
+local ipairs = ipairs
+local require = require
+local tostring = tostring
+local table_concat = table.concat
+
+local plugin_name = "ai-cache"
+
+local _M = {
+version = 0.1,
+priority = 1065,
+name = plugin_name,
+schema = schema.schema
+}
+
+
+local function layer_enabled(conf, name)
+local layers = conf.layers or { "exact", "semantic" }
+for _, l in ipairs(layers) do
+if l == name then return true end
+end
+return false
+end
+
+
+function _M.check_schema(conf)
+local ok, err = core.schema.check(schema.schema, conf)
+if not ok then
+return false, err
+end
+
+if layer_enabled(conf, "semantic") then
+if not (conf.semantic and conf.semantic.embedding) then
+return false, "semantic layer requires semantic.embedding to be
configured"
+end
+end
+
+core.utils.check_https({ "semantic.embedding.endpoint" }, conf,
plugin_name)
+
+return true
+end
+
+
+function _M.access(conf, ctx)
+-- Check bypass_on conditions
+if conf.bypass_on then
+local req_headers = ngx.req.get_headers()
+for _, rule in ipairs(conf.bypass_on) do
+if req_headers[rule.header] == rule.equals then
+ctx.ai_cache_bypass = true
+ctx.ai_cache_status = "BYPASS"
+return
+end
+end
+end
+
+local body_tab, err = core.request.get_json_request_body_table()
+if not body_tab then
+core.log.warn("ai-cache: failed to read request body: ", err or
"unknown error")
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local protocol_name = protocols.detect(body_tab, ctx)
+if not protocol_name then
+core.log.warn("ai-cache: could not detect AI protocol, skipping cache")
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local proto = protocols.get(protocol_name)
+local contents = proto.extract_request_content(body_tab)
+if not contents or #contents == 0 then
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local prompt_text = table_concat(contents, " ")
+local scope_hash = exact.compute_scope_hash(conf, ctx)
+local prompt_hash, hash_err = exact.compute_prompt_hash(prompt_text)
+if not prompt_hash then
+core.log.warn("ai-cache: failed to compute prompt hash: ", hash_err)
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local is_stream = body_tab.stream == true
+
+-- L1 exact lookup
+if layer_enabled(conf, "exact") then
+local cached_text, written_at, lookup_err = exact.get(conf,
scope_hash, prompt_hash)
+if lookup_err then
+core.log.warn("ai-cache: L1 lookup error: ", lookup_err)
+elseif cached_text then
+core.log.info("ai-cache: L1 hit for key: ", prompt_hash)
+ctx.ai_cache_status = "HIT-L1"
+ctx.ai_cache_written_at = written_at
+if is_stream then
+core.response.set_header("Content-Type", "text/event-stream")
+else
+core.response.set_header("Content-Type", "application/json")
+end
+-- TODO: rename build_deny_response to build_response_from_text in
a
+-- follow-up. We use it here to wrap cached text in the protocol's
+-- response shape, not for policy denial.
+return 200, proto.build_deny_response({
+stream = is_stream,
+text = cached_text,
+})
+end
+end
+
+-- L
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3183832655
##
apisix/plugins/ai-cache/semantic.lua:
##
@@ -0,0 +1,184 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local apisix_redis = require("apisix.utils.redis")
+local uuid = require("resty.jit-uuid")
+local ffi = require("ffi")
+
+local ffi_new = ffi.new
+local ffi_string = ffi.string
+local ngx_time = ngx.time
+local tostring = tostring
+local tonumber = tonumber
+local type = type
+
+local _M = {}
+
+
+local function index_name(dim)
+return "ai-cache-idx-" .. dim
+end
+
+
+local function key_prefix(dim)
+return "ai-cache:l2:" .. dim .. ":"
+end
+
+local function pack_vector(vec)
+local n = #vec
+local buf = ffi_new("float[?]", n)
+for i = 0, n - 1 do
+buf[i] = vec[i + 1]
+end
+return ffi_string(buf, n * 4)
+end
+
+local index_ready = {}
+
+local function ensure_index(red, dim)
+if index_ready[dim] then
+return true
+end
Review Comment:
fixed
##
apisix/plugins/ai-cache/schema.lua:
##
@@ -0,0 +1,127 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local redis_schema = require("apisix.utils.redis-schema")
+
+local _M = {}
+
+local embedding_schema = {
+type = "object",
+properties = {
+provider = {
+type = "string",
+enum = { "openai", "azure_openai" },
+},
+model = { type = "string" },
+endpoint = { type = "string" },
+api_key = { type = "string" },
+},
+required = { "provider", "endpoint", "api_key" },
+}
+
+local semantic_schema = {
+type = "object",
+properties = {
+similarity_threshold = {
+type = "number",
+minimum = 0,
+maximum = 1,
+default = 0.95,
+},
+top_k = {
+type = "integer",
+minimum = 1,
+default = 1,
+},
+ttl = {
+type = "integer",
+minimum = 1,
+default = 86400,
+},
+embedding = embedding_schema,
+},
+required = { "embedding" },
+}
+
+local exact_schema = {
+type = "object",
+properties = {
+ttl = {
+type = "integer",
+minimum = 1,
+default = 3600,
+},
+},
+}
+
+
+local bypass_item_schema = {
+type = "object",
+properties = {
+header = { type = "string" },
+equals = { type = "string" },
+},
+required = { "header", "equals" },
+}
+
+local headers_schema = {
+type = "object",
+properties = {
+cache_status = { type = "string", default = "X-AI-Cache-Status" },
+cache_similarity = { type = "string", default =
"X-AI-Cache-Similarity" },
+cache_age = { type = "string", default = "X-AI-Cache-Age" },
+},
+}
+
+_M.schema = {
+type = "object",
+properties = {
+layers = {
+type = "array",
+items = { type = "string", enum = { "exact", "semantic" } },
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat: ai-cache plugin [apisix]
Copilot commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3182998457
##
apisix/plugins/ai-cache/schema.lua:
##
@@ -0,0 +1,127 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local redis_schema = require("apisix.utils.redis-schema")
+
+local _M = {}
+
+local embedding_schema = {
+type = "object",
+properties = {
+provider = {
+type = "string",
+enum = { "openai", "azure_openai" },
+},
+model = { type = "string" },
+endpoint = { type = "string" },
+api_key = { type = "string" },
+},
+required = { "provider", "endpoint", "api_key" },
+}
+
+local semantic_schema = {
+type = "object",
+properties = {
+similarity_threshold = {
+type = "number",
+minimum = 0,
+maximum = 1,
+default = 0.95,
+},
+top_k = {
+type = "integer",
+minimum = 1,
+default = 1,
+},
+ttl = {
+type = "integer",
+minimum = 1,
+default = 86400,
+},
+embedding = embedding_schema,
+},
+required = { "embedding" },
+}
+
+local exact_schema = {
+type = "object",
+properties = {
+ttl = {
+type = "integer",
+minimum = 1,
+default = 3600,
+},
+},
+}
+
+
+local bypass_item_schema = {
+type = "object",
+properties = {
+header = { type = "string" },
+equals = { type = "string" },
+},
+required = { "header", "equals" },
+}
+
+local headers_schema = {
+type = "object",
+properties = {
+cache_status = { type = "string", default = "X-AI-Cache-Status" },
+cache_similarity = { type = "string", default =
"X-AI-Cache-Similarity" },
+cache_age = { type = "string", default = "X-AI-Cache-Age" },
+},
+}
+
+_M.schema = {
+type = "object",
+properties = {
+layers = {
+type = "array",
+items = { type = "string", enum = { "exact", "semantic" } },
Review Comment:
The schema allows `layers: []` (empty array), which results in a confusing
no-op cache configuration (always MISS, but still does key computation and
schedules a timer). Consider enforcing at least one layer via `minItems: 1`
and/or rejecting an empty `layers` array in check_schema.
##
apisix/plugins/ai-cache/semantic.lua:
##
@@ -0,0 +1,184 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local apisix_redis = require("apisix.utils.redis")
+local uuid = require("resty.jit-uuid")
+local ffi = require("ffi")
+
+local ffi_new = ffi.new
+local ffi_string = ffi.string
+local ngx_time = ngx.time
+local tostring = tostring
+local tonumber = tonumber
+local type = type
+
+local _M = {}
+
+
+local function index_name(dim)
+return "ai-cache-idx-" .. dim
+end
+
+
+local function key_prefix(dim)
+return "ai-cache:l2:" .. dim .. ":"
+end
+
+local function pack_vector(vec)
+local n = #vec
+local buf = ffi_new("float[?]", n)
+for i = 0, n - 1 do
+buf[i] = vec[i + 1]
+end
+return ffi_string(buf, n * 4)
+end
+
+local index_ready = {}
+
+local function ensure_index(red, dim)
+if index_ready[dim] then
+return true
+end
Review Comment:
`index_ready[dim]` is treated as a permanent truth. If the RediSearch index
is dropped (eg `FT.DR
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3171225045
##
apisix/plugins/ai-cache.lua:
##
@@ -0,0 +1,285 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local core = require("apisix.core")
+local schema= require("apisix.plugins.ai-cache.schema")
+local exact = require("apisix.plugins.ai-cache.exact")
+local semantic = require("apisix.plugins.ai-cache.semantic")
+local protocols = require("apisix.plugins.ai-protocols")
+local http = require("resty.http")
+local ngx_time = ngx.time
+local ngx_now = ngx.now
+local tostring = tostring
+local table_concat = table.concat
+
+local plugin_name = "ai-cache"
+
+local _M = {
+version = 0.1,
+priority = 1065,
+name = plugin_name,
+schema = schema.schema
+}
+
+
+local function layer_enabled(conf, name)
+local layers = conf.layers or { "exact", "semantic" }
+for _, l in ipairs(layers) do
+if l == name then return true end
+end
+return false
+end
+
+
+function _M.check_schema(conf)
+local ok, err = core.schema.check(schema.schema, conf)
+if not ok then
+return false, err
+end
+
+if layer_enabled(conf, "semantic") then
+if not (conf.semantic and conf.semantic.embedding) then
+return false, "semantic layer requires semantic.embedding to be
configured"
+end
+end
+
+core.utils.check_https({ "semantic.embedding.endpoint" }, conf,
plugin_name)
+
+return true
+end
+
+
+function _M.access(conf, ctx)
+-- Check bypass_on conditions
+if conf.bypass_on then
+local req_headers = ngx.req.get_headers()
+for _, rule in ipairs(conf.bypass_on) do
+if req_headers[rule.header] == rule.equals then
+ctx.ai_cache_bypass = true
+ctx.ai_cache_status = "BYPASS"
+return
+end
+end
+end
+
+local body_tab, err = core.request.get_json_request_body_table()
+if not body_tab then
+core.log.warn("ai-cache: failed to read request body: ", err or
"unknown error")
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local protocol_name = protocols.detect(body_tab, ctx)
+if not protocol_name then
+core.log.warn("ai-cache: could not detect AI protocol, skipping cache")
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local proto = protocols.get(protocol_name)
+local contents = proto.extract_request_content(body_tab)
+if not contents or #contents == 0 then
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local prompt_text = table_concat(contents, " ")
+local scope_hash = exact.compute_scope_hash(conf, ctx)
+local prompt_hash, hash_err = exact.compute_prompt_hash(prompt_text)
+if not prompt_hash then
+core.log.warn("ai-cache: failed to compute prompt hash: ", hash_err)
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local is_stream = body_tab.stream == true
+
+-- L1 exact lookup
+if layer_enabled(conf, "exact") then
+local cached_text, written_at, lookup_err = exact.get(conf,
scope_hash, prompt_hash)
+if lookup_err then
+core.log.warn("ai-cache: L1 lookup error: ", lookup_err)
+elseif cached_text then
+core.log.info("ai-cache: L1 hit for key: ", prompt_hash)
+ctx.ai_cache_status = "HIT-L1"
+ctx.ai_cache_written_at = written_at
+if is_stream then
+core.response.set_header("Content-Type", "text/event-stream")
+else
+core.response.set_header("Content-Type", "application/json")
+end
+return core.response.exit(200, proto.build_deny_response({
+stream = is_stream,
+text = cached_text,
+}))
+end
Review Comment:
`proto.build_deny_response` could be reuse here to simulate LLM response,
and could be rename to `proto.build_response_from_text` to prevent confusion.
added TODO message to be addressed in later PR to keep focus on ai-cache
implementation.
--
This is an automated message from the Apa
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3170913673
##
apisix/plugins/ai-cache.lua:
##
@@ -0,0 +1,285 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local core = require("apisix.core")
+local schema= require("apisix.plugins.ai-cache.schema")
+local exact = require("apisix.plugins.ai-cache.exact")
+local semantic = require("apisix.plugins.ai-cache.semantic")
+local protocols = require("apisix.plugins.ai-protocols")
+local http = require("resty.http")
+local ngx_time = ngx.time
+local ngx_now = ngx.now
+local tostring = tostring
+local table_concat = table.concat
+
+local plugin_name = "ai-cache"
+
+local _M = {
+version = 0.1,
+priority = 1065,
+name = plugin_name,
+schema = schema.schema
+}
+
+
+local function layer_enabled(conf, name)
+local layers = conf.layers or { "exact", "semantic" }
+for _, l in ipairs(layers) do
+if l == name then return true end
+end
+return false
+end
+
+
+function _M.check_schema(conf)
+local ok, err = core.schema.check(schema.schema, conf)
+if not ok then
+return false, err
+end
+
+if layer_enabled(conf, "semantic") then
+if not (conf.semantic and conf.semantic.embedding) then
+return false, "semantic layer requires semantic.embedding to be
configured"
+end
+end
+
+core.utils.check_https({ "semantic.embedding.endpoint" }, conf,
plugin_name)
+
+return true
+end
+
+
+function _M.access(conf, ctx)
+-- Check bypass_on conditions
+if conf.bypass_on then
+local req_headers = ngx.req.get_headers()
+for _, rule in ipairs(conf.bypass_on) do
+if req_headers[rule.header] == rule.equals then
+ctx.ai_cache_bypass = true
+ctx.ai_cache_status = "BYPASS"
+return
+end
+end
+end
+
+local body_tab, err = core.request.get_json_request_body_table()
+if not body_tab then
+core.log.warn("ai-cache: failed to read request body: ", err or
"unknown error")
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local protocol_name = protocols.detect(body_tab, ctx)
+if not protocol_name then
+core.log.warn("ai-cache: could not detect AI protocol, skipping cache")
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local proto = protocols.get(protocol_name)
+local contents = proto.extract_request_content(body_tab)
+if not contents or #contents == 0 then
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local prompt_text = table_concat(contents, " ")
+local scope_hash = exact.compute_scope_hash(conf, ctx)
+local prompt_hash, hash_err = exact.compute_prompt_hash(prompt_text)
+if not prompt_hash then
+core.log.warn("ai-cache: failed to compute prompt hash: ", hash_err)
+ctx.ai_cache_status = "MISS"
+return
+end
+
+local is_stream = body_tab.stream == true
+
+-- L1 exact lookup
+if layer_enabled(conf, "exact") then
+local cached_text, written_at, lookup_err = exact.get(conf,
scope_hash, prompt_hash)
+if lookup_err then
+core.log.warn("ai-cache: L1 lookup error: ", lookup_err)
+elseif cached_text then
+core.log.info("ai-cache: L1 hit for key: ", prompt_hash)
+ctx.ai_cache_status = "HIT-L1"
+ctx.ai_cache_written_at = written_at
+if is_stream then
+core.response.set_header("Content-Type", "text/event-stream")
+else
+core.response.set_header("Content-Type", "application/json")
+end
+return core.response.exit(200, proto.build_deny_response({
+stream = is_stream,
+text = cached_text,
+}))
+end
+end
+
+-- L2 semantic lookup
+if layer_enabled(conf, "semantic") then
+local emb_conf = conf.semantic.embedding
+local emb_driver = require("apisix.plugins.ai-cache.embeddings." ..
emb_conf.provider)
+local httpc = http.new()
+
+local t0 = ngx_now()
+local e
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3170892117
##
apisix/plugins/prometheus/exporter.lua:
##
@@ -260,6 +268,35 @@ function _M.http_init(prometheus_enabled_in_stream)
unpack(extra_labels("llm_active_connections"))},
llm_active_connections_exptime)
+metrics.ai_cache_hits = prometheus:counter("ai_cache_hits_total",
+"AI cache hit count by layer",
+{"route_id", "service_id", "consumer", "layer",
+unpack(extra_labels("ai_cache_hits"))},
+ai_cache_hits_exptime)
+
+metrics.ai_cache_misses = prometheus:counter("ai_cache_misses_total",
+"AI cache miss count",
+{"route_id", "service_id", "consumer",
+unpack(extra_labels("ai_cache_misses"))},
+ai_cache_misses_exptime)
+
+local ai_cache_embedding_latency_buckets = DEFAULT_BUCKETS
+if attr and attr.ai_cache_embedding_latency_buckets then
+ai_cache_embedding_latency_buckets =
attr.ai_cache_embedding_latency_buckets
+end
+metrics.ai_cache_embedding_latency =
prometheus:histogram("ai_cache_embedding_latency",
+"AI cache embedding API call latency in milliseconds",
+{"route_id", "service_id", "consumer", "provider",
+unpack(extra_labels("ai_cache_embedding_latency"))},
Review Comment:
Declined for consistency, `http_latency` and `llm_latency` didn't use ms.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3170882292
##
apisix/plugins/ai-cache/semantic.lua:
##
@@ -0,0 +1,180 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local apisix_redis = require("apisix.utils.redis")
+local uuid = require("resty.jit-uuid")
+local ffi = require("ffi")
+
+local ffi_new = ffi.new
+local ffi_string = ffi.string
+local ngx_time = ngx.time
+local tostring = tostring
+local tonumber = tonumber
+local type = type
+
+local _M = {}
+
+
+local function index_name(dim)
+return "ai-cache-idx-" .. dim
+end
+
+
+local function key_prefix(dim)
+return "ai-cache:l2:" .. dim .. ":"
+end
+
+local function pack_vector(vec)
+local n = #vec
+local buf = ffi_new("float[?]", n)
+for i = 0, n - 1 do
+buf[i] = vec[i + 1]
+end
+return ffi_string(buf, n * 4)
+end
+
+local index_ready = {}
+
+local function ensure_index(red, dim)
+if index_ready[dim] then
+return true
+end
+
+local _, err = red["FT.CREATE"](red,
+index_name(dim),
+"ON", "HASH",
+"PREFIX", "1", key_prefix(dim),
+"SCHEMA",
+"embedding", "VECTOR", "HNSW", "6",
+"TYPE", "FLOAT32",
+"DIM", tostring(dim),
+"DISTANCE_METRIC", "COSINE",
+"scope", "TAG",
+"created_at", "NUMERIC"
+)
+
+if err and not err:find("already exists") then
+return nil, "FT.CREATE failed: " .. err
+end
+
+index_ready[dim] = true
+return true
+end
+
+
+function _M.search(conf, scope_hash, embedding_vec, threshold)
+local red, err = apisix_redis.new(conf)
+if not red then
+return nil, nil, err
+end
+
+local ok, init_err = ensure_index(red, #embedding_vec)
+if not ok then
+red:set_keepalive(conf.redis_keepalive_timeout,
conf.redis_keepalive_pool)
+return nil, nil, init_err
+end
+
+local binary_vec = pack_vector(embedding_vec)
+
+local query
+if scope_hash == "" then
+query = "*=>[KNN 1 @embedding $vec AS dist]"
+else
+query = "@scope:{" .. scope_hash .. "}=>[KNN 1 @embedding $vec AS
dist]"
+end
+
+local res, search_err = red["FT.SEARCH"](red,
+index_name(#embedding_vec),
+query,
+"PARAMS", "2", "vec", binary_vec,
+"SORTBY", "dist", "ASC",
+"LIMIT", "0", "1",
+"RETURN", "2", "response", "dist",
+"DIALECT", "2"
+)
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3170857202
##
docs/en/latest/plugins/ai-cache.md:
##
@@ -0,0 +1,1085 @@
+---
+title: ai-cache
+keywords:
+ - Apache APISIX
+ - API Gateway
+ - Plugin
+ - ai-cache
+description: The ai-cache Plugin caches LLM responses in Redis so identical or
semantically similar prompts are served from cache, reducing latency and
upstream cost.
+---
+
+
+
+
+ https://docs.api7.ai/hub/ai-cache"; />
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## Description
+
+The `ai-cache` Plugin caches LLM responses in Redis so identical or
semantically similar prompts are served from cache instead of incurring another
upstream call. It supports two cache layers: an exact-match layer (`exact`)
keyed by a hash of the prompt, and a semantic layer (`semantic`) that compares
prompt embeddings via Redis Stack vector search. Either layer can be enabled
independently, and a hit on the semantic layer backfills the exact layer so
subsequent identical prompts return immediately.
+
+The Plugin should be used together with [ai-proxy](./ai-proxy.md) or
[ai-proxy-multi](./ai-proxy-multi.md) on the same Route. The semantic layer
requires Redis Stack with the RediSearch module and an embedding provider
(OpenAI or Azure OpenAI).
+
+## Plugin Attributes
+
+| Name | Type | Required | Default | Valid values | Description |
+| --- | --- | --- | --- | --- | --- |
+| `layers` | array[string] | False | `["exact", "semantic"]` | `"exact"`,
`"semantic"` | Cache layers to enable, queried in order. |
+| `exact.ttl` | integer | False | `3600` | ≥ 1 | Time-to-live in seconds for
exact-layer entries. |
+| `semantic.similarity_threshold` | number | False | `0.95` | 0–1 | Minimum
cosine similarity required for a semantic-layer hit. |
+| `semantic.ttl` | integer | False | `86400` | ≥ 1 | Time-to-live in seconds
for semantic-layer entries. |
+| `semantic.embedding.provider` | string | True (if semantic enabled) | |
`"openai"`, `"azure_openai"` | Embedding API provider. |
+| `semantic.embedding.endpoint` | string | True (if semantic enabled) | | |
HTTPS URL of the embedding API. |
+| `semantic.embedding.api_key` | string | True (if semantic enabled) | | | API
key for the embedding provider. Stored encrypted. |
+| `semantic.embedding.model` | string | False | | | Embedding model name. Uses
provider default if omitted. |
+| `cache_key.include_consumer` | boolean | False | `false` | | If `true`,
partition the cache by consumer name. |
+| `cache_key.include_vars` | array[string] | False | `[]` | | Additional
`ctx.var` names included in the cache key, for example `["$http_x_tenant_id"]`.
|
+| `bypass_on` | array[object] | False | | | List of `{header, equals}` rules.
If any matches, the request bypasses the cache. |
+| `max_cache_body_size` | integer | False | `1048576` | ≥ 1 | Maximum response
size in bytes to write to cache. Larger responses pass through but are not
cached. |
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3170843083
##
t/plugin/prometheus-ai-cache.t:
##
@@ -0,0 +1,305 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+$ENV{TEST_ENABLE_CONTROL_API_V1} = "0";
+
+if ($ENV{TEST_NGINX_CHECK_LEAK}) {
+$SkipReason = "unavailable for the hup tests";
+} else {
+$ENV{TEST_NGINX_USE_HUP} = 1;
+undef $ENV{TEST_NGINX_USE_STAP};
+}
+}
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+my ($block) = @_;
+
+if (!defined $block->request) {
+$block->set_value("request", "GET /t");
+}
+
+my $user_yaml_config = <<_EOC_;
+plugin_attr:
+prometheus:
+refresh_interval: 0.1
+plugins:
+ - ai-proxy
+ - ai-cache
+ - prometheus
+ - public-api
+_EOC_
+$block->set_value("extra_yaml_config", $user_yaml_config);
+
+if (!defined $block->http_config) {
+$block->set_value("http_config", <<_EOC_);
+server {
+listen 1990;
+default_type 'application/json';
+
+location /v1/embeddings {
+content_by_lua_block {
+local fixture_loader = require("lib.fixture_loader")
+local content, err =
fixture_loader.load("openai/embeddings-list.json")
+if not content then
+ngx.status = 500
+ngx.say(err)
+return
+end
+
+ngx.status = 200
+ngx.print(content)
+}
+}
+}
+_EOC_
+}
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: set up routes
+--- config
+location /t {
+content_by_lua_block {
+local t = require("lib.test_admin").test
+
+local routes = {
+{
+url = "/apisix/admin/routes/1",
+data = [[{
+"uri": "/chat",
+"plugins": {
+"prometheus": {},
+"ai-proxy": {
+"provider": "openai",
+"auth": {
+"header": {
+"Authorization": "Bearer test-key"
+}
+},
+"override": {
+"endpoint":
"http://127.0.0.1:1980/v1/chat/completions";
+}
+},
+"ai-cache": {
+"layers": ["exact"],
+"exact": { "ttl": 60 },
+"redis_host": "127.0.0.1",
+"bypass_on": [{"header": "X-Cache-Bypass",
"equals": "1"}]
+}
+}
+}]],
+},
+{
+url = "/apisix/admin/routes/2",
+data = [[{
+"uri": "/semantic",
+"plugins": {
+"prometheus": {},
+"ai-proxy": {
+"provider": "openai",
+"auth": {
+"header": {
+"Authorization": "Bearer test-key"
+}
+},
+"override": {
+"endpoint":
"http://127.0.0.1:1980/v1/chat/completions";
+}
+},
+"ai-cache": {
+"layers": ["exact", "semantic"],
+"exact": { "ttl": 60 },
+"semantic": {
+"similarity_threshold": 0.90,
+"ttl": 300,
+"embedding": {
+"provider": "openai",
+
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3170845676
##
t/plugin/ai-cache-scope.t:
##
@@ -0,0 +1,355 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+$ENV{TEST_ENABLE_CONTROL_API_V1} = "0";
+}
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+my ($block) = @_;
+
+if (!defined $block->request) {
+$block->set_value("request", "GET /t");
+}
+
+if (!$block->error_log && !$block->no_error_log) {
+$block->set_value("no_error_log", "[error]\n[alert]");
+}
+
+if (!defined $block->http_config) {
+$block->set_value("http_config", <<_EOC_);
+server {
+listen 1990;
+default_type 'application/json';
+
+location /v1/embeddings {
+content_by_lua_block {
+local fixture_loader = require("lib.fixture_loader")
+local content, err =
fixture_loader.load("openai/embeddings-list.json")
+if not content then
+ngx.status = 500
+ngx.say(err)
+return
+end
+
+ngx.status = 200
+ngx.print(content)
+}
+}
+}
+_EOC_
+}
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set up route with cache_key include_vars
+--- config
+location /t {
+content_by_lua_block {
+local t = require("lib.test_admin").test
+local code, body = t('/apisix/admin/routes/1',
+ngx.HTTP_PUT,
+[[{
+"uri": "/scoped",
+"plugins": {
+"ai-proxy": {
+"provider": "openai",
+"auth": {
+"header": {
+"Authorization": "Bearer test-key"
+}
+},
+"override": {
+"endpoint":
"http://127.0.0.1:1980/v1/chat/completions";
+}
+},
+"ai-cache": {
+"layers": ["exact"],
+"exact": { "ttl": 60 },
+"cache_key": {
+"include_vars": ["$http_x_tenant_id"]
+},
+"redis_host": "127.0.0.1"
+}
+}
+}]]
+)
+
+if code >= 300 then
+ngx.status = code
+end
+ngx.say(body)
+}
+}
+--- response_body
+passed
+
+
+
+=== TEST 2: tenant-a first request - MISS
+--- request
+POST /scoped
+{"messages":[{"role":"user","content":"scope test prompt"}]}
+--- more_headers
+Content-Type: application/json
+X-Tenant-Id: tenant-a
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 3: tenant-b same prompt - MISS (proves cache_key partitioning)
+--- request
+POST /scoped
+{"messages":[{"role":"user","content":"scope test prompt"}]}
+--- more_headers
+Content-Type: application/json
+X-Tenant-Id: tenant-b
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 1
+
+
+
+=== TEST 4: tenant-a same prompt again - HIT-L1
+--- request
+POST /scoped
+{"messages":[{"role":"user","content":"scope test prompt"}]}
+--- more_headers
+Content-Type: application/json
+X-Tenant-Id: tenant-a
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: HIT-L1
+
+
+
+=== TEST 5: set up consumers for include_consumer test
+--- config
+location /t {
+content_by_lua_block {
+local t = require("lib.test_admin").test
+
+local consumers = {
+{ username = "alice", key = "alice-key" },
+{ username = "bob", key = "bob-key" },
+}
+
+for _, c in ipairs(consumers) do
+local code, body = t('/apisix/admin/consumers',
+ng
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3170844453
##
t/plugin/ai-cache.t:
##
@@ -0,0 +1,845 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+$ENV{TEST_ENABLE_CONTROL_API_V1} = "0";
+}
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+my ($block) = @_;
+
+if (!defined $block->request) {
+$block->set_value("request", "GET /t");
+}
+
+if (!$block->error_log && !$block->no_error_log) {
+$block->set_value("no_error_log", "[error]\n[alert]");
+}
+
+if (!defined $block->http_config) {
+$block->set_value("http_config", <<_EOC_);
+server {
+listen 1990;
+default_type 'application/json';
+
+location /v1/embeddings {
+content_by_lua_block {
+local fixture_loader = require("lib.fixture_loader")
+local content, err =
fixture_loader.load("openai/embeddings-list.json")
+if not content then
+ngx.status = 500
+ngx.say(err)
+return
+end
+
+ngx.status = 200
+ngx.print(content)
+}
+}
+}
+_EOC_
+}
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: valid config - exact layer only
+--- config
+location /t {
+content_by_lua_block {
+local plugin = require("apisix.plugins.ai-cache")
+local ok, err = plugin.check_schema({
+layers = { "exact" },
+exact = { ttl = 600 },
+redis_host = "127.0.0.1",
+redis_port = 6379,
+})
+
+if not ok then
+ngx.say("failed")
+else
+ngx.say("passed")
+end
+}
+}
+--- response_body
+passed
+
+
+
+=== TEST 2: valid config - both layers with semantic embedding
+--- config
+location /t {
+content_by_lua_block {
+local plugin = require("apisix.plugins.ai-cache")
+local ok, err = plugin.check_schema({
+layers = { "exact", "semantic" },
+exact = { ttl = 3600 },
+semantic = {
+similarity_threshold = 0.95,
+ttl = 86400,
+embedding = {
+provider = "openai",
+endpoint = "https://api.openai.com/v1/embeddings";,
+api_key = "sk-test",
+},
+},
+redis_host = "127.0.0.1",
+redis_port = 6379,
+})
+
+if not ok then
+ngx.say(err)
+else
+ngx.say("passed")
+end
+}
+}
+--- response_body
+passed
+
+
+
+=== TEST 3: semantic without embedding config - should fail
+--- config
+location /t {
+content_by_lua_block {
+local plugin = require("apisix.plugins.ai-cache")
+local ok, err = plugin.check_schema({
+layers = { "semantic" },
+redis_host = "127.0.0.1",
+})
+if not ok then
+ngx.say("failed: ", err)
+else
+ngx.say("passed")
+end
+}
+}
+--- response_body
+failed: semantic layer requires semantic.embedding to be configured
+
+
+
+=== TEST 4: invalid layer value - should fail
+--- config
+location /t {
+content_by_lua_block {
+local plugin = require("apisix.plugins.ai-cache")
+local ok, err = plugin.check_schema({
+
Re: [PR] feat: ai-cache plugin [apisix]
janiussyafiq commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3170843083
##
t/plugin/prometheus-ai-cache.t:
##
@@ -0,0 +1,305 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+$ENV{TEST_ENABLE_CONTROL_API_V1} = "0";
+
+if ($ENV{TEST_NGINX_CHECK_LEAK}) {
+$SkipReason = "unavailable for the hup tests";
+} else {
+$ENV{TEST_NGINX_USE_HUP} = 1;
+undef $ENV{TEST_NGINX_USE_STAP};
+}
+}
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+my ($block) = @_;
+
+if (!defined $block->request) {
+$block->set_value("request", "GET /t");
+}
+
+my $user_yaml_config = <<_EOC_;
+plugin_attr:
+prometheus:
+refresh_interval: 0.1
+plugins:
+ - ai-proxy
+ - ai-cache
+ - prometheus
+ - public-api
+_EOC_
+$block->set_value("extra_yaml_config", $user_yaml_config);
+
+if (!defined $block->http_config) {
+$block->set_value("http_config", <<_EOC_);
+server {
+listen 1990;
+default_type 'application/json';
+
+location /v1/embeddings {
+content_by_lua_block {
+local fixture_loader = require("lib.fixture_loader")
+local content, err =
fixture_loader.load("openai/embeddings-list.json")
+if not content then
+ngx.status = 500
+ngx.say(err)
+return
+end
+
+ngx.status = 200
+ngx.print(content)
+}
+}
+}
+_EOC_
+}
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: set up routes
+--- config
+location /t {
+content_by_lua_block {
+local t = require("lib.test_admin").test
+
+local routes = {
+{
+url = "/apisix/admin/routes/1",
+data = [[{
+"uri": "/chat",
+"plugins": {
+"prometheus": {},
+"ai-proxy": {
+"provider": "openai",
+"auth": {
+"header": {
+"Authorization": "Bearer test-key"
+}
+},
+"override": {
+"endpoint":
"http://127.0.0.1:1980/v1/chat/completions";
+}
+},
+"ai-cache": {
+"layers": ["exact"],
+"exact": { "ttl": 60 },
+"redis_host": "127.0.0.1",
+"bypass_on": [{"header": "X-Cache-Bypass",
"equals": "1"}]
+}
+}
+}]],
+},
+{
+url = "/apisix/admin/routes/2",
+data = [[{
+"uri": "/semantic",
+"plugins": {
+"prometheus": {},
+"ai-proxy": {
+"provider": "openai",
+"auth": {
+"header": {
+"Authorization": "Bearer test-key"
+}
+},
+"override": {
+"endpoint":
"http://127.0.0.1:1980/v1/chat/completions";
+}
+},
+"ai-cache": {
+"layers": ["exact", "semantic"],
+"exact": { "ttl": 60 },
+"semantic": {
+"similarity_threshold": 0.90,
+"ttl": 300,
+"embedding": {
+"provider": "openai",
+
Re: [PR] feat: ai-cache plugin [apisix]
Copilot commented on code in PR #13308:
URL: https://github.com/apache/apisix/pull/13308#discussion_r3165074724
##
t/plugin/ai-cache-scope.t:
##
@@ -0,0 +1,355 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BEGIN {
+$ENV{TEST_ENABLE_CONTROL_API_V1} = "0";
+}
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+my ($block) = @_;
+
+if (!defined $block->request) {
+$block->set_value("request", "GET /t");
+}
+
+if (!$block->error_log && !$block->no_error_log) {
+$block->set_value("no_error_log", "[error]\n[alert]");
+}
+
+if (!defined $block->http_config) {
+$block->set_value("http_config", <<_EOC_);
+server {
+listen 1990;
+default_type 'application/json';
+
+location /v1/embeddings {
+content_by_lua_block {
+local fixture_loader = require("lib.fixture_loader")
+local content, err =
fixture_loader.load("openai/embeddings-list.json")
+if not content then
+ngx.status = 500
+ngx.say(err)
+return
+end
+
+ngx.status = 200
+ngx.print(content)
+}
+}
+}
+_EOC_
+}
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set up route with cache_key include_vars
+--- config
+location /t {
+content_by_lua_block {
+local t = require("lib.test_admin").test
+local code, body = t('/apisix/admin/routes/1',
+ngx.HTTP_PUT,
+[[{
+"uri": "/scoped",
+"plugins": {
+"ai-proxy": {
+"provider": "openai",
+"auth": {
+"header": {
+"Authorization": "Bearer test-key"
+}
+},
+"override": {
+"endpoint":
"http://127.0.0.1:1980/v1/chat/completions";
+}
+},
+"ai-cache": {
+"layers": ["exact"],
+"exact": { "ttl": 60 },
+"cache_key": {
+"include_vars": ["$http_x_tenant_id"]
+},
+"redis_host": "127.0.0.1"
+}
+}
+}]]
+)
+
+if code >= 300 then
+ngx.status = code
+end
+ngx.say(body)
+}
+}
+--- response_body
+passed
+
+
+
+=== TEST 2: tenant-a first request - MISS
+--- request
+POST /scoped
+{"messages":[{"role":"user","content":"scope test prompt"}]}
+--- more_headers
+Content-Type: application/json
+X-Tenant-Id: tenant-a
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+
+
+
+=== TEST 3: tenant-b same prompt - MISS (proves cache_key partitioning)
+--- request
+POST /scoped
+{"messages":[{"role":"user","content":"scope test prompt"}]}
+--- more_headers
+Content-Type: application/json
+X-Tenant-Id: tenant-b
+X-AI-Fixture: openai/chat-basic.json
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 1
+
+
+
+=== TEST 4: tenant-a same prompt again - HIT-L1
+--- request
+POST /scoped
+{"messages":[{"role":"user","content":"scope test prompt"}]}
+--- more_headers
+Content-Type: application/json
+X-Tenant-Id: tenant-a
+--- error_code: 200
+--- response_headers
+X-AI-Cache-Status: HIT-L1
+
+
+
+=== TEST 5: set up consumers for include_consumer test
+--- config
+location /t {
+content_by_lua_block {
+local t = require("lib.test_admin").test
+
+local consumers = {
+{ username = "alice", key = "alice-key" },
+{ username = "bob", key = "bob-key" },
+}
+
+for _, c in ipairs(consumers) do
+local code, body = t('/apisix/admin/consumers',
+ngx.HTT
