Copilot commented on code in PR #13308: URL: https://github.com/apache/apisix/pull/13308#discussion_r3165074724
########## t/plugin/ai-cache-scope.t: ########## @@ -0,0 +1,355 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BEGIN { + $ENV{TEST_ENABLE_CONTROL_API_V1} = "0"; +} + +use t::APISIX 'no_plan'; + +log_level("info"); +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->error_log && !$block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } + + if (!defined $block->http_config) { + $block->set_value("http_config", <<_EOC_); +server { + listen 1990; + default_type 'application/json'; + + location /v1/embeddings { + content_by_lua_block { + local fixture_loader = require("lib.fixture_loader") + local content, err = fixture_loader.load("openai/embeddings-list.json") + if not content then + ngx.status = 500 + ngx.say(err) + return + end + + ngx.status = 200 + ngx.print(content) + } + } +} +_EOC_ + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: set up route with cache_key include_vars +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/scoped", + "plugins": { + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact"], + "exact": { "ttl": 60 }, + "cache_key": { + "include_vars": ["$http_x_tenant_id"] + }, + "redis_host": "127.0.0.1" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 2: tenant-a first request - MISS +--- request +POST /scoped +{"messages":[{"role":"user","content":"scope test prompt"}]} +--- more_headers +Content-Type: application/json +X-Tenant-Id: tenant-a +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS + + + +=== TEST 3: tenant-b same prompt - MISS (proves cache_key partitioning) +--- request +POST /scoped +{"messages":[{"role":"user","content":"scope test prompt"}]} +--- more_headers +Content-Type: application/json +X-Tenant-Id: tenant-b +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS +--- wait: 1 + + + +=== TEST 4: tenant-a same prompt again - HIT-L1 +--- request +POST /scoped +{"messages":[{"role":"user","content":"scope test prompt"}]} +--- more_headers +Content-Type: application/json +X-Tenant-Id: tenant-a +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: HIT-L1 + + + +=== TEST 5: set up consumers for include_consumer test +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + local consumers = { + { username = "alice", key = "alice-key" }, + { username = "bob", key = "bob-key" }, + } + + for _, c in ipairs(consumers) do + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + string.format([[{ + "username": "%s", + "plugins": { "key-auth": { "key": "%s" } } + }]], c.username, c.key) + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + end + ngx.say("passed") + } + } +--- response_body +passed + + + +=== TEST 6: set up route with cache_key include_consumer + key-auth +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + [[{ + "uri": "/per-consumer", + "plugins": { + "key-auth": {}, + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact"], + "exact": { "ttl": 60 }, + "cache_key": { + "include_consumer": true + }, + "redis_host": "127.0.0.1" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 7: alice first request - MISS +--- request +POST /per-consumer +{"messages":[{"role":"user","content":"per-consumer prompt"}]} +--- more_headers +Content-Type: application/json +apikey: alice-key +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS +--- wait: 1 + + + +=== TEST 8: bob same prompt - MISS (proves include_consumer partitioning) +--- request +POST /per-consumer +{"messages":[{"role":"user","content":"per-consumer prompt"}]} +--- more_headers +Content-Type: application/json +apikey: bob-key +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS + + + +=== TEST 9: set up route with L2 semantic + cache_key include_vars +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/3', + ngx.HTTP_PUT, + [[{ + "uri": "/scoped-semantic", + "plugins": { + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact", "semantic"], + "exact": { "ttl": 60 }, + "semantic": { + "similarity_threshold": 0.90, + "ttl": 300, + "embedding": { + "provider": "openai", + "endpoint": "http://127.0.0.1:1990/v1/embeddings";, + "api_key": "test-key" + } + }, + "cache_key": { + "include_vars": ["$http_x_tenant_id"] + }, + "redis_host": "127.0.0.1" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 10: tenant-a first request - MISS, writes to L2 with scope=hash(tenant-a) +--- request +POST /scoped-semantic +{"messages":[{"role":"user","content":"What is the capital of France??"}]} +--- more_headers +Content-Type: application/json +X-Tenant-Id: tenant-a +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS +--- wait: 1 + + + +=== TEST 11: tenant-b same prompt - MISS (FT.SEARCH scope filter excludes tenant-a's entry) +--- request +POST /scoped-semantic +{"messages":[{"role":"user","content":"What is the capital of France??"}]} +--- more_headers +Content-Type: application/json +X-Tenant-Id: tenant-b +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS + + + +=== TEST 12: tenant-a paraphrase - HIT-L2 (scope filter finds tenant-a's entry) +--- request +POST /scoped-semantic +{"messages":[{"role":"user","content":"Name the capital city of France"}]} +--- more_headers +Content-Type: application/json +X-Tenant-Id: tenant-a +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: HIT-L2 Review Comment: This file’s semantic-scope tests rely on Redis Stack/RediSearch for vector search (`FT.CREATE`/`FT.SEARCH`). The repository’s CI uses vanilla `redis:latest`, so these tests will fail unless they’re skipped/gated when RediSearch isn’t available (or the CI Redis image is changed). ########## docs/en/latest/plugins/ai-cache.md: ########## @@ -0,0 +1,1085 @@ +--- +title: ai-cache +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-cache +description: The ai-cache Plugin caches LLM responses in Redis so identical or semantically similar prompts are served from cache, reducing latency and upstream cost. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +<head> + <link rel="canonical" href="https://docs.api7.ai/hub/ai-cache"; /> +</head> + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Description + +The `ai-cache` Plugin caches LLM responses in Redis so identical or semantically similar prompts are served from cache instead of incurring another upstream call. It supports two cache layers: an exact-match layer (`exact`) keyed by a hash of the prompt, and a semantic layer (`semantic`) that compares prompt embeddings via Redis Stack vector search. Either layer can be enabled independently, and a hit on the semantic layer backfills the exact layer so subsequent identical prompts return immediately. + +The Plugin should be used together with [ai-proxy](./ai-proxy.md) or [ai-proxy-multi](./ai-proxy-multi.md) on the same Route. The semantic layer requires Redis Stack with the RediSearch module and an embedding provider (OpenAI or Azure OpenAI). + +## Plugin Attributes + +| Name | Type | Required | Default | Valid values | Description | +| --- | --- | --- | --- | --- | --- | +| `layers` | array[string] | False | `["exact", "semantic"]` | `"exact"`, `"semantic"` | Cache layers to enable, queried in order. | +| `exact.ttl` | integer | False | `3600` | ≥ 1 | Time-to-live in seconds for exact-layer entries. | +| `semantic.similarity_threshold` | number | False | `0.95` | 0–1 | Minimum cosine similarity required for a semantic-layer hit. | +| `semantic.ttl` | integer | False | `86400` | ≥ 1 | Time-to-live in seconds for semantic-layer entries. | +| `semantic.embedding.provider` | string | True (if semantic enabled) | | `"openai"`, `"azure_openai"` | Embedding API provider. | +| `semantic.embedding.endpoint` | string | True (if semantic enabled) | | | HTTPS URL of the embedding API. | +| `semantic.embedding.api_key` | string | True (if semantic enabled) | | | API key for the embedding provider. Stored encrypted. | +| `semantic.embedding.model` | string | False | | | Embedding model name. Uses provider default if omitted. | +| `cache_key.include_consumer` | boolean | False | `false` | | If `true`, partition the cache by consumer name. | +| `cache_key.include_vars` | array[string] | False | `[]` | | Additional `ctx.var` names included in the cache key, for example `["$http_x_tenant_id"]`. | +| `bypass_on` | array[object] | False | | | List of `{header, equals}` rules. If any matches, the request bypasses the cache. | +| `max_cache_body_size` | integer | False | `1048576` | ≥ 1 | Maximum response size in bytes to write to cache. Larger responses pass through but are not cached. | Review Comment: The attributes table is missing `semantic.top_k` even though the plugin schema exposes it, and `semantic.embedding.endpoint` is described as an "HTTPS URL" even though the implementation only logs a warning for non-HTTPS (tests use `http://127.0.0.1/...`). Please align the docs with actual behavior (either document `top_k` + implement it, or remove it; and clarify that HTTPS is recommended rather than required unless you plan to enforce it). ########## apisix/plugins/ai-cache.lua: ########## @@ -0,0 +1,285 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") +local schema = require("apisix.plugins.ai-cache.schema") +local exact = require("apisix.plugins.ai-cache.exact") +local semantic = require("apisix.plugins.ai-cache.semantic") +local protocols = require("apisix.plugins.ai-protocols") +local http = require("resty.http") +local ngx_time = ngx.time +local ngx_now = ngx.now +local tostring = tostring +local table_concat = table.concat + +local plugin_name = "ai-cache" + +local _M = { + version = 0.1, + priority = 1065, + name = plugin_name, + schema = schema.schema +} + + +local function layer_enabled(conf, name) + local layers = conf.layers or { "exact", "semantic" } + for _, l in ipairs(layers) do + if l == name then return true end + end + return false +end + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema.schema, conf) + if not ok then + return false, err + end + + if layer_enabled(conf, "semantic") then + if not (conf.semantic and conf.semantic.embedding) then + return false, "semantic layer requires semantic.embedding to be configured" + end + end + + core.utils.check_https({ "semantic.embedding.endpoint" }, conf, plugin_name) + + return true +end + + +function _M.access(conf, ctx) + -- Check bypass_on conditions + if conf.bypass_on then + local req_headers = ngx.req.get_headers() + for _, rule in ipairs(conf.bypass_on) do + if req_headers[rule.header] == rule.equals then + ctx.ai_cache_bypass = true + ctx.ai_cache_status = "BYPASS" + return + end + end + end + + local body_tab, err = core.request.get_json_request_body_table() + if not body_tab then + core.log.warn("ai-cache: failed to read request body: ", err or "unknown error") + ctx.ai_cache_status = "MISS" + return + end + + local protocol_name = protocols.detect(body_tab, ctx) + if not protocol_name then + core.log.warn("ai-cache: could not detect AI protocol, skipping cache") + ctx.ai_cache_status = "MISS" + return + end + + local proto = protocols.get(protocol_name) + local contents = proto.extract_request_content(body_tab) + if not contents or #contents == 0 then + ctx.ai_cache_status = "MISS" + return + end + + local prompt_text = table_concat(contents, " ") + local scope_hash = exact.compute_scope_hash(conf, ctx) + local prompt_hash, hash_err = exact.compute_prompt_hash(prompt_text) + if not prompt_hash then + core.log.warn("ai-cache: failed to compute prompt hash: ", hash_err) + ctx.ai_cache_status = "MISS" + return + end + + local is_stream = body_tab.stream == true + + -- L1 exact lookup + if layer_enabled(conf, "exact") then + local cached_text, written_at, lookup_err = exact.get(conf, scope_hash, prompt_hash) + if lookup_err then + core.log.warn("ai-cache: L1 lookup error: ", lookup_err) + elseif cached_text then + core.log.info("ai-cache: L1 hit for key: ", prompt_hash) + ctx.ai_cache_status = "HIT-L1" + ctx.ai_cache_written_at = written_at + if is_stream then + core.response.set_header("Content-Type", "text/event-stream") + else + core.response.set_header("Content-Type", "application/json") + end + return core.response.exit(200, proto.build_deny_response({ + stream = is_stream, + text = cached_text, + })) + end + end + + -- L2 semantic lookup + if layer_enabled(conf, "semantic") then + local emb_conf = conf.semantic.embedding + local emb_driver = require("apisix.plugins.ai-cache.embeddings." .. emb_conf.provider) + local httpc = http.new() + + local t0 = ngx_now() + local embedding, _, emb_err = emb_driver.get_embeddings(emb_conf, prompt_text, httpc, true) + if not embedding then + core.log.warn("ai-cache: embedding fetch failed (degrading to MISS): ", emb_err) + ctx.ai_cache_embedding_failed = true + else + ctx.ai_cache_embedding_latency_ms = (ngx_now() - t0) * 1000 + ctx.ai_cache_embedding_provider = emb_conf.provider + ctx.ai_cache_embedding = embedding + + local threshold = conf.semantic.similarity_threshold or 0.95 + local cached_text, similarity, search_err = semantic.search( + conf, scope_hash, embedding, threshold + ) + + if search_err then + core.log.warn("ai-cache: L2 search error (degrading to MISS): ", search_err) + elseif cached_text then + core.log.info("ai-cache: L2 hit, similarity=", similarity) + + local l1_ttl = (conf.exact and conf.exact.ttl) or 3600 + local l1_err = exact.set(conf, scope_hash, prompt_hash, cached_text, l1_ttl) + + if l1_err then + core.log.warn("ai-cache: L2->L1 backfill failed: ", l1_err) + end + + ctx.ai_cache_status = "HIT-L2" + ctx.ai_cache_similarity = similarity + if is_stream then + core.response.set_header("Content-Type", "text/event-stream") + else + core.response.set_header("Content-Type", "application/json") + end + return core.response.exit(200, proto.build_deny_response({ + stream = is_stream, + text = cached_text, + })) + end + end + end + + ctx.ai_cache_status = "MISS" + ctx.ai_cache_scope_hash = scope_hash + ctx.ai_cache_prompt_hash = prompt_hash + ctx.ai_cache_prompt_text = prompt_text +end + + +function _M.header_filter(conf, ctx) + if not ctx.ai_cache_status then + return + end + + local status_header = (conf.headers and conf.headers.cache_status) + or "X-AI-Cache-Status" + ngx.header[status_header] = ctx.ai_cache_status + + if ctx.ai_cache_status == "HIT-L1" and ctx.ai_cache_written_at then + local age_header = (conf.headers and conf.headers.cache_age) + or "X-AI-Cache-Age" + ngx.header[age_header] = tostring(ngx_time() - ctx.ai_cache_written_at) + end + + if ctx.ai_cache_status == "HIT-L2" and ctx.ai_cache_similarity then + local sim_header = (conf.headers and conf.headers.cache_similarity) + or "X-AI-Cache-Similarity" + ngx.header[sim_header] = tostring(ctx.ai_cache_similarity) + end +end + + +function _M.log(conf, ctx) + if ctx.ai_cache_status ~= "MISS" then + return + end + + local upstream_status = core.response.get_upstream_status(ctx) or ngx.status + if not upstream_status or upstream_status < 200 or upstream_status >= 300 then + return + end + + local response_text = ctx.var.llm_response_text + if not response_text or response_text == "" then + return + end + + local max_size = conf.max_cache_body_size or 1048576 + if #response_text > max_size then + core.log.warn("ai-cache: response size ", #response_text, + " exceeds max_cache_body_size ", max_size, + ", skipping cache write") + return + end + + local exact_enabled = layer_enabled(conf, "exact") + local semantic_enabled = layer_enabled(conf, "semantic") + local ttl_exact = (conf.exact and conf.exact.ttl) or 3600 + local scope_hash = ctx.ai_cache_scope_hash + local prompt_hash = ctx.ai_cache_prompt_hash + local embedding = ctx.ai_cache_embedding + local prompt_text = ctx.ai_cache_prompt_text + Review Comment: `log()` assumes `ctx.ai_cache_scope_hash` / `ctx.ai_cache_prompt_hash` / `ctx.ai_cache_prompt_text` were set in `access()`, but there are multiple early-return paths that set `ctx.ai_cache_status = "MISS"` without populating these fields (eg request body parse failure / protocol detection failure). In those cases, `exact.set()` / embedding fetch will be called with nil values inside the timer and will error on string concatenation. Consider only setting `ai_cache_status` to MISS once the cache key has been computed, or add an explicit guard in `log()` to return when the key fields are missing. ```suggestion if (exact_enabled or semantic_enabled) and not scope_hash then core.log.warn("ai-cache: missing scope hash, skipping cache write") return end if exact_enabled and not prompt_hash then core.log.warn("ai-cache: missing prompt hash, skipping exact cache write") return end if semantic_enabled and not embedding and not prompt_text then core.log.warn("ai-cache: missing prompt text, skipping semantic cache write") return end ``` ########## apisix/plugins/prometheus/exporter.lua: ########## @@ -260,6 +268,35 @@ function _M.http_init(prometheus_enabled_in_stream) unpack(extra_labels("llm_active_connections"))}, llm_active_connections_exptime) + metrics.ai_cache_hits = prometheus:counter("ai_cache_hits_total", + "AI cache hit count by layer", + {"route_id", "service_id", "consumer", "layer", + unpack(extra_labels("ai_cache_hits"))}, + ai_cache_hits_exptime) + + metrics.ai_cache_misses = prometheus:counter("ai_cache_misses_total", + "AI cache miss count", + {"route_id", "service_id", "consumer", + unpack(extra_labels("ai_cache_misses"))}, + ai_cache_misses_exptime) + + local ai_cache_embedding_latency_buckets = DEFAULT_BUCKETS + if attr and attr.ai_cache_embedding_latency_buckets then + ai_cache_embedding_latency_buckets = attr.ai_cache_embedding_latency_buckets + end + metrics.ai_cache_embedding_latency = prometheus:histogram("ai_cache_embedding_latency", + "AI cache embedding API call latency in milliseconds", + {"route_id", "service_id", "consumer", "provider", + unpack(extra_labels("ai_cache_embedding_latency"))}, Review Comment: This introduces a new histogram `ai_cache_embedding_latency` but the name doesn’t include a base unit suffix even though the value observed is milliseconds (and this file’s own guidance asks new metrics to include units). Consider renaming to something like `ai_cache_embedding_latency_ms` (and updating tests/docs accordingly) to make the unit unambiguous. ```suggestion metrics.ai_cache_embedding_latency = prometheus:histogram("ai_cache_embedding_latency_ms", "AI cache embedding API call latency in milliseconds", {"route_id", "service_id", "consumer", "provider", unpack(extra_labels("ai_cache_embedding_latency_ms"))}, ``` ########## apisix/plugins/ai-cache.lua: ########## @@ -0,0 +1,285 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") +local schema = require("apisix.plugins.ai-cache.schema") +local exact = require("apisix.plugins.ai-cache.exact") +local semantic = require("apisix.plugins.ai-cache.semantic") +local protocols = require("apisix.plugins.ai-protocols") +local http = require("resty.http") +local ngx_time = ngx.time +local ngx_now = ngx.now +local tostring = tostring +local table_concat = table.concat + +local plugin_name = "ai-cache" + +local _M = { + version = 0.1, + priority = 1065, + name = plugin_name, + schema = schema.schema +} + + +local function layer_enabled(conf, name) + local layers = conf.layers or { "exact", "semantic" } + for _, l in ipairs(layers) do + if l == name then return true end + end + return false +end + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema.schema, conf) + if not ok then + return false, err + end + + if layer_enabled(conf, "semantic") then + if not (conf.semantic and conf.semantic.embedding) then + return false, "semantic layer requires semantic.embedding to be configured" + end + end + + core.utils.check_https({ "semantic.embedding.endpoint" }, conf, plugin_name) + + return true +end + + +function _M.access(conf, ctx) + -- Check bypass_on conditions + if conf.bypass_on then + local req_headers = ngx.req.get_headers() + for _, rule in ipairs(conf.bypass_on) do + if req_headers[rule.header] == rule.equals then + ctx.ai_cache_bypass = true + ctx.ai_cache_status = "BYPASS" + return + end + end + end + + local body_tab, err = core.request.get_json_request_body_table() + if not body_tab then + core.log.warn("ai-cache: failed to read request body: ", err or "unknown error") + ctx.ai_cache_status = "MISS" + return + end + + local protocol_name = protocols.detect(body_tab, ctx) + if not protocol_name then + core.log.warn("ai-cache: could not detect AI protocol, skipping cache") + ctx.ai_cache_status = "MISS" + return + end + + local proto = protocols.get(protocol_name) + local contents = proto.extract_request_content(body_tab) + if not contents or #contents == 0 then + ctx.ai_cache_status = "MISS" + return + end + + local prompt_text = table_concat(contents, " ") + local scope_hash = exact.compute_scope_hash(conf, ctx) + local prompt_hash, hash_err = exact.compute_prompt_hash(prompt_text) + if not prompt_hash then + core.log.warn("ai-cache: failed to compute prompt hash: ", hash_err) + ctx.ai_cache_status = "MISS" + return + end + + local is_stream = body_tab.stream == true + + -- L1 exact lookup + if layer_enabled(conf, "exact") then + local cached_text, written_at, lookup_err = exact.get(conf, scope_hash, prompt_hash) + if lookup_err then + core.log.warn("ai-cache: L1 lookup error: ", lookup_err) + elseif cached_text then + core.log.info("ai-cache: L1 hit for key: ", prompt_hash) + ctx.ai_cache_status = "HIT-L1" + ctx.ai_cache_written_at = written_at + if is_stream then + core.response.set_header("Content-Type", "text/event-stream") + else + core.response.set_header("Content-Type", "application/json") + end + return core.response.exit(200, proto.build_deny_response({ + stream = is_stream, + text = cached_text, + })) + end Review Comment: Cache hits are returned using `proto.build_deny_response(...)`. That helper is intended for policy denials and produces protocol-specific *error/deny* shapes for some protocols (eg `openai-embeddings` returns an `error` object), which can make cached responses invalid if this plugin is ever applied to non-chat endpoints. It would be safer to either (1) explicitly restrict ai-cache to chat-style protocols only, or (2) add a protocol method dedicated to building a successful cached response (and include required fields like `model`/`usage` where applicable). ########## apisix/plugins/ai-cache/semantic.lua: ########## @@ -0,0 +1,180 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local apisix_redis = require("apisix.utils.redis") +local uuid = require("resty.jit-uuid") +local ffi = require("ffi") + +local ffi_new = ffi.new +local ffi_string = ffi.string +local ngx_time = ngx.time +local tostring = tostring +local tonumber = tonumber +local type = type + +local _M = {} + + +local function index_name(dim) + return "ai-cache-idx-" .. dim +end + + +local function key_prefix(dim) + return "ai-cache:l2:" .. dim .. ":" +end + +local function pack_vector(vec) + local n = #vec + local buf = ffi_new("float[?]", n) + for i = 0, n - 1 do + buf[i] = vec[i + 1] + end + return ffi_string(buf, n * 4) +end + +local index_ready = {} + +local function ensure_index(red, dim) + if index_ready[dim] then + return true + end + + local _, err = red["FT.CREATE"](red, + index_name(dim), + "ON", "HASH", + "PREFIX", "1", key_prefix(dim), + "SCHEMA", + "embedding", "VECTOR", "HNSW", "6", + "TYPE", "FLOAT32", + "DIM", tostring(dim), + "DISTANCE_METRIC", "COSINE", + "scope", "TAG", + "created_at", "NUMERIC" + ) + + if err and not err:find("already exists") then + return nil, "FT.CREATE failed: " .. err + end + + index_ready[dim] = true + return true +end + + +function _M.search(conf, scope_hash, embedding_vec, threshold) + local red, err = apisix_redis.new(conf) + if not red then + return nil, nil, err + end + + local ok, init_err = ensure_index(red, #embedding_vec) + if not ok then + red:set_keepalive(conf.redis_keepalive_timeout, conf.redis_keepalive_pool) + return nil, nil, init_err + end + + local binary_vec = pack_vector(embedding_vec) + + local query + if scope_hash == "" then + query = "*=>[KNN 1 @embedding $vec AS dist]" + else + query = "@scope:{" .. scope_hash .. "}=>[KNN 1 @embedding $vec AS dist]" + end + + local res, search_err = red["FT.SEARCH"](red, + index_name(#embedding_vec), + query, + "PARAMS", "2", "vec", binary_vec, + "SORTBY", "dist", "ASC", + "LIMIT", "0", "1", + "RETURN", "2", "response", "dist", + "DIALECT", "2" + ) Review Comment: `semantic.top_k` exists in the schema, but the search query is hard-coded to `KNN 1` and `LIMIT 0 1`, so the configuration is ignored. Either remove `top_k` from the schema/docs or plumb it through (use `conf.semantic.top_k` in the query and iterate candidates until one meets `similarity_threshold`, or return the best match among top_k). ########## t/plugin/prometheus-ai-cache.t: ########## @@ -0,0 +1,305 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BEGIN { + $ENV{TEST_ENABLE_CONTROL_API_V1} = "0"; + + if ($ENV{TEST_NGINX_CHECK_LEAK}) { + $SkipReason = "unavailable for the hup tests"; + } else { + $ENV{TEST_NGINX_USE_HUP} = 1; + undef $ENV{TEST_NGINX_USE_STAP}; + } +} + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + + my $user_yaml_config = <<_EOC_; +plugin_attr: + prometheus: + refresh_interval: 0.1 +plugins: + - ai-proxy + - ai-cache + - prometheus + - public-api +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); + + if (!defined $block->http_config) { + $block->set_value("http_config", <<_EOC_); +server { + listen 1990; + default_type 'application/json'; + + location /v1/embeddings { + content_by_lua_block { + local fixture_loader = require("lib.fixture_loader") + local content, err = fixture_loader.load("openai/embeddings-list.json") + if not content then + ngx.status = 500 + ngx.say(err) + return + end + + ngx.status = 200 + ngx.print(content) + } + } +} +_EOC_ + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: set up routes +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + local routes = { + { + url = "/apisix/admin/routes/1", + data = [[{ + "uri": "/chat", + "plugins": { + "prometheus": {}, + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact"], + "exact": { "ttl": 60 }, + "redis_host": "127.0.0.1", + "bypass_on": [{"header": "X-Cache-Bypass", "equals": "1"}] + } + } + }]], + }, + { + url = "/apisix/admin/routes/2", + data = [[{ + "uri": "/semantic", + "plugins": { + "prometheus": {}, + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact", "semantic"], + "exact": { "ttl": 60 }, + "semantic": { + "similarity_threshold": 0.90, + "ttl": 300, + "embedding": { + "provider": "openai", + "endpoint": "http://127.0.0.1:1990/v1/embeddings";, + "api_key": "test-key" + } + }, + "redis_host": "127.0.0.1" + } + } + }]], + }, + { + url = "/apisix/admin/routes/metrics", + data = [[{ + "plugins": { + "public-api": {} + }, + "uri": "/apisix/prometheus/metrics" + }]], + }, + } + + for _, route in ipairs(routes) do + local code, body = t(route.url, ngx.HTTP_PUT, route.data) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + end + } + } +--- response_body eval +"passed\n" x 3 + + + +=== TEST 2: MISS request - upstream called +--- request +POST /chat +{"messages":[{"role":"user","content":"What is the meaning of life?"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS + + + +=== TEST 3: same request - HIT-L1 +--- request +POST /chat +{"messages":[{"role":"user","content":"What is the meaning of life?"}]} +--- more_headers +Content-Type: application/json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: HIT-L1 +--- wait: 1 + + + +=== TEST 4: verify miss counter +--- request +GET /apisix/prometheus/metrics +--- response_body_like eval +qr/apisix_ai_cache_misses_total\{route_id="1",service_id="",consumer=""\} 1/ + + + +=== TEST 5: verify hit counter with layer label +--- request +GET /apisix/prometheus/metrics +--- response_body_like eval +qr/apisix_ai_cache_hits_total\{route_id="1",service_id="",consumer="",layer="l1"\} 1/ + + + +=== TEST 6: BYPASS request - upstream called, no cache interaction +--- request +POST /chat +{"messages":[{"role":"user","content":"What is the meaning of life?"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +X-Cache-Bypass: 1 +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: BYPASS + + + +=== TEST 7: verify BYPASS did not increment misses counter +--- request +GET /apisix/prometheus/metrics +--- response_body_like eval +qr/apisix_ai_cache_misses_total\{route_id="1",service_id="",consumer=""\} 1\n/ + + + +=== TEST 8: cleanup Redis L2 state before semantic tests +--- config + location /t { + content_by_lua_block { + local redis = require("resty.redis") + local red = redis:new() + red:set_timeout(1000) + assert(red:connect("127.0.0.1", 6379)) + + red["FT.DROPINDEX"](red, "ai-cache-idx", "DD") + + local keys = red:keys("ai-cache:*") + if type(keys) == "table" and #keys > 0 then + red:del(unpack(keys)) + end + + red:close() + ngx.say("ok") + } + } +--- response_body +ok + + + +=== TEST 9: L2 first request - MISS, embedding API called +--- request +POST /semantic +{"messages":[{"role":"user","content":"What is the capital of France??"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS +--- wait: 1 + + + +=== TEST 10: L2 second request - different wording, HIT-L2 +--- request +POST /semantic +{"messages":[{"role":"user","content":"Name the capital city of France"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: HIT-L2 +--- wait: 1 + + + +=== TEST 11: verify hits counter with layer="l2" +--- request +GET /apisix/prometheus/metrics +--- response_body_like eval +qr/apisix_ai_cache_hits_total\{route_id="2",service_id="",consumer="",layer="l2"\} 1/ + + + +=== TEST 12: verify embedding latency histogram with provider label +--- request +GET /apisix/prometheus/metrics +--- response_body_like eval +qr/apisix_ai_cache_embedding_latency_count\{route_id="2",service_id="",consumer="",provider="openai"\} 2/ Review Comment: These semantic cache assertions (`HIT-L2`, embedding latency metrics) require Redis Stack/RediSearch, but the project’s default CI Redis container is `redis:latest` without RediSearch. Without gating, this test will fail in CI. Consider skipping the semantic part when `FT.SEARCH` is unavailable, or updating CI to run Redis Stack for the test suite. ########## t/plugin/ai-cache.t: ########## @@ -0,0 +1,845 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BEGIN { + $ENV{TEST_ENABLE_CONTROL_API_V1} = "0"; +} + +use t::APISIX 'no_plan'; + +log_level("info"); +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->error_log && !$block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } + + if (!defined $block->http_config) { + $block->set_value("http_config", <<_EOC_); +server { + listen 1990; + default_type 'application/json'; + + location /v1/embeddings { + content_by_lua_block { + local fixture_loader = require("lib.fixture_loader") + local content, err = fixture_loader.load("openai/embeddings-list.json") + if not content then + ngx.status = 500 + ngx.say(err) + return + end + + ngx.status = 200 + ngx.print(content) + } + } +} +_EOC_ + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: valid config - exact layer only +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.ai-cache") + local ok, err = plugin.check_schema({ + layers = { "exact" }, + exact = { ttl = 600 }, + redis_host = "127.0.0.1", + redis_port = 6379, + }) + + if not ok then + ngx.say("failed") + else + ngx.say("passed") + end + } + } +--- response_body +passed + + + +=== TEST 2: valid config - both layers with semantic embedding +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.ai-cache") + local ok, err = plugin.check_schema({ + layers = { "exact", "semantic" }, + exact = { ttl = 3600 }, + semantic = { + similarity_threshold = 0.95, + ttl = 86400, + embedding = { + provider = "openai", + endpoint = "https://api.openai.com/v1/embeddings";, + api_key = "sk-test", + }, + }, + redis_host = "127.0.0.1", + redis_port = 6379, + }) + + if not ok then + ngx.say(err) + else + ngx.say("passed") + end + } + } +--- response_body +passed + + + +=== TEST 3: semantic without embedding config - should fail +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.ai-cache") + local ok, err = plugin.check_schema({ + layers = { "semantic" }, + redis_host = "127.0.0.1", + }) + if not ok then + ngx.say("failed: ", err) + else + ngx.say("passed") + end + } + } +--- response_body +failed: semantic layer requires semantic.embedding to be configured + + + +=== TEST 4: invalid layer value - should fail +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.ai-cache") + local ok, err = plugin.check_schema({ + layers = { "invalid_layer" }, + }) + if not ok then + ngx.say("failed") + else + ngx.say("passed") + end + } + } +--- response_body +failed + + + +=== TEST 5: unsupported embedding provider - should fail +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.ai-cache") + local ok, err = plugin.check_schema({ + layers = { "semantic" }, + semantic = { + embedding = { + provider = "some-unknown-provider", + endpoint = "https://example.com/embeddings";, + api_key = "key", + }, + }, + }) + + if not ok then + ngx.say("failed") + else + ngx.say("passed") + end + } + } +--- response_body +failed + + + +=== TEST 6: similarity_threshold out of range - should fail +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.ai-cache") + local ok, err = plugin.check_schema({ + layers = { "semantic" }, + semantic = { + similarity_threshold = 1.5, + embedding = { + provider = "openai", + endpoint = "https://api.openai.com/v1/embeddings";, + api_key = "sk-test", + }, + }, + }) + + if not ok then + ngx.say("failed") + else + ngx.say("passed") + end + } + } +--- response_body +failed + + + +=== TEST 7: set up route for L1 cache tests +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/chat", + "plugins": { + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact"], + "exact": { "ttl": 60 }, + "redis_host": "127.0.0.1", + "bypass_on": [{"header": "X-Cache-Bypass", "equals": "1"}] + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 8: first request - cache MISS, upstream called +--- request +POST /chat +{"messages":[{"role":"user","content":"What is the answer to life?"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS +--- response_body_like eval +qr/content/ + + + +=== TEST 9: second identical request - cache HIT-L1, no upstream call +--- request +POST /chat +{"messages":[{"role":"user","content":"What is the answer to life?"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: HIT-L1 +--- response_body_like eval +qr/content/ +--- error_log +ai-cache: L1 hit for key + + + +=== TEST 10: bypass header - BYPASS, upstream called, not cached +--- request +POST /chat +{"messages":[{"role":"user","content":"What is the bypass question?"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +X-Cache-Bypass: 1 +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: BYPASS + + + +=== TEST 11: same prompt without bypass after bypass - still MISS (bypass did not cache) +--- request +POST /chat +{"messages":[{"role":"user","content":"What is the bypass question?"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: MISS + + + +=== TEST 12: set up route with two bypass rules +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/chat", + "plugins": { + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact"], + "exact": { "ttl": 60 }, + "redis_host": "127.0.0.1", + "bypass_on": [ + {"header": "X-Cache-Bypass", "equals": "1"}, + {"header": "X-Debug", "equals": "true"} + ] + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 13: first bypass rule matches - BYPASS +--- request +POST /chat +{"messages":[{"role":"user","content":"multi-rule bypass test"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +X-Cache-Bypass: 1 +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: BYPASS + + + +=== TEST 14: second bypass rule matches - BYPASS +--- request +POST /chat +{"messages":[{"role":"user","content":"multi-rule bypass test"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +X-Debug: true +--- error_code: 200 +--- response_headers +X-AI-Cache-Status: BYPASS + + + +=== TEST 15: set up route for 4xx test +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + [[{ + "uri": "/error", + "plugins": { + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact"], + "exact": { "ttl": 60 }, + "redis_host": "127.0.0.1" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 16: 4xx from upstream - not cached +--- request +POST /error +{"messages":[{"role":"user","content":"trigger an error please"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +X-AI-Fixture-Status: 400 +--- error_code: 400 +--- response_headers +X-AI-Cache-Status: MISS + + + +=== TEST 17: same prompt after 4xx - still MISS (4xx was not cached) +--- request +POST /error +{"messages":[{"role":"user","content":"trigger an error please"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-basic.json +X-AI-Fixture-Status: 400 +--- error_code: 400 +--- response_headers +X-AI-Cache-Status: MISS + + + +=== TEST 18: openai driver - parses embedding vector correctly +--- http_config +server { + listen 1990; + default_type 'application/json'; + + location /v1/embeddings { + content_by_lua_block { + local cjson = require("cjson.safe") + ngx.req.read_body() + local body = cjson.decode(ngx.req.get_body_data()) + + if ngx.req.get_headers()["Authorization"] ~= "Bearer test-key" then + ngx.status = 401 + ngx.say('{"error":"unauthorized"}') + return + end + + ngx.status = 200 + ngx.say(cjson.encode({ + data = { + { embedding = {0.1, 0.2, 0.3}, index = 0, object = "embedding" } + }, + model = body.model, + object = "list" + })) + } + } +} +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local driver = require("apisix.plugins.ai-cache.embeddings.openai") + + local httpc = http.new() + local conf = { + endpoint = "http://127.0.0.1:1990/v1/embeddings";, + api_key = "test-key", + model = "text-embedding-3-small", + } + + local embedding, status, err = driver.get_embeddings(conf, "hello world", httpc, false) + if not embedding then + ngx.say("error: ", err) + return + end + + if #embedding ~= 3 then + ngx.say("wrong length: ", #embedding) + return + end + + ngx.say("ok: ", embedding[1], " ", embedding[2], " ", embedding[3]) + } + } +--- response_body +ok: 0.1 0.2 0.3 + + + +=== TEST 19: openai driver - 429 from API return nil with status +--- http_config +server { + listen 1990; + default_type 'application/json'; + + location /v1/embeddings { + content_by_lua_block { + ngx.status = 429 + ngx.say('{"error":{"message":"rate limit exceeded","type":"requests"}}') + } + } +} +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local driver = require("apisix.plugins.ai-cache.embeddings.openai") + + local httpc = http.new() + local conf = { + endpoint = "http://127.0.0.1:1990/v1/embeddings";, + api_key = "test-key", + } + + local embedding, status, err = driver.get_embeddings(conf, "hello", httpc, false) + if embedding then + ngx.say("unexpected success") + return + end + + ngx.say("status: ", status) + } + } +--- response_body +status: 429 + + + +=== TEST 20: azure_openai driver - parses embedding vector correctly +--- http_config +server { + listen 1990; + default_type 'application/json'; + + location /embeddings { + content_by_lua_block { + local cjson = require("cjson.safe") + + if ngx.req.get_headers()["api-key"] ~= "azure-test-key" then + ngx.status = 401 + ngx.say('{"error":"unauthorized"}') + return + end + + ngx.status = 200 + ngx.say(cjson.encode({ + data = { + { embedding = {0.4, 0.5, 0.6}, index = 0, object = "embedding" } + }, + object = "list" + })) + } + } +} +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local driver = require("apisix.plugins.ai-cache.embeddings.azure_openai") + + local httpc = http.new() + local conf = { + endpoint = "http://127.0.0.1:1990/embeddings";, + api_key = "azure-test-key", + } + + local embedding, status, err = driver.get_embeddings(conf, "hello world", httpc, false) + if not embedding then + ngx.say("error: ", err) + return + end + + ngx.say("ok: ", embedding[1], " ", embedding[2], " ", embedding[3]) + } + } +--- response_body +ok: 0.4 0.5 0.6 + + + +=== TEST 21: openai driver - 500 from API returns nil with status +--- http_config +server { + listen 1990; + default_type 'application/json'; + + location /v1/embeddings { + content_by_lua_block { + ngx.status = 500 + ngx.say('{"error":{"message":"internal server error"}}') + } + } +} +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local driver = require("apisix.plugins.ai-cache.embeddings.openai") + + local httpc = http.new() + local conf = { + endpoint = "http://127.0.0.1:1990/v1/embeddings";, + api_key = "test-key", + } + + local embedding, status, err = driver.get_embeddings(conf, "hello", httpc, false) + if embedding then + ngx.say("unexpected success") + return + end + + ngx.say("status: ", status) + } + } +--- response_body +status: 500 + + + +=== TEST 22: clean up L2 state before semantic tests +--- config + location /t { + content_by_lua_block { + local redis = require("resty.redis") + local red = redis:new() + red:set_timeout(1000) + assert(red:connect("127.0.0.1", 6379)) + + red["FT.DROPINDEX"](red, "ai-cache-idx", "DD") + red["FT.DROPINDEX"](red, "ai-cache-idx-3", "DD") + + local keys = red:keys("ai-cache:*") + if type(keys) == "table" and #keys > 0 then + red:del(unpack(keys)) + end + + red:close() + ngx.say("ok") + } + } +--- response_body +ok + + + +=== TEST 23: set up route for L2 semantic cache tests +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/3', + ngx.HTTP_PUT, + [[{ + "uri": "/semantic", + "plugins": { + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer test-key" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions"; + } + }, + "ai-cache": { + "layers": ["exact", "semantic"], + "exact": { + "ttl": 60 + }, + "semantic": { + "similarity_threshold": 0.90, + "ttl": 300, + "embedding": { + "provider": "openai", + "endpoint": "http://127.0.0.1:1990/v1/embeddings";, + "api_key": "test-key" + } + }, + "redis_host": "127.0.0.1" + } + } + }]] Review Comment: The semantic-layer tests here assume Redis has RediSearch available (`FT.CREATE`/`FT.SEARCH`). The repo’s CI docker-compose uses `redis:latest` (vanilla Redis) which won’t support these commands, so these cases will consistently fail in CI unless the test is gated or the CI Redis image is switched to Redis Stack. Consider detecting RediSearch support at runtime and `skip` semantic tests when unavailable, or updating the test environment to provide Redis Stack. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
