yeganeahmadnejad commented on code in PR #13764:
URL: https://github.com/apache/apisix/pull/13764#discussion_r3685548917
##########
apisix/plugins/api-breaker.lua:
##########
@@ -195,8 +383,145 @@ function _M.access(conf, ctx)
return
end
+-- Ratio-based circuit breaker
+local function ratio_based_access(conf, ctx)
+ -- Check and reset sliding window first to ensure consistent state
+ check_and_reset_window(ctx, conf)
-function _M.log(conf, ctx)
+ local current_state = get_circuit_breaker_state(ctx)
+ local current_time = ngx.time()
+
+ -- Handle OPEN state
+ if current_state == OPEN then
+ local last_change_key = gen_last_state_change_key(ctx)
+ local last_change_time, err = shared_buffer:get(last_change_key)
+ if err then
+ core.log.warn("failed to get last change time: ", err)
+ return conf.break_response_code or 503,
+ conf.break_response_body or "Service temporarily
unavailable"
+ end
+
+ local wait_duration = conf.max_breaker_sec or 300
+ if last_change_time and (current_time - last_change_time) >=
wait_duration then
+ -- Use atomic operation to ensure only one request transitions to
HALF_OPEN
+ local transition_key = "cb-transition-" ..
core.request.get_host(ctx) .. ctx.var.uri
+ local transition_success
+ transition_success, err = shared_buffer:add(transition_key, 1, 1)
+
+ if err then
+ core.log.warn("failed to add transition lock: ", err)
+ end
+
+ if transition_success then
+ -- Transition to HALF_OPEN
+ set_circuit_breaker_state(ctx, HALF_OPEN)
+ -- Reset half-open counters
+ shared_buffer:set(gen_half_open_calls_key(ctx), 0)
+ shared_buffer:set(gen_half_open_success_key(ctx), 0)
+ core.log.info("Circuit breaker transitioned from OPEN to
HALF_OPEN")
+
+ -- Clean up transition lock
+ shared_buffer:delete(transition_key)
Review Comment:
Fixed in 6f9af8c1. The winning request now falls through into the same
admission code path used by later probes instead of returning early, so it's
counted as probe #1. The transition lock is left to expire on its own 1s TTL
instead of being deleted immediately, so a request racing in during that window
is conservatively rejected as still-open rather than risking another uncounted
admission.
##########
apisix/plugins/api-breaker.lua:
##########
@@ -133,13 +267,67 @@ local _M = {
schema = schema,
}
-
function _M.check_schema(conf)
return core.schema.check(schema, conf)
end
+-- Circuit breaker state management functions
+local function get_circuit_breaker_state(ctx)
+ local state_key = gen_state_key(ctx)
+ local state, err = shared_buffer:get(state_key)
+ if err then
+ core.log.warn("failed to get circuit breaker state: ", err)
+ return CLOSED
+ end
+ return state or CLOSED
+end
+
+local function set_circuit_breaker_state(ctx, state)
+ local state_key = gen_state_key(ctx)
+ local last_change_key = gen_last_state_change_key(ctx)
+ local current_time = ngx.time()
-function _M.access(conf, ctx)
+ shared_buffer:set(state_key, state)
+ shared_buffer:set(last_change_key, current_time)
+
+ core.log.info("Circuit breaker state changed to: ", state, " at: ",
current_time)
+end
+
+-- Sliding window management
+local function reset_sliding_window(ctx, current_time, window_size)
+ local window_start_key = gen_window_start_time_key(ctx)
+ local total_requests_key = gen_total_requests_key(ctx)
+ local unhealthy_key = gen_unhealthy_key(ctx)
+
+ shared_buffer:set(window_start_key, current_time)
+ shared_buffer:set(total_requests_key, 0)
Review Comment:
Fixed in 6f9af8c1. Replaced the single reset-on-expiry counter pair with
fixed time buckets named by their own epoch (cb-breq-<id>-<bucket_size>-<epoch>
/ cb-bfail-<id>-<bucket_size>-<epoch>). Aging out old data is now implicit — an
epoch outside the last N buckets just isn't summed — so there's no shared 'is
it time to reset' decision for concurrent requests to race on, and each
increment is a single atomic shared_buffer:incr. Added a harness-based
regression covering window aging (Scenario 6 in the linked test run) alongside
the existing sliding-window .t test.
##########
apisix/plugins/api-breaker.lua:
##########
@@ -61,70 +65,200 @@ local schema = {
type = "integer",
minimum = 3,
default = 300,
+ description = "Circuit breaker duration in seconds " ..
+ "(applies to both count and ratio policies)"
},
- unhealthy = {
- type = "object",
- properties = {
- http_statuses = {
- type = "array",
- minItems = 1,
- items = {
+ policy = {
+ type = "string",
+ enum = { "unhealthy-count", "unhealthy-ratio" },
+ default = "unhealthy-count",
+ }
+ },
+ required = { "break_response_code" },
+ ["if"] = {
+ properties = {
+ policy = {
+ enum = { "unhealthy-count" },
+ },
+ },
+ },
+ ["then"] = {
+ properties = {
+ unhealthy = {
+ type = "object",
+ properties = {
+ http_statuses = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "integer",
+ minimum = 500,
+ maximum = 599,
+ },
+ uniqueItems = true,
+ default = { 500 }
+ },
+ failures = {
type = "integer",
- minimum = 500,
- maximum = 599,
+ minimum = 1,
+ default = 3,
+ }
+ },
+ default = { http_statuses = { 500 }, failures = 3 }
+ },
+ healthy = {
+ type = "object",
+ properties = {
+ http_statuses = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "integer",
+ minimum = 200,
+ maximum = 499,
+ },
+ uniqueItems = true,
+ default = { 200 }
},
- uniqueItems = true,
- default = {500}
+ successes = {
+ type = "integer",
+ minimum = 1,
+ default = 3,
+ }
+ },
+ default = { http_statuses = { 200 }, successes = 3 }
+ }
+ }
+ },
+ ["else"] = {
+ ["if"] = {
+ properties = {
+ policy = {
+ enum = { "unhealthy-ratio" },
},
- failures = {
- type = "integer",
- minimum = 1,
- default = 3,
- }
},
- default = {http_statuses = {500}, failures = 3}
},
- healthy = {
- type = "object",
+ ["then"] = {
properties = {
- http_statuses = {
- type = "array",
- minItems = 1,
- items = {
- type = "integer",
- minimum = 200,
- maximum = 499,
+ unhealthy = {
+ type = "object",
+ properties = {
+ http_statuses = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "integer",
+ minimum = 500,
+ maximum = 599,
+ },
+ uniqueItems = true,
+ default = { 500 }
+ },
+ error_ratio = {
+ type = "number",
+ minimum = 0,
+ maximum = 1,
+ default = 0.5,
+ description = "Failure rate threshold to trigger
circuit breaker"
+ },
+ min_request_threshold = {
+ type = "integer",
+ minimum = 1,
+ default = 10,
+ description = "Minimum number of calls before " ..
+ "circuit breaker can be triggered"
+ },
+ sliding_window_size = {
+ type = "integer",
+ minimum = 10,
+ maximum = 3600,
+ default = 300,
+ description = "Size of the sliding window in
seconds"
+ },
+ half_open_max_calls = {
+ type = "integer",
+ minimum = 1,
+ maximum = 20,
+ default = 3,
+ description = "Number of permitted calls when " ..
+ "circuit breaker is half-open"
+ }
},
- uniqueItems = true,
- default = {200}
+ default = {
+ http_statuses = { 500 },
+ error_ratio = 0.5,
+ min_request_threshold = 10,
+ sliding_window_size = 300,
+ half_open_max_calls = 3
+ }
},
- successes = {
- type = "integer",
- minimum = 1,
- default = 3,
+ healthy = {
+ type = "object",
+ properties = {
+ http_statuses = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "integer",
+ minimum = 200,
+ maximum = 499,
+ },
+ uniqueItems = true,
+ default = { 200 }
+ },
+ success_ratio = {
+ type = "number",
+ minimum = 0,
+ maximum = 1,
+ default = 0.6,
+ description = "Success rate threshold to close
circuit breaker " ..
+ "from half-open state"
+ }
+ },
+ default = { http_statuses = { 200 }, success_ratio = 0.6 }
}
- },
- default = {http_statuses = {200}, successes = 3}
+ }
}
- },
- required = {"break_response_code"},
+ }
}
-
+-- Key generation functions (based on latest APISIX version)
local function gen_healthy_key(ctx)
return "healthy-" .. core.request.get_host(ctx) .. ctx.var.uri
end
-
local function gen_unhealthy_key(ctx)
return "unhealthy-" .. core.request.get_host(ctx) .. ctx.var.uri
end
-
local function gen_lasttime_key(ctx)
return "unhealthy-lasttime" .. core.request.get_host(ctx) .. ctx.var.uri
end
+-- New key generation functions for ratio policy
+local function gen_state_key(ctx)
+ return "cb-state-" .. core.request.get_host(ctx) .. ctx.var.uri
+end
+
+local function gen_total_requests_key(ctx)
+ return "cb-total-" .. core.request.get_host(ctx) .. ctx.var.uri
Review Comment:
Fixed in 6f9af8c1. Ratio-policy keys are now scoped by
conf_type/conf_id/conf_version (gen_breaker_id) instead of host+URI — same
approach limit-count/limit-conn use to bound cardinality and to start fresh on
a config change. Every key now also carries a bounded TTL (get_state_ttl /
get_bucket_ttl).
##########
apisix/plugins/api-breaker.lua:
##########
@@ -264,4 +589,107 @@ function _M.log(conf, ctx)
return
end
+-- Ratio-based logging
+local function ratio_based_log(conf, ctx)
+ local upstream_status = core.response.get_upstream_status(ctx)
+ if not upstream_status then
+ return
+ end
+
+ local current_state = get_circuit_breaker_state(ctx)
+
+ -- Increment total request counter
+ local total_requests_key = gen_total_requests_key(ctx)
+ local total_requests, err = shared_buffer:incr(total_requests_key, 1, 0)
+ if err then
+ core.log.warn("failed to increment total requests: ", err)
+ end
+
+ -- Handle response based on status
+ local is_failure = core.table.array_find(conf.unhealthy.http_statuses,
upstream_status)
+ local is_success = not is_failure and
+ core.table.array_find(conf.healthy.http_statuses, upstream_status)
+
+ if is_failure then
+ -- Increment failure counter
+ local unhealthy_key = gen_unhealthy_key(ctx)
+ local unhealthy_count, err = shared_buffer:incr(unhealthy_key, 1, 0)
+ if err then
+ core.log.warn("failed to increment unhealthy count: ", err)
+ end
+
+ core.log.info("Request failed - status: ", upstream_status,
+ " total: ", total_requests,
+ " failures: ", unhealthy_count)
+
+ -- If in HALF_OPEN state and got a failure, immediately go back to OPEN
+ if current_state == HALF_OPEN then
+ set_circuit_breaker_state(ctx, OPEN)
+ core.log.warn("Circuit breaker returned to OPEN state due to
failure in HALF_OPEN")
+ -- Clean up half-open counters
+ shared_buffer:delete(gen_half_open_calls_key(ctx))
+ shared_buffer:delete(gen_half_open_success_key(ctx))
+ end
+ elseif is_success then
Review Comment:
Fixed in 6f9af8c1. Introduced a dedicated half-open 'completed' counter,
incremented for every probe that reaches the log phase regardless of how its
status is classified (success/failure/neither). The close-vs-reopen decision
now fires once completed >= half_open_max_calls, so a run of unclassified
status codes can no longer strand the breaker in HALF_OPEN forever. Verified in
harness Scenario 5 (two unclassified 404 probes still resolve the half-open
evaluation).
##########
apisix/plugins/api-breaker.lua:
##########
@@ -133,13 +267,67 @@ local _M = {
schema = schema,
}
-
function _M.check_schema(conf)
return core.schema.check(schema, conf)
end
+-- Circuit breaker state management functions
+local function get_circuit_breaker_state(ctx)
+ local state_key = gen_state_key(ctx)
+ local state, err = shared_buffer:get(state_key)
+ if err then
+ core.log.warn("failed to get circuit breaker state: ", err)
+ return CLOSED
+ end
+ return state or CLOSED
+end
+
+local function set_circuit_breaker_state(ctx, state)
+ local state_key = gen_state_key(ctx)
+ local last_change_key = gen_last_state_change_key(ctx)
+ local current_time = ngx.time()
-function _M.access(conf, ctx)
+ shared_buffer:set(state_key, state)
+ shared_buffer:set(last_change_key, current_time)
Review Comment:
Fixed in 6f9af8c1. Every shared_buffer:set/incr call in the ratio path now
passes an exptime (get_state_ttl for state/half-open bookkeeping,
get_bucket_ttl for window buckets).
##########
apisix/plugins/api-breaker.lua:
##########
@@ -195,8 +383,145 @@ function _M.access(conf, ctx)
return
end
+-- Ratio-based circuit breaker
+local function ratio_based_access(conf, ctx)
+ -- Check and reset sliding window first to ensure consistent state
+ check_and_reset_window(ctx, conf)
-function _M.log(conf, ctx)
+ local current_state = get_circuit_breaker_state(ctx)
+ local current_time = ngx.time()
+
+ -- Handle OPEN state
+ if current_state == OPEN then
+ local last_change_key = gen_last_state_change_key(ctx)
+ local last_change_time, err = shared_buffer:get(last_change_key)
+ if err then
+ core.log.warn("failed to get last change time: ", err)
+ return conf.break_response_code or 503,
+ conf.break_response_body or "Service temporarily
unavailable"
+ end
+
+ local wait_duration = conf.max_breaker_sec or 300
+ if last_change_time and (current_time - last_change_time) >=
wait_duration then
+ -- Use atomic operation to ensure only one request transitions to
HALF_OPEN
+ local transition_key = "cb-transition-" ..
core.request.get_host(ctx) .. ctx.var.uri
+ local transition_success
+ transition_success, err = shared_buffer:add(transition_key, 1, 1)
+
+ if err then
+ core.log.warn("failed to add transition lock: ", err)
+ end
+
+ if transition_success then
+ -- Transition to HALF_OPEN
+ set_circuit_breaker_state(ctx, HALF_OPEN)
Review Comment:
Fixed in 6f9af8c1, same root cause as your comment above on the
half_open_max_calls enforcement — the transitioning request now falls through
into the shared admission/increment path instead of returning early, so it
counts as probe #1 rather than granting an extra n+1'th slot.
##########
docs/zh/latest/plugins/api-breaker.md:
##########
@@ -30,34 +29,70 @@ description: 本文介绍了 Apache APISIX api-breaker 插件的相关操作，
`api-breaker` 插件实现了 API 熔断功能，从而帮助我们保护上游业务服务。
+该插件支持两种熔断策略：
+
+- **按错误次数熔断（`unhealthy-count`）**：当连续失败次数达到阈值时触发熔断
+- **按错误比例熔断（`unhealthy-ratio`）**：当在滑动时间窗口内的错误率达到阈值时触发熔断
+
:::note 注意
-关于熔断超时逻辑，由代码逻辑自动按**触发不健康状态**的次数递增运算：
+**按错误次数熔断（`unhealthy-count`）**：
当上游服务返回 `unhealthy.http_statuses` 配置中的状态码（默认为 `500`），并达到 `unhealthy.failures`
预设次数时（默认为 3 次），则认为上游服务处于不健康状态。
第一次触发不健康状态时，熔断 2 秒。超过熔断时间后，将重新开始转发请求到上游服务，如果继续返回 `unhealthy.http_statuses`
状态码，记数再次达到 `unhealthy.failures` 预设次数时，熔断 4 秒。依次类推（2，4，8，16，……），直到达到预设的
`max_breaker_sec`值。
当上游服务处于不健康状态时，如果转发请求到上游服务并返回 `healthy.http_statuses` 配置中的状态码（默认为 `200`），并达到
`healthy.successes` 次时，则认为上游服务恢复至健康状态。
+**按错误比例熔断（`unhealthy-ratio`）**：
+
+该策略基于滑动时间窗口统计错误率。当在 `sliding_window_size` 时间窗口内，请求总数达到 `min_request_threshold`
且错误率超过 `error_ratio` 时，熔断器进入开启状态，持续 `max_breaker_sec` 秒。
+
+熔断器有三种状态：
+
+- **关闭（CLOSED）**：正常转发请求
+- **开启（OPEN）**：直接返回熔断响应，不转发请求
+- **半开启（HALF_OPEN）**：允许少量请求通过以测试服务是否恢复
+
:::
## 属性
-| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述
|
-| ----------------------- | -------------- | ------ | ---------- |
--------------- | -------------------------------- |
-| break_response_code | integer | 是 | | [200, ..., 599]
| 当上游服务处于不健康状态时返回的 HTTP 错误码。 |
-| break_response_body | string | 否 | |
| 当上游服务处于不健康状态时返回的 HTTP 响应体信息。 |
-| break_response_headers | array[object] | 否 | |
[{"key":"header_name","value":"can contain Nginx $var"}] | 当上游服务处于不健康状态时返回的
HTTP 响应头信息。该字段仅在配置了 `break_response_body` 属性时生效，并能够以 `$var` 的格式包含 APISIX 变量，比如
`{"key":"X-Client-Addr","value":"$remote_addr:$remote_port"}`。 |
-| max_breaker_sec | integer | 否 | 300 | >=3
| 上游服务熔断的最大持续时间，以秒为单位。 |
-| unhealthy.http_statuses | array[integer] | 否 | [500] | [500, ...,
599] | 上游服务处于不健康状态时的 HTTP 状态码。 |
-| unhealthy.failures | integer | 否 | 3 | >=1
| 上游服务在一定时间内触发不健康状态的异常请求次数。 |
-| healthy.http_statuses | array[integer] | 否 | [200] | [200, ...,
499] | 上游服务处于健康状态时的 HTTP 状态码。 |
-| healthy.successes | integer | 否 | 3 | >=1
| 上游服务触发健康状态的连续正常请求次数。 |
+| 名称 | 类型 | 必选项 | 默认值 | 有效值
| 描述
|
+| ---------------------- | ------------- | ------ | ----------------- |
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------------
|
+| break_response_code | integer | 是 | | [200,
..., 599]
| 当上游服务处于不健康状态时返回的 HTTP 错误码。
|
+| break_response_body | string | 否 | |
| 当上游服务处于不健康状态时返回的 HTTP 响应体信息。
|
+| break_response_headers | array[object] | 否 | |
[{"key":"header_name","value":"can contain Nginx $var"}] | 当上游服务处于不健康状态时返回的
HTTP 响应头信息。该字段仅在配置了 `break_response_body` 属性时生效，并能够以 `$var` 的格式包含 APISIX
变量，比如`{"key":"X-Client-Addr","value":"$remote_addr:$remote_port"}`。 |
|
+| max_breaker_sec | integer | 否 | 300 | >=3
| 上游服务熔断的最大持续时间，以秒为单位。适用于两种熔断策略。
|
+| policy | string | 否 | "unhealthy-count" |
["unhealthy-count", "unhealthy-ratio"]
| 熔断策略。`unhealthy-count`
为按错误次数熔断，`unhealthy-ratio` 为按错误比例熔断。 |
+
+### 按错误次数熔断（policy = "unhealthy-count"）
+
+| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述
|
+| ----------------------- | -------------- | ------ | ------ | ---------------
| -------------------------------------------------- |
+| unhealthy.http_statuses | array[integer] | 否 | [500] | [500, ..., 599]
| 上游服务处于不健康状态时的 HTTP 状态码。 |
+| unhealthy.failures | integer | 否 | 3 | >=1
| 上游服务在一定时间内触发不健康状态的异常请求次数。 |
+| healthy.http_statuses | array[integer] | 否 | [200] | [200, ..., 499]
| 上游服务处于健康状态时的 HTTP 状态码。 |
+| healthy.successes | integer | 否 | 3 | >=1
| 上游服务触发健康状态的连续正常请求次数。 |
+
+### 按错误比例熔断（policy = "unhealthy-ratio"）
+
+| 名称 | 类型 | 必选项 |
默认值 | 有效值 | 描述
|
+| ------------------------------------------------------ | -------------- |
------ | ------ | --------------- |
----------------------------------------------------------------------------------------
|
+| unhealthy.http_statuses | array[integer] | 否
| [500] | [500, ..., 599] | 上游服务处于不健康状态时的 HTTP 状态码。
|
+| unhealthy.error_ratio | number | 否
| 0.5 | [0, 1] | 触发熔断的错误率阈值。例如 0.5 表示错误率达到 50% 时触发熔断。
|
+| unhealthy.min_request_threshold | integer | 否
| 10 | >=1 | 在滑动时间窗口内触发熔断所需的最小请求数。只有请求数达到此阈值时才会评估错误率。
|
+| unhealthy.sliding_window_size | integer | 否
| 300 | [10, 3600] | 滑动时间窗口大小，以秒为单位。用于统计错误率的时间范围。
|
+| unhealthy.half_open_max_calls | integer | 否 | 3 | [1, 20]
| 在半开启状态下允许通过的请求数量。用于测试服务是否恢复正常。 |
+| healthy.http_statuses | array[integer] | 否
| [200] | [200, ..., 499] | 上游服务处于健康状态时的 HTTP 状态码。
|
+| healthy.successes | integer | 否
| 3 | >=1 | 上游服务触发健康状态的连续正常请求次数。
|
Review Comment:
Fixed in 6f9af8c1 — removed the stray healthy.successes row from the
unhealthy-ratio table (that field only applies to the unhealthy-count policy).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
