This is an automated email from the ASF dual-hosted git repository. AlinsRan pushed a commit to branch fix/healthcheck-stale-cleanup-test in repository https://gitbox.apache.org/repos/asf/apisix.git
commit b73da9dbc5d43c78d61c3722a450a37766b267bb Author: AlinsRan <[email protected]> AuthorDate: Mon Jun 29 18:14:48 2026 +0800 fix(healthcheck): purge stale targets for every checker Bump lua-resty-healthcheck-api7 from 3.2.1-0 to 3.2.2-0, which carries the stale-target cleanup and periodic-lock fixes (api7/lua-resty-healthcheck#55). Before the bump, the health-check library advanced a module-level cleanup timestamp inside its per-checker loop, so only the first checker was purged each window. With multiple health-checked upstreams the others kept their delayed_clear()-marked nodes forever -- still reported by the control API (apache/apisix#13385) and still actively probed (apache/apisix#13141). Add t/node/healthcheck-stale-cleanup.t: two health-checked upstreams each drop a node; the test asserts both removed nodes are gone from /v1/healthcheck. It fails on 3.2.1-0 (one stale node remains) and passes on 3.2.2-0. A single-upstream setup cannot reproduce the bug because the sole checker is always the "first" one cleaned. --- apisix-master-0.rockspec | 2 +- t/node/healthcheck-stale-cleanup.t | 101 +++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/apisix-master-0.rockspec b/apisix-master-0.rockspec index 2ff4f6c12..69ca2a12d 100644 --- a/apisix-master-0.rockspec +++ b/apisix-master-0.rockspec @@ -42,7 +42,7 @@ dependencies = { "lua-resty-ngxvar = 0.5.2-0", "lua-resty-jit-uuid = 0.0.7-2", "lua-resty-ksuid = 1.0.1-0", - "lua-resty-healthcheck-api7 = 3.2.1-0", + "lua-resty-healthcheck-api7 = 3.2.2-0", "api7-lua-resty-jwt = 0.2.6-0", "lua-resty-hmac-ffi = 0.06-1", "lua-resty-cookie = 0.4.1-1", diff --git a/t/node/healthcheck-stale-cleanup.t b/t/node/healthcheck-stale-cleanup.t new file mode 100644 index 000000000..25af18aa8 --- /dev/null +++ b/t/node/healthcheck-stale-cleanup.t @@ -0,0 +1,101 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('warn'); +no_root_location(); +no_shuffle(); + +run_tests(); + +__DATA__ + +=== TEST 1: stale targets are purged for every checker, not just the first one +# Two upstreams each have active health checks enabled. When a node is removed +# from each, the health-check manager marks the removed target via +# delayed_clear(); the health-check library must then purge it from the shm +# target list of *every* checker. A library bug cleaned only the first checker +# per window, so with multiple health-checked upstreams the others kept their +# deleted nodes forever -- still reported by the control API (apache/apisix#13385) +# and still actively probed (apache/apisix#13141). Reproduces only with multiple +# upstreams; a single-upstream setup always cleans (it is the "first" checker). +--- config +location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin").test + + local function put_route(id, uri, nodes) + local cfg = { + uri = uri, + upstream = { + type = "roundrobin", + nodes = nodes, + checks = { + active = { + type = "tcp", + healthy = { interval = 1, successes = 1 }, + unhealthy = { interval = 1, tcp_failures = 1 }, + }, + }, + }, + } + assert(t('/apisix/admin/routes/' .. id, ngx.HTTP_PUT, cfg) < 300) + end + + -- two upstreams, each health-checked, each with two nodes + put_route(1, "/hello1", { ["127.0.0.1:1980"] = 1, ["127.0.0.1:1981"] = 1 }) + put_route(2, "/hello2", { ["127.0.0.1:1980"] = 1, ["127.0.0.1:1982"] = 1 }) + + -- traffic instantiates both checkers and registers them with the + -- shared active-check timer + t('/hello1', ngx.HTTP_GET) + t('/hello2', ngx.HTTP_GET) + ngx.sleep(2) + + -- remove one node from each upstream: the manager calls delayed_clear() + -- on the removed targets and rebuilds each checker + put_route(1, "/hello1", { ["127.0.0.1:1980"] = 1 }) + put_route(2, "/hello2", { ["127.0.0.1:1980"] = 1 }) + t('/hello1', ngx.HTTP_GET) + t('/hello2', ngx.HTTP_GET) + + -- wait past DELAYED_CLEAR_TIMEOUT (10s) plus a cleanup window + ngx.sleep(15) + + local _, _, res = t('/v1/healthcheck', ngx.HTTP_GET) + res = json.decode(res) + + -- count the removed nodes (1981, 1982) still present across all checkers + local stale = 0 + for _, info in ipairs(res) do + for _, node in ipairs(info.nodes or {}) do + if node.port == 1981 or node.port == 1982 then + stale = stale + 1 + end + end + end + ngx.say("stale: ", stale) + } +} +--- request +GET /t +--- response_body +stale: 0 +--- ignore_error_log +--- timeout: 30
