This is an automated email from the ASF dual-hosted git repository.
nic-6443 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 8615934a9d test: stabilize plugin tests and cover CI bailouts (#13948)
8615934a9d is described below
commit 8615934a9d5575f4d4632bcbdb3cda941cc1d683
Author: Nic <[email protected]>
AuthorDate: Wed Sep 16 13:36:05 2026 +0800
test: stabilize plugin tests and cover CI bailouts (#13948)
---
ci/init-last-test-service.sh | 4 +-
ci/init-plugin-test-service.sh | 4 +-
docs/en/latest/internal/testing-framework.md | 23 +++++++++
t/cli/ci-bailout.t | 46 +++++++++++++++++
t/cli/ci-kafka-topics.t | 74 ++++++++++++++++++++++++++++
t/cli/test_etcd_sync_event_handle.sh | 6 +++
t/core/plugin-checker.t | 17 ++++---
t/plugin/elasticsearch-logger2.t | 7 ++-
t/plugin/kafka-logger.t | 49 +++++++++++-------
t/plugin/limit-count-redis-sentinel.t | 7 ++-
t/plugin/limit-count-sliding.t | 42 +++++++++++++---
t/plugin/loggly.t | 19 +++++++
12 files changed, 260 insertions(+), 38 deletions(-)
diff --git a/ci/init-last-test-service.sh b/ci/init-last-test-service.sh
index bc90a25a37..bbf8afbd21 100755
--- a/ci/init-last-test-service.sh
+++ b/ci/init-last-test-service.sh
@@ -18,8 +18,8 @@
# A broker that is not registered in ZooKeeper yet rejects topic creation with
# "Replication factor: 1 larger than available brokers: 0". The failure used
to be
-# silent, and the topic was then auto-created with the default single
partition on
-# first produce, which quietly broke tests that expect a specific partition
layout.
+# silent, and the topic was then auto-created with the broker's default
partition
+# count, which quietly broke tests that expect a specific partition layout.
create_kafka_topic() {
local container="$1"
local zookeeper="$2"
diff --git a/ci/init-plugin-test-service.sh b/ci/init-plugin-test-service.sh
index 3a14d4770d..71c63dcc67 100755
--- a/ci/init-plugin-test-service.sh
+++ b/ci/init-plugin-test-service.sh
@@ -18,8 +18,8 @@
# A broker that is not registered in ZooKeeper yet rejects topic creation with
# "Replication factor: 1 larger than available brokers: 0". The failure used
to be
-# silent, and the topic was then auto-created with the default single
partition on
-# first produce, which quietly broke tests that expect a specific partition
layout.
+# silent, and the topic was then auto-created with the broker's default
partition
+# count, which quietly broke tests that expect a specific partition layout.
create_kafka_topic() {
local container="$1"
local zookeeper="$2"
diff --git a/docs/en/latest/internal/testing-framework.md
b/docs/en/latest/internal/testing-framework.md
index 7fcdf01e4d..1c6a355f0f 100644
--- a/docs/en/latest/internal/testing-framework.md
+++ b/docs/en/latest/internal/testing-framework.md
@@ -29,6 +29,11 @@ you need to write a shell script in the t/cli directory to
test it. You can refe
If you want to test the others, you need to write test code based on the
framework.
+Run service-backed tests with the services and initialization scripts selected
by
+that test's CI job. A running container does not guarantee that initialization
+has succeeded: for example, Kafka topic creation must finish with the expected
+partition count before a test sends messages.
+
Here, we briefly describe how to do simple testing based on this framework.
## Test file
@@ -111,6 +116,11 @@ GET /index.html
no valid upstream node
```
+An Admin API write can finish before a worker's configuration watcher applies
the
+update. If a test sends a request immediately after changing plugin metadata,
+wait for the expected values in the worker's metadata cache before sending it.
+Use a bounded wait and fail explicitly if the update does not arrive.
+
## Preparing the upstream
To test the code, we need to provide a mock upstream.
@@ -318,6 +328,19 @@ Assume your current work directory is the root of the
apisix source code.
The Nginx configuration and logs generated by the test cases are located in
the t/servroot directory. The Nginx configuration template for testing is
located in t/APISIX.pm.
+### Replacing module functions in tests
+
+APISIX background timers can run while a test yields, including while it waits
for configuration watchers to start. When replacing a shared module function,
restrict the replacement to the test's own inputs and delegate other calls to
the original function. Restore the original function after the test operation.
+
+### External service state
+
+Check cleanup responses as well as setup responses. A failed cleanup request
can
+leave configuration active for later test cases while the current case still
passes.
+
+Reloading Nginx or resetting etcd does not clear Redis. Tests that count all
keys in a database must clear the dedicated test Redis instance before creating
their counters, so unexpired keys from earlier cases cannot change the result.
+
+For asynchronous delivery, poll for the message produced by the current test
within a deadline rather than sleeping a fixed duration. When checking a
rate-limit burst within one window, start early in the window and verify that
the requests finish before it ends.
+
### Running only some test cases
Three notes can be used to control which parts of the tests are executed.
diff --git a/t/cli/ci-bailout.t b/t/cli/ci-bailout.t
new file mode 100644
index 0000000000..3a357f260a
--- /dev/null
+++ b/t/cli/ci-bailout.t
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use strict;
+use warnings;
+use Test::More;
+use File::Temp qw(tempfile);
+
+sub check_result {
+ my ($name, $log, $expected) = @_;
+ my ($file, $path) = tempfile(UNLINK => 1);
+ print {$file} $log;
+ close $file;
+
+ open my $runner, "-|", "bash", "-c",
+ 'exec 2>&1; source ci/common.sh; set +x; '
+ . 'fail_on_bailout "$1"; rerun_flaky_tests "$1"',
+ "ci-bailout", $path or die "cannot run CI result checks: $!";
+ my $output = do { local $/; <$runner> };
+ close $runner;
+ is($? >> 8, $expected, $name) or diag($output);
+}
+
+check_result("a completed passing suite succeeds", "Result: PASS\n", 0);
+check_result("bailout overrides a passing summary",
+ "Bailout called. Further testing stopped: nginx failed to start\n"
+ . "All tests successful.\nResult: PASS\n", 1);
+check_result("a bailed-out failing suite is not retried as a partial suite",
+ "Bailout called. Further testing stopped: nginx failed to start\n"
+ . "Result: FAIL\n", 1);
+
+done_testing();
diff --git a/t/cli/ci-kafka-topics.t b/t/cli/ci-kafka-topics.t
new file mode 100644
index 0000000000..fe36b3e3fe
--- /dev/null
+++ b/t/cli/ci-kafka-topics.t
@@ -0,0 +1,74 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use strict;
+use warnings;
+use Test::More;
+use File::Temp qw(tempdir);
+
+my $dir = tempdir(CLEANUP => 1);
+sub write_file {
+ my ($path, $body) = @_;
+ open my $file, ">", $path or die "cannot write $path: $!";
+ print {$file} $body;
+ close $file;
+}
+
+write_file("$dir/docker", <<'SH');
+#!/usr/bin/env bash
+count=$(cat "$KAFKA_TEST_CALLS")
+count=$((count + 1))
+printf '%s' "$count" > "$KAFKA_TEST_CALLS"
+if [ "$count" -le "$KAFKA_TEST_FAILURES" ]; then
+ echo 'Replication factor: 1 larger than available brokers: 0.'
+ exit 1
+fi
+SH
+write_file("$dir/sleep", "#!/usr/bin/env bash\nexit 0\n");
+chmod 0755, "$dir/docker", "$dir/sleep";
+
+for my $script (qw(ci/init-plugin-test-service.sh
ci/init-last-test-service.sh)) {
+ for my $failures (2, 30) {
+ write_file("$dir/calls", "0");
+ local $ENV{PATH} = "$dir:$ENV{PATH}";
+ local $ENV{KAFKA_TEST_CALLS} = "$dir/calls";
+ local $ENV{KAFKA_TEST_FAILURES} = $failures;
+ open my $runner, "-|", "bash", "-c",
+ 'exec 2>&1; source "$1" test; '
+ . 'create_kafka_topic kafka zookeeper:2181 1 test2; '
+ . 'echo "topic initialization completed"',
+ "ci-kafka", $script or die "cannot run $script: $!";
+ my $output = do { local $/; <$runner> };
+ close $runner;
+ my $exit = $? >> 8;
+ open my $calls, "<", "$dir/calls" or die "cannot read calls: $!";
+ my $count = <$calls>;
+ close $calls;
+ if ($failures == 2) {
+ is($exit, 0, "$script retries until topic creation succeeds") or
diag($output);
+ is($count, 3, "$script stops retrying after success");
+ like($output, qr/topic initialization completed/, "$script
continues after success");
+ } else {
+ is($exit, 1, "$script fails when the broker stays unavailable") or
diag($output);
+ is($count, 30, "$script bounds topic creation attempts");
+ unlike($output, qr/topic initialization completed/,
+ "$script does not hide initialization failure behind a later
command");
+ }
+ }
+}
+
+done_testing();
diff --git a/t/cli/test_etcd_sync_event_handle.sh
b/t/cli/test_etcd_sync_event_handle.sh
index 2e608f4b07..1f5a0549ed 100755
--- a/t/cli/test_etcd_sync_event_handle.sh
+++ b/t/cli/test_etcd_sync_event_handle.sh
@@ -72,6 +72,12 @@ sleep 3
project_compose_ci=ci/pod/docker-compose.common.yml make ci-env-stop
project_compose_ci=ci/pod/docker-compose.common.yml make ci-env-up
+# Wait for etcd to be ready after restart
+for i in $(seq 1 30); do
+ etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6-sync endpoint
health 2>/dev/null && break
+ sleep 1
+done
+
# Make some changes when APISIX cannot be synchronized
# Authentication ensures that only etcdctl can access etcd at this time
etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6-sync put
/apisix/routes/1
'{"uri":"/1","plugins":{"fault-injection":{"abort":{"http_status":204}}}}'
diff --git a/t/core/plugin-checker.t b/t/core/plugin-checker.t
index 8e44ce5c85..20d9641a27 100644
--- a/t/core/plugin-checker.t
+++ b/t/core/plugin-checker.t
@@ -124,16 +124,21 @@ plugin [missing] is not enabled and will be skipped
local etcd = require("apisix.core.etcd")
local original_get_format = etcd.get_format
local etcd_cli = {}
+ local mock_response = {
+ status = 200,
+ headers = {},
+ body = {header = {revision = 1}, kvs = {}},
+ }
function etcd_cli.readdir()
- return {
- status = 200,
- headers = {},
- body = {header = {revision = 1}, kvs = {}},
- }
+ return mock_response
end
- etcd.get_format = function(res)
+ etcd.get_format = function(res, ...)
+ if res ~= mock_response then
+ return original_get_format(res, ...)
+ end
+
res.body = {
node = {
key = "/apisix/plugins",
diff --git a/t/plugin/elasticsearch-logger2.t b/t/plugin/elasticsearch-logger2.t
index dec188fd85..80d6c2ac64 100644
--- a/t/plugin/elasticsearch-logger2.t
+++ b/t/plugin/elasticsearch-logger2.t
@@ -136,7 +136,12 @@ location /t {
keepalive_pool = 1,
})
ngx.sleep(2)
- t('/apisix/admin/plugin_metadata/elasticsearch-logger',
ngx.HTTP_DELETE)
+ local code, body =
t('/apisix/admin/plugin_metadata/elasticsearch-logger', ngx.HTTP_DELETE)
+ if not code or code >= 300 then
+ ngx.status = code or ngx.HTTP_INTERNAL_SERVER_ERROR
+ ngx.say(body)
+ return
+ end
}
}
--- error_log
diff --git a/t/plugin/kafka-logger.t b/t/plugin/kafka-logger.t
index 61f17422e8..9ed14872a6 100644
--- a/t/plugin/kafka-logger.t
+++ b/t/plugin/kafka-logger.t
@@ -930,7 +930,8 @@ done
"producer_type": "sync",
"timeout" : 1,
"batch_max_size": 1,
- "api_version": 2
+ "api_version": 2,
+ "log_format": {"marker": "$http_x_kafka_test"}
}
},
"upstream": {
@@ -962,25 +963,37 @@ done
end
offset = tonumber(tostring(offset):match("^%-?%d+"))
- -- hit the route to send the log to kafka
- t('/hello', ngx.HTTP_GET)
- ngx.sleep(2)
-
- local data, err = consumer:fetch("test2", 0, offset)
- if not data then
- ngx.say("failed to fetch message: ", err)
- return
- end
- local message = data.records[1]
- if not message then
- ngx.say("no message fetched")
+ local marker = ngx.worker.pid() .. ":" .. ngx.now()
+ local code, body = t('/hello', ngx.HTTP_GET, nil, nil,
+ {["X-Kafka-Test"] = marker})
+ if code ~= 200 then
+ ngx.say("failed to trigger log: ", body)
return
end
- if tonumber(message.timestamp) > 0 then
- ngx.say("message timestamp is stored")
- else
- ngx.say("invalid message timestamp: ",
tostring(message.timestamp))
- end
+
+ -- Batch processing and broker delivery can finish after the
response.
+ local json = require("apisix.core").json
+ local deadline = ngx.now() + 5
+ repeat
+ local data, err = consumer:fetch("test2", 0, offset)
+ if not data then
+ ngx.say("failed to fetch message: ", err)
+ return
+ end
+ for _, message in ipairs(data.records) do
+ local entry = json.decode(message.value)
+ if entry and entry.marker == marker then
+ if tonumber(message.timestamp) > 0 then
+ ngx.say("message timestamp is stored")
+ else
+ ngx.say("invalid message timestamp: ",
tostring(message.timestamp))
+ end
+ return
+ end
+ end
+ ngx.sleep(0.05)
+ until ngx.now() >= deadline
+ ngx.say("matching log entry not fetched")
}
}
--- timeout: 10
diff --git a/t/plugin/limit-count-redis-sentinel.t
b/t/plugin/limit-count-redis-sentinel.t
index e8b1d001ea..c1ccc811bc 100644
--- a/t/plugin/limit-count-redis-sentinel.t
+++ b/t/plugin/limit-count-redis-sentinel.t
@@ -514,6 +514,12 @@ invalid username-password pair
--- config
location /t {
content_by_lua_block {
+ local redis = require "resty.redis"
+ local red = redis:new()
+ assert(red:connect("127.0.0.1", 6479))
+ assert(red:flushall())
+ assert(red:close())
+
local t = require("lib.test_admin").test
for i, db in ipairs({1, 2}) do
local code, body = t('/apisix/admin/routes/' .. i,
@@ -575,7 +581,6 @@ invalid username-password pair
-- each database must contain only its own route's counter,
-- tracking exactly the 2 requests sent to that route
- local redis = require "resty.redis"
for db = 1, 2 do
local red = redis:new()
red:set_timeout(1000)
diff --git a/t/plugin/limit-count-sliding.t b/t/plugin/limit-count-sliding.t
index d853169438..65fd3ac1a7 100644
--- a/t/plugin/limit-count-sliding.t
+++ b/t/plugin/limit-count-sliding.t
@@ -32,6 +32,26 @@ add_block_preprocessor(sub {
}
});
+our $burst_config = <<'_EOC_';
+ location /t {
+ content_by_lua_block {
+ local http = require("resty.http")
+ local window = 5
+ while ngx.now() % window >= 1 do
+ ngx.sleep(0.05)
+ end
+ local window_id = math.floor(ngx.now() / window)
+ local uri = "http://127.0.0.1:"; .. ngx.var.server_port .. "/hello"
+ for i = 1, 3 do
+ local res = assert(http.new():request_uri(uri))
+ ngx.say(res.status)
+ end
+ assert(math.floor(ngx.now() / window) == window_id,
+ "request burst crossed the rate-limit window")
+ }
+ }
+_EOC_
+
run_tests;
__DATA__
@@ -77,10 +97,13 @@ passed
=== TEST 2: up the limit
---- pipelined_requests eval
-["GET /hello", "GET /hello", "GET /hello"]
---- error_code eval
-[200, 200, 503]
+--- timeout: 10
+--- config eval
+$::burst_config
+--- response_body
+200
+200
+503
@@ -196,10 +219,13 @@ passed
=== TEST 5: up the limit
---- pipelined_requests eval
-["GET /hello", "GET /hello", "GET /hello"]
---- error_code eval
-[200, 200, 503]
+--- timeout: 10
+--- config eval
+$::burst_config
+--- response_body
+200
+200
+503
diff --git a/t/plugin/loggly.t b/t/plugin/loggly.t
index 69c3ddc427..edc8c1386c 100644
--- a/t/plugin/loggly.t
+++ b/t/plugin/loggly.t
@@ -617,6 +617,24 @@ qr/message received: <14>1 [\d\-T:.]+Z [\d.]+ apisix [\d]+
- \[tok\@41058 tag="a
ngx.say("fail")
return
end
+
+ -- The Admin API write can return before the worker's metadata
watch updates.
+ local plugin = require("apisix.plugin")
+ local synced
+ for _ = 1, 50 do
+ local metadata = plugin.plugin_metadata("loggly")
+ local format = metadata and metadata.value.log_format
+ if format and format.host == "$host" and format.client ==
"$remote_addr" then
+ synced = true
+ break
+ end
+ ngx.sleep(0.1)
+ end
+ if not synced then
+ ngx.status = 500
+ ngx.say("loggly metadata did not propagate")
+ return
+ end
ngx.say(body)
local code, _, body = t("/opentracing?foo=bar", "GET")
@@ -628,6 +646,7 @@ qr/message received: <14>1 [\d\-T:.]+Z [\d.]+ apisix [\d]+
- \[tok\@41058 tag="a
ngx.print(body)
}
}
+--- timeout: 10
--- response_body
passed
opentracing
