Add a selftest to verify that ZCRX notification are properly delivered to userspace and that the shared-memory notification stats (copy_count, copy_bytes) are correctly incremented when zero-copy RX falls back to copying or when it runs out of buffers.
The test registers a notification descriptor during IORING_REGISTER_ZCRX_IFQ with a stats region placed after the refill queue entries. A new -n flag verifies that the copy fallback is triggered and -b/-a flags allows to check for out of buffer notification. To reliably trigger copy fallback, the Python test uses a new single_no_flow() setup variant that configures tcp-data-split and RSS but without ethtool flow rule. Without flow steering, traffic arrives on non-zcrx queues as regular pages, forcing the kernel copy-fallback path in io_zcrx_copy_frag(). Out-of-buffer notification is verified by using a smaller receive area and by avoiding recycling the buffers so that the kernel runs out of buffer quickly. Signed-off-by: Clément Léger <[email protected]> --- .../selftests/drivers/net/hw/iou-zcrx.c | 112 ++++++++++++++++-- .../selftests/drivers/net/hw/iou-zcrx.py | 49 +++++++- 2 files changed, 149 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/drivers/net/hw/iou-zcrx.c b/tools/testing/selftests/drivers/net/hw/iou-zcrx.c index 240d13dbc54e..3c95e6460c24 100644 --- a/tools/testing/selftests/drivers/net/hw/iou-zcrx.c +++ b/tools/testing/selftests/drivers/net/hw/iou-zcrx.c @@ -52,7 +52,27 @@ struct t_io_uring_zcrx_ifq_reg { struct io_uring_zcrx_offsets offsets; __u32 zcrx_id; __u32 rx_buf_len; - __u64 __resv[3]; + __u64 notif_desc; + __u64 __resv[2]; +}; + +#define ZCRX_NOTIF_NO_BUFFERS (1 << 0) +#define ZCRX_NOTIF_COPY (1 << 1) +#define ZCRX_NOTIF_DESC_FLAG_STATS (1 << 0) + +#define NOTIF_USER_DATA 3 + +struct t_zcrx_notification_desc { + __u64 user_data; + __u32 type_mask; + __u32 flags; + __u64 stats_offset; + __u64 __resv2[9]; +}; + +struct t_io_uring_zcrx_notif_stats { + __u64 copy_count; + __u64 copy_bytes; }; static long page_size; @@ -84,7 +104,10 @@ static int cfg_oneshot_recvs; static int cfg_send_size = SEND_SIZE; static struct sockaddr_in6 cfg_addr; static unsigned int cfg_rx_buf_len; +static size_t cfg_area_size; static bool cfg_dry_run; +static bool cfg_copy_fallback; +static bool cfg_no_buffers; static char *payload; static void *area_ptr; @@ -95,6 +118,8 @@ static unsigned long area_token; static int connfd; static bool stop; static size_t received; +static unsigned int notif_received_mask; +static size_t notif_stats_offset; static unsigned long gettimeofday_ms(void) { @@ -142,6 +167,7 @@ static void setup_zcrx(struct io_uring *ring) { unsigned int ifindex; unsigned int rq_entries = 4096; + size_t area_size = cfg_area_size ? cfg_area_size : AREA_SIZE; int ret; ifindex = if_nametoindex(cfg_ifname); @@ -150,7 +176,7 @@ static void setup_zcrx(struct io_uring *ring) if (cfg_rx_buf_len && cfg_rx_buf_len != page_size) { area_ptr = mmap(NULL, - AREA_SIZE, + area_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB | MAP_HUGE_2MB, @@ -162,7 +188,7 @@ static void setup_zcrx(struct io_uring *ring) } } else { area_ptr = mmap(NULL, - AREA_SIZE, + area_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, @@ -172,6 +198,12 @@ static void setup_zcrx(struct io_uring *ring) } ring_size = get_refill_ring_size(rq_entries); + + if (cfg_copy_fallback) { + notif_stats_offset = ring_size; + ring_size += ALIGN_UP(sizeof(struct t_io_uring_zcrx_notif_stats), page_size); + } + ring_ptr = mmap(NULL, ring_size, PROT_READ | PROT_WRITE, @@ -187,10 +219,11 @@ static void setup_zcrx(struct io_uring *ring) struct io_uring_zcrx_area_reg area_reg = { .addr = (__u64)(unsigned long)area_ptr, - .len = AREA_SIZE, + .len = area_size, .flags = 0, }; + struct t_zcrx_notification_desc notif_desc; struct t_io_uring_zcrx_ifq_reg reg = { .if_idx = ifindex, .if_rxq = cfg_queue_id, @@ -200,11 +233,32 @@ static void setup_zcrx(struct io_uring *ring) .rx_buf_len = cfg_rx_buf_len, }; + if (cfg_copy_fallback || cfg_no_buffers) { + __u32 type_mask = 0; + + if (cfg_copy_fallback) + type_mask = ZCRX_NOTIF_COPY; + if (cfg_no_buffers) + type_mask = ZCRX_NOTIF_NO_BUFFERS; + + memset(¬if_desc, 0, sizeof(notif_desc)); + notif_desc.user_data = NOTIF_USER_DATA; + notif_desc.type_mask = type_mask; + if (cfg_copy_fallback) { + notif_desc.flags = ZCRX_NOTIF_DESC_FLAG_STATS; + notif_desc.stats_offset = notif_stats_offset; + } + reg.notif_desc = (__u64)(unsigned long)¬if_desc; + } + ret = io_uring_register_ifq(ring, (void *)®); if (cfg_rx_buf_len && (ret == -EINVAL || ret == -EOPNOTSUPP || ret == -ERANGE)) { printf("Large chunks are not supported %i\n", ret); exit(SKIP_CODE); + } else if ((cfg_copy_fallback || cfg_no_buffers) && ret == -EINVAL) { + printf("Notifications not supported %i\n", ret); + exit(SKIP_CODE); } else if (ret) { error(1, 0, "io_uring_register_ifq(): %d", ret); } @@ -304,10 +358,13 @@ static void process_recvzc(struct io_uring *ring, struct io_uring_cqe *cqe) } received += n; - rqe = &rq_ring.rqes[(rq_ring.rq_tail & rq_mask)]; - rqe->off = (rcqe->off & ~IORING_ZCRX_AREA_MASK) | area_token; - rqe->len = cqe->res; - io_uring_smp_store_release(rq_ring.ktail, ++rq_ring.rq_tail); + /* Skip ring refill so that we ran out of buffers quickly */ + if (!cfg_no_buffers) { + rqe = &rq_ring.rqes[(rq_ring.rq_tail & rq_mask)]; + rqe->off = (rcqe->off & ~IORING_ZCRX_AREA_MASK) | area_token; + rqe->len = cqe->res; + io_uring_smp_store_release(rq_ring.ktail, ++rq_ring.rq_tail); + } } static void server_loop(struct io_uring *ring) @@ -324,8 +381,15 @@ static void server_loop(struct io_uring *ring) process_accept(ring, cqe); else if (cqe->user_data == 2) process_recvzc(ring, cqe); - else + else if ((cfg_copy_fallback || cfg_no_buffers) && + cqe->user_data == NOTIF_USER_DATA) { + notif_received_mask |= cqe->res; + if (cfg_no_buffers && + (cqe->res & ZCRX_NOTIF_NO_BUFFERS)) + stop = true; + } else { error(1, 0, "unknown cqe"); + } count++; } io_uring_cq_advance(ring, count); @@ -374,6 +438,23 @@ static void run_server(void) if (!stop) error(1, 0, "test failed\n"); + + if (cfg_copy_fallback) { + struct t_io_uring_zcrx_notif_stats *stats = + (void *)((char *)ring_ptr + notif_stats_offset); + + if (!(notif_received_mask & ZCRX_NOTIF_COPY)) + error(1, 0, "expected copy fallback notification"); + if (!IO_URING_READ_ONCE(stats->copy_count)) + error(1, 0, "expected copy_count > 0"); + if (!IO_URING_READ_ONCE(stats->copy_bytes)) + error(1, 0, "expected copy_bytes > 0"); + } + + if (cfg_no_buffers) { + if (!(notif_received_mask & ZCRX_NOTIF_NO_BUFFERS)) + error(1, 0, "expected no-buffers notification"); + } } static void run_client(void) @@ -425,7 +506,7 @@ static void parse_opts(int argc, char **argv) usage(argv[0]); cfg_payload_len = max_payload_len; - while ((c = getopt(argc, argv, "sch:p:l:i:q:o:z:x:d")) != -1) { + while ((c = getopt(argc, argv, "sch:p:l:i:q:o:z:x:a:dnb")) != -1) { switch (c) { case 's': if (cfg_client) @@ -466,8 +547,19 @@ static void parse_opts(int argc, char **argv) case 'd': cfg_dry_run = true; break; + case 'n': + cfg_copy_fallback = true; + break; + case 'b': + cfg_no_buffers = true; + break; + case 'a': + cfg_area_size = strtoul(optarg, NULL, 0) * page_size; + break; } } + if (cfg_copy_fallback && cfg_no_buffers) + error(1, 0, "Pass one of -n or -b"); if (cfg_server && addr) error(1, 0, "Receiver cannot have -h specified"); diff --git a/tools/testing/selftests/drivers/net/hw/iou-zcrx.py b/tools/testing/selftests/drivers/net/hw/iou-zcrx.py index e81724cb5542..f7f1cbff5959 100755 --- a/tools/testing/selftests/drivers/net/hw/iou-zcrx.py +++ b/tools/testing/selftests/drivers/net/hw/iou-zcrx.py @@ -41,7 +41,9 @@ def set_flow_rule_rss(cfg, rss_ctx_id): return int(values) -def single(cfg): +def single_no_flow(cfg): + """Like single() but without a flow rule.""" + channels = cfg.ethnl.channels_get({'header': {'dev-index': cfg.ifindex}}) channels = channels['combined-count'] if channels < 2: @@ -65,6 +67,9 @@ def single(cfg): ethtool(f"-X {cfg.ifname} equal {cfg.target}") defer(ethtool, f"-X {cfg.ifname} default") +def single(cfg): + single_no_flow(cfg) + flow_rule_id = set_flow_rule(cfg) defer(ethtool, f"-N {cfg.ifname} delete {flow_rule_id}") @@ -130,6 +135,26 @@ def test_zcrx_oneshot(cfg, setup) -> None: cmd(tx_cmd, host=cfg.remote) +@ksft_variants([ + KsftNamedVariant("single", single_no_flow), +]) +def test_zcrx_notif(cfg, setup) -> None: + """Test zcrx copy fallback notification. + + Omits the flow rule so traffic arrives on non-zcrx queues as regular + pages, forcing the kernel copy-fallback path. Asserts that the + ZCRX_NOTIF_COPY notification CQE is delivered.""" + + cfg.require_ipver('6') + + setup(cfg) + rx_cmd = f"{cfg.bin_local} -s -p {cfg.port} -i {cfg.ifname} -q {cfg.target} -n" + tx_cmd = f"{cfg.bin_remote} -c -h {cfg.addr_v['6']} -p {cfg.port} -l 12840" + with bkg(rx_cmd, exit_wait=True): + wait_port_listen(cfg.port, proto="tcp") + cmd(tx_cmd, host=cfg.remote) + + def test_zcrx_large_chunks(cfg) -> None: """Test zcrx with large buffer chunks.""" @@ -157,6 +182,25 @@ def test_zcrx_large_chunks(cfg) -> None: cmd(tx_cmd, host=cfg.remote) +@ksft_variants([ + KsftNamedVariant("single", single), +]) +def test_zcrx_notif_no_buffers(cfg, setup) -> None: + """Test zcrx out-of-buffer notification. + + Skips buffer refill so the pool is quickly exhausted, triggering + a ZCRX_NOTIF_NO_BUFFERS notification CQE.""" + + cfg.require_ipver('6') + + setup(cfg) + rx_cmd = f"{cfg.bin_local} -s -p {cfg.port} -i {cfg.ifname} -q {cfg.target} -b -a 64" + tx_cmd = f"{cfg.bin_remote} -c -h {cfg.addr_v['6']} -p {cfg.port} -l 12840" + with bkg(rx_cmd, exit_wait=True): + wait_port_listen(cfg.port, proto="tcp") + cmd(tx_cmd, host=cfg.remote, fail=False) + + def main() -> None: with NetDrvEpEnv(__file__) as cfg: cfg.bin_local = path.abspath(path.dirname(__file__) + "/../../../drivers/net/hw/iou-zcrx") @@ -166,7 +210,8 @@ def main() -> None: cfg.netnl = NetdevFamily() cfg.port = rand_port() ksft_run(globs=globals(), cases=[test_zcrx, test_zcrx_oneshot, - test_zcrx_large_chunks], args=(cfg, )) + test_zcrx_large_chunks, test_zcrx_notif, + test_zcrx_notif_no_buffers], args=(cfg, )) ksft_exit() -- 2.52.0
