On Wed, Jul 24, 2024 at 6:49 AM Mina Almasry wrote:
>
> On Tue, Jul 9, 2024 at 8:37 AM Taehee Yoo wrote:
> ...
> > Reproducer:
> > ./ncdevmem -f -l -p 5201 -v 7 -t 0 -q 2 &
> > sleep 10
> > modprobe -rv bnxt_en
> > killall ncdevmem
> >
> >
On Tue, Jul 9, 2024 at 5:08 AM Mina Almasry wrote:
>
Hi Mina, Thanks a lot for your reply!
> On Thu, Jul 4, 2024 at 10:57 AM Taehee Yoo wrote:
> >
> > I found several locking warnings while testing.
> >
>
> Thanks for Testing Taehee! And sorry for the late r
start+0x3f4/0x5a0
[ 1135.719686] Modules linked in: 8021q garp mrp xt_nat xt_tcpudp veth
xt_conntrack nft_chain_nat xt_MASQUERADE nf_nat nf
_conntrack_netlink nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4
xfrm_user xt_addrtype nft_compat nf_tables br_netfilter bri
dge stp llc qrtr crct10dif_pclmul overlay crc32_generic crc32_pclmul
crc32c_intel ghash_clmulni_intel sha512_ssse3 sha256
_ssse3 sha1_ssse3 xts amdgpu cts wmi_bmof aesni_intel amdxcp
i2c_algo_bit crypto_simd drm_ttm_helper cryptd ttm drm_exec
bnxt_en ionic gpu_sched drm_suballoc_helper drm_buddy ptp video
drm_display_helper drm_kms_helper wmi cfg80211 drm drm_pa
nel_orientation_quirks backlight nfnetlink bpf_preload ip_tables x_tables
[ 1135.779526] CPU: 3 PID: 1644 Comm: ncdevmem Tainted: G W
6.10.0-rc5+ #43 6e089cf25edb5a71cabb8ab97c9df
bf7e96b1a3a
[ 1135.791882] Hardware name: ASUS System Product Name/PRIME Z690-P
D4, BIOS 0603 11/01/2021
[ 1135.800781] RIP: 0010:netdev_rx_queue_restart+0x3f4/0x5a0
[ 1135.806905] Code: d0 0f 1f 00 48 89 df e8 9a ce a9 fe 4c 89 f7 e8
92 ce a9 fe 48 83 c4 08 44 89 e0 5b 5d 41 5c 41 5d 4
1 5e 41 5f c3 cc cc cc cc <0f> 0b e9 05 fd ff ff 44 89 fe 48 c7 c7 80
70 fc 86 e8 46 7c 38 fe
[ 1135.826382] RSP: 0018:888224c2fbb0 EFLAGS: 00010246
[ 1135.832339] RAX: RBX: c0bb5c80 RCX: 842d81c3
[ 1135.840185] RDX: 11e6fc44 RSI: 0008 RDI: 8f37e220
[ 1135.848028] RBP: 88814b864000 R08: R09: fbfff1e6fc44
[ 1135.855875] R10: 8f37e227 R11: R12: 888224c2fc28
[ 1135.863716] R13: 88814b864be0 R14: ed102265e337 R15: 0001
[ 1135.871560] FS: 7fe9b77b4740() GS:1ba0()
knlGS:
[ 1135.880364] CS: 0010 DS: ES: CR0: 80050033
[ 1135.886824] CR2: 7f5f22a21f50 CR3: 0001094a CR4: 007506f0
[ 1135.894670] PKRU: 5554
[ 1135.898090] Call Trace:
[ 1135.901255]
[ 1135.904073] ? __warn+0xc8/0x2f0
[ 1135.908020] ? netdev_rx_queue_restart+0x3f4/0x5a0
[ 1135.913524] ? report_bug+0x326/0x3c0
[ 1135.917908] ? handle_bug+0x3c/0x70
[ 1135.922112] ? exc_invalid_op+0x14/0x50
[ 1135.926671] ? asm_exc_invalid_op+0x16/0x20
[ 1135.931591] ? mutex_is_locked+0x13/0x50
[ 1135.936238] ? netdev_rx_queue_restart+0x3f4/0x5a0
[ 1135.941748] net_devmem_unbind_dmabuf+0x2a3/0x440
[ 1135.947179] ? __pfx_lock_release+0x10/0x10
[ 1135.952081] ? __pfx_net_devmem_unbind_dmabuf+0x10/0x10
[ 1135.958040] netdev_nl_sock_priv_destroy+0x72/0xc0
[ 1135.963561] genl_release+0xed/0x190
[ 1135.967851] ? __pfx_genl_release+0x10/0x10
[ 1135.972755] ? mark_held_locks+0xa5/0xf0
[ 1135.977392] ? __local_bh_enable_ip+0xa5/0x120
[ 1135.982561] ? __pfx_genl_release+0x10/0x10
[ 1135.987464] netlink_release+0x839/0x18f0
Thanks!
Taehee Yoo
ligned_frags=%lu, non_page_aligned_frags=%lu\n",
> + page_aligned_frags, non_page_aligned_frags);
> +
> + fprintf(stdout, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n",
> + page_aligned_frags, non_page_aligned_frags);
> +
> +cleanup:
> +
> + munmap(buf_mem, dmabuf_size);
> + close(client_fd);
> + close(socket_fd);
> + close(buf);
> + close(memfd);
> + close(devfd);
> + ynl_sock_destroy(ys);
> +
> + return 0;
> +}
> +
> +void run_devmem_tests(void)
> +{
> + struct netdev_queue_dmabuf *queues;
> + int devfd, memfd, buf;
> + struct ynl_sock *ys;
> + size_t dmabuf_size;
> + size_t i = 0;
> +
> + dmabuf_size = getpagesize() * NUM_PAGES;
> +
> + create_udmabuf(, , , dmabuf_size);
> +
> + /* Configure RSS to divert all traffic from our devmem queues */
> + configure_rss();
> +
> + sleep(1);
> +
> + queues = malloc(sizeof(*queues) * num_queues);
> +
> + for (i = 0; i < num_queues; i++) {
> + queues[i]._present.type = 1;
> + queues[i]._present.idx = 1;
> + queues[i].type = NETDEV_QUEUE_TYPE_RX;
> + queues[i].idx = start_queue + i;
> + }
> +
> + if (bind_rx_queue(ifindex, buf, queues, num_queues, ))
> + error(1, 0, "Failed to bind\n");
> +
> + /* Closing the netlink socket does an implicit unbind */
> + ynl_sock_destroy(ys);
> +}
> +
> +int main(int argc, char *argv[])
> +{
> + int is_server = 0, opt;
> +
> + while ((opt = getopt(argc, argv, "ls:c:p:v:q:f:n:i:d:")) != -1) {
I think 't' option should be added here.
> + switch (opt) {
> + case 'l':
> + is_server = 1;
> + break;
> + case 's':
> + server_ip = optarg;
> + break;
> + case 'c':
> + client_ip = optarg;
> + break;
> + case 'p':
> + port = optarg;
> + break;
> + case 'v':
> + do_validation = atoll(optarg);
> + break;
> + case 'q':
> + num_queues = atoi(optarg);
> + break;
> + case 't':
> + start_queue = atoi(optarg);
> + break;
> + case 'f':
> + ifname = optarg;
> + break;
> + case 'd':
> + ifindex = atoi(optarg);
How about using if_nametoindex() instead of 'd' option?
> + break;
> + case 'i':
> + iterations = atoll(optarg);
I couldn't find a use of this variable.
> + break;
> + case '?':
> + printf("unknown option: %c\n", optopt);
> + break;
> + }
> + }
> +
> + for (; optind < argc; optind++)
> + printf("extra arguments: %s\n", argv[optind]);
> +
> + run_devmem_tests();
> +
> + if (is_server)
> + return do_server();
> +
> + return 0;
> +}
> --
> 2.45.2.803.g4e1b14247a-goog
>
>
Thanks a lot!
Taehee Yoo
goto err_unlock;
> +
> + nla_for_each_attr(attr, genlmsg_data(info->genlhdr),
> + genlmsg_len(info->genlhdr), rem) {
> + if (nla_type(attr) != NETDEV_A_BIND_DMABUF_QUEUES)
> + continue;
> +
> + err = nla_parse_nested(
> + tb, ARRAY_SIZE(netdev_queue_dmabuf_nl_policy) - 1,
> attr,
> + netdev_queue_dmabuf_nl_policy, info->extack);
> + if (err < 0)
> + goto err_unbind;
> +
> + rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_DMABUF_IDX]);
> +
> + err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx,
> + out_binding);
> + if (err)
> + goto err_unbind;
> + }
> +
> + sock_binding_list = genl_sk_priv_get(_nl_family,
> +NETLINK_CB(skb).sk);
> + if (IS_ERR(sock_binding_list)) {
> + err = PTR_ERR(sock_binding_list);
> + goto err_unbind;
> + }
> +
> + list_add(_binding->list, sock_binding_list);
> +
> + rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
> + if (!rsp) {
> + err = -ENOMEM;
> + goto err_unbind;
> + }
> +
> + hdr = genlmsg_iput(rsp, info);
> + if (!hdr) {
> + err = -EMSGSIZE;
> + goto err_genlmsg_free;
> + }
> +
> + nla_put_u32(rsp, NETDEV_A_BIND_DMABUF_DMABUF_ID, out_binding->id);
> + genlmsg_end(rsp, hdr);
> +
> + rtnl_unlock();
> +
> + return genlmsg_reply(rsp, info);
> +
> +err_genlmsg_free:
> + nlmsg_free(rsp);
> +err_unbind:
> + net_devmem_unbind_dmabuf(out_binding);
> +err_unlock:
> + rtnl_unlock();
> + return err;
> }
>
> static int netdev_genl_netdevice_event(struct notifier_block *nb,
> @@ -771,3 +854,17 @@ static int __init netdev_genl_init(void)
> }
>
> subsys_initcall(netdev_genl_init);
> +
> +void netdev_nl_sock_priv_init(struct list_head *priv)
> +{
> + INIT_LIST_HEAD(priv);
> +}
> +
> +void netdev_nl_sock_priv_destroy(struct list_head *priv)
> +{
> + struct net_devmem_dmabuf_binding *binding;
> + struct net_devmem_dmabuf_binding *temp;
> +
> + list_for_each_entry_safe(binding, temp, priv, list)
> + net_devmem_unbind_dmabuf(binding);
> +}
> --
> 2.45.2.803.g4e1b14247a-goog
>
>
Thanks a lot!
Taehee Yoo
