On Wed, Apr 26, 2023 at 11:17:37PM +0300, Vitaliy Makkoveev wrote:
> Route timers and route labels protected by corresponding mutexes. `ifa'
> uses references counting for protection. No protection required for `rt'
> passed to rt_mpls_clear() because only current thread owns it.
>
> ok?
>
> Index: sys/net/route.c
> ===================================================================
> RCS file: /cvs/src/sys/net/route.c,v
> retrieving revision 1.418
> diff -u -p -r1.418 route.c
> --- sys/net/route.c 26 Apr 2023 16:09:44 -0000 1.418
> +++ sys/net/route.c 26 Apr 2023 20:11:16 -0000
> @@ -497,7 +497,6 @@ rtfree(struct rtentry *rt)
> KASSERT(!RT_ROOT(rt));
> atomic_dec_int(&rttrash);
>
> - KERNEL_LOCK();
> rt_timer_remove_all(rt);
> ifafree(rt->rt_ifa);
> rtlabel_unref(rt->rt_labelid);
> @@ -506,7 +505,6 @@ rtfree(struct rtentry *rt)
> #endif
> free(rt->rt_gateway, M_RTABLE, ROUNDUP(rt->rt_gateway->sa_len));
> free(rt_key(rt), M_RTABLE, rt_key(rt)->sa_len);
> - KERNEL_UNLOCK();
>
> pool_put(&rtentry_pool, rt);
> }
After running stress test successfully with this diff, next day
machine crashed while compiling a new kernel. It is unclear whether
it is related to the diff. The softdep in ps is problably processing
make output via ssh. Looks like recursive kernel stack overflow.
[-- MARK -- Fri Apr 28 13:25:00 2023]
kernel: protection fault trap, code=0
Stopped at rt_if_linkstate_change+0x21: movl 0x58(%rdi),%eax
ddb{3}>
ddb{3}> trace
rt_if_linkstate_change(c000000000000000,ffff800000784050,0) at rt_if_linkstate_
change+0x21
rtable_walk_helper(fffffd8746df9350,ffff8000247c98a0) at rtable_walk_helper+0x5
3
art_table_walk(ffff800000390900,fffffd8830272550,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x205
art_table_walk(ffff800000390900,fffffd8830272530,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec548,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec568,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec5c8,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec648,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec688,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec6c8,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec788,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec7a8,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec7c8,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec7e8,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec808,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8745fec828,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8746decdc0,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_table_walk(ffff800000390900,fffffd8746decea0,ffffffff813bde40,ffff8000247c9
8a0) at art_table_walk+0x26c
art_walk(ffff800000390900,ffffffff813bde40,ffff8000247c98a0) at art_walk+0xd1
rtable_walk(0,18,ffff8000247c9938,ffffffff813c2b70,ffff800000784050) at rtable_
walk+0xa4
art_walk(ffff800000390900,ffffffff813bde40,ffff8000247c98a0) at art_walk+0xd1
rtable_walk(0,18,ffff8000247c9938,ffffffff813c2b70,ffff800000784050) at rtable_
walk+0xa4
rt_if_track(ffff800000784050) at rt_if_track+0xdb
if_linkstate_task(3) at if_linkstate_task+0xb0
taskq_thread(ffff800000036180) at taskq_thread+0x100
end trace frame: 0x0, count: -23
ddb{3}> show panic
the kernel did not panic
ddb{3}> show register
rdi 0xc000000000000000
rsi 0xffff800000784050
rbp 0xffff8000247c8dc0
rbx 0xfffffd8746df9350
rdx 0
rcx 0xffff80002251c710
rax 0xc000000000000000
r8 0x8
r9 0x4
r10 0x12
r11 0xe5d48e0613cd564e
r12 0xffff8000247c8dd0
r13 0xfffffd8830272550
r14 0xffff8000247c98a0
r15 0xc000000000000000
rip 0xffffffff813c2b91 rt_if_linkstate_change+0x21
cs 0x8
rflags 0x10286 __ALIGN_SIZE+0xf286
rsp 0xffff8000247c8d60
ss 0
rt_if_linkstate_change+0x21: movl 0x58(%rdi),%eax
ddb{3}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
54044 278626 14147 0 7 0x3 arp
14147 281297 48383 0 3 0x10008b sigsusp timeout
68485 464481 24635 0 2 0x3 cc
24635 123811 11134 0 3 0x10008b sigsusp sh
92304 333371 93014 0 2 0x3 cc
93014 60581 11134 0 3 0x10008b sigsusp sh
48369 298699 36744 0 2 0x3 cc
36744 19282 11134 0 3 0x10008b sigsusp sh
69130 443067 39578 0 7 0x3 cc
39578 43217 11134 0 3 0x10008b sigsusp sh
34899 440228 31189 0 2 0x3 cc
31189 515744 11134 0 3 0x10008b sigsusp sh
22583 212506 37076 0 7 0x3 cc
37076 2679 11134 0 3 0x10008b sigsusp sh
95169 406065 94286 0 2 0x3 cc
94286 369995 11134 0 3 0x10008b sigsusp sh
84618 162660 78605 0 7 0x3 cc
78605 283757 11134 0 3 0x10008b sigsusp sh
11134 326791 47280 0 3 0x10008b sigsusp make
15617 273881 13388 0 3 0x100083 ttyin ksh
47280 13997 13388 0 3 0x10008b sigsusp ksh
23630 345318 88977 0 3 0x100083 kqread tmux
88977 276563 55388 0 3 0x10008b sigsusp ksh
55388 151577 87454 0 3 0x9a kqread sshd
15480 8431 66142 0 3 0x100083 ttyin ksh
66142 42569 87454 0 3 0x9a kqread sshd
86009 430157 13388 0 3 0x100083 ttyin ksh
48383 121548 13388 0 3 0x10008b sigsusp ksh
13388 39922 1 0 3 0x100080 kqread tmux
15579 328714 0 0 3 0x14200 bored sosplice
76541 522891 1 0 3 0x100083 ttyin getty
46282 145613 1 0 3 0x100098 kqread cron
38280 449926 1 99 3 0x1100090 kqread sndiod
71509 185468 1 110 3 0x100090 kqread sndiod
18019 240833 39769 95 3 0x1100092 kqread smtpd
90534 225699 39769 103 3 0x1100092 kqread smtpd
63215 241112 39769 95 3 0x1100092 kqread smtpd
31591 133135 39769 95 3 0x100092 kqread smtpd
7010 284594 39769 95 3 0x1100092 kqread smtpd
16629 223245 39769 95 3 0x1100092 kqread smtpd
39769 232162 1 0 3 0x100080 kqread smtpd
78666 386316 7107 89 3 0x1100092 kqread relayd
68039 15578 7107 89 3 0x1100092 kqread relayd
36675 9476 7107 89 3 0x1100092 kqread relayd
366 363913 7107 89 3 0x1100092 kqread relayd
22383 211332 7107 89 3 0x1100092 kqread relayd
5429 118908 7107 89 3 0x1100092 kqread relayd
87324 495270 7107 89 3 0x1100092 kqread relayd
97321 350625 7107 89 3 0x1100092 kqread relayd
7107 170630 1 0 3 0x80 kqread relayd
20131 107836 54962 91 7 0x12 snmpd_metrics
54962 484362 1 0 3 0x100080 kqread snmpd
25469 328526 1 91 3 0x1100092 kqread snmpd
87454 304519 1 0 3 0x88 kqread sshd
88041 353542 0 0 3 0x14280 nfsidl nfsio
33661 69348 0 0 3 0x14280 nfsidl nfsio
4906 29973 0 0 3 0x14280 nfsidl nfsio
56338 501608 0 0 3 0x14280 nfsidl nfsio
43739 356618 1 0 3 0x100080 kqread ntpd
75045 81844 77856 83 3 0x100092 kqread ntpd
77856 128330 1 83 3 0x1100092 kqread ntpd
98958 231170 21584 74 3 0x1100092 bpf pflogd
21584 427610 1 0 3 0x80 netio pflogd
49485 125301 30425 73 3 0x1100090 kqread syslogd
30425 365569 1 0 3 0x100082 netio syslogd
59467 198560 28024 77 7 0x100012 dhcpleased
9716 376946 28024 77 3 0x100092 kqread dhcpleased
28024 465318 1 0 3 0x80 kqread dhcpleased
41224 514677 66676 115 7 0x100012 slaacd
25235 279438 66676 115 3 0x100092 kqread slaacd
66676 369311 1 0 3 0x100080 kqread slaacd
56468 167420 0 0 3 0x14200 bored smr
9359 95306 0 0 2 0x14200 zerothread
36151 36152 0 0 3 0x14200 aiodoned aiodoned
4186 491459 0 0 3 0x14200 syncer update
2296 260176 0 0 3 0x14200 cleaner cleaner
14222 66730 0 0 3 0x14200 reaper reaper
13161 137510 0 0 3 0x14200 pgdaemon pagedaemon
18227 374801 0 0 3 0x14200 usbtsk usbtask
12254 75529 0 0 3 0x14200 usbatsk usbatsk
32112 220223 0 0 3 0x40014200 acpi0 acpi0
84986 206194 0 0 3 0x40014200 idle7
65936 8777 0 0 3 0x40014200 idle6
95579 438230 0 0 3 0x40014200 idle5
86138 259143 0 0 3 0x40014200 idle4
23213 463290 0 0 3 0x40014200 idle3
47905 271216 0 0 3 0x40014200 idle2
12000 253171 0 0 3 0x40014200 idle1
55632 435627 0 0 3 0x14200 bored sensors
*30374 197019 0 0 7 0x14200 softnet
97395 314588 0 0 3 0x14200 bored softnet
13707 147072 0 0 3 0x14200 bored softnet
1764 515477 0 0 3 0x14200 bored softnet
46234 501954 0 0 3 0x14200 bored systqmp
93391 98530 0 0 3 0x14200 bored systq
87333 16097 0 0 3 0x40014200 bored softclock
66847 163540 0 0 3 0x40014200 idle0
1 154810 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{0}> trace
x86_ipi_db(ffffffff83213ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff83453000) at __mp_lock+0x6e
intr_handler(ffff800024a518c0,ffff800000381400) at intr_handler+0x48
Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f
Xspllower() at Xspllower+0x1d
ifiq_input(ffff80000078e450,ffff800024a51a40) at ifiq_input+0x196
em_rxeof(ffff80000078d200) at em_rxeof+0x44e
em_intr(ffff80000078e000) at em_intr+0x93
intr_handler(ffff800024a51b80,ffff80000038ec00) at intr_handler+0x72
Xintr_ioapic_edge27_untramp() at Xintr_ioapic_edge27_untramp+0x18f
_kernel_lock() at _kernel_lock+0xb9
softintr_dispatch(0) at softintr_dispatch+0x4d
Xsoftclock() at Xsoftclock+0x23
_kernel_lock() at _kernel_lock+0xb9
syscall(ffff800024a51e50) at syscall+0x39b
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7a79a69fcf50, count: -18
ddb{1}> trace
x86_ipi_db(ffff800022509ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb6
sys_kbind(ffff800024841b78,ffff800024a5ce90,ffff800024a5cef0) at sys_kbind+0x22
4
syscall(ffff800024a5cf60) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x73c954bbdbc8, count: -7
ddb{2}> trace
x86_ipi_db(ffff800022512ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
end of kernel
end trace frame: 0x710afcf775a0, count: -3
ddb{4}> trace
x86_ipi_db(ffff800022524ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
end of kernel
end trace frame: 0x74f999744f40, count: -3
ddb{5}> trace
x86_ipi_db(ffff80002252dff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb0
syscall(ffff800024a63810) at syscall+0x39b
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x77c9518a37e0, count: -6
ddb{6}> trace
x86_ipi_db(ffff800022536ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb0
syscall(ffff8000248c5460) at syscall+0x39b
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x75c56168b570, count: -6
ddb{7}> trace
x86_ipi_db(ffff80002253fff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb9
syscall(ffff800024889a60) at syscall+0x39b
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x73731fa87df0, count: -6
