On 22/12/25(Mon) 14:59, Martin Pieuchot wrote:
> "K R" can reproduce a hang on a multiple sockets amd64 that has been
> first reported in a different thread:
> https://marc.info/?l=openbsd-tech&m=176631121132731&w=2
>
> The reports seem to always contain a CPU spinning for `tlb_shoot_wait'
> inside pmap_kremove().
>
> In two these reports, including the one below, instead of a hang, a CPU
> faulted inside Xipi_invlrange_pcid executing the following instruction:
>
> 00000000000006e0 <Xipi_invlrange_pcid>:
> [...]
> 717: 66 0f 38 82 0c 24 invpcid (%rsp),%rcx
Similar fault:
kernel: protection fault trap, code=0
Stopped at Xipi_invlrange_pcid+0x37:
ddb{0}> show reg
rdi 0xffffffff82ad6880 kernel_lock
rsi 0x1
rbp 0xffff800055ee1b70
rbx 0xffff800001d28100
rdx 0xffff800041d7f000
rcx 0
rax 0x800000000000
r8 0
r9 0
r10 0
r11 0x81df3a8602ca569e
r12 0xffff800055ee1bd0
r13 0xffff800001d28100
r14 0xffffffff82a11ff0 cpu_info_full_primary+0x1ff0
r15 0
rip 0xffffffff82578717 Xipi_invlrange_pcid+0x37
cs 0x8
rflags 0x10007 __ALIGN_SIZE+0xf007
rsp 0xffff800055ee1b10
ss 0
Xipi_invlrange_pcid+0x37:
ddb{0}> tr
Xipi_invlrange_pcid() at Xipi_invlrange_pcid+0x37
intr_handler(ffff800055ee1bd0,ffff800001d28100) at intr_handler+0x71
Xintr_ioapic_edge27_untramp() at Xintr_ioapic_edge27_untramp+0x18f
end of kernel
end trace frame: 0x1cfb4b782b0, count: -3
ddb{0}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
407487 54688 0 0x3 0x4000000 10 rustc
397294 54897 0 0x3 0x4000000 14 rustc
470955 65708 0 0x3 0x4000000 13 rustc
334248 26786 0 0x3 0x4000000 5 rustc
*310343 26028 0 0x3 0x4000000 0 rustc
212198 96410 0 0x3 0x4000000 8 rustc
23365 79502 0 0x10000013 0 4K perl
516669 89480 0 0x3 0x4000000 15 rustc
20254 15120 0 0x3 0x4000000 6 rustc
349282 35624 0 0x14000 0x200 2 softnet0
ddb{0}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
54688 135907 55280 0 3 0x83 fsleep rustc
54688 320829 55280 0 3 0x4000083 piperd rustc
54688 407487 55280 0 7 0x4000003 rustc
54688 15548 55280 0 3 0x4000083 fsleep rustc
55280 452830 51455 0 3 0x83 wait rustc
54897 521097 33989 0 3 0x83 fsleep rustc
54897 389406 33989 0 3 0x4000083 piperd rustc
54897 397294 33989 0 7 0x4000003 rustc
54897 175368 33989 0 3 0x4000083 fsleep rustc
33989 422360 51455 0 3 0x83 wait rustc
65708 63354 83064 0 3 0x83 fsleep rustc
65708 355299 83064 0 3 0x4000083 piperd rustc
65708 79695 83064 0 3 0x4000083 fsleep rustc
65708 314644 83064 0 3 0x4000083 fsleep rustc
65708 392661 83064 0 3 0x4000083 fsleep rustc
65708 495785 83064 0 3 0x4000083 piperd rustc
65708 470955 83064 0 7 0x4000003 rustc
83064 454726 51455 0 3 0x83 wait rustc
26786 344467 35071 0 3 0x83 fsleep rustc
26786 349403 35071 0 3 0x4000083 piperd rustc
26786 147950 35071 0 3 0x4000083 fsleep rustc
26786 335363 35071 0 3 0x4000083 fsleep rustc
26786 245714 35071 0 3 0x4000083 fsleep rustc
26786 393236 35071 0 3 0x4000083 piperd rustc
26786 334248 35071 0 7 0x4000003 rustc
26028 407393 2147 0 3 0x83 fsleep rustc
26028 176067 2147 0 3 0x4000083 piperd rustc
26028 234391 2147 0 3 0x4000083 fsleep rustc
26028 62448 2147 0 3 0x4000083 fsleep rustc
26028 49770 2147 0 3 0x4000083 piperd rustc
26028 419393 2147 0 3 0x4000083 fsleep rustc
*26028 310343 2147 0 7 0x4000003 rustc
96410 478405 29722 0 3 0x83 fsleep rustc
96410 359663 29722 0 3 0x4000083 piperd rustc
96410 400538 29722 0 3 0x4000083 fsleep rustc
96410 240099 29722 0 3 0x4000083 fsleep rustc
96410 158893 29722 0 3 0x4000083 piperd rustc
96410 522894 29722 0 3 0x4000083 fsleep rustc
96410 212198 29722 0 7 0x4000003 rustc
2147 35635 51455 0 3 0x83 wait rustc
35071 401005 51455 0 3 0x83 wait rustc
29722 100754 51455 0 3 0x83 wait rustc
79502 23365 94881 0 7 0x10000013 perl
89480 492448 56519 0 3 0x83 fsleep rustc
89480 23920 56519 0 3 0x4000083 piperd rustc
89480 207907 56519 0 3 0x4000083 fsleep rustc
89480 127270 56519 0 3 0x4000083 fsleep rustc
89480 483606 56519 0 3 0x4000083 piperd rustc
89480 381513 56519 0 3 0x4000083 fsleep rustc
89480 409781 56519 0 3 0x4000083 fsleep rustc
89480 162126 56519 0 3 0x4000003 vmmaplk rustc
89480 19713 56519 0 3 0x4000083 fsleep rustc
89480 225086 56519 0 3 0x4000083 fsleep rustc
89480 114437 56519 0 3 0x4000083 fsleep rustc
89480 516669 56519 0 7 0x4000003 rustc
89480 183743 56519 0 3 0x4000003 vmmaplk rustc
89480 280602 56519 0 3 0x4000003 vmmaplk rustc
56519 279409 51455 0 3 0x83 wait rustc
15120 23895 70220 0 3 0x83 fsleep rustc
15120 310786 70220 0 3 0x4000083 piperd rustc
15120 501181 70220 0 3 0x4000083 fsleep rustc
15120 420670 70220 0 3 0x4000083 fsleep rustc
15120 123668 70220 0 3 0x4000083 piperd rustc
15120 209268 70220 0 3 0x4000083 fsleep rustc
15120 20254 70220 0 7 0x4000003 rustc
15120 251409 70220 0 3 0x4000003 vmmaplk rustc
70220 60972 51455 0 3 0x83 wait rustc
51455 361282 30803 0 3 0x83 fsleep cargo
51455 398749 30803 0 3 0x4000083 piperd cargo
51455 511984 30803 0 3 0x4000083 kqread cargo
51455 501854 30803 0 3 0x4000083 kqread cargo
51455 258538 30803 0 3 0x4000083 kqread cargo
51455 83542 30803 0 3 0x4000083 kqread cargo
51455 11274 30803 0 3 0x4000083 kqread cargo
51455 38509 30803 0 3 0x4000083 kqread cargo
51455 101605 30803 0 3 0x4000083 kqread cargo
51455 99005 30803 0 3 0x4000083 kqread cargo
30803 321185 91099 0 3 0x83 piperd bootstrap
91099 120989 79003 0 3 0x83 wait python3.13
79003 37051 82238 0 3 0x10008b sigsusp make
82238 76371 32780 0 3 0x2010008b sigsusp make
32780 431734 50813 0 3 0x2010008b sigsusp sh
50813 61592 49098 0 3 0x2010008b sigsusp make
49098 462502 94881 0 3 0x2010008b sigsusp sh
95270 431384 1 0 3 0x100083 ttyin getty
6959 325305 1 0 3 0x100083 ttyin getty
98660 135510 1 0 3 0x100083 ttyin getty
35198 21654 1 0 3 0x100083 ttyin getty
23795 223443 1 0 3 0x100083 ttyin getty
94881 290596 1 0 3 0x10008b sigsusp ksh
88426 215172 1 0 3 0x100098 kqread cron
90201 7888 1 99 3 0x1100090 kqread sndiod
33122 514734 1 110 3 0x100090 kqread sndiod
30125 74478 82589 95 3 0x1100092 kqread smtpd
14010 120242 82589 103 3 0x1100092 kqread smtpd
80759 181180 82589 95 3 0x1100092 kqread smtpd
13142 428326 82589 95 3 0x100092 kqread smtpd
59795 208483 82589 95 3 0x1100092 kqread smtpd
73766 153899 82589 95 3 0x1100092 kqread smtpd
82589 253788 1 0 3 0x100080 kqread smtpd
95443 369358 1 0 3 0x88 kqread sshd
14365 281457 1 0 3 0x100080 kqread ntpd
46821 516801 43313 83 3 0x100092 kqread ntpd
43313 330539 1 83 3 0x1100092 kqread ntpd
12539 190425 70332 74 3 0x1100092 bpf pflogd
70332 87312 1 0 3 0x80 sbwait pflogd
60560 244473 71838 73 3 0x1100090 kqread syslogd
71838 504035 1 0 3 0x100082 sbwait syslogd
88481 215984 1 0 3 0x100080 kqread resolvd
75083 316917 86475 77 3 0x100092 kqread dhcpleased
44625 12730 86475 77 3 0x100092 kqread dhcpleased
86475 180428 1 0 3 0x80 kqread dhcpleased
62406 437453 61737 115 3 0x100092 kqread slaacd
71934 261570 61737 115 3 0x100092 kqread slaacd
61737 265316 1 0 3 0x100080 kqread slaacd
39149 134898 0 0 3 0x14200 bored smr
37857 124960 0 0 3 0x14200 pgzero zerothread
56401 335601 0 0 3 0x14200 aiodoned aiodoned
67082 114287 0 0 3 0x14200 syncer update
85990 395862 0 0 3 0x14200 cleaner cleaner
57401 15651 0 0 3 0x14200 reaper reaper
66476 432284 0 0 3 0x14200 pgdaemon pagedaemon
19181 224371 0 0 3 0x14200 bored wsdisplay0
39227 332364 0 0 3 0x14200 usbtsk usbtask
27176 245944 0 0 3 0x14200 usbatsk usbatsk
45477 497400 0 0 3 0x40014200 acpi0 acpi0
54260 8456 0 0 7 0x40014200 idle31
51214 380596 0 0 7 0x40014200 idle30
10542 132558 0 0 7 0x40014200 idle29
64191 128390 0 0 7 0x40014200 idle28
45416 498903 0 0 7 0x40014200 idle27
71458 252444 0 0 7 0x40014200 idle26
26422 294473 0 0 7 0x40014200 idle25
15180 359787 0 0 7 0x40014200 idle24
79379 138864 0 0 7 0x40014200 idle23
67890 129352 0 0 7 0x40014200 idle22
23299 404214 0 0 7 0x40014200 idle21
37681 121781 0 0 7 0x40014200 idle20
11106 263205 0 0 7 0x40014200 idle19
76553 6351 0 0 7 0x40014200 idle18
78638 209898 0 0 7 0x40014200 idle17
82036 257171 0 0 7 0x40014200 idle16
52407 237175 0 0 3 0x40014200 idle15
4564 438375 0 0 3 0x40014200 idle14
21037 112870 0 0 3 0x40014200 idle13
9185 414996 0 0 7 0x40014200 idle12
78701 266313 0 0 7 0x40014200 idle11
92023 183380 0 0 3 0x40014200 idle10
35504 461859 0 0 7 0x40014200 idle9
14565 496134 0 0 3 0x40014200 idle8
51076 428392 0 0 7 0x40014200 idle7
21252 33158 0 0 3 0x40014200 idle6
26178 149706 0 0 3 0x40014200 idle5
88040 420462 0 0 3 0x40014200 idle4
22669 364050 0 0 7 0x40014200 idle3
67097 16788 0 0 3 0x40014200 idle2
37702 47142 0 0 7 0x40014200 idle1
33160 287027 0 0 3 0x14200 bored sensors
29090 110254 0 0 3 0x14200 bored softnet7
65734 211196 0 0 3 0x14200 bored softnet6
85046 151498 0 0 3 0x14200 bored softnet5
15647 20515 0 0 3 0x14200 bored softnet4
40272 11238 0 0 3 0x14200 bored softnet3
96561 429307 0 0 3 0x14200 bored softnet2
68853 44704 0 0 3 0x14200 bored softnet1
35624 349282 0 0 7 0x14200 softnet0
25850 25522 0 0 3 0x14200 bored systqmp
41688 149502 0 0 3 0x14200 bored systq
23008 245163 0 0 3 0x14200 tmoslp softclockmp
88961 404565 0 0 3 0x40014200 tmoslp softclock
19378 218615 0 0 3 0x40014200 idle0
1 471535 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{0}> mach ddb 0t2
Stopped at x86_ipi_db+0x16: leave
ddb{2}> tr
x86_ipi_db(ffff8000552c3ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
_kernel_lock() at _kernel_lock+0xb2
rt_clone(ffff800055bdad58,ffff800055bdade8,0) at rt_clone+0x64
rtalloc(ffff800055bdade8,1,0) at rtalloc+0x69
in_arpinput(ffff80000192e048,fffffd805907fe00) at in_arpinput+0x171
arpintr() at arpintr+0xb7
if_netisr(0) at if_netisr+0xd5
taskq_thread(ffff800000037000) at taskq_thread+0x129
end trace frame: 0x0, count: -10
ddb{2}> mach ddb 0t4
Stopped at x86_ipi_db+0x16: leave
ddb{4}> tr
x86_ipi_db(ffff8000552d5ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_kremove(ffff80004fa6f000,10000) at pmap_kremove+0xa2
buf_unmap(fffffd93ff708af8) at buf_unmap+0xbf
buf_map(fffffd942cf48c40) at buf_map+0x147
buf_get(fffffd93eac3f9a0,29,4000) at buf_get+0x360
getblk(fffffd93eac3f9a0,29,4000,0,ffffffffffffffff) at getblk+0x7b
ffs2_balloc(fffffd93dc659700,a4000,4000,fffffda07f7d8af8,0,ffff800056469a18)
at ffs2_balloc+0xe93
ffs_write(ffff800056469a98) at ffs_write+0x21d
VOP_WRITE(fffffd93eac3f9a0,ffff800056469bf8,1,fffffda07f7d8af8) at
VOP_WRITE+0x45
vn_write(fffffd941f135630,ffff800056469bf8,0) at vn_write+0xd8
dofilewritev(ffff800055fd4808,6,ffff800056469bf8,0,ffff800056469c90)
at dofilewritev+0x171
sys_write(ffff800055fd4808,ffff800056469d10,ffff800056469c90) at sys_write+0x55
syscall(ffff800056469d10) at syscall+0x5f9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x75b42c60f490, count: -16
ddb{4}> mach ddb 0t5
Stopped at x86_ipi_db+0x16: leave
ddb{5}> tr
x86_ipi_db(ffff8000552deff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_do_remove(fffffd95c385e8a8,d73f04e4000,d73f04e6000,0) at
pmap_do_remove+0x3d2
uvm_unmap_kill_entry_withlock(fffffd9609cb6308,fffffd93972dd4a0,1) at
uvm_unmap_kill_entry_withlock+0x133
uvm_unmap_remove(fffffd9609cb6308,d73f04e4000,d73f04e6000,ffff800056353d10,0,1,48c2f91c03d12620)
at uvm_unmap_remove+0x32f
sys_munmap(ffff800056004298,ffff800056353e10,ffff800056353d90) at
sys_munmap+0x10b
syscall(ffff800056353e10) at syscall+0x5f9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xd7388b0e430, count: -9
ddb{5}> mach ddb 0t6
Stopped at x86_ipi_db+0x16: leave
ddb{6}> tr
x86_ipi_db(ffff8000552e7ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffda07ffdf288,11b0d8ee000,13b816a000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff80005637ba78,ffff80005637bab0,ffff80005637b9f0) at
uvm_fault_lower+0x255
uvm_fault(fffffd95daee95d0,11b0d8ee000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff80005637bbf0,11b0d8ee000) at upageflttrap+0x6c
usertrap(ffff80005637bbf0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x11a4d69d340, count: -9
ddb{6}> mach ddb 0t8
Stopped at x86_ipi_db+0x16: leave
ddb{8}> tr
x86_ipi_db(ffff8000552f9ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_do_remove(fffffd95c385e2c0,1d7441af000,1d7441b0000,0) at
pmap_do_remove+0x592
uvm_unmap_kill_entry_withlock(fffffd9609cb6e88,fffffd95be7e5660,1) at
uvm_unmap_kill_entry_withlock+0x133
uvm_unmap_remove(fffffd9609cb6e88,1d7441af000,1d7441b0000,ffff800055ec9b50,0,1,48c2f91c03d12620)
at uvm_unmap_remove+0x32f
sys_munmap(ffff800055fd42d8,ffff800055ec9c50,ffff800055ec9bd0) at
sys_munmap+0x10b
syscall(ffff800055ec9c50) at syscall+0x5f9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x1d8086fcac0, count: -9
ddb{8}> mach ddb 0t10
Stopped at x86_ipi_db+0x16: leave
ddb{10}> tr
x86_ipi_db(ffff80005530bff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd95c385eb30,8dc807fc000,13fa7c7000,3,21) at pmap_enter+0x662
uvm_fault_lower(ffff80005639d008,ffff80005639d040,ffff80005639cf80) at
uvm_fault_lower+0x255
uvm_fault(fffffd95daee92f0,8dc807fc000,0,1) at uvm_fault+0x1c5
upageflttrap(ffff80005639d180,8dc807fc07c) at upageflttrap+0x6c
usertrap(ffff80005639d180) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x8dc1fe3c1d0, count: -9
ddb{10}> mach ddb 0t13
Stopped at x86_ipi_db+0x16: leave
ddb{13}> tr
x86_ipi_db(ffff800055326ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd95c385e1e8,f0f4694f000,13671f1000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff8000564b9648,ffff8000564b9680,ffff8000564b95c0) at
uvm_fault_lower+0x255
uvm_fault(fffffd9ca04c6a18,f0f4694f000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff8000564b97c0,f0f4694f000) at upageflttrap+0x6c
usertrap(ffff8000564b97c0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0xf0f0c8ace80, count: -9
ddb{13}> mach ddb 0t14
Stopped at x86_ipi_db+0x16: leave
ddb{14}> tr
x86_ipi_db(ffff80005532fff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd95c385e398,9060188d000,1a479f6000,1,20) at pmap_enter+0x662
uvm_fault_lower_lookup(ffff8000564bf0c8,ffff8000564bf100,ffff8000564bf040)
at uvm_fault_lower_lookup+0x126
uvm_fault_lower(ffff8000564bf0c8,ffff8000564bf100,ffff8000564bf040) at
uvm_fault_lower+0x5c
uvm_fault(fffffd95daee9b90,9060188e000,0,1) at uvm_fault+0x1c5
upageflttrap(ffff8000564bf240,9060188ebda) at upageflttrap+0x6c
usertrap(ffff8000564bf240) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x905b460a970, count: -10
ddb{14}> mach ddb 0t15
Stopped at x86_ipi_db+0x16: leave
ddb{15}> tr
x86_ipi_db(ffff800055338ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_do_remove(fffffd95c385e7d0,df10f70d000,df10f70e000,0) at
pmap_do_remove+0x592
uvm_unmap_kill_entry_withlock(fffffd9609cb65e8,fffffd938ea657f8,1) at
uvm_unmap_kill_entry_withlock+0x133
uvm_unmap_remove(fffffd9609cb65e8,df10f70d000,df10f70e000,ffff8000563651c0,0,1,48c2f91c03d12620)
at uvm_unmap_remove+0x32f
sys_munmap(ffff800056005c88,ffff8000563652c0,ffff800056365240) at
sys_munmap+0x10b
syscall(ffff8000563652c0) at syscall+0x5f9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0xdf1b545df40, count: -9
ddb{15}>
[...]
